Merge remote-tracking branch 'upstream/master'

.gitignore

@@ -2,3 +2,6 @@ old.go
 old_test.go
 file.txt
 user_agents.txt
+ua2.go
+settings.json
+benchmarks.txt

is.go

@@ -79,3 +79,9 @@ func (ua UserAgent) IsTwitterbot() bool {
 func (ua UserAgent) IsFacebookbot() bool {
 	return ua.Name == FacebookExternalHit
 }
+
+// IsUnknown returns true if the package can't determine the user agent reliably.
+// Fields like Name, OS, etc. might still have values.
+func (ua UserAgent) IsUnknown() bool {
+	return !ua.Mobile && !ua.Tablet && !ua.Desktop && !ua.Bot
+}

ua.go

@@ -31,7 +31,9 @@ const (
 	MacOS        = "macOS"
 	IOS          = "iOS"
 	Linux        = "Linux"
+	FreeBSD      = "FreeBSD"
 	ChromeOS     = "ChromeOS"
+	BlackBerry   = "BlackBerry"
 
 	Opera            = "Opera"
 	OperaMini        = "Opera Mini"
@@ -73,6 +75,8 @@ func Parse(userAgent string) UserAgent {
 		}
 	}
 
+	//fmt.Printf("%+v\n", tokens)
+
 	// OS lookup
 	switch {
 	case tokens.exists("Android"):
@@ -114,10 +118,20 @@ func Parse(userAgent string) UserAgent {
 		ua.OSVersion = tokens.get(Linux)
 		ua.Desktop = true
 
+	case tokens.exists("FreeBSD"):
+		ua.OS = FreeBSD
+		ua.OSVersion = tokens.get(FreeBSD)
+		ua.Desktop = true
+
 	case tokens.exists("CrOS"):
 		ua.OS = ChromeOS
 		ua.OSVersion = tokens.get("CrOS")
 		ua.Desktop = true
+
+	case tokens.exists("BlackBerry"):
+		ua.OS = BlackBerry
+		ua.OSVersion = tokens.get("BlackBerry")
+		ua.Mobile = true
 	}
 
 	switch {
@@ -127,6 +141,12 @@ func Parse(userAgent string) UserAgent {
 		ua.Bot = true
 		ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari")
 
+	case tokens.existsAny("GoogleProber", "GoogleProducer"):
+		if name := tokens.findBestMatch(false); name != "" {
+			ua.Name = name
+		}
+		ua.Bot = true
+
 	case tokens.exists("Applebot"):
 		ua.Name = Applebot
 		ua.Version = tokens.get(Applebot)
@@ -243,6 +263,7 @@ func Parse(userAgent string) UserAgent {
 	case tokens.exists("FBAN"):
 		ua.Name = FacebookApp
 		ua.Version = tokens.get("FBAN")
+
 	case tokens.exists("FB_IAB"):
 		ua.Name = FacebookApp
 		ua.Version = tokens.get("FBAV")
@@ -260,6 +281,15 @@ func Parse(userAgent string) UserAgent {
 		ua.Version = tokens.get("HuaweiBrowser")
 		ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari")
 
+	case tokens.exists("BlackBerry"):
+		ua.Name = "BlackBerry"
+		ua.Version = tokens.get("Version")
+
+	case tokens.exists("NetFront"):
+		ua.Name = "NetFront"
+		ua.Version = tokens.get("NetFront")
+		ua.Mobile = true
+
 	// if chrome and Safari defined, find any other token sent descr
 	case tokens.exists(Chrome) && tokens.exists(Safari):
 		name := tokens.findBestMatch(true)
@@ -303,7 +333,10 @@ func Parse(userAgent string) UserAgent {
 				ua.Name = ua.String
 			}
 			ua.Bot = strings.Contains(strings.ToLower(ua.Name), "bot")
-			ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari")
+			// If mobile flag has already been set, don't override it.
+			if !ua.Mobile {
+				ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari")
+			}
 		}
 	}
 
@@ -379,6 +412,9 @@ func parse(userAgent string) properties {
 		case (parOpen || braOpen) && c == 59: // ;
 			addToken()
 
+		case c == 59: // ;
+			addToken()
+
 		case c == 40: // (
 			addToken()
 			parOpen = true
@@ -390,6 +426,16 @@ func parse(userAgent string) properties {
 			addToken()
 			braOpen = false
 
+		case c == 58: // :
+			if bytes.HasSuffix(buff.Bytes(), []byte("http")) || bytes.HasSuffix(buff.Bytes(), []byte("https")) {
+				// If we are part of a URL just write the character.
+				buff.WriteByte(c)
+			} else if i != len(bua)-1 && bua[i+1] != ' ' {
+				// If the following character is not a space, change to a space.
+				buff.WriteByte(' ')
+			}
+			// Otherwise don't write as its probably a badly formatted key value separator.
+
 		case slash && c == 32:
 			addToken()
 
@@ -401,7 +447,11 @@ func parse(userAgent string) properties {
 				buff.WriteByte(c)
 				isURL = true
 			} else {
-				slash = true
+				if ignore(buff.String()) {
+					buff.Reset()
+				} else {
+					slash = true
+				}
 			}
 
 		default:
@@ -424,7 +474,7 @@ func checkVer(s string) (name, v string) {
 	switch s[:i] {
 	case "Linux", "Windows NT", "Windows Phone OS", "MSIE", "Android":
 		return s[:i], s[i+1:]
-	case "CrOS x86_64", "CrOS aarch64":
+	case "CrOS x86_64", "CrOS aarch64", "CrOS armv7l":
 		j := strings.LastIndex(s[:i], " ")
 		return s[:j], s[j+1 : i]
 	default:
@@ -443,7 +493,7 @@ func checkVer(s string) (name, v string) {
 // ignore retursn true if token should be ignored
 func ignore(s string) bool {
 	switch s {
-	case "KHTML, like Gecko", "U", "compatible", "Mozilla", "WOW64", "en", "en-us", "en-gb", "ru-ru":
+	case "KHTML, like Gecko", "U", "compatible", "Mozilla", "WOW64", "en", "en-us", "en-gb", "ru-ru", "Browser":
 		return true
 	default:
 		return false

ua_test.go

@@ -70,12 +70,16 @@ var testTable = [][]string{
 	// Windows phone
 	{"Mozilla/4.0 (compatible; MSIE 7.0; Windows Phone OS 7.0; Trident/3.1; IEMobile/7.0; NOKIA; Lumia 630)", ua.InternetExplorer, "7.0", "mobile", ua.WindowsPhone},
 
+	// FreeBSD
+	{"Mozilla/5.0 (compatible; Konqueror/4.5; FreeBSD) KHTML/4.5.4 (like Gecko)", "Konqueror", "4.5", "desktop", "FreeBSD"},
+
 	// Bots
 	{"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", ua.Googlebot, "2.1", "mobile", "Android", "Nexus 5X"},
 	{"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", ua.Googlebot, "2.1", "bot", ""},
 	{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "Applebot", "0.1", "bot", ""},
 	{"Twitterbot/1.0", ua.Twitterbot, "1.0", ua.Applebot, ""},
 	{"facebookexternalhit/1.1", ua.FacebookExternalHit, "1.1", "bot", ""},
+	{"facebookcatalog/1.0", "facebookcatalog", "1.0", "bot", ""},
 	{"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html", "SemrushBot", "7~bl", "bot", ""},
 	{"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.268", "YandexBot", "3.0", "bot", ""},
 	{"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)", "Discordbot", "2.0", "bot", ""},
@@ -84,6 +88,8 @@ var testTable = [][]string{
 	{"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.1.0.0 Mobile Safari/537.36 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "Bingbot", "2.0", "bot", ua.Android}, // new bingbot mobile
 	{"Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)  tands-prod-eng.hlfs-prod---sieve.hlfs-desktop/1681336006-0", "Yahoo Ad monitoring", "", "bot", ""},
 	{"Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1681346790-0", "Yahoo Ad monitoring", "", "bot", ""},
+	{"GoogleProber", "GoogleProber", "", "bot", ""},
+	{"GoogleProducer; (+http://goo.gl/7y4SX)", "GoogleProducer", "", "bot", ""},
 
 	// Google ads bots
 	{"Mozilla/5.0 (Linux; Android 4.0.0; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; Mediapartners-Google) Chrome/104.0.0.0 Mobile Safari/537.36", ua.GoogleAdsBot, "", "bot", ua.Android},
@@ -117,10 +123,18 @@ var testTable = [][]string{
 	{"Go-http-client/1.1", "Go-http-client", "1.1", "", ""},
 	{"Wget/1.12 (linux-gnu)", "Wget", "1.12", "", ""},
 	{"Wget/1.17.1 (darwin15.2.0)", "Wget", "1.17.1", "", ""},
+	{"Seafile/9.0.2 (Linux)", "Seafile", "9.0.2", "", "Linux"},
 
 	// unstandard stuff
 	{"BUbiNG (+http://law.di.unimi.it/BUbiNG.html)", "BUbiNG", "", "", ""},
 	//{"Aweme 8.2.0 rv:82017 (iPhone6,2; iOS 12.4; zh_CN) Cronet", "Aweme", "", "", ""},
+	{"surveyon/3.1.0 Mobile (Android: 6.0.1; MODEL:SM-G532G; PRODUCT:grandppltedx; MANUFACTURER:samsung;)", "surveyon", "3.1.0", "mobile", ua.Android},
+	{"surveyon/3.1.0 Mobile (Android: 9; MODEL:CPH1923; PRODUCT:CPH1923; MANUFACTURER:OPPO;)", "surveyon", "3.1.0", "mobile", ua.Android},
+	{"surveyon/3.1.0 Mobile (Android: 13; MODEL:SM-M127F; PRODUCT:m12nnxx; MANUFACTURER:samsung;)", "surveyon", "3.1.0", "mobile", ua.Android},
+	{"surveyon/2.9.5 (iPhone; CPU iPhone OS 12_5_7 like Mac OS X)", "surveyon", "2.9.5", "mobile", ua.IOS},
+	{"Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.187 Mobile Safari/534.11+", "BlackBerry", "7.0.0.187", "mobile", "BlackBerry"},
+	{"Mozilla/5.0 (X11; CrOS armv7l 13099.110.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.136 Safari/537.36", ua.Chrome, "84.0.4147.136", "desktop", ua.ChromeOS},
+	{"SonyEricssonK310iv/R4DA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.1.13.0", "NetFront", "3.3", "mobile", ""},
 
 	// Device names
 	{"Mozilla/5.0 (Linux; Android 10; 8092) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36", "Chrome", "112.0.0.0", "mobile", ua.Android, "8092"},
@@ -135,6 +149,12 @@ var testTable = [][]string{
 	{"Mozilla/5.0 (Linux; Android 9; m5621 Build/PPR2.180905.006.A1; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.158 Safari/537.36", ua.Chrome, "66.0.3359.158", "mobile", ua.Android, "m5621"},
 	{"Mozilla/5.0 (Linux; Android 10; meanIT_X20 Build/QP1A.190711.020) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.5481.153 Safari/537.36", ua.Chrome, "110.0.5481.153", "mobile", ua.Android, "meanIT_X20"},
 	{"Mozilla/5.0 (Linux; Android 10;)", "Mozilla/5.0 (Linux; Android 10;)", "", "mobile", ua.Android},
+	// {`() { ignored; }; echo Content-Type: text/plain ; echo ; echo "bash_cve_2014_6271_rce Output : $((70+91))"`, "", "mobile", ua.Android},
+	//{`${jndi:ldap://log4shell-generic-8ZnJfq2XFL3GWyaLyOpT${lower:ten}.w.nessus.org/nessus}`, "", "mobile", ua.Android},
+	//
+
+	//
+	// ${jndi:ldap://log4shell-generic-8ZnJfq2XFL3GWyaLyOpT${lower:ten}.w.nessus.org/nessus}
 
 	// TODO:
 	// Mozilla/5.0 (Linux; U; Android 13; sr-rs; V2206 Build/TP1A.220624.014) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/61.0.3163.128 Mobile Safari/537.36 XiaoMi/Mint Browser/3.9.3
@@ -190,6 +210,11 @@ func TestParse(t *testing.T) {
 	}
 }
 
+func TestSingle(t *testing.T) {
+	agent := ua.Parse("SonyEricssonK310iv/R4DA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.1.13.0")
+	fmt.Printf("\n%+v\n", agent)
+}
+
 var testUA ua.UserAgent
 
 func BenchmarkUserAgent(b *testing.B) {
@@ -258,7 +283,6 @@ func ExampleParse() {
 		if ua.URL != "" {
 			fmt.Println(ua.URL)
 		}
-
 	}
 
 }