initial code for classifier complete
This commit is contained in:
+363
-8
@@ -17,10 +17,13 @@ import (
|
|||||||
"encoding/json/v2"
|
"encoding/json/v2"
|
||||||
"fmt"
|
"fmt"
|
||||||
"maps"
|
"maps"
|
||||||
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"regexp"
|
"regexp"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Constants reflecting confidence levels.
|
// Constants reflecting confidence levels.
|
||||||
@@ -71,19 +74,38 @@ type agentDef struct {
|
|||||||
patterns []*regexp.Regexp // compiled user agent expressions
|
patterns []*regexp.Regexp // compiled user agent expressions
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RawClientDef is the client definition data loaded from the http-clients.json file.
|
||||||
|
type RawClientDef struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
UserAgent string `json:"userAgent"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// clientdef is the definition for a compiled client.
|
||||||
|
type clientDef struct {
|
||||||
|
r *RawClientDef
|
||||||
|
pattern *regexp.Regexp
|
||||||
|
}
|
||||||
|
|
||||||
//go:embed agents.json
|
//go:embed agents.json
|
||||||
var initAgents []byte
|
var initAgents []byte
|
||||||
|
|
||||||
// compiledAgents is the compiled list of agents.
|
// compiledAgents is the compiled list of agents.
|
||||||
var compiledAgents []agentDef
|
var compiledAgents []agentDef
|
||||||
|
|
||||||
|
//go:embed http-clients.json
|
||||||
|
var initClients []byte
|
||||||
|
|
||||||
|
// compiledClients is the compiled list of clients.
|
||||||
|
var compiledClients []clientDef
|
||||||
|
|
||||||
// AmInitAgentClass is the initialization function.
|
// AmInitAgentClass is the initialization function.
|
||||||
func AmInitAgentClass() {
|
func AmInitAgentClass(ctxname any) func() {
|
||||||
// load the raw agent list
|
// load the raw agent list
|
||||||
var rawAgents []RawAgentDef
|
var rawAgents []RawAgentDef
|
||||||
err := json.Unmarshal(initAgents, &rawAgents)
|
err := json.Unmarshal(initAgents, &rawAgents)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(fmt.Sprintf("agent load failure: %v", err))
|
panic(fmt.Sprintf("agent definition load failure: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
// build the compiled agent list
|
// build the compiled agent list
|
||||||
@@ -100,6 +122,20 @@ func AmInitAgentClass() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// load the raw clients list
|
||||||
|
var rawClients []RawClientDef
|
||||||
|
err = json.Unmarshal(initClients, &rawClients)
|
||||||
|
if err != nil {
|
||||||
|
panic(fmt.Sprintf("client definition load failure: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
// build the compiled client list
|
||||||
|
compiledClients = make([]clientDef, len(rawClients))
|
||||||
|
for i, v := range rawClients {
|
||||||
|
compiledClients[i].r = &(rawClients[i])
|
||||||
|
compiledClients[i].pattern = regexp.MustCompile("(?i)" + v.UserAgent)
|
||||||
|
}
|
||||||
|
|
||||||
// get the list of IP range files
|
// get the list of IP range files
|
||||||
urls := make(map[string]struct{})
|
urls := make(map[string]struct{})
|
||||||
for _, ra := range rawAgents {
|
for _, ra := range rawAgents {
|
||||||
@@ -109,7 +145,36 @@ func AmInitAgentClass() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
loadIPRanges(slices.Collect(maps.Keys(urls)))
|
return loadIPRanges(slices.Collect(maps.Keys(urls)), ctxname)
|
||||||
|
}
|
||||||
|
|
||||||
|
type ClassifyOptions struct {
|
||||||
|
MaxAge int // maximum age of IP range files
|
||||||
|
Now *time.Time // "now" timestamp (for testing)
|
||||||
|
HeadersComplete bool // look at complete headers?
|
||||||
|
}
|
||||||
|
|
||||||
|
func AmClassifyOptions() ClassifyOptions {
|
||||||
|
return ClassifyOptions{
|
||||||
|
MaxAge: 180,
|
||||||
|
Now: nil,
|
||||||
|
HeadersComplete: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *ClassifyOptions) WithMaxAge(maxAge int) *ClassifyOptions {
|
||||||
|
o.MaxAge = maxAge
|
||||||
|
return o
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *ClassifyOptions) WithNow(now *time.Time) *ClassifyOptions {
|
||||||
|
o.Now = now
|
||||||
|
return o
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *ClassifyOptions) WithHeadersComplete(flag bool) *ClassifyOptions {
|
||||||
|
o.HeadersComplete = flag
|
||||||
|
return o
|
||||||
}
|
}
|
||||||
|
|
||||||
// AgentMatch is the return from findAgent.
|
// AgentMatch is the return from findAgent.
|
||||||
@@ -120,6 +185,29 @@ type AgentMatch struct {
|
|||||||
Weak bool // is this a weak match?
|
Weak bool // is this a weak match?
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var textBrowserExpr = regexp.MustCompile(`^(Lynx|ELinks|Links|w3m)\/`)
|
||||||
|
var browserExpr = regexp.MustCompile(`^Mozilla\/[4-6]\.0 \(|^Opera\/\d`)
|
||||||
|
var browserEngineExpr = regexp.MustCompile(`\(KHTML,? ?like Gecko|KHTML\/[\d.]+ \(like Gecko|Gecko\/\d|\) like Gecko|Trident\/\d|MSIE \d|Presto\/\d|Chromium\/\d`)
|
||||||
|
var botWordExpr = regexp.MustCompile(`(?:[a-z0-9-]|\b)[Bb]ot(?:\b|\/)`)
|
||||||
|
var emailExpr = regexp.MustCompile(`(?i)[\w.+-]+@[\w-]+(\.[\w-]+)*\.[a-z]{2,}`)
|
||||||
|
var automationExpr = regexp.MustCompile(`(?i)crawl|spider|scrap|headless|https?:\/\/|\+http|(?:^|[^a-z])(?:archive|observ|monitor|check(?:er)?\b|preview|verif|render|insight|console|transcod|scanner|synthetic)|lighthouse|selenium|cypress|webdriver|\bsplash\b|gomezagent|rigor|speedcurve|\bptst\b|dareboost|dejaclick|favicon|siteimprove|onetrust|datanyze|feeder\.co|rss\b|weblight|duplexweb|ips-agent`)
|
||||||
|
var compatibleExpr = regexp.MustCompile(`compatible;`)
|
||||||
|
var enginesExpr = regexp.MustCompile(`MSIE|Trident|Konqueror`)
|
||||||
|
|
||||||
|
// isBrowser retruns true if the user agent indicates a browser.
|
||||||
|
func isBrowser(ua string) bool {
|
||||||
|
if textBrowserExpr.MatchString(ua) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if !browserExpr.MatchString(ua) || !browserEngineExpr.MatchString(ua) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if botWordExpr.MatchString(ua) || emailExpr.MatchString(ua) || automationExpr.MatchString(ua) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return !compatibleExpr.MatchString(ua) || enginesExpr.MatchString(ua)
|
||||||
|
}
|
||||||
|
|
||||||
// normalizeHeaders joins header values and smashes case on header names.
|
// normalizeHeaders joins header values and smashes case on header names.
|
||||||
func normalizeHeaders(headers http.Header) map[string]string {
|
func normalizeHeaders(headers http.Header) map[string]string {
|
||||||
rc := make(map[string]string)
|
rc := make(map[string]string)
|
||||||
@@ -132,6 +220,77 @@ func normalizeHeaders(headers http.Header) map[string]string {
|
|||||||
return rc
|
return rc
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// rankedAcceptData is the type of data returned from rankAccept.
|
||||||
|
type rankedAcceptData struct {
|
||||||
|
mtype string // MIME type
|
||||||
|
q1000 int // quality value multiplied by 1000
|
||||||
|
ndx int // original index in Accept string
|
||||||
|
}
|
||||||
|
|
||||||
|
// rankAccept turns an Accept string into a list of ranked accept members.
|
||||||
|
func rankAccept(accept string) []rankedAcceptData {
|
||||||
|
components := strings.Split(accept, ",")
|
||||||
|
ranks := make([]rankedAcceptData, len(components))
|
||||||
|
for i, com := range components {
|
||||||
|
subparts := strings.Split(com, ";")
|
||||||
|
ranks[i].mtype = subparts[0]
|
||||||
|
ranks[i].q1000 = 1000
|
||||||
|
ranks[i].ndx = i
|
||||||
|
if len(subparts) > 1 {
|
||||||
|
for _, part := range subparts[1:] {
|
||||||
|
part = strings.TrimSpace(part)
|
||||||
|
if strings.HasPrefix(part, "q=") {
|
||||||
|
parsed, err := strconv.ParseFloat(part[2:], 64)
|
||||||
|
if err != nil {
|
||||||
|
ranks[i].q1000 = int(math.Floor(1000.0 * parsed))
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ranks2 := make([]rankedAcceptData, 0, len(ranks))
|
||||||
|
for i := range ranks {
|
||||||
|
if ranks[i].q1000 > 0 {
|
||||||
|
ranks2 = append(ranks2, ranks[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slices.SortFunc(ranks2, func(a, b rankedAcceptData) int {
|
||||||
|
t := b.q1000 - a.q1000
|
||||||
|
if t == 0 {
|
||||||
|
t = b.ndx - a.ndx
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
})
|
||||||
|
return ranks2
|
||||||
|
}
|
||||||
|
|
||||||
|
// ranksAboveHtml returns true if any of the given MIME types ranks above HTML in the given Accept: string.
|
||||||
|
func ranksAboveHtml(accept string, types []string) bool {
|
||||||
|
ranked := rankAccept(accept)
|
||||||
|
md := slices.IndexFunc(ranked, func(d rankedAcceptData) bool {
|
||||||
|
return slices.Contains(types, d.mtype)
|
||||||
|
})
|
||||||
|
if md < 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
html := slices.IndexFunc(ranked, func(d rankedAcceptData) bool {
|
||||||
|
return d.mtype == "text/html"
|
||||||
|
})
|
||||||
|
if html < 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return md < html
|
||||||
|
}
|
||||||
|
|
||||||
|
// markdownTypes are the MIME types associated with Markdown.
|
||||||
|
var markdownTypes = []string{"text/markdown", "text/x-markdown"}
|
||||||
|
|
||||||
|
// prefersMarkdown returns true if the given Accept: string prefers Markdown over HTML.
|
||||||
|
func prefersMarkdown(accept string) bool {
|
||||||
|
return ranksAboveHtml(accept, markdownTypes)
|
||||||
|
}
|
||||||
|
|
||||||
// normalizeAcceptExpr is how we close the gaps around delimiters in an Accept: expression.
|
// normalizeAcceptExpr is how we close the gaps around delimiters in an Accept: expression.
|
||||||
var normalizeAcceptExpr = regexp.MustCompile(`\s*([,;=])\s*`)
|
var normalizeAcceptExpr = regexp.MustCompile(`\s*([,;=])\s*`)
|
||||||
|
|
||||||
@@ -223,9 +382,10 @@ func findAgent(ua string, h map[string]string) *AgentMatch {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromAgent(match *AgentMatch, rawIP string) Classification {
|
// fromAgent turns an AgentMatch into a Classification.
|
||||||
|
func fromAgent(match *AgentMatch, rawIP string, opts ClassifyOptions) *Classification {
|
||||||
agent := match.Agent
|
agent := match.Agent
|
||||||
base := Classification{
|
base := new(Classification{
|
||||||
Category: agent.Category,
|
Category: agent.Category,
|
||||||
Kind: agent.Kind,
|
Kind: agent.Kind,
|
||||||
Name: agent.Name,
|
Name: agent.Name,
|
||||||
@@ -234,7 +394,7 @@ func fromAgent(match *AgentMatch, rawIP string) Classification {
|
|||||||
Verified: true,
|
Verified: true,
|
||||||
Confidence: ConfidenceMedium,
|
Confidence: ConfidenceMedium,
|
||||||
Reasons: make([]string, 1),
|
Reasons: make([]string, 1),
|
||||||
}
|
})
|
||||||
base.Reasons[0] = match.Reason
|
base.Reasons[0] = match.Reason
|
||||||
if agent.RunsOn == "user" {
|
if agent.RunsOn == "user" {
|
||||||
base.Reasons = append(base.Reasons, fmt.Sprintf("%s runs on the user's machine, so IP verification does not apply", agent.Name))
|
base.Reasons = append(base.Reasons, fmt.Sprintf("%s runs on the user's machine, so IP verification does not apply", agent.Name))
|
||||||
@@ -249,9 +409,140 @@ func fromAgent(match *AgentMatch, rawIP string) Classification {
|
|||||||
if agent.Verify.IpRanges != nil {
|
if agent.Verify.IpRanges != nil {
|
||||||
urls = append(urls, agent.Verify.IpRanges...)
|
urls = append(urls, agent.Verify.IpRanges...)
|
||||||
}
|
}
|
||||||
|
now := opts.Now
|
||||||
|
if now == nil {
|
||||||
|
tmp := time.Now()
|
||||||
|
now = &tmp
|
||||||
|
}
|
||||||
|
maxDuration := time.Hour * time.Duration(24*opts.MaxAge)
|
||||||
|
impersonation := false
|
||||||
|
validRangeFiles := false
|
||||||
|
for _, url := range urls {
|
||||||
|
cidrs, creation, err := getIPRangeData(url)
|
||||||
|
if err != nil {
|
||||||
|
base.Reasons = append(base.Reasons, fmt.Sprintf("unable to get range file %s: %v", url, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
validRangeFiles = true
|
||||||
|
if inRanges(*ip, cidrs) {
|
||||||
|
base.Verified = true
|
||||||
|
base.Confidence = ConfidenceHigh
|
||||||
|
base.Reasons = append(base.Reasons, "%s is in %s's published ranges", *ip, agent.Operator)
|
||||||
|
return base
|
||||||
|
}
|
||||||
|
if now.Sub(*creation) <= maxDuration {
|
||||||
|
impersonation = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !validRangeFiles {
|
||||||
|
return base
|
||||||
|
}
|
||||||
|
if impersonation {
|
||||||
|
base.Category = "bot"
|
||||||
|
base.Kind = "impersonator"
|
||||||
|
base.Verified = false
|
||||||
|
base.Confidence = ConfidenceHigh
|
||||||
|
base.Reasons = append(base.Reasons, fmt.Sprintf("claims to be %s but %s is outside %s's published ranges", agent.Name, *ip, agent.Operator))
|
||||||
|
return base
|
||||||
|
}
|
||||||
|
base.Reasons = append(base.Reasons, fmt.Sprintf("%s's range file is older than %d days, so a mismatch is not counted as spoofing", agent.Operator, opts.MaxAge))
|
||||||
|
return base
|
||||||
}
|
}
|
||||||
|
|
||||||
func AmClassify(headers http.Header, ipAddr string, userAgent string) Classification {
|
// fake returns a Classification identifying the request as a fake browser.
|
||||||
|
func fake(reason string, confidence int) *Classification {
|
||||||
|
rc := new(Classification{
|
||||||
|
Category: "bot",
|
||||||
|
Kind: "fake-browser",
|
||||||
|
Verified: false,
|
||||||
|
Confidence: confidence,
|
||||||
|
Reasons: make([]string, 1),
|
||||||
|
})
|
||||||
|
rc.Reasons[0] = reason
|
||||||
|
return rc
|
||||||
|
}
|
||||||
|
|
||||||
|
// majorVersion extracts a browser's major version from a regular expression.
|
||||||
|
func majorVersion(ua string, re *regexp.Regexp) int {
|
||||||
|
m := re.FindStringSubmatch(ua)
|
||||||
|
if m != nil {
|
||||||
|
v, err := strconv.Atoi(m[1])
|
||||||
|
if err == nil {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
var firefoxBrowserExpr = regexp.MustCompile(`Firefox\/`)
|
||||||
|
var firefoxVersionExpr = regexp.MustCompile(`Firefox\/(\d+)`)
|
||||||
|
var chromeBrowserExpr = regexp.MustCompile(`Chrome\/`)
|
||||||
|
var chromeVersionExpr = regexp.MustCompile(`Chrome\/(\d+)`)
|
||||||
|
var safariBrowserExpr = regexp.MustCompile(`Version\/[\d.]+.*Safari\/`)
|
||||||
|
var safariVersionExpr = regexp.MustCompile(`Version\/(\d+)`)
|
||||||
|
|
||||||
|
func sendsFetchMetadata(ua string) bool {
|
||||||
|
if firefoxBrowserExpr.MatchString(ua) {
|
||||||
|
return majorVersion(ua, firefoxVersionExpr) >= 90
|
||||||
|
}
|
||||||
|
if chromeBrowserExpr.MatchString(ua) {
|
||||||
|
return majorVersion(ua, chromeVersionExpr) >= 80
|
||||||
|
}
|
||||||
|
if safariBrowserExpr.MatchString(ua) {
|
||||||
|
return majorVersion(ua, safariVersionExpr) >= 17
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
var plainTextType = []string{"text/plain"}
|
||||||
|
|
||||||
|
func fromBrowser(ua string, h map[string]string, opts ClassifyOptions) *Classification {
|
||||||
|
accept, ok := h["accept"]
|
||||||
|
if ok {
|
||||||
|
if ranksAboveHtml(accept, plainTextType) {
|
||||||
|
return fake("Accept header ranks text/plain above HTML, which browsers never do", ConfidenceLow)
|
||||||
|
}
|
||||||
|
if strings.Contains(accept, "application/metalink") {
|
||||||
|
return fake("Accept header asks for metalink, which download tools send and browsers do not", ConfidenceMedium)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !opts.HeadersComplete {
|
||||||
|
rc := new(Classification{
|
||||||
|
Category: "human",
|
||||||
|
Verified: false,
|
||||||
|
Confidence: ConfidenceLow,
|
||||||
|
Reasons: make([]string, 1),
|
||||||
|
})
|
||||||
|
rc.Reasons[0] = "browser user agent; headers not checked"
|
||||||
|
return rc
|
||||||
|
}
|
||||||
|
_, ok = h["sec-ch-ua"]
|
||||||
|
if ok && !chromeBrowserExpr.MatchString(ua) {
|
||||||
|
return fake("sec-ch-ua sent with a non-Chromium user agent", ConfidenceHigh)
|
||||||
|
}
|
||||||
|
_, ok = h["sec-fetch-mode"]
|
||||||
|
if !ok && sendsFetchMetadata(ua) {
|
||||||
|
return fake("browser user agent without sec-fetch-* headers", ConfidenceMedium)
|
||||||
|
}
|
||||||
|
_, ok = h["accept-language"]
|
||||||
|
if !ok {
|
||||||
|
return fake("browser user agent without Accept-Language", ConfidenceLow)
|
||||||
|
}
|
||||||
|
rc := new(Classification{
|
||||||
|
Category: "human",
|
||||||
|
Verified: false,
|
||||||
|
Confidence: ConfidenceMedium,
|
||||||
|
Reasons: make([]string, 1),
|
||||||
|
})
|
||||||
|
rc.Reasons[0] = "browser user agent with consistent browser headers"
|
||||||
|
return rc
|
||||||
|
}
|
||||||
|
|
||||||
|
func AmClassify(headers http.Header, ipAddr string, userAgent string, opts ...ClassifyOptions) *Classification {
|
||||||
|
myopts := AmClassifyOptions()
|
||||||
|
if len(opts) > 0 {
|
||||||
|
myopts = opts[0]
|
||||||
|
}
|
||||||
h := normalizeHeaders(headers)
|
h := normalizeHeaders(headers)
|
||||||
ua := userAgent
|
ua := userAgent
|
||||||
if ua == "" {
|
if ua == "" {
|
||||||
@@ -260,8 +551,72 @@ func AmClassify(headers http.Header, ipAddr string, userAgent string) Classifica
|
|||||||
ua = t
|
ua = t
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ua == "" {
|
||||||
|
rc := new(Classification{
|
||||||
|
Verified: false,
|
||||||
|
Confidence: ConfidenceLow,
|
||||||
|
Reasons: make([]string, 1),
|
||||||
|
})
|
||||||
|
if len(h) == 0 {
|
||||||
|
rc.Category = "unknown"
|
||||||
|
rc.Reasons[0] = "no user agent or headers to classify"
|
||||||
|
} else {
|
||||||
|
rc.Category = "bot"
|
||||||
|
rc.Kind = "unidentified"
|
||||||
|
rc.Reasons[0] = "request has no user agent, which browsers always send"
|
||||||
|
}
|
||||||
|
return rc
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look for a matching agent.
|
||||||
match := findAgent(ua, h)
|
match := findAgent(ua, h)
|
||||||
if match != nil {
|
if match != nil {
|
||||||
return fromAgent(match, ipAddr)
|
return fromAgent(match, ipAddr, myopts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check the Accept: line; if the request prefers Markdown to HTML, we're some kind of agent.
|
||||||
|
acc, ok := h["accept"]
|
||||||
|
if ok {
|
||||||
|
if prefersMarkdown(acc) {
|
||||||
|
rc := new(Classification{
|
||||||
|
Category: "agent",
|
||||||
|
Kind: "unidentified",
|
||||||
|
Verified: false,
|
||||||
|
Confidence: ConfidenceMedium,
|
||||||
|
Reasons: make([]string, 1),
|
||||||
|
})
|
||||||
|
rc.Reasons[0] = "Accept header prefers markdown over HTML, which browsers never do"
|
||||||
|
return rc
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Look for an HTTP client.
|
||||||
|
ndx := slices.IndexFunc(compiledClients, func(d clientDef) bool {
|
||||||
|
return d.pattern.MatchString(ua)
|
||||||
|
})
|
||||||
|
if ndx >= 0 {
|
||||||
|
rc := new(Classification{
|
||||||
|
Category: "bot",
|
||||||
|
Kind: compiledClients[ndx].r.Kind,
|
||||||
|
Name: compiledClients[ndx].r.Name,
|
||||||
|
Verified: false,
|
||||||
|
Confidence: ConfidenceMedium,
|
||||||
|
Reasons: make([]string, 1),
|
||||||
|
})
|
||||||
|
rc.Reasons[0] = fmt.Sprintf("user agent is %s", compiledClients[ndx].r.Name)
|
||||||
|
return rc
|
||||||
|
}
|
||||||
|
if isBrowser(ua) {
|
||||||
|
return fromBrowser(ua, h, myopts)
|
||||||
|
}
|
||||||
|
rc := new(Classification{
|
||||||
|
Category: "bot",
|
||||||
|
Kind: "unidentified",
|
||||||
|
Verified: false,
|
||||||
|
Confidence: ConfidenceLow,
|
||||||
|
Reasons: make([]string, 1),
|
||||||
|
})
|
||||||
|
rc.Reasons[0] = "user agent is not a browser and matches no known agent, crawler or client"
|
||||||
|
return rc
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
[
|
||||||
|
{ "name": "axios", "kind": "http-client", "userAgent": "^axios/" },
|
||||||
|
{ "name": "curl", "kind": "http-client", "userAgent": "^curl/" },
|
||||||
|
{ "name": "Wget", "kind": "http-client", "userAgent": "^Wget/" },
|
||||||
|
{ "name": "Node.js", "kind": "http-client", "userAgent": "^(node|undici|node-fetch)(/|$)" },
|
||||||
|
{ "name": "Bun", "kind": "http-client", "userAgent": "^Bun/" },
|
||||||
|
{ "name": "Deno", "kind": "http-client", "userAgent": "^Deno/" },
|
||||||
|
{ "name": "Python requests", "kind": "http-client", "userAgent": "^python-requests/" },
|
||||||
|
{ "name": "Python httpx", "kind": "http-client", "userAgent": "^python-httpx/" },
|
||||||
|
{ "name": "Python urllib", "kind": "http-client", "userAgent": "^Python-urllib/" },
|
||||||
|
{ "name": "Python aiohttp", "kind": "http-client", "userAgent": "^Python/[\\d.]+ aiohttp/" },
|
||||||
|
{ "name": "Go net/http", "kind": "http-client", "userAgent": "^Go-http-client/" },
|
||||||
|
{ "name": "OkHttp", "kind": "http-client", "userAgent": "^okhttp/" },
|
||||||
|
{ "name": "Java", "kind": "http-client", "userAgent": "^(Java/|Apache-HttpClient/)" },
|
||||||
|
{ "name": "Guzzle", "kind": "http-client", "userAgent": "^GuzzleHttp/" },
|
||||||
|
{ "name": "Rust reqwest", "kind": "http-client", "userAgent": "^reqwest/" },
|
||||||
|
{ "name": "lua-resty-http", "kind": "http-client", "userAgent": "^lua-resty-http/" },
|
||||||
|
{ "name": "Colly", "kind": "http-client", "userAgent": "^colly\\b" },
|
||||||
|
{ "name": "Scrapy", "kind": "http-client", "userAgent": "\\bScrapy/" },
|
||||||
|
{ "name": "n8n", "kind": "http-client", "userAgent": "^n8n$" },
|
||||||
|
{ "name": "Postman", "kind": "http-client", "userAgent": "^PostmanRuntime/" },
|
||||||
|
{ "name": "Headless Chrome", "kind": "headless", "userAgent": "\\bHeadlessChrome/" },
|
||||||
|
{ "name": "PhantomJS", "kind": "headless", "userAgent": "\\bPhantomJS/" },
|
||||||
|
{ "name": "Playwright", "kind": "headless", "userAgent": "\\bPlaywright/" },
|
||||||
|
{ "name": "PowerShell", "kind": "http-client", "userAgent": "PowerShell/" },
|
||||||
|
{ "name": "Ruby", "kind": "http-client", "userAgent": "^Ruby$" },
|
||||||
|
{ "name": "got", "kind": "http-client", "userAgent": "^got\\b" },
|
||||||
|
{ "name": "libwww-perl", "kind": "http-client", "userAgent": "\\blibwww-perl/" },
|
||||||
|
{ "name": "aria2", "kind": "http-client", "userAgent": "^aria2/" },
|
||||||
|
{ "name": "HTTrack", "kind": "http-client", "userAgent": "\\bHTTrack\\b" },
|
||||||
|
{ "name": "Crawl4AI", "kind": "http-client", "userAgent": "^crawl4ai" },
|
||||||
|
{ "name": "Apple CFNetwork (native app)", "kind": "http-client", "userAgent": "\\bCFNetwork/" },
|
||||||
|
{ "name": "Android Dalvik (native app)", "kind": "http-client", "userAgent": "\\bDalvik/" },
|
||||||
|
{ "name": "Amazon CloudFront", "kind": "http-client", "userAgent": "^Amazon CloudFront$" },
|
||||||
|
{ "name": "Vercel Edge Functions", "kind": "http-client", "userAgent": "^Vercel Edge Functions$" },
|
||||||
|
{ "name": "Google Cloud Scheduler and Tasks", "kind": "http-client", "userAgent": "^Google-Cloud-(Scheduler|Tasks)$" }
|
||||||
|
]
|
||||||
+18
-12
@@ -15,24 +15,19 @@ package agentclass
|
|||||||
import (
|
import (
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ParsedIP is the structure that holds a parsed IP address.
|
|
||||||
type ParsedIP struct {
|
|
||||||
V6 bool // is this a v6 address?
|
|
||||||
Value netip.Addr // the actual address
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseIP turns a string IP address into a ParsedIP. Returns nil if address is not parseable.
|
// parseIP turns a string IP address into a ParsedIP. Returns nil if address is not parseable.
|
||||||
func parseIP(ip string) *ParsedIP {
|
func parseIP(ip string) *netip.Addr {
|
||||||
s := strings.TrimSpace(ip)
|
s := strings.TrimSpace(ip)
|
||||||
v6 := strings.Index(s, ":") >= 0
|
v6 := strings.Contains(s, ":")
|
||||||
addy, err := netip.ParseAddr(s)
|
addy, err := netip.ParseAddr(s)
|
||||||
if err != nil || addy.Is6() != v6 {
|
if err != nil || addy.Is6() != v6 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return new(ParsedIP{V6: v6, Value: addy})
|
return &addy
|
||||||
}
|
}
|
||||||
|
|
||||||
// bracketedExpr recognizes an IP address with brackets.
|
// bracketedExpr recognizes an IP address with brackets.
|
||||||
@@ -44,7 +39,7 @@ var v4WithPortExpr = regexp.MustCompile(`^(\d{1,3}(?:\.\d{1,3}){3}):\d+$`)
|
|||||||
// normalizeIP filters a supplied IP address. Returns nil if the address is not parseable.
|
// normalizeIP filters a supplied IP address. Returns nil if the address is not parseable.
|
||||||
func normalizeIP(ip string) *string {
|
func normalizeIP(ip string) *string {
|
||||||
s := strings.TrimSpace(ip)
|
s := strings.TrimSpace(ip)
|
||||||
if s == "" || strings.Index(s, ",") >= 0 {
|
if s == "" || strings.Contains(s, ",") {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
bracketed := bracketedExpr.FindStringSubmatch(s)
|
bracketed := bracketedExpr.FindStringSubmatch(s)
|
||||||
@@ -59,8 +54,19 @@ func normalizeIP(ip string) *string {
|
|||||||
if parsed == nil {
|
if parsed == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if parsed.V6 && parsed.Value.Is4In6() {
|
if parsed.Is4In6() {
|
||||||
return new(netip.AddrFrom4(parsed.Value.As4()).String())
|
return new(netip.AddrFrom4(parsed.As4()).String())
|
||||||
}
|
}
|
||||||
return new(candidate)
|
return new(candidate)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func inRanges(ip string, cidrs []netip.Prefix) bool {
|
||||||
|
normalized := normalizeIP(ip)
|
||||||
|
if normalized == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
addy := parseIP(*normalized)
|
||||||
|
return slices.IndexFunc(cidrs, func(c netip.Prefix) bool {
|
||||||
|
return c.Contains(*addy)
|
||||||
|
}) >= 0
|
||||||
|
}
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
"encoding/json/v2"
|
"encoding/json/v2"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -41,7 +42,7 @@ type ipRangeData struct {
|
|||||||
loaded bool
|
loaded bool
|
||||||
loadError error
|
loadError error
|
||||||
creation *time.Time
|
creation *time.Time
|
||||||
cidrs []string
|
cidrs []netip.Prefix
|
||||||
}
|
}
|
||||||
|
|
||||||
var ipRangeStore map[string]*ipRangeData
|
var ipRangeStore map[string]*ipRangeData
|
||||||
@@ -49,11 +50,12 @@ var ipRangeStore map[string]*ipRangeData
|
|||||||
func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
|
func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
|
||||||
rd.m.Lock()
|
rd.m.Lock()
|
||||||
defer rd.m.Unlock()
|
defer rd.m.Unlock()
|
||||||
|
LOADLOOP:
|
||||||
for counter := NUM_RETRIES; counter > 0; counter-- {
|
for counter := NUM_RETRIES; counter > 0; counter-- {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
rd.loadError = ctx.Err()
|
rd.loadError = ctx.Err()
|
||||||
break
|
break LOADLOOP
|
||||||
default:
|
default:
|
||||||
client := &http.Client{Timeout: TIMEOUT * time.Second}
|
client := &http.Client{Timeout: TIMEOUT * time.Second}
|
||||||
resp, err := client.Get(url)
|
resp, err := client.Get(url)
|
||||||
@@ -61,7 +63,7 @@ func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
|
|||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
rd.loadError = ctx.Err()
|
rd.loadError = ctx.Err()
|
||||||
break
|
break LOADLOOP
|
||||||
default:
|
default:
|
||||||
if err == nil {
|
if err == nil {
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
@@ -74,9 +76,25 @@ func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
|
|||||||
if parsed.Prefixes != nil {
|
if parsed.Prefixes != nil {
|
||||||
lx = len(parsed.Prefixes)
|
lx = len(parsed.Prefixes)
|
||||||
}
|
}
|
||||||
|
rd.cidrs = make([]netip.Prefix, lx)
|
||||||
|
if lx > 0 {
|
||||||
|
for i, p := range parsed.Prefixes {
|
||||||
|
if p.IPv4Prefix == "" {
|
||||||
|
rd.cidrs[i], err = netip.ParsePrefix(p.IPv6Prefix)
|
||||||
|
} else {
|
||||||
|
rd.cidrs[i], err = netip.ParsePrefix(p.IPv4Prefix)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
rd.loadError = err
|
||||||
|
break LOADLOOP
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rd.loadError = nil
|
||||||
|
break LOADLOOP
|
||||||
} else {
|
} else {
|
||||||
rd.loadError = err
|
rd.loadError = err
|
||||||
|
break LOADLOOP
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
|
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
|
||||||
@@ -87,16 +105,55 @@ func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
if rd.loadError == nil {
|
||||||
|
rd.loadError = ctx.Err()
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
// fall through
|
||||||
|
}
|
||||||
rd.loaded = true
|
rd.loaded = true
|
||||||
rd.c.Signal()
|
rd.c.Signal()
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadIPRanges(urls []string, ctxname any) {
|
var loadRangesCtx context.Context
|
||||||
|
|
||||||
|
func loadIPRanges(urls []string, ctxname any) func() {
|
||||||
ipRangeStore = make(map[string]*ipRangeData)
|
ipRangeStore = make(map[string]*ipRangeData)
|
||||||
ctx, cancelfunc := context.WithCancel(context.Background())
|
ctx, cancelfunc := context.WithCancel(context.Background())
|
||||||
ctx = context.WithValue(ctx, ctxname, "loadIPRanges")
|
loadRangesCtx = context.WithValue(ctx, ctxname, "loadIPRanges")
|
||||||
for _, url := range urls {
|
for _, url := range urls {
|
||||||
rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil})
|
rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil})
|
||||||
rdat.c = sync.NewCond(&(rdat.m))
|
rdat.c = sync.NewCond(&(rdat.m))
|
||||||
|
ipRangeStore[url] = rdat
|
||||||
|
go loadIPRangeData(loadRangesCtx, url, rdat)
|
||||||
|
}
|
||||||
|
return func() {
|
||||||
|
cancelfunc()
|
||||||
|
for _, v := range ipRangeStore {
|
||||||
|
v.m.Lock()
|
||||||
|
if !v.loaded {
|
||||||
|
v.loadError = context.Canceled
|
||||||
|
v.c.Signal()
|
||||||
|
}
|
||||||
|
v.m.Unlock()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func getIPRangeData(url string) ([]netip.Prefix, *time.Time, error) {
|
||||||
|
rdat, ok := ipRangeStore[url]
|
||||||
|
if !ok {
|
||||||
|
rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil})
|
||||||
|
rdat.c = sync.NewCond(&(rdat.m))
|
||||||
|
ipRangeStore[url] = rdat
|
||||||
|
go loadIPRangeData(loadRangesCtx, url, rdat)
|
||||||
|
}
|
||||||
|
rdat.m.Lock()
|
||||||
|
defer rdat.m.Unlock()
|
||||||
|
for !rdat.loaded {
|
||||||
|
rdat.c.Wait()
|
||||||
|
}
|
||||||
|
return rdat.cidrs, rdat.creation, rdat.loadError
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user