initial code for classifier complete

This commit is contained in:
amy
2026-11-15 16:54:44 -07:00
parent d239f39f9a
commit 0d576e4b55
4 changed files with 481 additions and 26 deletions
+363 -8
View File
@@ -17,10 +17,13 @@ import (
"encoding/json/v2"
"fmt"
"maps"
"math"
"net/http"
"regexp"
"slices"
"strconv"
"strings"
"time"
)
// Constants reflecting confidence levels.
@@ -71,19 +74,38 @@ type agentDef struct {
patterns []*regexp.Regexp // compiled user agent expressions
}
// RawClientDef is the client definition data loaded from the http-clients.json file.
type RawClientDef struct {
Name string `json:"name"`
Kind string `json:"kind"`
UserAgent string `json:"userAgent"`
}
// clientdef is the definition for a compiled client.
type clientDef struct {
r *RawClientDef
pattern *regexp.Regexp
}
//go:embed agents.json
var initAgents []byte
// compiledAgents is the compiled list of agents.
var compiledAgents []agentDef
//go:embed http-clients.json
var initClients []byte
// compiledClients is the compiled list of clients.
var compiledClients []clientDef
// AmInitAgentClass is the initialization function.
func AmInitAgentClass() {
func AmInitAgentClass(ctxname any) func() {
// load the raw agent list
var rawAgents []RawAgentDef
err := json.Unmarshal(initAgents, &rawAgents)
if err != nil {
panic(fmt.Sprintf("agent load failure: %v", err))
panic(fmt.Sprintf("agent definition load failure: %v", err))
}
// build the compiled agent list
@@ -100,6 +122,20 @@ func AmInitAgentClass() {
}
}
// load the raw clients list
var rawClients []RawClientDef
err = json.Unmarshal(initClients, &rawClients)
if err != nil {
panic(fmt.Sprintf("client definition load failure: %v", err))
}
// build the compiled client list
compiledClients = make([]clientDef, len(rawClients))
for i, v := range rawClients {
compiledClients[i].r = &(rawClients[i])
compiledClients[i].pattern = regexp.MustCompile("(?i)" + v.UserAgent)
}
// get the list of IP range files
urls := make(map[string]struct{})
for _, ra := range rawAgents {
@@ -109,7 +145,36 @@ func AmInitAgentClass() {
}
}
}
loadIPRanges(slices.Collect(maps.Keys(urls)))
return loadIPRanges(slices.Collect(maps.Keys(urls)), ctxname)
}
type ClassifyOptions struct {
MaxAge int // maximum age of IP range files
Now *time.Time // "now" timestamp (for testing)
HeadersComplete bool // look at complete headers?
}
func AmClassifyOptions() ClassifyOptions {
return ClassifyOptions{
MaxAge: 180,
Now: nil,
HeadersComplete: false,
}
}
func (o *ClassifyOptions) WithMaxAge(maxAge int) *ClassifyOptions {
o.MaxAge = maxAge
return o
}
func (o *ClassifyOptions) WithNow(now *time.Time) *ClassifyOptions {
o.Now = now
return o
}
func (o *ClassifyOptions) WithHeadersComplete(flag bool) *ClassifyOptions {
o.HeadersComplete = flag
return o
}
// AgentMatch is the return from findAgent.
@@ -120,6 +185,29 @@ type AgentMatch struct {
Weak bool // is this a weak match?
}
var textBrowserExpr = regexp.MustCompile(`^(Lynx|ELinks|Links|w3m)\/`)
var browserExpr = regexp.MustCompile(`^Mozilla\/[4-6]\.0 \(|^Opera\/\d`)
var browserEngineExpr = regexp.MustCompile(`\(KHTML,? ?like Gecko|KHTML\/[\d.]+ \(like Gecko|Gecko\/\d|\) like Gecko|Trident\/\d|MSIE \d|Presto\/\d|Chromium\/\d`)
var botWordExpr = regexp.MustCompile(`(?:[a-z0-9-]|\b)[Bb]ot(?:\b|\/)`)
var emailExpr = regexp.MustCompile(`(?i)[\w.+-]+@[\w-]+(\.[\w-]+)*\.[a-z]{2,}`)
var automationExpr = regexp.MustCompile(`(?i)crawl|spider|scrap|headless|https?:\/\/|\+http|(?:^|[^a-z])(?:archive|observ|monitor|check(?:er)?\b|preview|verif|render|insight|console|transcod|scanner|synthetic)|lighthouse|selenium|cypress|webdriver|\bsplash\b|gomezagent|rigor|speedcurve|\bptst\b|dareboost|dejaclick|favicon|siteimprove|onetrust|datanyze|feeder\.co|rss\b|weblight|duplexweb|ips-agent`)
var compatibleExpr = regexp.MustCompile(`compatible;`)
var enginesExpr = regexp.MustCompile(`MSIE|Trident|Konqueror`)
// isBrowser retruns true if the user agent indicates a browser.
func isBrowser(ua string) bool {
if textBrowserExpr.MatchString(ua) {
return true
}
if !browserExpr.MatchString(ua) || !browserEngineExpr.MatchString(ua) {
return false
}
if botWordExpr.MatchString(ua) || emailExpr.MatchString(ua) || automationExpr.MatchString(ua) {
return false
}
return !compatibleExpr.MatchString(ua) || enginesExpr.MatchString(ua)
}
// normalizeHeaders joins header values and smashes case on header names.
func normalizeHeaders(headers http.Header) map[string]string {
rc := make(map[string]string)
@@ -132,6 +220,77 @@ func normalizeHeaders(headers http.Header) map[string]string {
return rc
}
// rankedAcceptData is the type of data returned from rankAccept.
type rankedAcceptData struct {
mtype string // MIME type
q1000 int // quality value multiplied by 1000
ndx int // original index in Accept string
}
// rankAccept turns an Accept string into a list of ranked accept members.
func rankAccept(accept string) []rankedAcceptData {
components := strings.Split(accept, ",")
ranks := make([]rankedAcceptData, len(components))
for i, com := range components {
subparts := strings.Split(com, ";")
ranks[i].mtype = subparts[0]
ranks[i].q1000 = 1000
ranks[i].ndx = i
if len(subparts) > 1 {
for _, part := range subparts[1:] {
part = strings.TrimSpace(part)
if strings.HasPrefix(part, "q=") {
parsed, err := strconv.ParseFloat(part[2:], 64)
if err != nil {
ranks[i].q1000 = int(math.Floor(1000.0 * parsed))
}
break
}
}
}
}
ranks2 := make([]rankedAcceptData, 0, len(ranks))
for i := range ranks {
if ranks[i].q1000 > 0 {
ranks2 = append(ranks2, ranks[i])
}
}
slices.SortFunc(ranks2, func(a, b rankedAcceptData) int {
t := b.q1000 - a.q1000
if t == 0 {
t = b.ndx - a.ndx
}
return t
})
return ranks2
}
// ranksAboveHtml returns true if any of the given MIME types ranks above HTML in the given Accept: string.
func ranksAboveHtml(accept string, types []string) bool {
ranked := rankAccept(accept)
md := slices.IndexFunc(ranked, func(d rankedAcceptData) bool {
return slices.Contains(types, d.mtype)
})
if md < 0 {
return false
}
html := slices.IndexFunc(ranked, func(d rankedAcceptData) bool {
return d.mtype == "text/html"
})
if html < 0 {
return true
}
return md < html
}
// markdownTypes are the MIME types associated with Markdown.
var markdownTypes = []string{"text/markdown", "text/x-markdown"}
// prefersMarkdown returns true if the given Accept: string prefers Markdown over HTML.
func prefersMarkdown(accept string) bool {
return ranksAboveHtml(accept, markdownTypes)
}
// normalizeAcceptExpr is how we close the gaps around delimiters in an Accept: expression.
var normalizeAcceptExpr = regexp.MustCompile(`\s*([,;=])\s*`)
@@ -223,9 +382,10 @@ func findAgent(ua string, h map[string]string) *AgentMatch {
return nil
}
func fromAgent(match *AgentMatch, rawIP string) Classification {
// fromAgent turns an AgentMatch into a Classification.
func fromAgent(match *AgentMatch, rawIP string, opts ClassifyOptions) *Classification {
agent := match.Agent
base := Classification{
base := new(Classification{
Category: agent.Category,
Kind: agent.Kind,
Name: agent.Name,
@@ -234,7 +394,7 @@ func fromAgent(match *AgentMatch, rawIP string) Classification {
Verified: true,
Confidence: ConfidenceMedium,
Reasons: make([]string, 1),
}
})
base.Reasons[0] = match.Reason
if agent.RunsOn == "user" {
base.Reasons = append(base.Reasons, fmt.Sprintf("%s runs on the user's machine, so IP verification does not apply", agent.Name))
@@ -249,9 +409,140 @@ func fromAgent(match *AgentMatch, rawIP string) Classification {
if agent.Verify.IpRanges != nil {
urls = append(urls, agent.Verify.IpRanges...)
}
now := opts.Now
if now == nil {
tmp := time.Now()
now = &tmp
}
maxDuration := time.Hour * time.Duration(24*opts.MaxAge)
impersonation := false
validRangeFiles := false
for _, url := range urls {
cidrs, creation, err := getIPRangeData(url)
if err != nil {
base.Reasons = append(base.Reasons, fmt.Sprintf("unable to get range file %s: %v", url, err))
continue
}
validRangeFiles = true
if inRanges(*ip, cidrs) {
base.Verified = true
base.Confidence = ConfidenceHigh
base.Reasons = append(base.Reasons, "%s is in %s's published ranges", *ip, agent.Operator)
return base
}
if now.Sub(*creation) <= maxDuration {
impersonation = true
}
}
if !validRangeFiles {
return base
}
if impersonation {
base.Category = "bot"
base.Kind = "impersonator"
base.Verified = false
base.Confidence = ConfidenceHigh
base.Reasons = append(base.Reasons, fmt.Sprintf("claims to be %s but %s is outside %s's published ranges", agent.Name, *ip, agent.Operator))
return base
}
base.Reasons = append(base.Reasons, fmt.Sprintf("%s's range file is older than %d days, so a mismatch is not counted as spoofing", agent.Operator, opts.MaxAge))
return base
}
func AmClassify(headers http.Header, ipAddr string, userAgent string) Classification {
// fake returns a Classification identifying the request as a fake browser.
func fake(reason string, confidence int) *Classification {
rc := new(Classification{
Category: "bot",
Kind: "fake-browser",
Verified: false,
Confidence: confidence,
Reasons: make([]string, 1),
})
rc.Reasons[0] = reason
return rc
}
// majorVersion extracts a browser's major version from a regular expression.
func majorVersion(ua string, re *regexp.Regexp) int {
m := re.FindStringSubmatch(ua)
if m != nil {
v, err := strconv.Atoi(m[1])
if err == nil {
return v
}
}
return 0
}
var firefoxBrowserExpr = regexp.MustCompile(`Firefox\/`)
var firefoxVersionExpr = regexp.MustCompile(`Firefox\/(\d+)`)
var chromeBrowserExpr = regexp.MustCompile(`Chrome\/`)
var chromeVersionExpr = regexp.MustCompile(`Chrome\/(\d+)`)
var safariBrowserExpr = regexp.MustCompile(`Version\/[\d.]+.*Safari\/`)
var safariVersionExpr = regexp.MustCompile(`Version\/(\d+)`)
func sendsFetchMetadata(ua string) bool {
if firefoxBrowserExpr.MatchString(ua) {
return majorVersion(ua, firefoxVersionExpr) >= 90
}
if chromeBrowserExpr.MatchString(ua) {
return majorVersion(ua, chromeVersionExpr) >= 80
}
if safariBrowserExpr.MatchString(ua) {
return majorVersion(ua, safariVersionExpr) >= 17
}
return false
}
var plainTextType = []string{"text/plain"}
func fromBrowser(ua string, h map[string]string, opts ClassifyOptions) *Classification {
accept, ok := h["accept"]
if ok {
if ranksAboveHtml(accept, plainTextType) {
return fake("Accept header ranks text/plain above HTML, which browsers never do", ConfidenceLow)
}
if strings.Contains(accept, "application/metalink") {
return fake("Accept header asks for metalink, which download tools send and browsers do not", ConfidenceMedium)
}
}
if !opts.HeadersComplete {
rc := new(Classification{
Category: "human",
Verified: false,
Confidence: ConfidenceLow,
Reasons: make([]string, 1),
})
rc.Reasons[0] = "browser user agent; headers not checked"
return rc
}
_, ok = h["sec-ch-ua"]
if ok && !chromeBrowserExpr.MatchString(ua) {
return fake("sec-ch-ua sent with a non-Chromium user agent", ConfidenceHigh)
}
_, ok = h["sec-fetch-mode"]
if !ok && sendsFetchMetadata(ua) {
return fake("browser user agent without sec-fetch-* headers", ConfidenceMedium)
}
_, ok = h["accept-language"]
if !ok {
return fake("browser user agent without Accept-Language", ConfidenceLow)
}
rc := new(Classification{
Category: "human",
Verified: false,
Confidence: ConfidenceMedium,
Reasons: make([]string, 1),
})
rc.Reasons[0] = "browser user agent with consistent browser headers"
return rc
}
func AmClassify(headers http.Header, ipAddr string, userAgent string, opts ...ClassifyOptions) *Classification {
myopts := AmClassifyOptions()
if len(opts) > 0 {
myopts = opts[0]
}
h := normalizeHeaders(headers)
ua := userAgent
if ua == "" {
@@ -260,8 +551,72 @@ func AmClassify(headers http.Header, ipAddr string, userAgent string) Classifica
ua = t
}
}
if ua == "" {
rc := new(Classification{
Verified: false,
Confidence: ConfidenceLow,
Reasons: make([]string, 1),
})
if len(h) == 0 {
rc.Category = "unknown"
rc.Reasons[0] = "no user agent or headers to classify"
} else {
rc.Category = "bot"
rc.Kind = "unidentified"
rc.Reasons[0] = "request has no user agent, which browsers always send"
}
return rc
}
// Look for a matching agent.
match := findAgent(ua, h)
if match != nil {
return fromAgent(match, ipAddr)
return fromAgent(match, ipAddr, myopts)
}
// Check the Accept: line; if the request prefers Markdown to HTML, we're some kind of agent.
acc, ok := h["accept"]
if ok {
if prefersMarkdown(acc) {
rc := new(Classification{
Category: "agent",
Kind: "unidentified",
Verified: false,
Confidence: ConfidenceMedium,
Reasons: make([]string, 1),
})
rc.Reasons[0] = "Accept header prefers markdown over HTML, which browsers never do"
return rc
}
}
// Look for an HTTP client.
ndx := slices.IndexFunc(compiledClients, func(d clientDef) bool {
return d.pattern.MatchString(ua)
})
if ndx >= 0 {
rc := new(Classification{
Category: "bot",
Kind: compiledClients[ndx].r.Kind,
Name: compiledClients[ndx].r.Name,
Verified: false,
Confidence: ConfidenceMedium,
Reasons: make([]string, 1),
})
rc.Reasons[0] = fmt.Sprintf("user agent is %s", compiledClients[ndx].r.Name)
return rc
}
if isBrowser(ua) {
return fromBrowser(ua, h, myopts)
}
rc := new(Classification{
Category: "bot",
Kind: "unidentified",
Verified: false,
Confidence: ConfidenceLow,
Reasons: make([]string, 1),
})
rc.Reasons[0] = "user agent is not a browser and matches no known agent, crawler or client"
return rc
}