next part of work, including developing range loader

This commit is contained in:
amy
2026-11-15 11:17:18 -07:00
parent 4078d62efc
commit d239f39f9a
3 changed files with 171 additions and 34 deletions
+61 -32
View File
@@ -16,50 +16,53 @@ import (
_ "embed" _ "embed"
"encoding/json/v2" "encoding/json/v2"
"fmt" "fmt"
"maps"
"net/http" "net/http"
"regexp" "regexp"
"slices" "slices"
"strings" "strings"
) )
// Constants reflecting confidence levels.
const ( const (
ConfidenceHigh = iota ConfidenceHigh = iota // high confidence
ConfidenceMedium ConfidenceMedium // medium confidence
ConfidenceLow ConfidenceLow // low confidence
) )
// Classification is the return value from AmClassify, showing what a particular HTTP request is classified as.
type Classification struct { type Classification struct {
Category string Category string // category of requester: agent, ai-crawler, crawler, bot, human, unknown
Kind string Kind string // what kind of requester is it?
Id string Id string // actual identifier
Name string Name string // name of requester
Operator string Operator string // what organization operates this requester?
Version string Version string // version of requester, where applicable
Verified bool Verified bool // true if all this is verified, false if not
Confidence int Confidence int // confidence level (above)
Reasons []string Reasons []string // why did we classify the request this way?
} }
// RawAgentDef is the agent definition data loaded from the agents.json file. // RawAgentDef is the agent definition data loaded from the agents.json file.
type RawAgentDef struct { type RawAgentDef struct {
Id string `json:"id"` Id string `json:"id"` // agent ID
Name string `json:"name"` Name string `json:"name"` // agent name
Operator string `json:"operator"` Operator string `json:"operator"` // agent operating organization
Category string `json:"category"` Category string `json:"category"` // category of agent
Kind string `json:"kind"` Kind string `json:"kind"` // kind (sub-category) of agent
RunsOn string `json:"runsOn"` RunsOn string `json:"runsOn"` // does it run on a user machine or an operator machine?
UserAgents []string `json:"userAgents"` UserAgents []string `json:"userAgents"` // regexps to recognize its User-Agent
Accept []string `json:"accept"` Accept []string `json:"accept"` // Accept: string(s) the agent sends
SignatureAgent []string `json:"signatureAgent"` SignatureAgent []string `json:"signatureAgent"` // what agent(s) we can find in a request signature
Verify struct { Verify struct {
IpRanges []string `json:"ipRanges"` IpRanges []string `json:"ipRanges"` // where can we find what IP ranges this agent uses?
ReverseDNS []string `json:"reverseDns"` ReverseDNS []string `json:"reverseDns"` // what should the agent's IP address resolve to?
WebBotAuth string `json:"webBotAuth"` WebBotAuth string `json:"webBotAuth"` // where can I find the authentication?
} `json:"verify"` } `json:"verify"`
Docs string `json:"docs"` Docs string `json:"docs"` // URL to documentation
Examples []string `json:"examples"` Examples []string `json:"examples"` // examples of strings that match
Notes string `json:"notes"` Notes string `json:"notes"` // notes
LastVerified string `json:"lastVerified"` LastVerified string `json:"lastVerified"` // when was this information last verified?
} }
// agentDef is the definition for a compiled agent. // agentDef is the definition for a compiled agent.
@@ -74,12 +77,16 @@ var initAgents []byte
// compiledAgents is the compiled list of agents. // compiledAgents is the compiled list of agents.
var compiledAgents []agentDef var compiledAgents []agentDef
// AmInitAgentClass is the initialization function.
func AmInitAgentClass() { func AmInitAgentClass() {
// load the raw agent list
var rawAgents []RawAgentDef var rawAgents []RawAgentDef
err := json.Unmarshal(initAgents, &rawAgents) err := json.Unmarshal(initAgents, &rawAgents)
if err != nil { if err != nil {
panic(fmt.Sprintf("agent load failure: %v", err)) panic(fmt.Sprintf("agent load failure: %v", err))
} }
// build the compiled agent list
compiledAgents = make([]agentDef, len(rawAgents)) compiledAgents = make([]agentDef, len(rawAgents))
for i, v := range rawAgents { for i, v := range rawAgents {
compiledAgents[i].r = &(rawAgents[i]) compiledAgents[i].r = &(rawAgents[i])
@@ -92,15 +99,28 @@ func AmInitAgentClass() {
compiledAgents[i].patterns = make([]*regexp.Regexp, 0) compiledAgents[i].patterns = make([]*regexp.Regexp, 0)
} }
} }
// get the list of IP range files
urls := make(map[string]struct{})
for _, ra := range rawAgents {
if ra.Verify.IpRanges != nil {
for _, url := range ra.Verify.IpRanges {
urls[url] = struct{}{}
}
}
}
loadIPRanges(slices.Collect(maps.Keys(urls)))
} }
// AgentMatch is the return from findAgent.
type AgentMatch struct { type AgentMatch struct {
Agent *RawAgentDef Agent *RawAgentDef // pointer to raw agent definition
Version string Version string // version of agent, where applicable
Reason string Reason string // reason we matched this agent
Weak bool Weak bool // is this a weak match?
} }
// normalizeHeaders joins header values and smashes case on header names.
func normalizeHeaders(headers http.Header) map[string]string { func normalizeHeaders(headers http.Header) map[string]string {
rc := make(map[string]string) rc := make(map[string]string)
if headers == nil { if headers == nil {
@@ -112,16 +132,24 @@ func normalizeHeaders(headers http.Header) map[string]string {
return rc return rc
} }
// normalizeAcceptExpr is how we close the gaps around delimiters in an Accept: expression.
var normalizeAcceptExpr = regexp.MustCompile(`\s*([,;=])\s*`) var normalizeAcceptExpr = regexp.MustCompile(`\s*([,;=])\s*`)
// normalizeAccept squeezes the whitespace out of an Accept: expression.
func normalizeAccept(accept string) string { func normalizeAccept(accept string) string {
return strings.TrimSpace(normalizeAcceptExpr.ReplaceAllString(strings.ToLower(accept), "$1")) return strings.TrimSpace(normalizeAcceptExpr.ReplaceAllString(strings.ToLower(accept), "$1"))
} }
// signatureAgentHostExpr recognizes the "host" in a URL.
var signatureAgentHostExpr = regexp.MustCompile(`(?i)https?:\/\/([^/"\s]+)`) var signatureAgentHostExpr = regexp.MustCompile(`(?i)https?:\/\/([^/"\s]+)`)
// signatureAgentHostBare1 is used to trim up a bare host name.
var signatureAgentHostBare1 = regexp.MustCompile(`^[\w-]+=`) var signatureAgentHostBare1 = regexp.MustCompile(`^[\w-]+=`)
// signatureAgentHostBare2 is used to trim up a bare host name.
var signatureAgentHostBare2 = regexp.MustCompile(`"`) var signatureAgentHostBare2 = regexp.MustCompile(`"`)
// signatureAgentHost locates the host associated with a request signature.
func signatureAgentHost(value string) *string { func signatureAgentHost(value string) *string {
m := signatureAgentHostExpr.FindStringSubmatch(value) m := signatureAgentHostExpr.FindStringSubmatch(value)
if m != nil { if m != nil {
@@ -134,6 +162,7 @@ func signatureAgentHost(value string) *string {
return nil return nil
} }
// findAgent matches the request against the list of compiled agents to locate a match. Returns nil on failure.
func findAgent(ua string, h map[string]string) *AgentMatch { func findAgent(ua string, h map[string]string) *AgentMatch {
for _, ca := range compiledAgents { for _, ca := range compiledAgents {
for _, p := range ca.patterns { for _, p := range ca.patterns {
+8 -2
View File
@@ -18,11 +18,13 @@ import (
"strings" "strings"
) )
// ParsedIP is the structure that holds a parsed IP address.
type ParsedIP struct { type ParsedIP struct {
V6 bool V6 bool // is this a v6 address?
Value netip.Addr Value netip.Addr // the actual address
} }
// parseIP turns a string IP address into a ParsedIP. Returns nil if address is not parseable.
func parseIP(ip string) *ParsedIP { func parseIP(ip string) *ParsedIP {
s := strings.TrimSpace(ip) s := strings.TrimSpace(ip)
v6 := strings.Index(s, ":") >= 0 v6 := strings.Index(s, ":") >= 0
@@ -33,9 +35,13 @@ func parseIP(ip string) *ParsedIP {
return new(ParsedIP{V6: v6, Value: addy}) return new(ParsedIP{V6: v6, Value: addy})
} }
// bracketedExpr recognizes an IP address with brackets.
var bracketedExpr = regexp.MustCompile(`^\[([^\]]+)\](?::\d+)?$`) var bracketedExpr = regexp.MustCompile(`^\[([^\]]+)\](?::\d+)?$`)
// v4WithPortExpr recognizes an IPv4 address with a port.
var v4WithPortExpr = regexp.MustCompile(`^(\d{1,3}(?:\.\d{1,3}){3}):\d+$`) var v4WithPortExpr = regexp.MustCompile(`^(\d{1,3}(?:\.\d{1,3}){3}):\d+$`)
// normalizeIP filters a supplied IP address. Returns nil if the address is not parseable.
func normalizeIP(ip string) *string { func normalizeIP(ip string) *string {
s := strings.TrimSpace(ip) s := strings.TrimSpace(ip)
if s == "" || strings.Index(s, ",") >= 0 { if s == "" || strings.Index(s, ",") >= 0 {
+102
View File
@@ -0,0 +1,102 @@
/*
* Amsterdam Web Communities System
* Copyright (c) 2025-2026 Erbosoft Metaverse Design Solutions, All Rights Reserved
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*
* SPDX-License-Identifier: MPL-2.0
*/
// Package agentclass contains support for agent request classification.
package agentclass
import (
"context"
"encoding/json/jsontext"
"encoding/json/v2"
"fmt"
"net/http"
"sync"
"time"
)
const NUM_RETRIES = 3
const TIMEOUT = 10
type RangeDataPrefix struct {
IPv4Prefix string `json:"ipv4Prefix"`
IPv6Prefix string `json:"ipv6Prefix"`
}
type RangeDataFormat struct {
CreationTime *time.Time `json:"creationTime"`
Prefixes []RangeDataPrefix `json:"prefixes"`
}
type ipRangeData struct {
m sync.Mutex
c *sync.Cond
loaded bool
loadError error
creation *time.Time
cidrs []string
}
var ipRangeStore map[string]*ipRangeData
func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
rd.m.Lock()
defer rd.m.Unlock()
for counter := NUM_RETRIES; counter > 0; counter-- {
select {
case <-ctx.Done():
rd.loadError = ctx.Err()
break
default:
client := &http.Client{Timeout: TIMEOUT * time.Second}
resp, err := client.Get(url)
select {
case <-ctx.Done():
resp.Body.Close()
rd.loadError = ctx.Err()
break
default:
if err == nil {
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
decoder := jsontext.NewDecoder(resp.Body)
var parsed RangeDataFormat
err = json.UnmarshalDecode(decoder, &parsed)
if err == nil {
lx := 0
if parsed.Prefixes != nil {
lx = len(parsed.Prefixes)
}
} else {
rd.loadError = err
}
} else {
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
}
} else {
rd.loadError = err
}
}
}
}
rd.loaded = true
rd.c.Signal()
}
func loadIPRanges(urls []string, ctxname any) {
ipRangeStore = make(map[string]*ipRangeData)
ctx, cancelfunc := context.WithCancel(context.Background())
ctx = context.WithValue(ctx, ctxname, "loadIPRanges")
for _, url := range urls {
rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil})
rdat.c = sync.NewCond(&(rdat.m))
}
}