next part of work, including developing range loader
This commit is contained in:
+61
-32
@@ -16,50 +16,53 @@ import (
|
|||||||
_ "embed"
|
_ "embed"
|
||||||
"encoding/json/v2"
|
"encoding/json/v2"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"maps"
|
||||||
"net/http"
|
"net/http"
|
||||||
"regexp"
|
"regexp"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Constants reflecting confidence levels.
|
||||||
const (
|
const (
|
||||||
ConfidenceHigh = iota
|
ConfidenceHigh = iota // high confidence
|
||||||
ConfidenceMedium
|
ConfidenceMedium // medium confidence
|
||||||
ConfidenceLow
|
ConfidenceLow // low confidence
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Classification is the return value from AmClassify, showing what a particular HTTP request is classified as.
|
||||||
type Classification struct {
|
type Classification struct {
|
||||||
Category string
|
Category string // category of requester: agent, ai-crawler, crawler, bot, human, unknown
|
||||||
Kind string
|
Kind string // what kind of requester is it?
|
||||||
Id string
|
Id string // actual identifier
|
||||||
Name string
|
Name string // name of requester
|
||||||
Operator string
|
Operator string // what organization operates this requester?
|
||||||
Version string
|
Version string // version of requester, where applicable
|
||||||
Verified bool
|
Verified bool // true if all this is verified, false if not
|
||||||
Confidence int
|
Confidence int // confidence level (above)
|
||||||
Reasons []string
|
Reasons []string // why did we classify the request this way?
|
||||||
}
|
}
|
||||||
|
|
||||||
// RawAgentDef is the agent definition data loaded from the agents.json file.
|
// RawAgentDef is the agent definition data loaded from the agents.json file.
|
||||||
type RawAgentDef struct {
|
type RawAgentDef struct {
|
||||||
Id string `json:"id"`
|
Id string `json:"id"` // agent ID
|
||||||
Name string `json:"name"`
|
Name string `json:"name"` // agent name
|
||||||
Operator string `json:"operator"`
|
Operator string `json:"operator"` // agent operating organization
|
||||||
Category string `json:"category"`
|
Category string `json:"category"` // category of agent
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"` // kind (sub-category) of agent
|
||||||
RunsOn string `json:"runsOn"`
|
RunsOn string `json:"runsOn"` // does it run on a user machine or an operator machine?
|
||||||
UserAgents []string `json:"userAgents"`
|
UserAgents []string `json:"userAgents"` // regexps to recognize its User-Agent
|
||||||
Accept []string `json:"accept"`
|
Accept []string `json:"accept"` // Accept: string(s) the agent sends
|
||||||
SignatureAgent []string `json:"signatureAgent"`
|
SignatureAgent []string `json:"signatureAgent"` // what agent(s) we can find in a request signature
|
||||||
Verify struct {
|
Verify struct {
|
||||||
IpRanges []string `json:"ipRanges"`
|
IpRanges []string `json:"ipRanges"` // where can we find what IP ranges this agent uses?
|
||||||
ReverseDNS []string `json:"reverseDns"`
|
ReverseDNS []string `json:"reverseDns"` // what should the agent's IP address resolve to?
|
||||||
WebBotAuth string `json:"webBotAuth"`
|
WebBotAuth string `json:"webBotAuth"` // where can I find the authentication?
|
||||||
} `json:"verify"`
|
} `json:"verify"`
|
||||||
Docs string `json:"docs"`
|
Docs string `json:"docs"` // URL to documentation
|
||||||
Examples []string `json:"examples"`
|
Examples []string `json:"examples"` // examples of strings that match
|
||||||
Notes string `json:"notes"`
|
Notes string `json:"notes"` // notes
|
||||||
LastVerified string `json:"lastVerified"`
|
LastVerified string `json:"lastVerified"` // when was this information last verified?
|
||||||
}
|
}
|
||||||
|
|
||||||
// agentDef is the definition for a compiled agent.
|
// agentDef is the definition for a compiled agent.
|
||||||
@@ -74,12 +77,16 @@ var initAgents []byte
|
|||||||
// compiledAgents is the compiled list of agents.
|
// compiledAgents is the compiled list of agents.
|
||||||
var compiledAgents []agentDef
|
var compiledAgents []agentDef
|
||||||
|
|
||||||
|
// AmInitAgentClass is the initialization function.
|
||||||
func AmInitAgentClass() {
|
func AmInitAgentClass() {
|
||||||
|
// load the raw agent list
|
||||||
var rawAgents []RawAgentDef
|
var rawAgents []RawAgentDef
|
||||||
err := json.Unmarshal(initAgents, &rawAgents)
|
err := json.Unmarshal(initAgents, &rawAgents)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(fmt.Sprintf("agent load failure: %v", err))
|
panic(fmt.Sprintf("agent load failure: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// build the compiled agent list
|
||||||
compiledAgents = make([]agentDef, len(rawAgents))
|
compiledAgents = make([]agentDef, len(rawAgents))
|
||||||
for i, v := range rawAgents {
|
for i, v := range rawAgents {
|
||||||
compiledAgents[i].r = &(rawAgents[i])
|
compiledAgents[i].r = &(rawAgents[i])
|
||||||
@@ -92,15 +99,28 @@ func AmInitAgentClass() {
|
|||||||
compiledAgents[i].patterns = make([]*regexp.Regexp, 0)
|
compiledAgents[i].patterns = make([]*regexp.Regexp, 0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// get the list of IP range files
|
||||||
|
urls := make(map[string]struct{})
|
||||||
|
for _, ra := range rawAgents {
|
||||||
|
if ra.Verify.IpRanges != nil {
|
||||||
|
for _, url := range ra.Verify.IpRanges {
|
||||||
|
urls[url] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loadIPRanges(slices.Collect(maps.Keys(urls)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AgentMatch is the return from findAgent.
|
||||||
type AgentMatch struct {
|
type AgentMatch struct {
|
||||||
Agent *RawAgentDef
|
Agent *RawAgentDef // pointer to raw agent definition
|
||||||
Version string
|
Version string // version of agent, where applicable
|
||||||
Reason string
|
Reason string // reason we matched this agent
|
||||||
Weak bool
|
Weak bool // is this a weak match?
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// normalizeHeaders joins header values and smashes case on header names.
|
||||||
func normalizeHeaders(headers http.Header) map[string]string {
|
func normalizeHeaders(headers http.Header) map[string]string {
|
||||||
rc := make(map[string]string)
|
rc := make(map[string]string)
|
||||||
if headers == nil {
|
if headers == nil {
|
||||||
@@ -112,16 +132,24 @@ func normalizeHeaders(headers http.Header) map[string]string {
|
|||||||
return rc
|
return rc
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// normalizeAcceptExpr is how we close the gaps around delimiters in an Accept: expression.
|
||||||
var normalizeAcceptExpr = regexp.MustCompile(`\s*([,;=])\s*`)
|
var normalizeAcceptExpr = regexp.MustCompile(`\s*([,;=])\s*`)
|
||||||
|
|
||||||
|
// normalizeAccept squeezes the whitespace out of an Accept: expression.
|
||||||
func normalizeAccept(accept string) string {
|
func normalizeAccept(accept string) string {
|
||||||
return strings.TrimSpace(normalizeAcceptExpr.ReplaceAllString(strings.ToLower(accept), "$1"))
|
return strings.TrimSpace(normalizeAcceptExpr.ReplaceAllString(strings.ToLower(accept), "$1"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// signatureAgentHostExpr recognizes the "host" in a URL.
|
||||||
var signatureAgentHostExpr = regexp.MustCompile(`(?i)https?:\/\/([^/"\s]+)`)
|
var signatureAgentHostExpr = regexp.MustCompile(`(?i)https?:\/\/([^/"\s]+)`)
|
||||||
|
|
||||||
|
// signatureAgentHostBare1 is used to trim up a bare host name.
|
||||||
var signatureAgentHostBare1 = regexp.MustCompile(`^[\w-]+=`)
|
var signatureAgentHostBare1 = regexp.MustCompile(`^[\w-]+=`)
|
||||||
|
|
||||||
|
// signatureAgentHostBare2 is used to trim up a bare host name.
|
||||||
var signatureAgentHostBare2 = regexp.MustCompile(`"`)
|
var signatureAgentHostBare2 = regexp.MustCompile(`"`)
|
||||||
|
|
||||||
|
// signatureAgentHost locates the host associated with a request signature.
|
||||||
func signatureAgentHost(value string) *string {
|
func signatureAgentHost(value string) *string {
|
||||||
m := signatureAgentHostExpr.FindStringSubmatch(value)
|
m := signatureAgentHostExpr.FindStringSubmatch(value)
|
||||||
if m != nil {
|
if m != nil {
|
||||||
@@ -134,6 +162,7 @@ func signatureAgentHost(value string) *string {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// findAgent matches the request against the list of compiled agents to locate a match. Returns nil on failure.
|
||||||
func findAgent(ua string, h map[string]string) *AgentMatch {
|
func findAgent(ua string, h map[string]string) *AgentMatch {
|
||||||
for _, ca := range compiledAgents {
|
for _, ca := range compiledAgents {
|
||||||
for _, p := range ca.patterns {
|
for _, p := range ca.patterns {
|
||||||
|
|||||||
+8
-2
@@ -18,11 +18,13 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ParsedIP is the structure that holds a parsed IP address.
|
||||||
type ParsedIP struct {
|
type ParsedIP struct {
|
||||||
V6 bool
|
V6 bool // is this a v6 address?
|
||||||
Value netip.Addr
|
Value netip.Addr // the actual address
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseIP turns a string IP address into a ParsedIP. Returns nil if address is not parseable.
|
||||||
func parseIP(ip string) *ParsedIP {
|
func parseIP(ip string) *ParsedIP {
|
||||||
s := strings.TrimSpace(ip)
|
s := strings.TrimSpace(ip)
|
||||||
v6 := strings.Index(s, ":") >= 0
|
v6 := strings.Index(s, ":") >= 0
|
||||||
@@ -33,9 +35,13 @@ func parseIP(ip string) *ParsedIP {
|
|||||||
return new(ParsedIP{V6: v6, Value: addy})
|
return new(ParsedIP{V6: v6, Value: addy})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// bracketedExpr recognizes an IP address with brackets.
|
||||||
var bracketedExpr = regexp.MustCompile(`^\[([^\]]+)\](?::\d+)?$`)
|
var bracketedExpr = regexp.MustCompile(`^\[([^\]]+)\](?::\d+)?$`)
|
||||||
|
|
||||||
|
// v4WithPortExpr recognizes an IPv4 address with a port.
|
||||||
var v4WithPortExpr = regexp.MustCompile(`^(\d{1,3}(?:\.\d{1,3}){3}):\d+$`)
|
var v4WithPortExpr = regexp.MustCompile(`^(\d{1,3}(?:\.\d{1,3}){3}):\d+$`)
|
||||||
|
|
||||||
|
// normalizeIP filters a supplied IP address. Returns nil if the address is not parseable.
|
||||||
func normalizeIP(ip string) *string {
|
func normalizeIP(ip string) *string {
|
||||||
s := strings.TrimSpace(ip)
|
s := strings.TrimSpace(ip)
|
||||||
if s == "" || strings.Index(s, ",") >= 0 {
|
if s == "" || strings.Index(s, ",") >= 0 {
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
/*
|
||||||
|
* Amsterdam Web Communities System
|
||||||
|
* Copyright (c) 2025-2026 Erbosoft Metaverse Design Solutions, All Rights Reserved
|
||||||
|
*
|
||||||
|
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: MPL-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Package agentclass contains support for agent request classification.
|
||||||
|
package agentclass
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json/jsontext"
|
||||||
|
"encoding/json/v2"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const NUM_RETRIES = 3
|
||||||
|
const TIMEOUT = 10
|
||||||
|
|
||||||
|
type RangeDataPrefix struct {
|
||||||
|
IPv4Prefix string `json:"ipv4Prefix"`
|
||||||
|
IPv6Prefix string `json:"ipv6Prefix"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type RangeDataFormat struct {
|
||||||
|
CreationTime *time.Time `json:"creationTime"`
|
||||||
|
Prefixes []RangeDataPrefix `json:"prefixes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ipRangeData struct {
|
||||||
|
m sync.Mutex
|
||||||
|
c *sync.Cond
|
||||||
|
loaded bool
|
||||||
|
loadError error
|
||||||
|
creation *time.Time
|
||||||
|
cidrs []string
|
||||||
|
}
|
||||||
|
|
||||||
|
var ipRangeStore map[string]*ipRangeData
|
||||||
|
|
||||||
|
func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
|
||||||
|
rd.m.Lock()
|
||||||
|
defer rd.m.Unlock()
|
||||||
|
for counter := NUM_RETRIES; counter > 0; counter-- {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
rd.loadError = ctx.Err()
|
||||||
|
break
|
||||||
|
default:
|
||||||
|
client := &http.Client{Timeout: TIMEOUT * time.Second}
|
||||||
|
resp, err := client.Get(url)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
resp.Body.Close()
|
||||||
|
rd.loadError = ctx.Err()
|
||||||
|
break
|
||||||
|
default:
|
||||||
|
if err == nil {
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode == http.StatusOK {
|
||||||
|
decoder := jsontext.NewDecoder(resp.Body)
|
||||||
|
var parsed RangeDataFormat
|
||||||
|
err = json.UnmarshalDecode(decoder, &parsed)
|
||||||
|
if err == nil {
|
||||||
|
lx := 0
|
||||||
|
if parsed.Prefixes != nil {
|
||||||
|
lx = len(parsed.Prefixes)
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
rd.loadError = err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
rd.loadError = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rd.loaded = true
|
||||||
|
rd.c.Signal()
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadIPRanges(urls []string, ctxname any) {
|
||||||
|
ipRangeStore = make(map[string]*ipRangeData)
|
||||||
|
ctx, cancelfunc := context.WithCancel(context.Background())
|
||||||
|
ctx = context.WithValue(ctx, ctxname, "loadIPRanges")
|
||||||
|
for _, url := range urls {
|
||||||
|
rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil})
|
||||||
|
rdat.c = sync.NewCond(&(rdat.m))
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user