Compare commits
6
Commits
main
...
agentclass
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
26fc9ca3e7 | ||
|
|
8c7c7c9f91 | ||
|
|
0d576e4b55 | ||
|
|
d239f39f9a | ||
|
|
4078d62efc | ||
|
|
b20a2a4f6c |
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,644 @@
|
||||
/*
|
||||
* Amsterdam Web Communities System
|
||||
* Copyright (c) 2025-2026 Erbosoft Metaverse Design Solutions, All Rights Reserved
|
||||
*
|
||||
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
*
|
||||
* SPDX-License-Identifier: MPL-2.0
|
||||
*/
|
||||
/* This file contains code adapted from Mintlify agent-traffic, https://github.com/mintlify/agent-traffic
|
||||
* ---
|
||||
* Copyright (c) 2026 Mintlify, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
// Package agentclass contains support for agent request classification.
|
||||
package agentclass
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json/v2"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Constants reflecting confidence levels.
|
||||
const (
|
||||
ConfidenceHigh = iota // high confidence
|
||||
ConfidenceMedium // medium confidence
|
||||
ConfidenceLow // low confidence
|
||||
)
|
||||
|
||||
// Classification is the return value from AmClassify, showing what a particular HTTP request is classified as.
|
||||
type Classification struct {
|
||||
Category string // category of requester: agent, ai-crawler, crawler, bot, human, unknown
|
||||
Kind string // what kind of requester is it?
|
||||
Id string // actual identifier
|
||||
Name string // name of requester
|
||||
Operator string // what organization operates this requester?
|
||||
Version string // version of requester, where applicable
|
||||
Verified bool // true if all this is verified, false if not
|
||||
Confidence int // confidence level (above)
|
||||
Reasons []string // why did we classify the request this way?
|
||||
}
|
||||
|
||||
// RawAgentDef is the agent definition data loaded from the agents.json file.
|
||||
type RawAgentDef struct {
|
||||
Id string `json:"id"` // agent ID
|
||||
Name string `json:"name"` // agent name
|
||||
Operator string `json:"operator"` // agent operating organization
|
||||
Category string `json:"category"` // category of agent
|
||||
Kind string `json:"kind"` // kind (sub-category) of agent
|
||||
RunsOn string `json:"runsOn"` // does it run on a user machine or an operator machine?
|
||||
UserAgents []string `json:"userAgents"` // regexps to recognize its User-Agent
|
||||
Accept []string `json:"accept"` // Accept: string(s) the agent sends
|
||||
SignatureAgent []string `json:"signatureAgent"` // what agent(s) we can find in a request signature
|
||||
Verify struct {
|
||||
IpRanges []string `json:"ipRanges"` // where can we find what IP ranges this agent uses?
|
||||
ReverseDNS []string `json:"reverseDns"` // what should the agent's IP address resolve to?
|
||||
WebBotAuth string `json:"webBotAuth"` // where can I find the authentication?
|
||||
} `json:"verify"`
|
||||
Docs string `json:"docs"` // URL to documentation
|
||||
Examples []string `json:"examples"` // examples of strings that match
|
||||
Notes string `json:"notes"` // notes
|
||||
LastVerified string `json:"lastVerified"` // when was this information last verified?
|
||||
}
|
||||
|
||||
// agentDef is the definition for a compiled agent.
|
||||
type agentDef struct {
|
||||
r *RawAgentDef // all basic agent information
|
||||
patterns []*regexp.Regexp // compiled user agent expressions
|
||||
}
|
||||
|
||||
// RawClientDef is the client definition data loaded from the http-clients.json file.
|
||||
type RawClientDef struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
UserAgent string `json:"userAgent"`
|
||||
}
|
||||
|
||||
// clientdef is the definition for a compiled client.
|
||||
type clientDef struct {
|
||||
r *RawClientDef
|
||||
pattern *regexp.Regexp
|
||||
}
|
||||
|
||||
//go:embed agents.json
|
||||
var initAgents []byte
|
||||
|
||||
// compiledAgents is the compiled list of agents.
|
||||
var compiledAgents []agentDef
|
||||
|
||||
//go:embed http-clients.json
|
||||
var initClients []byte
|
||||
|
||||
// compiledClients is the compiled list of clients.
|
||||
var compiledClients []clientDef
|
||||
|
||||
// AmInitAgentClass is the initialization function.
|
||||
func AmInitAgentClass(ctxname any) func() {
|
||||
// load the raw agent list
|
||||
var rawAgents []RawAgentDef
|
||||
err := json.Unmarshal(initAgents, &rawAgents)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("agent definition load failure: %v", err))
|
||||
}
|
||||
|
||||
// build the compiled agent list
|
||||
compiledAgents = make([]agentDef, len(rawAgents))
|
||||
for i, v := range rawAgents {
|
||||
compiledAgents[i].r = &(rawAgents[i])
|
||||
if v.UserAgents != nil {
|
||||
compiledAgents[i].patterns = make([]*regexp.Regexp, len(v.UserAgents))
|
||||
for j, p := range v.UserAgents {
|
||||
compiledAgents[i].patterns[j] = regexp.MustCompile("(?i)" + p)
|
||||
}
|
||||
} else {
|
||||
compiledAgents[i].patterns = make([]*regexp.Regexp, 0)
|
||||
}
|
||||
}
|
||||
|
||||
// load the raw clients list
|
||||
var rawClients []RawClientDef
|
||||
err = json.Unmarshal(initClients, &rawClients)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("client definition load failure: %v", err))
|
||||
}
|
||||
|
||||
// build the compiled client list
|
||||
compiledClients = make([]clientDef, len(rawClients))
|
||||
for i, v := range rawClients {
|
||||
compiledClients[i].r = &(rawClients[i])
|
||||
compiledClients[i].pattern = regexp.MustCompile("(?i)" + v.UserAgent)
|
||||
}
|
||||
|
||||
// get the list of IP range files
|
||||
urls := make(map[string]struct{})
|
||||
for _, ra := range rawAgents {
|
||||
if ra.Verify.IpRanges != nil {
|
||||
for _, url := range ra.Verify.IpRanges {
|
||||
urls[url] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
return loadIPRanges(slices.Collect(maps.Keys(urls)), ctxname)
|
||||
}
|
||||
|
||||
type ClassifyOptions struct {
|
||||
MaxAge int // maximum age of IP range files
|
||||
Now *time.Time // "now" timestamp (for testing)
|
||||
HeadersComplete bool // look at complete headers?
|
||||
}
|
||||
|
||||
func AmClassifyOptions() ClassifyOptions {
|
||||
return ClassifyOptions{
|
||||
MaxAge: 180,
|
||||
Now: nil,
|
||||
HeadersComplete: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ClassifyOptions) WithMaxAge(maxAge int) *ClassifyOptions {
|
||||
o.MaxAge = maxAge
|
||||
return o
|
||||
}
|
||||
|
||||
func (o *ClassifyOptions) WithNow(now *time.Time) *ClassifyOptions {
|
||||
o.Now = now
|
||||
return o
|
||||
}
|
||||
|
||||
func (o *ClassifyOptions) WithHeadersComplete(flag bool) *ClassifyOptions {
|
||||
o.HeadersComplete = flag
|
||||
return o
|
||||
}
|
||||
|
||||
// AgentMatch is the return from findAgent.
|
||||
type AgentMatch struct {
|
||||
Agent *RawAgentDef // pointer to raw agent definition
|
||||
Version string // version of agent, where applicable
|
||||
Reason string // reason we matched this agent
|
||||
Weak bool // is this a weak match?
|
||||
}
|
||||
|
||||
var textBrowserExpr = regexp.MustCompile(`^(Lynx|ELinks|Links|w3m)\/`)
|
||||
var browserExpr = regexp.MustCompile(`^Mozilla\/[4-6]\.0 \(|^Opera\/\d`)
|
||||
var browserEngineExpr = regexp.MustCompile(`\(KHTML,? ?like Gecko|KHTML\/[\d.]+ \(like Gecko|Gecko\/\d|\) like Gecko|Trident\/\d|MSIE \d|Presto\/\d|Chromium\/\d`)
|
||||
var botWordExpr = regexp.MustCompile(`(?:[a-z0-9-]|\b)[Bb]ot(?:\b|\/)`)
|
||||
var emailExpr = regexp.MustCompile(`(?i)[\w.+-]+@[\w-]+(\.[\w-]+)*\.[a-z]{2,}`)
|
||||
var automationExpr = regexp.MustCompile(`(?i)crawl|spider|scrap|headless|https?:\/\/|\+http|(?:^|[^a-z])(?:archive|observ|monitor|check(?:er)?\b|preview|verif|render|insight|console|transcod|scanner|synthetic)|lighthouse|selenium|cypress|webdriver|\bsplash\b|gomezagent|rigor|speedcurve|\bptst\b|dareboost|dejaclick|favicon|siteimprove|onetrust|datanyze|feeder\.co|rss\b|weblight|duplexweb|ips-agent`)
|
||||
var compatibleExpr = regexp.MustCompile(`compatible;`)
|
||||
var enginesExpr = regexp.MustCompile(`MSIE|Trident|Konqueror`)
|
||||
|
||||
// isBrowser retruns true if the user agent indicates a browser.
|
||||
func isBrowser(ua string) bool {
|
||||
if textBrowserExpr.MatchString(ua) {
|
||||
return true
|
||||
}
|
||||
if !browserExpr.MatchString(ua) || !browserEngineExpr.MatchString(ua) {
|
||||
return false
|
||||
}
|
||||
if botWordExpr.MatchString(ua) || emailExpr.MatchString(ua) || automationExpr.MatchString(ua) {
|
||||
return false
|
||||
}
|
||||
return !compatibleExpr.MatchString(ua) || enginesExpr.MatchString(ua)
|
||||
}
|
||||
|
||||
// normalizeHeaders joins header values and smashes case on header names.
|
||||
func normalizeHeaders(headers http.Header) map[string]string {
|
||||
rc := make(map[string]string)
|
||||
if headers == nil {
|
||||
return rc
|
||||
}
|
||||
for k, v := range headers {
|
||||
rc[strings.ToLower(k)] = strings.Join(v, ", ")
|
||||
}
|
||||
return rc
|
||||
}
|
||||
|
||||
// rankedAcceptData is the type of data returned from rankAccept.
|
||||
type rankedAcceptData struct {
|
||||
mtype string // MIME type
|
||||
q1000 int // quality value multiplied by 1000
|
||||
ndx int // original index in Accept string
|
||||
}
|
||||
|
||||
// rankAccept turns an Accept string into a list of ranked accept members.
|
||||
func rankAccept(accept string) []rankedAcceptData {
|
||||
components := strings.Split(accept, ",")
|
||||
ranks := make([]rankedAcceptData, len(components))
|
||||
for i, com := range components {
|
||||
subparts := strings.Split(com, ";")
|
||||
ranks[i].mtype = subparts[0]
|
||||
ranks[i].q1000 = 1000
|
||||
ranks[i].ndx = i
|
||||
if len(subparts) > 1 {
|
||||
for _, part := range subparts[1:] {
|
||||
part = strings.TrimSpace(part)
|
||||
if strings.HasPrefix(part, "q=") {
|
||||
parsed, err := strconv.ParseFloat(part[2:], 64)
|
||||
if err != nil {
|
||||
ranks[i].q1000 = int(math.Floor(1000.0 * parsed))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ranks2 := make([]rankedAcceptData, 0, len(ranks))
|
||||
for i := range ranks {
|
||||
if ranks[i].q1000 > 0 {
|
||||
ranks2 = append(ranks2, ranks[i])
|
||||
}
|
||||
}
|
||||
slices.SortFunc(ranks2, func(a, b rankedAcceptData) int {
|
||||
t := b.q1000 - a.q1000
|
||||
if t == 0 {
|
||||
t = b.ndx - a.ndx
|
||||
}
|
||||
return t
|
||||
})
|
||||
return ranks2
|
||||
}
|
||||
|
||||
// ranksAboveHtml returns true if any of the given MIME types ranks above HTML in the given Accept: string.
|
||||
func ranksAboveHtml(accept string, types []string) bool {
|
||||
ranked := rankAccept(accept)
|
||||
md := slices.IndexFunc(ranked, func(d rankedAcceptData) bool {
|
||||
return slices.Contains(types, d.mtype)
|
||||
})
|
||||
if md < 0 {
|
||||
return false
|
||||
}
|
||||
html := slices.IndexFunc(ranked, func(d rankedAcceptData) bool {
|
||||
return d.mtype == "text/html"
|
||||
})
|
||||
if html < 0 {
|
||||
return true
|
||||
}
|
||||
return md < html
|
||||
}
|
||||
|
||||
// markdownTypes are the MIME types associated with Markdown.
|
||||
var markdownTypes = []string{"text/markdown", "text/x-markdown"}
|
||||
|
||||
// prefersMarkdown returns true if the given Accept: string prefers Markdown over HTML.
|
||||
func prefersMarkdown(accept string) bool {
|
||||
return ranksAboveHtml(accept, markdownTypes)
|
||||
}
|
||||
|
||||
// normalizeAcceptExpr is how we close the gaps around delimiters in an Accept: expression.
|
||||
var normalizeAcceptExpr = regexp.MustCompile(`\s*([,;=])\s*`)
|
||||
|
||||
// normalizeAccept squeezes the whitespace out of an Accept: expression.
|
||||
func normalizeAccept(accept string) string {
|
||||
return strings.TrimSpace(normalizeAcceptExpr.ReplaceAllString(strings.ToLower(accept), "$1"))
|
||||
}
|
||||
|
||||
// signatureAgentHostExpr recognizes the "host" in a URL.
|
||||
var signatureAgentHostExpr = regexp.MustCompile(`(?i)https?:\/\/([^/"\s]+)`)
|
||||
|
||||
// signatureAgentHostBare1 is used to trim up a bare host name.
|
||||
var signatureAgentHostBare1 = regexp.MustCompile(`^[\w-]+=`)
|
||||
|
||||
// signatureAgentHostBare2 is used to trim up a bare host name.
|
||||
var signatureAgentHostBare2 = regexp.MustCompile(`"`)
|
||||
|
||||
// signatureAgentHost locates the host associated with a request signature.
|
||||
func signatureAgentHost(value string) *string {
|
||||
m := signatureAgentHostExpr.FindStringSubmatch(value)
|
||||
if m != nil {
|
||||
return new(strings.ToLower(m[1]))
|
||||
}
|
||||
bare := strings.TrimSpace(signatureAgentHostBare2.ReplaceAllString(signatureAgentHostBare1.ReplaceAllString(value, ""), ""))
|
||||
if bare != "" {
|
||||
return new(strings.ToLower(bare))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// findAgent matches the request against the list of compiled agents to locate a match. Returns nil on failure.
|
||||
func findAgent(ua string, h map[string]string) *AgentMatch {
|
||||
for _, ca := range compiledAgents {
|
||||
for _, p := range ca.patterns {
|
||||
m := p.FindStringSubmatch(ua)
|
||||
if m != nil {
|
||||
ndx := p.SubexpIndex("version")
|
||||
vers := ""
|
||||
if ndx > 0 {
|
||||
vers = m[ndx]
|
||||
}
|
||||
return new(AgentMatch{Agent: ca.r, Version: vers, Reason: fmt.Sprintf("user agent matches %s", ca.r.Name), Weak: false})
|
||||
}
|
||||
}
|
||||
}
|
||||
accept, ok := h["accept"]
|
||||
if ok {
|
||||
accept = normalizeAccept(accept)
|
||||
p := slices.IndexFunc(compiledAgents[:], func(ad agentDef) bool {
|
||||
if ad.r.Accept == nil {
|
||||
return false
|
||||
}
|
||||
for _, t := range ad.r.Accept {
|
||||
if normalizeAccept(t) == accept {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
})
|
||||
if p >= 0 {
|
||||
rc := compiledAgents[p]
|
||||
return new(AgentMatch{Agent: rc.r, Version: "", Reason: fmt.Sprintf("Accept header matches %s's fingerprint", rc.r.Name), Weak: false})
|
||||
}
|
||||
}
|
||||
_, isSigned := h["signed"]
|
||||
if isSigned {
|
||||
_, isSigned = h["signature-input"]
|
||||
}
|
||||
if isSigned {
|
||||
sig, ok := h["signature-agent"]
|
||||
if ok {
|
||||
psig := signatureAgentHost(sig)
|
||||
if psig != nil {
|
||||
p := slices.IndexFunc(compiledAgents[:], func(ad agentDef) bool {
|
||||
if ad.r.SignatureAgent == nil {
|
||||
return false
|
||||
}
|
||||
return slices.Index(ad.r.SignatureAgent[:], *psig) >= 0
|
||||
})
|
||||
if p >= 0 {
|
||||
rc := compiledAgents[p]
|
||||
return new(AgentMatch{Agent: rc.r, Version: "",
|
||||
Reason: fmt.Sprintf("signed request names %s in Signature-Agent; the signature itself is not verified", *psig),
|
||||
Weak: true})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fromAgent turns an AgentMatch into a Classification.
|
||||
func fromAgent(match *AgentMatch, rawIP string, opts ClassifyOptions) *Classification {
|
||||
agent := match.Agent
|
||||
base := new(Classification{
|
||||
Category: agent.Category,
|
||||
Kind: agent.Kind,
|
||||
Name: agent.Name,
|
||||
Operator: agent.Operator,
|
||||
Version: match.Version,
|
||||
Verified: true,
|
||||
Confidence: ConfidenceMedium,
|
||||
Reasons: make([]string, 1),
|
||||
})
|
||||
base.Reasons[0] = match.Reason
|
||||
if agent.RunsOn == "user" {
|
||||
base.Reasons = append(base.Reasons, fmt.Sprintf("%s runs on the user's machine, so IP verification does not apply", agent.Name))
|
||||
return base
|
||||
}
|
||||
ip := normalizeIP(rawIP)
|
||||
if ip == nil {
|
||||
base.Reasons = append(base.Reasons, fmt.Sprintf("%s is not a single valid address, so it was not checked", rawIP))
|
||||
return base
|
||||
}
|
||||
urls := make([]string, 0)
|
||||
if agent.Verify.IpRanges != nil {
|
||||
urls = append(urls, agent.Verify.IpRanges...)
|
||||
}
|
||||
now := opts.Now
|
||||
if now == nil {
|
||||
tmp := time.Now()
|
||||
now = &tmp
|
||||
}
|
||||
maxDuration := time.Hour * time.Duration(24*opts.MaxAge)
|
||||
impersonation := false
|
||||
validRangeFiles := false
|
||||
for _, url := range urls {
|
||||
cidrs, creation, err := getIPRangeData(url)
|
||||
if err != nil {
|
||||
base.Reasons = append(base.Reasons, fmt.Sprintf("unable to get range file %s: %v", url, err))
|
||||
continue
|
||||
}
|
||||
validRangeFiles = true
|
||||
if inRanges(*ip, cidrs) {
|
||||
base.Verified = true
|
||||
base.Confidence = ConfidenceHigh
|
||||
base.Reasons = append(base.Reasons, "%s is in %s's published ranges", *ip, agent.Operator)
|
||||
return base
|
||||
}
|
||||
if now.Sub(*creation) <= maxDuration {
|
||||
impersonation = true
|
||||
}
|
||||
}
|
||||
if !validRangeFiles {
|
||||
return base
|
||||
}
|
||||
if impersonation {
|
||||
base.Category = "bot"
|
||||
base.Kind = "impersonator"
|
||||
base.Verified = false
|
||||
base.Confidence = ConfidenceHigh
|
||||
base.Reasons = append(base.Reasons, fmt.Sprintf("claims to be %s but %s is outside %s's published ranges", agent.Name, *ip, agent.Operator))
|
||||
return base
|
||||
}
|
||||
base.Reasons = append(base.Reasons, fmt.Sprintf("%s's range file is older than %d days, so a mismatch is not counted as spoofing", agent.Operator, opts.MaxAge))
|
||||
return base
|
||||
}
|
||||
|
||||
// fake returns a Classification identifying the request as a fake browser.
|
||||
func fake(reason string, confidence int) *Classification {
|
||||
rc := new(Classification{
|
||||
Category: "bot",
|
||||
Kind: "fake-browser",
|
||||
Verified: false,
|
||||
Confidence: confidence,
|
||||
Reasons: make([]string, 1),
|
||||
})
|
||||
rc.Reasons[0] = reason
|
||||
return rc
|
||||
}
|
||||
|
||||
// majorVersion extracts a browser's major version from a regular expression.
|
||||
func majorVersion(ua string, re *regexp.Regexp) int {
|
||||
m := re.FindStringSubmatch(ua)
|
||||
if m != nil {
|
||||
v, err := strconv.Atoi(m[1])
|
||||
if err == nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var firefoxBrowserExpr = regexp.MustCompile(`Firefox\/`)
|
||||
var firefoxVersionExpr = regexp.MustCompile(`Firefox\/(\d+)`)
|
||||
var chromeBrowserExpr = regexp.MustCompile(`Chrome\/`)
|
||||
var chromeVersionExpr = regexp.MustCompile(`Chrome\/(\d+)`)
|
||||
var safariBrowserExpr = regexp.MustCompile(`Version\/[\d.]+.*Safari\/`)
|
||||
var safariVersionExpr = regexp.MustCompile(`Version\/(\d+)`)
|
||||
|
||||
func sendsFetchMetadata(ua string) bool {
|
||||
if firefoxBrowserExpr.MatchString(ua) {
|
||||
return majorVersion(ua, firefoxVersionExpr) >= 90
|
||||
}
|
||||
if chromeBrowserExpr.MatchString(ua) {
|
||||
return majorVersion(ua, chromeVersionExpr) >= 80
|
||||
}
|
||||
if safariBrowserExpr.MatchString(ua) {
|
||||
return majorVersion(ua, safariVersionExpr) >= 17
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var plainTextType = []string{"text/plain"}
|
||||
|
||||
func fromBrowser(ua string, h map[string]string, opts ClassifyOptions) *Classification {
|
||||
accept, ok := h["accept"]
|
||||
if ok {
|
||||
if ranksAboveHtml(accept, plainTextType) {
|
||||
return fake("Accept header ranks text/plain above HTML, which browsers never do", ConfidenceLow)
|
||||
}
|
||||
if strings.Contains(accept, "application/metalink") {
|
||||
return fake("Accept header asks for metalink, which download tools send and browsers do not", ConfidenceMedium)
|
||||
}
|
||||
}
|
||||
if !opts.HeadersComplete {
|
||||
rc := new(Classification{
|
||||
Category: "human",
|
||||
Verified: false,
|
||||
Confidence: ConfidenceLow,
|
||||
Reasons: make([]string, 1),
|
||||
})
|
||||
rc.Reasons[0] = "browser user agent; headers not checked"
|
||||
return rc
|
||||
}
|
||||
_, ok = h["sec-ch-ua"]
|
||||
if ok && !chromeBrowserExpr.MatchString(ua) {
|
||||
return fake("sec-ch-ua sent with a non-Chromium user agent", ConfidenceHigh)
|
||||
}
|
||||
_, ok = h["sec-fetch-mode"]
|
||||
if !ok && sendsFetchMetadata(ua) {
|
||||
return fake("browser user agent without sec-fetch-* headers", ConfidenceMedium)
|
||||
}
|
||||
_, ok = h["accept-language"]
|
||||
if !ok {
|
||||
return fake("browser user agent without Accept-Language", ConfidenceLow)
|
||||
}
|
||||
rc := new(Classification{
|
||||
Category: "human",
|
||||
Verified: false,
|
||||
Confidence: ConfidenceMedium,
|
||||
Reasons: make([]string, 1),
|
||||
})
|
||||
rc.Reasons[0] = "browser user agent with consistent browser headers"
|
||||
return rc
|
||||
}
|
||||
|
||||
func AmClassify(headers http.Header, ipAddr string, userAgent string, opts ...ClassifyOptions) *Classification {
|
||||
myopts := AmClassifyOptions()
|
||||
if len(opts) > 0 {
|
||||
myopts = opts[0]
|
||||
}
|
||||
h := normalizeHeaders(headers)
|
||||
ua := userAgent
|
||||
if ua == "" {
|
||||
t, ok := h["user-agent"]
|
||||
if ok {
|
||||
ua = t
|
||||
}
|
||||
}
|
||||
|
||||
if ua == "" {
|
||||
rc := new(Classification{
|
||||
Verified: false,
|
||||
Confidence: ConfidenceLow,
|
||||
Reasons: make([]string, 1),
|
||||
})
|
||||
if len(h) == 0 {
|
||||
rc.Category = "unknown"
|
||||
rc.Reasons[0] = "no user agent or headers to classify"
|
||||
} else {
|
||||
rc.Category = "bot"
|
||||
rc.Kind = "unidentified"
|
||||
rc.Reasons[0] = "request has no user agent, which browsers always send"
|
||||
}
|
||||
return rc
|
||||
}
|
||||
|
||||
// Look for a matching agent.
|
||||
match := findAgent(ua, h)
|
||||
if match != nil {
|
||||
return fromAgent(match, ipAddr, myopts)
|
||||
}
|
||||
|
||||
// Check the Accept: line; if the request prefers Markdown to HTML, we're some kind of agent.
|
||||
acc, ok := h["accept"]
|
||||
if ok {
|
||||
if prefersMarkdown(acc) {
|
||||
rc := new(Classification{
|
||||
Category: "agent",
|
||||
Kind: "unidentified",
|
||||
Verified: false,
|
||||
Confidence: ConfidenceMedium,
|
||||
Reasons: make([]string, 1),
|
||||
})
|
||||
rc.Reasons[0] = "Accept header prefers markdown over HTML, which browsers never do"
|
||||
return rc
|
||||
}
|
||||
}
|
||||
|
||||
// Look for an HTTP client.
|
||||
ndx := slices.IndexFunc(compiledClients, func(d clientDef) bool {
|
||||
return d.pattern.MatchString(ua)
|
||||
})
|
||||
if ndx >= 0 {
|
||||
rc := new(Classification{
|
||||
Category: "bot",
|
||||
Kind: compiledClients[ndx].r.Kind,
|
||||
Name: compiledClients[ndx].r.Name,
|
||||
Verified: false,
|
||||
Confidence: ConfidenceMedium,
|
||||
Reasons: make([]string, 1),
|
||||
})
|
||||
rc.Reasons[0] = fmt.Sprintf("user agent is %s", compiledClients[ndx].r.Name)
|
||||
return rc
|
||||
}
|
||||
if isBrowser(ua) {
|
||||
return fromBrowser(ua, h, myopts)
|
||||
}
|
||||
rc := new(Classification{
|
||||
Category: "bot",
|
||||
Kind: "unidentified",
|
||||
Verified: false,
|
||||
Confidence: ConfidenceLow,
|
||||
Reasons: make([]string, 1),
|
||||
})
|
||||
rc.Reasons[0] = "user agent is not a browser and matches no known agent, crawler or client"
|
||||
return rc
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
[
|
||||
{ "name": "axios", "kind": "http-client", "userAgent": "^axios/" },
|
||||
{ "name": "curl", "kind": "http-client", "userAgent": "^curl/" },
|
||||
{ "name": "Wget", "kind": "http-client", "userAgent": "^Wget/" },
|
||||
{ "name": "Node.js", "kind": "http-client", "userAgent": "^(node|undici|node-fetch)(/|$)" },
|
||||
{ "name": "Bun", "kind": "http-client", "userAgent": "^Bun/" },
|
||||
{ "name": "Deno", "kind": "http-client", "userAgent": "^Deno/" },
|
||||
{ "name": "Python requests", "kind": "http-client", "userAgent": "^python-requests/" },
|
||||
{ "name": "Python httpx", "kind": "http-client", "userAgent": "^python-httpx/" },
|
||||
{ "name": "Python urllib", "kind": "http-client", "userAgent": "^Python-urllib/" },
|
||||
{ "name": "Python aiohttp", "kind": "http-client", "userAgent": "^Python/[\\d.]+ aiohttp/" },
|
||||
{ "name": "Go net/http", "kind": "http-client", "userAgent": "^Go-http-client/" },
|
||||
{ "name": "OkHttp", "kind": "http-client", "userAgent": "^okhttp/" },
|
||||
{ "name": "Java", "kind": "http-client", "userAgent": "^(Java/|Apache-HttpClient/)" },
|
||||
{ "name": "Guzzle", "kind": "http-client", "userAgent": "^GuzzleHttp/" },
|
||||
{ "name": "Rust reqwest", "kind": "http-client", "userAgent": "^reqwest/" },
|
||||
{ "name": "lua-resty-http", "kind": "http-client", "userAgent": "^lua-resty-http/" },
|
||||
{ "name": "Colly", "kind": "http-client", "userAgent": "^colly\\b" },
|
||||
{ "name": "Scrapy", "kind": "http-client", "userAgent": "\\bScrapy/" },
|
||||
{ "name": "n8n", "kind": "http-client", "userAgent": "^n8n$" },
|
||||
{ "name": "Postman", "kind": "http-client", "userAgent": "^PostmanRuntime/" },
|
||||
{ "name": "Headless Chrome", "kind": "headless", "userAgent": "\\bHeadlessChrome/" },
|
||||
{ "name": "PhantomJS", "kind": "headless", "userAgent": "\\bPhantomJS/" },
|
||||
{ "name": "Playwright", "kind": "headless", "userAgent": "\\bPlaywright/" },
|
||||
{ "name": "PowerShell", "kind": "http-client", "userAgent": "PowerShell/" },
|
||||
{ "name": "Ruby", "kind": "http-client", "userAgent": "^Ruby$" },
|
||||
{ "name": "got", "kind": "http-client", "userAgent": "^got\\b" },
|
||||
{ "name": "libwww-perl", "kind": "http-client", "userAgent": "\\blibwww-perl/" },
|
||||
{ "name": "aria2", "kind": "http-client", "userAgent": "^aria2/" },
|
||||
{ "name": "HTTrack", "kind": "http-client", "userAgent": "\\bHTTrack\\b" },
|
||||
{ "name": "Crawl4AI", "kind": "http-client", "userAgent": "^crawl4ai" },
|
||||
{ "name": "Apple CFNetwork (native app)", "kind": "http-client", "userAgent": "\\bCFNetwork/" },
|
||||
{ "name": "Android Dalvik (native app)", "kind": "http-client", "userAgent": "\\bDalvik/" },
|
||||
{ "name": "Amazon CloudFront", "kind": "http-client", "userAgent": "^Amazon CloudFront$" },
|
||||
{ "name": "Vercel Edge Functions", "kind": "http-client", "userAgent": "^Vercel Edge Functions$" },
|
||||
{ "name": "Google Cloud Scheduler and Tasks", "kind": "http-client", "userAgent": "^Google-Cloud-(Scheduler|Tasks)$" }
|
||||
]
|
||||
@@ -0,0 +1,94 @@
|
||||
/*
|
||||
* Amsterdam Web Communities System
|
||||
* Copyright (c) 2025-2026 Erbosoft Metaverse Design Solutions, All Rights Reserved
|
||||
*
|
||||
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
*
|
||||
* SPDX-License-Identifier: MPL-2.0
|
||||
*/
|
||||
/* This file contains code adapted from Mintlify agent-traffic, https://github.com/mintlify/agent-traffic
|
||||
* ---
|
||||
* Copyright (c) 2026 Mintlify, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
// Package agentclass contains support for agent request classification.
|
||||
package agentclass
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// parseIP turns a string IP address into a ParsedIP. Returns nil if address is not parseable.
|
||||
func parseIP(ip string) *netip.Addr {
|
||||
s := strings.TrimSpace(ip)
|
||||
v6 := strings.Contains(s, ":")
|
||||
addy, err := netip.ParseAddr(s)
|
||||
if err != nil || addy.Is6() != v6 {
|
||||
return nil
|
||||
}
|
||||
return &addy
|
||||
}
|
||||
|
||||
// bracketedExpr recognizes an IP address with brackets.
|
||||
var bracketedExpr = regexp.MustCompile(`^\[([^\]]+)\](?::\d+)?$`)
|
||||
|
||||
// v4WithPortExpr recognizes an IPv4 address with a port.
|
||||
var v4WithPortExpr = regexp.MustCompile(`^(\d{1,3}(?:\.\d{1,3}){3}):\d+$`)
|
||||
|
||||
// normalizeIP filters a supplied IP address. Returns nil if the address is not parseable.
|
||||
func normalizeIP(ip string) *string {
|
||||
s := strings.TrimSpace(ip)
|
||||
if s == "" || strings.Contains(s, ",") {
|
||||
return nil
|
||||
}
|
||||
bracketed := bracketedExpr.FindStringSubmatch(s)
|
||||
v4WithPort := v4WithPortExpr.FindStringSubmatch(s)
|
||||
candidate := s
|
||||
if bracketed != nil {
|
||||
candidate = bracketed[1]
|
||||
} else if v4WithPort != nil {
|
||||
candidate = v4WithPort[1]
|
||||
}
|
||||
parsed := parseIP(candidate)
|
||||
if parsed == nil {
|
||||
return nil
|
||||
}
|
||||
if parsed.Is4In6() {
|
||||
return new(netip.AddrFrom4(parsed.As4()).String())
|
||||
}
|
||||
return new(candidate)
|
||||
}
|
||||
|
||||
func inRanges(ip string, cidrs []netip.Prefix) bool {
|
||||
normalized := normalizeIP(ip)
|
||||
if normalized == nil {
|
||||
return false
|
||||
}
|
||||
addy := parseIP(*normalized)
|
||||
return slices.IndexFunc(cidrs, func(c netip.Prefix) bool {
|
||||
return c.Contains(*addy)
|
||||
}) >= 0
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
/*
|
||||
* Amsterdam Web Communities System
|
||||
* Copyright (c) 2025-2026 Erbosoft Metaverse Design Solutions, All Rights Reserved
|
||||
*
|
||||
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
*
|
||||
* SPDX-License-Identifier: MPL-2.0
|
||||
*/
|
||||
/* This file contains code adapted from Mintlify agent-traffic, https://github.com/mintlify/agent-traffic
|
||||
* ---
|
||||
* Copyright (c) 2026 Mintlify, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
// Package agentclass contains support for agent request classification.
|
||||
package agentclass
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
"encoding/json/v2"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.erbosoft.com/amy/amsterdam/config"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Format for IP range data inside the RangeDataFormat.
|
||||
type RangeDataPrefix struct {
|
||||
IPv4Prefix string `json:"ipv4Prefix"` // prefix, in IPv4 format
|
||||
IPv6Prefix string `json:"ipv6Prefix"` // prefix, in IPv6 format
|
||||
}
|
||||
|
||||
// Format of IP range data files exported by bot providers.
|
||||
type RangeDataFormat struct {
|
||||
CreationTime *time.Time `json:"creationTime"` // creation date and time
|
||||
Prefixes []RangeDataPrefix `json:"prefixes"` // the prefixes
|
||||
}
|
||||
|
||||
// Stored IP range data for a single URL.
|
||||
type ipRangeData struct {
|
||||
m sync.Mutex // mutex on the range data
|
||||
c *sync.Cond // signals when load complete
|
||||
loaded bool // set when load operation completed
|
||||
loadError error // error during load, or nil
|
||||
lastMod string // Last-Modified header from last fetch
|
||||
creation *time.Time // creation date and time
|
||||
cidrs []netip.Prefix // the prefixes from this file
|
||||
}
|
||||
|
||||
// Contains all the data for loaded IP range files.
|
||||
var ipRangeStore map[string]*ipRangeData
|
||||
|
||||
// loadIPRangeData is the goroutine that fetches range data from a URL and parses it.
|
||||
func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
|
||||
rd.m.Lock()
|
||||
defer rd.m.Unlock()
|
||||
LOADLOOP:
|
||||
for counter := config.GlobalConfig.Tuning.IPRangeLoadAttempts; counter > 0; counter-- {
|
||||
select {
|
||||
case <-ctx.Done(): // cancellation check on every loop
|
||||
rd.loadError = ctx.Err()
|
||||
break LOADLOOP
|
||||
default:
|
||||
// build the HTTP request
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
rd.loadError = err
|
||||
break LOADLOOP
|
||||
}
|
||||
if rd.lastMod != "" {
|
||||
req.Header.Set("If-Modified-Since", rd.lastMod)
|
||||
}
|
||||
|
||||
// build a client and use it to load the file
|
||||
client := &http.Client{Timeout: time.Duration(config.GlobalConfig.Tuning.Timeouts.IPRangeLoad) * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
select {
|
||||
case <-ctx.Done(): // we might have been canceled while waiting for the GET
|
||||
resp.Body.Close()
|
||||
rd.loadError = ctx.Err()
|
||||
break LOADLOOP
|
||||
default:
|
||||
// interpret error and status codes
|
||||
if err != nil {
|
||||
rd.loadError = err
|
||||
continue LOADLOOP
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusNotModified {
|
||||
rd.loadError = nil
|
||||
break LOADLOOP
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
|
||||
continue LOADLOOP
|
||||
}
|
||||
|
||||
// load Last-Modified date
|
||||
lastmod := ""
|
||||
if lm, ok := resp.Header["Last-Modified"]; ok {
|
||||
lastmod = lm[0]
|
||||
}
|
||||
|
||||
// parse the JSON data retrieved
|
||||
decoder := jsontext.NewDecoder(resp.Body)
|
||||
var parsed RangeDataFormat
|
||||
err = json.UnmarshalDecode(decoder, &parsed)
|
||||
if err != nil {
|
||||
rd.loadError = err
|
||||
break LOADLOOP
|
||||
}
|
||||
|
||||
// turn the parsed data into our internal data
|
||||
lx := 0
|
||||
if parsed.Prefixes != nil {
|
||||
lx = len(parsed.Prefixes)
|
||||
}
|
||||
rd.cidrs = make([]netip.Prefix, lx)
|
||||
if lx > 0 {
|
||||
for i, p := range parsed.Prefixes {
|
||||
if p.IPv4Prefix == "" {
|
||||
rd.cidrs[i], err = netip.ParsePrefix(p.IPv6Prefix)
|
||||
} else {
|
||||
rd.cidrs[i], err = netip.ParsePrefix(p.IPv4Prefix)
|
||||
}
|
||||
if err != nil {
|
||||
rd.loadError = err
|
||||
break LOADLOOP
|
||||
}
|
||||
}
|
||||
}
|
||||
rd.loadError = nil
|
||||
rd.lastMod = lastmod
|
||||
rd.creation = parsed.CreationTime
|
||||
break LOADLOOP
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done(): // one more cancel check before ending
|
||||
if rd.loadError == nil {
|
||||
rd.loadError = ctx.Err()
|
||||
}
|
||||
default:
|
||||
// fall through
|
||||
}
|
||||
rd.loaded = true // finished loading, let another routine go
|
||||
rd.c.Signal()
|
||||
log.Debugf("loaded IP range document at %s", url)
|
||||
}
|
||||
|
||||
// prepareRefresh prepares some ipRangeData for being refreshed.
|
||||
func prepareRefresh(rd *ipRangeData) {
|
||||
rd.m.Lock()
|
||||
defer rd.m.Unlock()
|
||||
for !rd.loaded {
|
||||
rd.c.Wait()
|
||||
}
|
||||
rd.loaded = false
|
||||
}
|
||||
|
||||
// refreshRanges runs periodically to refresh the range data from the various sites.
|
||||
func refreshRanges(ctx context.Context, interval time.Duration) {
|
||||
tick := time.NewTicker(interval)
|
||||
REFRESHLOOP:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
tick.Stop()
|
||||
break REFRESHLOOP
|
||||
case <-tick.C:
|
||||
mymap := make(map[string]*ipRangeData)
|
||||
maps.Copy(mymap, ipRangeStore)
|
||||
for url, rd := range mymap {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
tick.Stop()
|
||||
break REFRESHLOOP
|
||||
default:
|
||||
prepareRefresh(rd)
|
||||
go loadIPRangeData(ctx, url, rd)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loadRangesCtx is the context used for running the range load and refresh data.
|
||||
var loadRangesCtx context.Context = nil
|
||||
|
||||
// loadIPRanges begins loading the initial IP ranges.
|
||||
func loadIPRanges(urls []string, ctxname any) func() {
|
||||
ipRangeStore = make(map[string]*ipRangeData)
|
||||
ctx, cancelfunc := context.WithCancel(context.Background())
|
||||
loadRangesCtx = context.WithValue(ctx, ctxname, "loadIPRanges")
|
||||
for _, url := range urls {
|
||||
rdat := new(ipRangeData{loaded: false, loadError: nil, lastMod: "", creation: nil})
|
||||
rdat.c = sync.NewCond(&(rdat.m))
|
||||
ipRangeStore[url] = rdat
|
||||
go loadIPRangeData(loadRangesCtx, url, rdat)
|
||||
}
|
||||
log.Infof("loadIPRanges: started loading %d IP range files", len(urls))
|
||||
dur, err := time.ParseDuration(config.GlobalConfig.Tuning.IPRangeRefresh)
|
||||
if err != nil {
|
||||
log.Errorf("invalid IP range refresh duration \"%s\", using default", config.GlobalConfig.Tuning.IPRangeRefresh)
|
||||
dur = 24 * time.Hour
|
||||
}
|
||||
go refreshRanges(loadRangesCtx, dur)
|
||||
return func() {
|
||||
cancelfunc()
|
||||
for _, v := range ipRangeStore {
|
||||
v.m.Lock()
|
||||
if !v.loaded {
|
||||
v.loadError = context.Canceled
|
||||
v.c.Signal()
|
||||
}
|
||||
v.m.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getIPRangeData gets the IP range data for a specified URL, which may have been cached already.
|
||||
func getIPRangeData(url string) ([]netip.Prefix, *time.Time, error) {
|
||||
rdat, ok := ipRangeStore[url]
|
||||
if !ok {
|
||||
rdat := new(ipRangeData{loaded: false, loadError: nil, lastMod: "", creation: nil})
|
||||
rdat.c = sync.NewCond(&(rdat.m))
|
||||
ipRangeStore[url] = rdat
|
||||
go loadIPRangeData(loadRangesCtx, url, rdat)
|
||||
}
|
||||
rdat.m.Lock()
|
||||
defer rdat.m.Unlock()
|
||||
for !rdat.loaded {
|
||||
rdat.c.Wait()
|
||||
}
|
||||
return rdat.cidrs, rdat.creation, rdat.loadError
|
||||
}
|
||||
@@ -182,12 +182,15 @@ type AmConfig struct {
|
||||
} `yaml:"posting"`
|
||||
Tuning struct {
|
||||
WorkerTasks int `yaml:"workerTasks"`
|
||||
IPRangeLoadAttempts int `yaml:"ipRangeLoadAttempts"`
|
||||
IPRangeRefresh string `yaml:"ipRangeRefresh"`
|
||||
Timeouts struct {
|
||||
HttpRead int `yaml:"httpRead"`
|
||||
HttpWrite int `yaml:"httpWrite"`
|
||||
HttpIdle int `yaml:"httpIdle"`
|
||||
PageExecute int `yaml:"pageExecute"`
|
||||
PageRender int `yaml:"pageRender"`
|
||||
IPRangeLoad int `yaml:"ipRangeLoad"`
|
||||
} `yaml:"timeouts"`
|
||||
Queues struct {
|
||||
AuditWrites int `yaml:"auditWrites"`
|
||||
|
||||
@@ -102,12 +102,15 @@ posting:
|
||||
- "image/png"
|
||||
tuning:
|
||||
workerTasks: 4
|
||||
ipRangeLoadAttempts: 3
|
||||
ipRangeRefresh: "24h"
|
||||
timeouts:
|
||||
httpRead: 30
|
||||
httpWrite: 30
|
||||
httpIdle: 120
|
||||
pageExecute: 15
|
||||
pageRender: 15
|
||||
ipRangeLoad: 10
|
||||
queues:
|
||||
auditWrites: 16
|
||||
contextRecycle: 16
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.erbosoft.com/amy/amsterdam/agentclass"
|
||||
"git.erbosoft.com/amy/amsterdam/config"
|
||||
"git.erbosoft.com/amy/amsterdam/database"
|
||||
"git.erbosoft.com/amy/amsterdam/email"
|
||||
@@ -58,6 +59,17 @@ func myIPAddress() net.IP {
|
||||
return localAddr.IP
|
||||
}
|
||||
|
||||
// ClassifyInjector injects AmClassify onto the request.
|
||||
func ClassifyInjector(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
opts := agentclass.AmClassifyOptions()
|
||||
opts.HeadersComplete = true
|
||||
return func(c *echo.Context) error {
|
||||
classif := agentclass.AmClassify(c.Request().Header, c.RealIP(), "", opts)
|
||||
c.Set("am.classification", classif)
|
||||
return next(c)
|
||||
}
|
||||
}
|
||||
|
||||
// setupEcho creates, configures, and returns a new Echo instance.
|
||||
func setupEcho() *echo.Echo {
|
||||
e := echo.New()
|
||||
@@ -72,7 +84,7 @@ func setupEcho() *echo.Echo {
|
||||
} else {
|
||||
log.Warn("WARNING: --debug-panic in effect - DO NOT use this in production!")
|
||||
}
|
||||
e.Use(LoggingMiddleware)
|
||||
e.Use(LoggingMiddleware, ClassifyInjector)
|
||||
|
||||
// set up the rate limiter from the configuration
|
||||
rateLimiter := AmSetupRateLimiter()
|
||||
@@ -257,6 +269,8 @@ func main() {
|
||||
closer = email.SetupMailSender()
|
||||
defer closer()
|
||||
htmlcheck.SetupDicts()
|
||||
closer = agentclass.AmInitAgentClass(CONTEXTVAL_NAME)
|
||||
defer closer()
|
||||
closer = ui.SetupUILayer()
|
||||
defer closer()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user