next bit of work on agentclass (including normalizeIP)
This commit is contained in:
+142
-7
@@ -18,9 +18,28 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ConfidenceHigh = iota
|
||||||
|
ConfidenceMedium
|
||||||
|
ConfidenceLow
|
||||||
|
)
|
||||||
|
|
||||||
|
type Classification struct {
|
||||||
|
Category string
|
||||||
|
Kind string
|
||||||
|
Id string
|
||||||
|
Name string
|
||||||
|
Operator string
|
||||||
|
Version string
|
||||||
|
Verified bool
|
||||||
|
Confidence int
|
||||||
|
Reasons []string
|
||||||
|
}
|
||||||
|
|
||||||
// RawAgentDef is the agent definition data loaded from the agents.json file.
|
// RawAgentDef is the agent definition data loaded from the agents.json file.
|
||||||
type RawAgentDef struct {
|
type RawAgentDef struct {
|
||||||
Id string `json:"id"`
|
Id string `json:"id"`
|
||||||
@@ -64,13 +83,24 @@ func AmInitAgentClass() {
|
|||||||
compiledAgents = make([]agentDef, len(rawAgents))
|
compiledAgents = make([]agentDef, len(rawAgents))
|
||||||
for i, v := range rawAgents {
|
for i, v := range rawAgents {
|
||||||
compiledAgents[i].r = &(rawAgents[i])
|
compiledAgents[i].r = &(rawAgents[i])
|
||||||
compiledAgents[i].patterns = make([]*regexp.Regexp, len(v.UserAgents))
|
if v.UserAgents != nil {
|
||||||
for j, p := range v.UserAgents {
|
compiledAgents[i].patterns = make([]*regexp.Regexp, len(v.UserAgents))
|
||||||
compiledAgents[i].patterns[j] = regexp.MustCompile("(?i)" + p)
|
for j, p := range v.UserAgents {
|
||||||
|
compiledAgents[i].patterns[j] = regexp.MustCompile("(?i)" + p)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
compiledAgents[i].patterns = make([]*regexp.Regexp, 0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type AgentMatch struct {
|
||||||
|
Agent *RawAgentDef
|
||||||
|
Version string
|
||||||
|
Reason string
|
||||||
|
Weak bool
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeHeaders(headers http.Header) map[string]string {
|
func normalizeHeaders(headers http.Header) map[string]string {
|
||||||
rc := make(map[string]string)
|
rc := make(map[string]string)
|
||||||
if headers == nil {
|
if headers == nil {
|
||||||
@@ -82,15 +112,117 @@ func normalizeHeaders(headers http.Header) map[string]string {
|
|||||||
return rc
|
return rc
|
||||||
}
|
}
|
||||||
|
|
||||||
func findAgent(ua string, h map[string]string) {
|
var normalizeAcceptExpr = regexp.MustCompile(`\s*([,;=])\s*`)
|
||||||
|
|
||||||
|
func normalizeAccept(accept string) string {
|
||||||
|
return strings.TrimSpace(normalizeAcceptExpr.ReplaceAllString(strings.ToLower(accept), "$1"))
|
||||||
|
}
|
||||||
|
|
||||||
|
var signatureAgentHostExpr = regexp.MustCompile(`(?i)https?:\/\/([^/"\s]+)`)
|
||||||
|
var signatureAgentHostBare1 = regexp.MustCompile(`^[\w-]+=`)
|
||||||
|
var signatureAgentHostBare2 = regexp.MustCompile(`"`)
|
||||||
|
|
||||||
|
func signatureAgentHost(value string) *string {
|
||||||
|
m := signatureAgentHostExpr.FindStringSubmatch(value)
|
||||||
|
if m != nil {
|
||||||
|
return new(strings.ToLower(m[1]))
|
||||||
|
}
|
||||||
|
bare := strings.TrimSpace(signatureAgentHostBare2.ReplaceAllString(signatureAgentHostBare1.ReplaceAllString(value, ""), ""))
|
||||||
|
if bare != "" {
|
||||||
|
return new(strings.ToLower(bare))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func findAgent(ua string, h map[string]string) *AgentMatch {
|
||||||
for _, ca := range compiledAgents {
|
for _, ca := range compiledAgents {
|
||||||
for _, p := range ca.patterns {
|
for _, p := range ca.patterns {
|
||||||
|
m := p.FindStringSubmatch(ua)
|
||||||
|
if m != nil {
|
||||||
|
ndx := p.SubexpIndex("version")
|
||||||
|
vers := ""
|
||||||
|
if ndx > 0 {
|
||||||
|
vers = m[ndx]
|
||||||
|
}
|
||||||
|
return new(AgentMatch{Agent: ca.r, Version: vers, Reason: fmt.Sprintf("user agent matches %s", ca.r.Name), Weak: false})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
accept, ok := h["accept"]
|
||||||
|
if ok {
|
||||||
|
accept = normalizeAccept(accept)
|
||||||
|
p := slices.IndexFunc(compiledAgents[:], func(ad agentDef) bool {
|
||||||
|
if ad.r.Accept == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, t := range ad.r.Accept {
|
||||||
|
if normalizeAccept(t) == accept {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
})
|
||||||
|
if p >= 0 {
|
||||||
|
rc := compiledAgents[p]
|
||||||
|
return new(AgentMatch{Agent: rc.r, Version: "", Reason: fmt.Sprintf("Accept header matches %s's fingerprint", rc.r.Name), Weak: false})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, isSigned := h["signed"]
|
||||||
|
if isSigned {
|
||||||
|
_, isSigned = h["signature-input"]
|
||||||
|
}
|
||||||
|
if isSigned {
|
||||||
|
sig, ok := h["signature-agent"]
|
||||||
|
if ok {
|
||||||
|
psig := signatureAgentHost(sig)
|
||||||
|
if psig != nil {
|
||||||
|
p := slices.IndexFunc(compiledAgents[:], func(ad agentDef) bool {
|
||||||
|
if ad.r.SignatureAgent == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return slices.Index(ad.r.SignatureAgent[:], *psig) >= 0
|
||||||
|
})
|
||||||
|
if p >= 0 {
|
||||||
|
rc := compiledAgents[p]
|
||||||
|
return new(AgentMatch{Agent: rc.r, Version: "",
|
||||||
|
Reason: fmt.Sprintf("signed request names %s in Signature-Agent; the signature itself is not verified", *psig),
|
||||||
|
Weak: true})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromAgent(match *AgentMatch, rawIP string) Classification {
|
||||||
|
agent := match.Agent
|
||||||
|
base := Classification{
|
||||||
|
Category: agent.Category,
|
||||||
|
Kind: agent.Kind,
|
||||||
|
Name: agent.Name,
|
||||||
|
Operator: agent.Operator,
|
||||||
|
Version: match.Version,
|
||||||
|
Verified: true,
|
||||||
|
Confidence: ConfidenceMedium,
|
||||||
|
Reasons: make([]string, 1),
|
||||||
|
}
|
||||||
|
base.Reasons[0] = match.Reason
|
||||||
|
if agent.RunsOn == "user" {
|
||||||
|
base.Reasons = append(base.Reasons, fmt.Sprintf("%s runs on the user's machine, so IP verification does not apply", agent.Name))
|
||||||
|
return base
|
||||||
|
}
|
||||||
|
ip := normalizeIP(rawIP)
|
||||||
|
if ip == nil {
|
||||||
|
base.Reasons = append(base.Reasons, fmt.Sprintf("%s is not a single valid address, so it was not checked", rawIP))
|
||||||
|
return base
|
||||||
|
}
|
||||||
|
urls := make([]string, 0)
|
||||||
|
if agent.Verify.IpRanges != nil {
|
||||||
|
urls = append(urls, agent.Verify.IpRanges...)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func AmClassify(headers http.Header, userAgent string) {
|
func AmClassify(headers http.Header, ipAddr string, userAgent string) Classification {
|
||||||
h := normalizeHeaders(headers)
|
h := normalizeHeaders(headers)
|
||||||
ua := userAgent
|
ua := userAgent
|
||||||
if ua == "" {
|
if ua == "" {
|
||||||
@@ -99,5 +231,8 @@ func AmClassify(headers http.Header, userAgent string) {
|
|||||||
ua = t
|
ua = t
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
findAgent(ua, h)
|
match := findAgent(ua, h)
|
||||||
|
if match != nil {
|
||||||
|
return fromAgent(match, ipAddr)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
/*
|
||||||
|
* Amsterdam Web Communities System
|
||||||
|
* Copyright (c) 2025-2026 Erbosoft Metaverse Design Solutions, All Rights Reserved
|
||||||
|
*
|
||||||
|
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: MPL-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Package agentclass contains support for agent request classification.
|
||||||
|
package agentclass
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/netip"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ParsedIP struct {
|
||||||
|
V6 bool
|
||||||
|
Value netip.Addr
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseIP(ip string) *ParsedIP {
|
||||||
|
s := strings.TrimSpace(ip)
|
||||||
|
v6 := strings.Index(s, ":") >= 0
|
||||||
|
addy, err := netip.ParseAddr(s)
|
||||||
|
if err != nil || addy.Is6() != v6 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return new(ParsedIP{V6: v6, Value: addy})
|
||||||
|
}
|
||||||
|
|
||||||
|
var bracketedExpr = regexp.MustCompile(`^\[([^\]]+)\](?::\d+)?$`)
|
||||||
|
var v4WithPortExpr = regexp.MustCompile(`^(\d{1,3}(?:\.\d{1,3}){3}):\d+$`)
|
||||||
|
|
||||||
|
func normalizeIP(ip string) *string {
|
||||||
|
s := strings.TrimSpace(ip)
|
||||||
|
if s == "" || strings.Index(s, ",") >= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
bracketed := bracketedExpr.FindStringSubmatch(s)
|
||||||
|
v4WithPort := v4WithPortExpr.FindStringSubmatch(s)
|
||||||
|
candidate := s
|
||||||
|
if bracketed != nil {
|
||||||
|
candidate = bracketed[1]
|
||||||
|
} else if v4WithPort != nil {
|
||||||
|
candidate = v4WithPort[1]
|
||||||
|
}
|
||||||
|
parsed := parseIP(candidate)
|
||||||
|
if parsed == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if parsed.V6 && parsed.Value.Is4In6() {
|
||||||
|
return new(netip.AddrFrom4(parsed.Value.As4()).String())
|
||||||
|
}
|
||||||
|
return new(candidate)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user