Files
amsterdam/agentclass/rangeloader.go
T

263 lines
7.9 KiB
Go

/*
* Amsterdam Web Communities System
* Copyright (c) 2025-2026 Erbosoft Metaverse Design Solutions, All Rights Reserved
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*
* SPDX-License-Identifier: MPL-2.0
*/
/* This file contains code adapted from Mintlify agent-traffic, https://github.com/mintlify/agent-traffic
* ---
* Copyright (c) 2026 Mintlify, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
// Package agentclass contains support for agent request classification.
package agentclass
import (
"context"
"encoding/json/jsontext"
"encoding/json/v2"
"fmt"
"maps"
"net/http"
"net/netip"
"sync"
"time"
"git.erbosoft.com/amy/amsterdam/config"
log "github.com/sirupsen/logrus"
)
// Format for IP range data inside the RangeDataFormat.
type RangeDataPrefix struct {
IPv4Prefix string `json:"ipv4Prefix"` // prefix, in IPv4 format
IPv6Prefix string `json:"ipv6Prefix"` // prefix, in IPv6 format
}
// Format of IP range data files exported by bot providers.
type RangeDataFormat struct {
CreationTime *time.Time `json:"creationTime"` // creation date and time
Prefixes []RangeDataPrefix `json:"prefixes"` // the prefixes
}
// Stored IP range data for a single URL.
type ipRangeData struct {
m sync.Mutex // mutex on the range data
c *sync.Cond // signals when load complete
loaded bool // set when load operation completed
loadError error // error during load, or nil
lastMod string // Last-Modified header from last fetch
creation *time.Time // creation date and time
cidrs []netip.Prefix // the prefixes from this file
}
// Contains all the data for loaded IP range files.
var ipRangeStore map[string]*ipRangeData
// loadIPRangeData is the goroutine that fetches range data from a URL and parses it.
func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
rd.m.Lock()
defer rd.m.Unlock()
LOADLOOP:
for counter := config.GlobalConfig.Tuning.IPRangeLoadAttempts; counter > 0; counter-- {
select {
case <-ctx.Done(): // cancellation check on every loop
rd.loadError = ctx.Err()
break LOADLOOP
default:
// build the HTTP request
req, err := http.NewRequest("GET", url, nil)
if err != nil {
rd.loadError = err
break LOADLOOP
}
if rd.lastMod != "" {
req.Header.Set("If-Modified-Since", rd.lastMod)
}
// build a client and use it to load the file
client := &http.Client{Timeout: time.Duration(config.GlobalConfig.Tuning.Timeouts.IPRangeLoad) * time.Second}
resp, err := client.Do(req)
select {
case <-ctx.Done(): // we might have been canceled while waiting for the GET
resp.Body.Close()
rd.loadError = ctx.Err()
break LOADLOOP
default:
// interpret error and status codes
if err != nil {
rd.loadError = err
continue LOADLOOP
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotModified {
rd.loadError = nil
break LOADLOOP
}
if resp.StatusCode != http.StatusOK {
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
continue LOADLOOP
}
// load Last-Modified date
lastmod := ""
if lm, ok := resp.Header["Last-Modified"]; ok {
lastmod = lm[0]
}
// parse the JSON data retrieved
decoder := jsontext.NewDecoder(resp.Body)
var parsed RangeDataFormat
err = json.UnmarshalDecode(decoder, &parsed)
if err != nil {
rd.loadError = err
break LOADLOOP
}
// turn the parsed data into our internal data
lx := 0
if parsed.Prefixes != nil {
lx = len(parsed.Prefixes)
}
rd.cidrs = make([]netip.Prefix, lx)
if lx > 0 {
for i, p := range parsed.Prefixes {
if p.IPv4Prefix == "" {
rd.cidrs[i], err = netip.ParsePrefix(p.IPv6Prefix)
} else {
rd.cidrs[i], err = netip.ParsePrefix(p.IPv4Prefix)
}
if err != nil {
rd.loadError = err
break LOADLOOP
}
}
}
rd.loadError = nil
rd.lastMod = lastmod
rd.creation = parsed.CreationTime
break LOADLOOP
}
}
}
select {
case <-ctx.Done(): // one more cancel check before ending
if rd.loadError == nil {
rd.loadError = ctx.Err()
}
default:
// fall through
}
rd.loaded = true // finished loading, let another routine go
rd.c.Signal()
log.Debugf("loaded IP range document at %s", url)
}
// prepareRefresh prepares some ipRangeData for being refreshed.
func prepareRefresh(rd *ipRangeData) {
rd.m.Lock()
defer rd.m.Unlock()
for !rd.loaded {
rd.c.Wait()
}
rd.loaded = false
}
// refreshRanges runs periodically to refresh the range data from the various sites.
func refreshRanges(ctx context.Context, interval time.Duration) {
tick := time.NewTicker(interval)
REFRESHLOOP:
for {
select {
case <-ctx.Done():
tick.Stop()
break REFRESHLOOP
case <-tick.C:
mymap := make(map[string]*ipRangeData)
maps.Copy(mymap, ipRangeStore)
for url, rd := range mymap {
select {
case <-ctx.Done():
tick.Stop()
break REFRESHLOOP
default:
prepareRefresh(rd)
go loadIPRangeData(ctx, url, rd)
}
}
}
}
}
// loadRangesCtx is the context used for running the range load and refresh data.
var loadRangesCtx context.Context = nil
// loadIPRanges begins loading the initial IP ranges.
func loadIPRanges(urls []string, ctxname any) func() {
ipRangeStore = make(map[string]*ipRangeData)
ctx, cancelfunc := context.WithCancel(context.Background())
loadRangesCtx = context.WithValue(ctx, ctxname, "loadIPRanges")
for _, url := range urls {
rdat := new(ipRangeData{loaded: false, loadError: nil, lastMod: "", creation: nil})
rdat.c = sync.NewCond(&(rdat.m))
ipRangeStore[url] = rdat
go loadIPRangeData(loadRangesCtx, url, rdat)
}
log.Infof("loadIPRanges: started loading %d IP range files", len(urls))
dur, err := time.ParseDuration(config.GlobalConfig.Tuning.IPRangeRefresh)
if err != nil {
log.Errorf("invalid IP range refresh duration \"%s\", using default", config.GlobalConfig.Tuning.IPRangeRefresh)
dur = 24 * time.Hour
}
go refreshRanges(loadRangesCtx, dur)
return func() {
cancelfunc()
for _, v := range ipRangeStore {
v.m.Lock()
if !v.loaded {
v.loadError = context.Canceled
v.c.Signal()
}
v.m.Unlock()
}
}
}
// getIPRangeData gets the IP range data for a specified URL, which may have been cached already.
func getIPRangeData(url string) ([]netip.Prefix, *time.Time, error) {
rdat, ok := ipRangeStore[url]
if !ok {
rdat := new(ipRangeData{loaded: false, loadError: nil, lastMod: "", creation: nil})
rdat.c = sync.NewCond(&(rdat.m))
ipRangeStore[url] = rdat
go loadIPRangeData(loadRangesCtx, url, rdat)
}
rdat.m.Lock()
defer rdat.m.Unlock()
for !rdat.loaded {
rdat.c.Wait()
}
return rdat.cidrs, rdat.creation, rdat.loadError
}