added auto-refresh on a daily basis (configurable)

This commit is contained in:
amy
2026-11-15 17:41:38 -07:00
parent 0d576e4b55
commit 8c7c7c9f91
3 changed files with 121 additions and 52 deletions
+95 -32
View File
@@ -17,61 +17,93 @@ import (
"encoding/json/jsontext" "encoding/json/jsontext"
"encoding/json/v2" "encoding/json/v2"
"fmt" "fmt"
"maps"
"net/http" "net/http"
"net/netip" "net/netip"
"sync" "sync"
"time" "time"
"git.erbosoft.com/amy/amsterdam/config"
"github.com/labstack/gommon/log"
) )
const NUM_RETRIES = 3 // Format for IP range data inside the RangeDataFormat.
const TIMEOUT = 10
type RangeDataPrefix struct { type RangeDataPrefix struct {
IPv4Prefix string `json:"ipv4Prefix"` IPv4Prefix string `json:"ipv4Prefix"` // prefix, in IPv4 format
IPv6Prefix string `json:"ipv6Prefix"` IPv6Prefix string `json:"ipv6Prefix"` // prefix, in IPv6 format
} }
// Format of IP range data files exported by bot providers.
type RangeDataFormat struct { type RangeDataFormat struct {
CreationTime *time.Time `json:"creationTime"` CreationTime *time.Time `json:"creationTime"` // creation date and time
Prefixes []RangeDataPrefix `json:"prefixes"` Prefixes []RangeDataPrefix `json:"prefixes"` // the prefixes
} }
// Stored IP range data for a single URL.
type ipRangeData struct { type ipRangeData struct {
m sync.Mutex m sync.Mutex // mutex on the range data
c *sync.Cond c *sync.Cond // signals when load complete
loaded bool loaded bool // set when load operation completed
loadError error loadError error // error during load, or nil
creation *time.Time lastMod string // Last-Modified header from last fetch
cidrs []netip.Prefix creation *time.Time // creation date and time
cidrs []netip.Prefix // the prefixes from this file
} }
// Contains all the data for loaded IP range files.
var ipRangeStore map[string]*ipRangeData var ipRangeStore map[string]*ipRangeData
// loadIPRangeData is the goroutine that fetches range data from a URL and parses it.
func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) { func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
rd.m.Lock() rd.m.Lock()
defer rd.m.Unlock() defer rd.m.Unlock()
LOADLOOP: LOADLOOP:
for counter := NUM_RETRIES; counter > 0; counter-- { for counter := config.GlobalConfig.Tuning.IPRangeLoadAttempts; counter > 0; counter-- {
select { select {
case <-ctx.Done(): case <-ctx.Done():
rd.loadError = ctx.Err() rd.loadError = ctx.Err()
break LOADLOOP break LOADLOOP
default: default:
client := &http.Client{Timeout: TIMEOUT * time.Second} req, err := http.NewRequest("GET", url, nil)
resp, err := client.Get(url) if err != nil {
rd.loadError = err
break LOADLOOP
}
if rd.lastMod != "" {
req.Header.Set("If-Modified-Since", rd.lastMod)
}
client := &http.Client{Timeout: time.Duration(config.GlobalConfig.Tuning.Timeouts.IPRangeLoad) * time.Second}
resp, err := client.Do(req)
select { select {
case <-ctx.Done(): case <-ctx.Done():
resp.Body.Close() resp.Body.Close()
rd.loadError = ctx.Err() rd.loadError = ctx.Err()
break LOADLOOP break LOADLOOP
default: default:
if err == nil { if err != nil {
rd.loadError = err
continue LOADLOOP
}
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode == http.StatusOK { if resp.StatusCode == http.StatusNotModified {
rd.loadError = nil
break LOADLOOP
}
if resp.StatusCode != http.StatusOK {
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
continue LOADLOOP
}
lastmod := ""
if lm, ok := resp.Header["Last-Modified"]; ok {
lastmod = lm[0]
}
decoder := jsontext.NewDecoder(resp.Body) decoder := jsontext.NewDecoder(resp.Body)
var parsed RangeDataFormat var parsed RangeDataFormat
err = json.UnmarshalDecode(decoder, &parsed) err = json.UnmarshalDecode(decoder, &parsed)
if err == nil { if err != nil {
rd.loadError = err
break LOADLOOP
}
lx := 0 lx := 0
if parsed.Prefixes != nil { if parsed.Prefixes != nil {
lx = len(parsed.Prefixes) lx = len(parsed.Prefixes)
@@ -91,17 +123,8 @@ LOADLOOP:
} }
} }
rd.loadError = nil rd.loadError = nil
rd.lastMod = lastmod
break LOADLOOP break LOADLOOP
} else {
rd.loadError = err
break LOADLOOP
}
} else {
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
}
} else {
rd.loadError = err
}
} }
} }
} }
@@ -117,18 +140,58 @@ LOADLOOP:
rd.c.Signal() rd.c.Signal()
} }
var loadRangesCtx context.Context func prepareRefresh(rd *ipRangeData) {
rd.m.Lock()
defer rd.m.Unlock()
for !rd.loaded {
rd.c.Wait()
}
rd.loaded = false
}
func refreshRanges(ctx context.Context, interval time.Duration) {
tick := time.NewTicker(interval)
REFRESHLOOP:
for {
select {
case <-ctx.Done():
tick.Stop()
break REFRESHLOOP
case <-tick.C:
mymap := make(map[string]*ipRangeData)
maps.Copy(mymap, ipRangeStore)
for url, rd := range mymap {
select {
case <-ctx.Done():
tick.Stop()
break REFRESHLOOP
default:
prepareRefresh(rd)
go loadIPRangeData(ctx, url, rd)
}
}
}
}
}
var loadRangesCtx context.Context = nil
func loadIPRanges(urls []string, ctxname any) func() { func loadIPRanges(urls []string, ctxname any) func() {
ipRangeStore = make(map[string]*ipRangeData) ipRangeStore = make(map[string]*ipRangeData)
ctx, cancelfunc := context.WithCancel(context.Background()) ctx, cancelfunc := context.WithCancel(context.Background())
loadRangesCtx = context.WithValue(ctx, ctxname, "loadIPRanges") loadRangesCtx = context.WithValue(ctx, ctxname, "loadIPRanges")
for _, url := range urls { for _, url := range urls {
rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil}) rdat := new(ipRangeData{loaded: false, loadError: nil, lastMod: "", creation: nil})
rdat.c = sync.NewCond(&(rdat.m)) rdat.c = sync.NewCond(&(rdat.m))
ipRangeStore[url] = rdat ipRangeStore[url] = rdat
go loadIPRangeData(loadRangesCtx, url, rdat) go loadIPRangeData(loadRangesCtx, url, rdat)
} }
dur, err := time.ParseDuration(config.GlobalConfig.Tuning.IPRangeRefresh)
if err != nil {
log.Errorf("invalid IP range refresh duration \"%s\", using default", config.GlobalConfig.Tuning.IPRangeRefresh)
dur = 24 * time.Hour
}
go refreshRanges(loadRangesCtx, dur)
return func() { return func() {
cancelfunc() cancelfunc()
for _, v := range ipRangeStore { for _, v := range ipRangeStore {
@@ -145,7 +208,7 @@ func loadIPRanges(urls []string, ctxname any) func() {
func getIPRangeData(url string) ([]netip.Prefix, *time.Time, error) { func getIPRangeData(url string) ([]netip.Prefix, *time.Time, error) {
rdat, ok := ipRangeStore[url] rdat, ok := ipRangeStore[url]
if !ok { if !ok {
rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil}) rdat := new(ipRangeData{loaded: false, loadError: nil, lastMod: "", creation: nil})
rdat.c = sync.NewCond(&(rdat.m)) rdat.c = sync.NewCond(&(rdat.m))
ipRangeStore[url] = rdat ipRangeStore[url] = rdat
go loadIPRangeData(loadRangesCtx, url, rdat) go loadIPRangeData(loadRangesCtx, url, rdat)
+3
View File
@@ -182,12 +182,15 @@ type AmConfig struct {
} `yaml:"posting"` } `yaml:"posting"`
Tuning struct { Tuning struct {
WorkerTasks int `yaml:"workerTasks"` WorkerTasks int `yaml:"workerTasks"`
IPRangeLoadAttempts int `yaml:"ipRangeLoadAttempts"`
IPRangeRefresh string `yaml:"ipRangeRefresh"`
Timeouts struct { Timeouts struct {
HttpRead int `yaml:"httpRead"` HttpRead int `yaml:"httpRead"`
HttpWrite int `yaml:"httpWrite"` HttpWrite int `yaml:"httpWrite"`
HttpIdle int `yaml:"httpIdle"` HttpIdle int `yaml:"httpIdle"`
PageExecute int `yaml:"pageExecute"` PageExecute int `yaml:"pageExecute"`
PageRender int `yaml:"pageRender"` PageRender int `yaml:"pageRender"`
IPRangeLoad int `yaml:"ipRangeLoad"`
} `yaml:"timeouts"` } `yaml:"timeouts"`
Queues struct { Queues struct {
AuditWrites int `yaml:"auditWrites"` AuditWrites int `yaml:"auditWrites"`
+3
View File
@@ -102,12 +102,15 @@ posting:
- "image/png" - "image/png"
tuning: tuning:
workerTasks: 4 workerTasks: 4
ipRangeLoadAttempts: 3
ipRangeRefresh: "24h"
timeouts: timeouts:
httpRead: 30 httpRead: 30
httpWrite: 30 httpWrite: 30
httpIdle: 120 httpIdle: 120
pageExecute: 15 pageExecute: 15
pageRender: 15 pageRender: 15
ipRangeLoad: 10
queues: queues:
auditWrites: 16 auditWrites: 16
contextRecycle: 16 contextRecycle: 16