diff --git a/agentclass/rangeloader.go b/agentclass/rangeloader.go index 1ee0c79..1b339fc 100644 --- a/agentclass/rangeloader.go +++ b/agentclass/rangeloader.go @@ -17,91 +17,114 @@ import ( "encoding/json/jsontext" "encoding/json/v2" "fmt" + "maps" "net/http" "net/netip" "sync" "time" + + "git.erbosoft.com/amy/amsterdam/config" + "github.com/labstack/gommon/log" ) -const NUM_RETRIES = 3 -const TIMEOUT = 10 - +// Format for IP range data inside the RangeDataFormat. type RangeDataPrefix struct { - IPv4Prefix string `json:"ipv4Prefix"` - IPv6Prefix string `json:"ipv6Prefix"` + IPv4Prefix string `json:"ipv4Prefix"` // prefix, in IPv4 format + IPv6Prefix string `json:"ipv6Prefix"` // prefix, in IPv6 format } +// Format of IP range data files exported by bot providers. type RangeDataFormat struct { - CreationTime *time.Time `json:"creationTime"` - Prefixes []RangeDataPrefix `json:"prefixes"` + CreationTime *time.Time `json:"creationTime"` // creation date and time + Prefixes []RangeDataPrefix `json:"prefixes"` // the prefixes } +// Stored IP range data for a single URL. type ipRangeData struct { - m sync.Mutex - c *sync.Cond - loaded bool - loadError error - creation *time.Time - cidrs []netip.Prefix + m sync.Mutex // mutex on the range data + c *sync.Cond // signals when load complete + loaded bool // set when load operation completed + loadError error // error during load, or nil + lastMod string // Last-Modified header from last fetch + creation *time.Time // creation date and time + cidrs []netip.Prefix // the prefixes from this file } +// Contains all the data for loaded IP range files. var ipRangeStore map[string]*ipRangeData +// loadIPRangeData is the goroutine that fetches range data from a URL and parses it. func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) { rd.m.Lock() defer rd.m.Unlock() LOADLOOP: - for counter := NUM_RETRIES; counter > 0; counter-- { + for counter := config.GlobalConfig.Tuning.IPRangeLoadAttempts; counter > 0; counter-- { select { case <-ctx.Done(): rd.loadError = ctx.Err() break LOADLOOP default: - client := &http.Client{Timeout: TIMEOUT * time.Second} - resp, err := client.Get(url) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + rd.loadError = err + break LOADLOOP + } + if rd.lastMod != "" { + req.Header.Set("If-Modified-Since", rd.lastMod) + } + client := &http.Client{Timeout: time.Duration(config.GlobalConfig.Tuning.Timeouts.IPRangeLoad) * time.Second} + resp, err := client.Do(req) select { case <-ctx.Done(): resp.Body.Close() rd.loadError = ctx.Err() break LOADLOOP default: - if err == nil { - defer resp.Body.Close() - if resp.StatusCode == http.StatusOK { - decoder := jsontext.NewDecoder(resp.Body) - var parsed RangeDataFormat - err = json.UnmarshalDecode(decoder, &parsed) - if err == nil { - lx := 0 - if parsed.Prefixes != nil { - lx = len(parsed.Prefixes) - } - rd.cidrs = make([]netip.Prefix, lx) - if lx > 0 { - for i, p := range parsed.Prefixes { - if p.IPv4Prefix == "" { - rd.cidrs[i], err = netip.ParsePrefix(p.IPv6Prefix) - } else { - rd.cidrs[i], err = netip.ParsePrefix(p.IPv4Prefix) - } - if err != nil { - rd.loadError = err - break LOADLOOP - } - } - } - rd.loadError = nil - break LOADLOOP + if err != nil { + rd.loadError = err + continue LOADLOOP + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotModified { + rd.loadError = nil + break LOADLOOP + } + if resp.StatusCode != http.StatusOK { + rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode) + continue LOADLOOP + } + lastmod := "" + if lm, ok := resp.Header["Last-Modified"]; ok { + lastmod = lm[0] + } + decoder := jsontext.NewDecoder(resp.Body) + var parsed RangeDataFormat + err = json.UnmarshalDecode(decoder, &parsed) + if err != nil { + rd.loadError = err + break LOADLOOP + } + lx := 0 + if parsed.Prefixes != nil { + lx = len(parsed.Prefixes) + } + rd.cidrs = make([]netip.Prefix, lx) + if lx > 0 { + for i, p := range parsed.Prefixes { + if p.IPv4Prefix == "" { + rd.cidrs[i], err = netip.ParsePrefix(p.IPv6Prefix) } else { + rd.cidrs[i], err = netip.ParsePrefix(p.IPv4Prefix) + } + if err != nil { rd.loadError = err break LOADLOOP } - } else { - rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode) } - } else { - rd.loadError = err } + rd.loadError = nil + rd.lastMod = lastmod + break LOADLOOP } } } @@ -117,18 +140,58 @@ LOADLOOP: rd.c.Signal() } -var loadRangesCtx context.Context +func prepareRefresh(rd *ipRangeData) { + rd.m.Lock() + defer rd.m.Unlock() + for !rd.loaded { + rd.c.Wait() + } + rd.loaded = false +} + +func refreshRanges(ctx context.Context, interval time.Duration) { + tick := time.NewTicker(interval) +REFRESHLOOP: + for { + select { + case <-ctx.Done(): + tick.Stop() + break REFRESHLOOP + case <-tick.C: + mymap := make(map[string]*ipRangeData) + maps.Copy(mymap, ipRangeStore) + for url, rd := range mymap { + select { + case <-ctx.Done(): + tick.Stop() + break REFRESHLOOP + default: + prepareRefresh(rd) + go loadIPRangeData(ctx, url, rd) + } + } + } + } +} + +var loadRangesCtx context.Context = nil func loadIPRanges(urls []string, ctxname any) func() { ipRangeStore = make(map[string]*ipRangeData) ctx, cancelfunc := context.WithCancel(context.Background()) loadRangesCtx = context.WithValue(ctx, ctxname, "loadIPRanges") for _, url := range urls { - rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil}) + rdat := new(ipRangeData{loaded: false, loadError: nil, lastMod: "", creation: nil}) rdat.c = sync.NewCond(&(rdat.m)) ipRangeStore[url] = rdat go loadIPRangeData(loadRangesCtx, url, rdat) } + dur, err := time.ParseDuration(config.GlobalConfig.Tuning.IPRangeRefresh) + if err != nil { + log.Errorf("invalid IP range refresh duration \"%s\", using default", config.GlobalConfig.Tuning.IPRangeRefresh) + dur = 24 * time.Hour + } + go refreshRanges(loadRangesCtx, dur) return func() { cancelfunc() for _, v := range ipRangeStore { @@ -145,7 +208,7 @@ func loadIPRanges(urls []string, ctxname any) func() { func getIPRangeData(url string) ([]netip.Prefix, *time.Time, error) { rdat, ok := ipRangeStore[url] if !ok { - rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil}) + rdat := new(ipRangeData{loaded: false, loadError: nil, lastMod: "", creation: nil}) rdat.c = sync.NewCond(&(rdat.m)) ipRangeStore[url] = rdat go loadIPRangeData(loadRangesCtx, url, rdat) diff --git a/config/config.go b/config/config.go index ac51292..f39b01e 100644 --- a/config/config.go +++ b/config/config.go @@ -181,13 +181,16 @@ type AmConfig struct { } `yaml:"uploads"` } `yaml:"posting"` Tuning struct { - WorkerTasks int `yaml:"workerTasks"` - Timeouts struct { + WorkerTasks int `yaml:"workerTasks"` + IPRangeLoadAttempts int `yaml:"ipRangeLoadAttempts"` + IPRangeRefresh string `yaml:"ipRangeRefresh"` + Timeouts struct { HttpRead int `yaml:"httpRead"` HttpWrite int `yaml:"httpWrite"` HttpIdle int `yaml:"httpIdle"` PageExecute int `yaml:"pageExecute"` PageRender int `yaml:"pageRender"` + IPRangeLoad int `yaml:"ipRangeLoad"` } `yaml:"timeouts"` Queues struct { AuditWrites int `yaml:"auditWrites"` diff --git a/config/default.yaml b/config/default.yaml index e8aaa38..9ce9a66 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -102,12 +102,15 @@ posting: - "image/png" tuning: workerTasks: 4 + ipRangeLoadAttempts: 3 + ipRangeRefresh: "24h" timeouts: httpRead: 30 httpWrite: 30 httpIdle: 120 pageExecute: 15 pageRender: 15 + ipRangeLoad: 10 queues: auditWrites: 16 contextRecycle: 16