added auto-refresh on a daily basis (configurable)
This commit is contained in:
+95
-32
@@ -17,61 +17,93 @@ import (
|
|||||||
"encoding/json/jsontext"
|
"encoding/json/jsontext"
|
||||||
"encoding/json/v2"
|
"encoding/json/v2"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"maps"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.erbosoft.com/amy/amsterdam/config"
|
||||||
|
"github.com/labstack/gommon/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
const NUM_RETRIES = 3
|
// Format for IP range data inside the RangeDataFormat.
|
||||||
const TIMEOUT = 10
|
|
||||||
|
|
||||||
type RangeDataPrefix struct {
|
type RangeDataPrefix struct {
|
||||||
IPv4Prefix string `json:"ipv4Prefix"`
|
IPv4Prefix string `json:"ipv4Prefix"` // prefix, in IPv4 format
|
||||||
IPv6Prefix string `json:"ipv6Prefix"`
|
IPv6Prefix string `json:"ipv6Prefix"` // prefix, in IPv6 format
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Format of IP range data files exported by bot providers.
|
||||||
type RangeDataFormat struct {
|
type RangeDataFormat struct {
|
||||||
CreationTime *time.Time `json:"creationTime"`
|
CreationTime *time.Time `json:"creationTime"` // creation date and time
|
||||||
Prefixes []RangeDataPrefix `json:"prefixes"`
|
Prefixes []RangeDataPrefix `json:"prefixes"` // the prefixes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stored IP range data for a single URL.
|
||||||
type ipRangeData struct {
|
type ipRangeData struct {
|
||||||
m sync.Mutex
|
m sync.Mutex // mutex on the range data
|
||||||
c *sync.Cond
|
c *sync.Cond // signals when load complete
|
||||||
loaded bool
|
loaded bool // set when load operation completed
|
||||||
loadError error
|
loadError error // error during load, or nil
|
||||||
creation *time.Time
|
lastMod string // Last-Modified header from last fetch
|
||||||
cidrs []netip.Prefix
|
creation *time.Time // creation date and time
|
||||||
|
cidrs []netip.Prefix // the prefixes from this file
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Contains all the data for loaded IP range files.
|
||||||
var ipRangeStore map[string]*ipRangeData
|
var ipRangeStore map[string]*ipRangeData
|
||||||
|
|
||||||
|
// loadIPRangeData is the goroutine that fetches range data from a URL and parses it.
|
||||||
func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
|
func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
|
||||||
rd.m.Lock()
|
rd.m.Lock()
|
||||||
defer rd.m.Unlock()
|
defer rd.m.Unlock()
|
||||||
LOADLOOP:
|
LOADLOOP:
|
||||||
for counter := NUM_RETRIES; counter > 0; counter-- {
|
for counter := config.GlobalConfig.Tuning.IPRangeLoadAttempts; counter > 0; counter-- {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
rd.loadError = ctx.Err()
|
rd.loadError = ctx.Err()
|
||||||
break LOADLOOP
|
break LOADLOOP
|
||||||
default:
|
default:
|
||||||
client := &http.Client{Timeout: TIMEOUT * time.Second}
|
req, err := http.NewRequest("GET", url, nil)
|
||||||
resp, err := client.Get(url)
|
if err != nil {
|
||||||
|
rd.loadError = err
|
||||||
|
break LOADLOOP
|
||||||
|
}
|
||||||
|
if rd.lastMod != "" {
|
||||||
|
req.Header.Set("If-Modified-Since", rd.lastMod)
|
||||||
|
}
|
||||||
|
client := &http.Client{Timeout: time.Duration(config.GlobalConfig.Tuning.Timeouts.IPRangeLoad) * time.Second}
|
||||||
|
resp, err := client.Do(req)
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
rd.loadError = ctx.Err()
|
rd.loadError = ctx.Err()
|
||||||
break LOADLOOP
|
break LOADLOOP
|
||||||
default:
|
default:
|
||||||
if err == nil {
|
if err != nil {
|
||||||
|
rd.loadError = err
|
||||||
|
continue LOADLOOP
|
||||||
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
if resp.StatusCode == http.StatusOK {
|
if resp.StatusCode == http.StatusNotModified {
|
||||||
|
rd.loadError = nil
|
||||||
|
break LOADLOOP
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
|
||||||
|
continue LOADLOOP
|
||||||
|
}
|
||||||
|
lastmod := ""
|
||||||
|
if lm, ok := resp.Header["Last-Modified"]; ok {
|
||||||
|
lastmod = lm[0]
|
||||||
|
}
|
||||||
decoder := jsontext.NewDecoder(resp.Body)
|
decoder := jsontext.NewDecoder(resp.Body)
|
||||||
var parsed RangeDataFormat
|
var parsed RangeDataFormat
|
||||||
err = json.UnmarshalDecode(decoder, &parsed)
|
err = json.UnmarshalDecode(decoder, &parsed)
|
||||||
if err == nil {
|
if err != nil {
|
||||||
|
rd.loadError = err
|
||||||
|
break LOADLOOP
|
||||||
|
}
|
||||||
lx := 0
|
lx := 0
|
||||||
if parsed.Prefixes != nil {
|
if parsed.Prefixes != nil {
|
||||||
lx = len(parsed.Prefixes)
|
lx = len(parsed.Prefixes)
|
||||||
@@ -91,17 +123,8 @@ LOADLOOP:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
rd.loadError = nil
|
rd.loadError = nil
|
||||||
|
rd.lastMod = lastmod
|
||||||
break LOADLOOP
|
break LOADLOOP
|
||||||
} else {
|
|
||||||
rd.loadError = err
|
|
||||||
break LOADLOOP
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
rd.loadError = err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -117,18 +140,58 @@ LOADLOOP:
|
|||||||
rd.c.Signal()
|
rd.c.Signal()
|
||||||
}
|
}
|
||||||
|
|
||||||
var loadRangesCtx context.Context
|
func prepareRefresh(rd *ipRangeData) {
|
||||||
|
rd.m.Lock()
|
||||||
|
defer rd.m.Unlock()
|
||||||
|
for !rd.loaded {
|
||||||
|
rd.c.Wait()
|
||||||
|
}
|
||||||
|
rd.loaded = false
|
||||||
|
}
|
||||||
|
|
||||||
|
func refreshRanges(ctx context.Context, interval time.Duration) {
|
||||||
|
tick := time.NewTicker(interval)
|
||||||
|
REFRESHLOOP:
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
tick.Stop()
|
||||||
|
break REFRESHLOOP
|
||||||
|
case <-tick.C:
|
||||||
|
mymap := make(map[string]*ipRangeData)
|
||||||
|
maps.Copy(mymap, ipRangeStore)
|
||||||
|
for url, rd := range mymap {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
tick.Stop()
|
||||||
|
break REFRESHLOOP
|
||||||
|
default:
|
||||||
|
prepareRefresh(rd)
|
||||||
|
go loadIPRangeData(ctx, url, rd)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var loadRangesCtx context.Context = nil
|
||||||
|
|
||||||
func loadIPRanges(urls []string, ctxname any) func() {
|
func loadIPRanges(urls []string, ctxname any) func() {
|
||||||
ipRangeStore = make(map[string]*ipRangeData)
|
ipRangeStore = make(map[string]*ipRangeData)
|
||||||
ctx, cancelfunc := context.WithCancel(context.Background())
|
ctx, cancelfunc := context.WithCancel(context.Background())
|
||||||
loadRangesCtx = context.WithValue(ctx, ctxname, "loadIPRanges")
|
loadRangesCtx = context.WithValue(ctx, ctxname, "loadIPRanges")
|
||||||
for _, url := range urls {
|
for _, url := range urls {
|
||||||
rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil})
|
rdat := new(ipRangeData{loaded: false, loadError: nil, lastMod: "", creation: nil})
|
||||||
rdat.c = sync.NewCond(&(rdat.m))
|
rdat.c = sync.NewCond(&(rdat.m))
|
||||||
ipRangeStore[url] = rdat
|
ipRangeStore[url] = rdat
|
||||||
go loadIPRangeData(loadRangesCtx, url, rdat)
|
go loadIPRangeData(loadRangesCtx, url, rdat)
|
||||||
}
|
}
|
||||||
|
dur, err := time.ParseDuration(config.GlobalConfig.Tuning.IPRangeRefresh)
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("invalid IP range refresh duration \"%s\", using default", config.GlobalConfig.Tuning.IPRangeRefresh)
|
||||||
|
dur = 24 * time.Hour
|
||||||
|
}
|
||||||
|
go refreshRanges(loadRangesCtx, dur)
|
||||||
return func() {
|
return func() {
|
||||||
cancelfunc()
|
cancelfunc()
|
||||||
for _, v := range ipRangeStore {
|
for _, v := range ipRangeStore {
|
||||||
@@ -145,7 +208,7 @@ func loadIPRanges(urls []string, ctxname any) func() {
|
|||||||
func getIPRangeData(url string) ([]netip.Prefix, *time.Time, error) {
|
func getIPRangeData(url string) ([]netip.Prefix, *time.Time, error) {
|
||||||
rdat, ok := ipRangeStore[url]
|
rdat, ok := ipRangeStore[url]
|
||||||
if !ok {
|
if !ok {
|
||||||
rdat := new(ipRangeData{loaded: false, loadError: nil, creation: nil})
|
rdat := new(ipRangeData{loaded: false, loadError: nil, lastMod: "", creation: nil})
|
||||||
rdat.c = sync.NewCond(&(rdat.m))
|
rdat.c = sync.NewCond(&(rdat.m))
|
||||||
ipRangeStore[url] = rdat
|
ipRangeStore[url] = rdat
|
||||||
go loadIPRangeData(loadRangesCtx, url, rdat)
|
go loadIPRangeData(loadRangesCtx, url, rdat)
|
||||||
|
|||||||
@@ -182,12 +182,15 @@ type AmConfig struct {
|
|||||||
} `yaml:"posting"`
|
} `yaml:"posting"`
|
||||||
Tuning struct {
|
Tuning struct {
|
||||||
WorkerTasks int `yaml:"workerTasks"`
|
WorkerTasks int `yaml:"workerTasks"`
|
||||||
|
IPRangeLoadAttempts int `yaml:"ipRangeLoadAttempts"`
|
||||||
|
IPRangeRefresh string `yaml:"ipRangeRefresh"`
|
||||||
Timeouts struct {
|
Timeouts struct {
|
||||||
HttpRead int `yaml:"httpRead"`
|
HttpRead int `yaml:"httpRead"`
|
||||||
HttpWrite int `yaml:"httpWrite"`
|
HttpWrite int `yaml:"httpWrite"`
|
||||||
HttpIdle int `yaml:"httpIdle"`
|
HttpIdle int `yaml:"httpIdle"`
|
||||||
PageExecute int `yaml:"pageExecute"`
|
PageExecute int `yaml:"pageExecute"`
|
||||||
PageRender int `yaml:"pageRender"`
|
PageRender int `yaml:"pageRender"`
|
||||||
|
IPRangeLoad int `yaml:"ipRangeLoad"`
|
||||||
} `yaml:"timeouts"`
|
} `yaml:"timeouts"`
|
||||||
Queues struct {
|
Queues struct {
|
||||||
AuditWrites int `yaml:"auditWrites"`
|
AuditWrites int `yaml:"auditWrites"`
|
||||||
|
|||||||
@@ -102,12 +102,15 @@ posting:
|
|||||||
- "image/png"
|
- "image/png"
|
||||||
tuning:
|
tuning:
|
||||||
workerTasks: 4
|
workerTasks: 4
|
||||||
|
ipRangeLoadAttempts: 3
|
||||||
|
ipRangeRefresh: "24h"
|
||||||
timeouts:
|
timeouts:
|
||||||
httpRead: 30
|
httpRead: 30
|
||||||
httpWrite: 30
|
httpWrite: 30
|
||||||
httpIdle: 120
|
httpIdle: 120
|
||||||
pageExecute: 15
|
pageExecute: 15
|
||||||
pageRender: 15
|
pageRender: 15
|
||||||
|
ipRangeLoad: 10
|
||||||
queues:
|
queues:
|
||||||
auditWrites: 16
|
auditWrites: 16
|
||||||
contextRecycle: 16
|
contextRecycle: 16
|
||||||
|
|||||||
Reference in New Issue
Block a user