fully integrated agent classification
This commit is contained in:
@@ -8,6 +8,28 @@
|
|||||||
*
|
*
|
||||||
* SPDX-License-Identifier: MPL-2.0
|
* SPDX-License-Identifier: MPL-2.0
|
||||||
*/
|
*/
|
||||||
|
/* This file contains code adapted from Mintlify agent-traffic, https://github.com/mintlify/agent-traffic
|
||||||
|
* ---
|
||||||
|
* Copyright (c) 2026 Mintlify, Inc.
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
* of this software and associated documentation files (the "Software"), to deal
|
||||||
|
* in the Software without restriction, including without limitation the rights
|
||||||
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the Software is
|
||||||
|
* furnished to do so, subject to the following conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be included in all
|
||||||
|
* copies or substantial portions of the Software.
|
||||||
|
*
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
* SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
// Package agentclass contains support for agent request classification.
|
// Package agentclass contains support for agent request classification.
|
||||||
package agentclass
|
package agentclass
|
||||||
|
|||||||
@@ -8,6 +8,28 @@
|
|||||||
*
|
*
|
||||||
* SPDX-License-Identifier: MPL-2.0
|
* SPDX-License-Identifier: MPL-2.0
|
||||||
*/
|
*/
|
||||||
|
/* This file contains code adapted from Mintlify agent-traffic, https://github.com/mintlify/agent-traffic
|
||||||
|
* ---
|
||||||
|
* Copyright (c) 2026 Mintlify, Inc.
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
* of this software and associated documentation files (the "Software"), to deal
|
||||||
|
* in the Software without restriction, including without limitation the rights
|
||||||
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the Software is
|
||||||
|
* furnished to do so, subject to the following conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be included in all
|
||||||
|
* copies or substantial portions of the Software.
|
||||||
|
*
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
* SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
// Package agentclass contains support for agent request classification.
|
// Package agentclass contains support for agent request classification.
|
||||||
package agentclass
|
package agentclass
|
||||||
|
|||||||
@@ -8,6 +8,28 @@
|
|||||||
*
|
*
|
||||||
* SPDX-License-Identifier: MPL-2.0
|
* SPDX-License-Identifier: MPL-2.0
|
||||||
*/
|
*/
|
||||||
|
/* This file contains code adapted from Mintlify agent-traffic, https://github.com/mintlify/agent-traffic
|
||||||
|
* ---
|
||||||
|
* Copyright (c) 2026 Mintlify, Inc.
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
* of this software and associated documentation files (the "Software"), to deal
|
||||||
|
* in the Software without restriction, including without limitation the rights
|
||||||
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the Software is
|
||||||
|
* furnished to do so, subject to the following conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be included in all
|
||||||
|
* copies or substantial portions of the Software.
|
||||||
|
*
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
* SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
// Package agentclass contains support for agent request classification.
|
// Package agentclass contains support for agent request classification.
|
||||||
package agentclass
|
package agentclass
|
||||||
@@ -24,7 +46,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.erbosoft.com/amy/amsterdam/config"
|
"git.erbosoft.com/amy/amsterdam/config"
|
||||||
"github.com/labstack/gommon/log"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Format for IP range data inside the RangeDataFormat.
|
// Format for IP range data inside the RangeDataFormat.
|
||||||
@@ -60,10 +82,11 @@ func loadIPRangeData(ctx context.Context, url string, rd *ipRangeData) {
|
|||||||
LOADLOOP:
|
LOADLOOP:
|
||||||
for counter := config.GlobalConfig.Tuning.IPRangeLoadAttempts; counter > 0; counter-- {
|
for counter := config.GlobalConfig.Tuning.IPRangeLoadAttempts; counter > 0; counter-- {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done(): // cancellation check on every loop
|
||||||
rd.loadError = ctx.Err()
|
rd.loadError = ctx.Err()
|
||||||
break LOADLOOP
|
break LOADLOOP
|
||||||
default:
|
default:
|
||||||
|
// build the HTTP request
|
||||||
req, err := http.NewRequest("GET", url, nil)
|
req, err := http.NewRequest("GET", url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rd.loadError = err
|
rd.loadError = err
|
||||||
@@ -72,14 +95,17 @@ LOADLOOP:
|
|||||||
if rd.lastMod != "" {
|
if rd.lastMod != "" {
|
||||||
req.Header.Set("If-Modified-Since", rd.lastMod)
|
req.Header.Set("If-Modified-Since", rd.lastMod)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// build a client and use it to load the file
|
||||||
client := &http.Client{Timeout: time.Duration(config.GlobalConfig.Tuning.Timeouts.IPRangeLoad) * time.Second}
|
client := &http.Client{Timeout: time.Duration(config.GlobalConfig.Tuning.Timeouts.IPRangeLoad) * time.Second}
|
||||||
resp, err := client.Do(req)
|
resp, err := client.Do(req)
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done(): // we might have been canceled while waiting for the GET
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
rd.loadError = ctx.Err()
|
rd.loadError = ctx.Err()
|
||||||
break LOADLOOP
|
break LOADLOOP
|
||||||
default:
|
default:
|
||||||
|
// interpret error and status codes
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rd.loadError = err
|
rd.loadError = err
|
||||||
continue LOADLOOP
|
continue LOADLOOP
|
||||||
@@ -93,10 +119,14 @@ LOADLOOP:
|
|||||||
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
|
rd.loadError = fmt.Errorf("failed to get file with status %d", resp.StatusCode)
|
||||||
continue LOADLOOP
|
continue LOADLOOP
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// load Last-Modified date
|
||||||
lastmod := ""
|
lastmod := ""
|
||||||
if lm, ok := resp.Header["Last-Modified"]; ok {
|
if lm, ok := resp.Header["Last-Modified"]; ok {
|
||||||
lastmod = lm[0]
|
lastmod = lm[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parse the JSON data retrieved
|
||||||
decoder := jsontext.NewDecoder(resp.Body)
|
decoder := jsontext.NewDecoder(resp.Body)
|
||||||
var parsed RangeDataFormat
|
var parsed RangeDataFormat
|
||||||
err = json.UnmarshalDecode(decoder, &parsed)
|
err = json.UnmarshalDecode(decoder, &parsed)
|
||||||
@@ -104,6 +134,8 @@ LOADLOOP:
|
|||||||
rd.loadError = err
|
rd.loadError = err
|
||||||
break LOADLOOP
|
break LOADLOOP
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// turn the parsed data into our internal data
|
||||||
lx := 0
|
lx := 0
|
||||||
if parsed.Prefixes != nil {
|
if parsed.Prefixes != nil {
|
||||||
lx = len(parsed.Prefixes)
|
lx = len(parsed.Prefixes)
|
||||||
@@ -124,22 +156,25 @@ LOADLOOP:
|
|||||||
}
|
}
|
||||||
rd.loadError = nil
|
rd.loadError = nil
|
||||||
rd.lastMod = lastmod
|
rd.lastMod = lastmod
|
||||||
|
rd.creation = parsed.CreationTime
|
||||||
break LOADLOOP
|
break LOADLOOP
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done(): // one more cancel check before ending
|
||||||
if rd.loadError == nil {
|
if rd.loadError == nil {
|
||||||
rd.loadError = ctx.Err()
|
rd.loadError = ctx.Err()
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
// fall through
|
// fall through
|
||||||
}
|
}
|
||||||
rd.loaded = true
|
rd.loaded = true // finished loading, let another routine go
|
||||||
rd.c.Signal()
|
rd.c.Signal()
|
||||||
|
log.Debugf("loaded IP range document at %s", url)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// prepareRefresh prepares some ipRangeData for being refreshed.
|
||||||
func prepareRefresh(rd *ipRangeData) {
|
func prepareRefresh(rd *ipRangeData) {
|
||||||
rd.m.Lock()
|
rd.m.Lock()
|
||||||
defer rd.m.Unlock()
|
defer rd.m.Unlock()
|
||||||
@@ -149,6 +184,7 @@ func prepareRefresh(rd *ipRangeData) {
|
|||||||
rd.loaded = false
|
rd.loaded = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refreshRanges runs periodically to refresh the range data from the various sites.
|
||||||
func refreshRanges(ctx context.Context, interval time.Duration) {
|
func refreshRanges(ctx context.Context, interval time.Duration) {
|
||||||
tick := time.NewTicker(interval)
|
tick := time.NewTicker(interval)
|
||||||
REFRESHLOOP:
|
REFRESHLOOP:
|
||||||
@@ -174,8 +210,10 @@ REFRESHLOOP:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// loadRangesCtx is the context used for running the range load and refresh data.
|
||||||
var loadRangesCtx context.Context = nil
|
var loadRangesCtx context.Context = nil
|
||||||
|
|
||||||
|
// loadIPRanges begins loading the initial IP ranges.
|
||||||
func loadIPRanges(urls []string, ctxname any) func() {
|
func loadIPRanges(urls []string, ctxname any) func() {
|
||||||
ipRangeStore = make(map[string]*ipRangeData)
|
ipRangeStore = make(map[string]*ipRangeData)
|
||||||
ctx, cancelfunc := context.WithCancel(context.Background())
|
ctx, cancelfunc := context.WithCancel(context.Background())
|
||||||
@@ -186,6 +224,7 @@ func loadIPRanges(urls []string, ctxname any) func() {
|
|||||||
ipRangeStore[url] = rdat
|
ipRangeStore[url] = rdat
|
||||||
go loadIPRangeData(loadRangesCtx, url, rdat)
|
go loadIPRangeData(loadRangesCtx, url, rdat)
|
||||||
}
|
}
|
||||||
|
log.Infof("loadIPRanges: started loading %d IP range files", len(urls))
|
||||||
dur, err := time.ParseDuration(config.GlobalConfig.Tuning.IPRangeRefresh)
|
dur, err := time.ParseDuration(config.GlobalConfig.Tuning.IPRangeRefresh)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("invalid IP range refresh duration \"%s\", using default", config.GlobalConfig.Tuning.IPRangeRefresh)
|
log.Errorf("invalid IP range refresh duration \"%s\", using default", config.GlobalConfig.Tuning.IPRangeRefresh)
|
||||||
@@ -205,6 +244,7 @@ func loadIPRanges(urls []string, ctxname any) func() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getIPRangeData gets the IP range data for a specified URL, which may have been cached already.
|
||||||
func getIPRangeData(url string) ([]netip.Prefix, *time.Time, error) {
|
func getIPRangeData(url string) ([]netip.Prefix, *time.Time, error) {
|
||||||
rdat, ok := ipRangeStore[url]
|
rdat, ok := ipRangeStore[url]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.erbosoft.com/amy/amsterdam/agentclass"
|
||||||
"git.erbosoft.com/amy/amsterdam/config"
|
"git.erbosoft.com/amy/amsterdam/config"
|
||||||
"git.erbosoft.com/amy/amsterdam/database"
|
"git.erbosoft.com/amy/amsterdam/database"
|
||||||
"git.erbosoft.com/amy/amsterdam/email"
|
"git.erbosoft.com/amy/amsterdam/email"
|
||||||
@@ -58,6 +59,17 @@ func myIPAddress() net.IP {
|
|||||||
return localAddr.IP
|
return localAddr.IP
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ClassifyInjector injects AmClassify onto the request.
|
||||||
|
func ClassifyInjector(next echo.HandlerFunc) echo.HandlerFunc {
|
||||||
|
opts := agentclass.AmClassifyOptions()
|
||||||
|
opts.HeadersComplete = true
|
||||||
|
return func(c *echo.Context) error {
|
||||||
|
classif := agentclass.AmClassify(c.Request().Header, c.RealIP(), "", opts)
|
||||||
|
c.Set("am.classification", classif)
|
||||||
|
return next(c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// setupEcho creates, configures, and returns a new Echo instance.
|
// setupEcho creates, configures, and returns a new Echo instance.
|
||||||
func setupEcho() *echo.Echo {
|
func setupEcho() *echo.Echo {
|
||||||
e := echo.New()
|
e := echo.New()
|
||||||
@@ -72,7 +84,7 @@ func setupEcho() *echo.Echo {
|
|||||||
} else {
|
} else {
|
||||||
log.Warn("WARNING: --debug-panic in effect - DO NOT use this in production!")
|
log.Warn("WARNING: --debug-panic in effect - DO NOT use this in production!")
|
||||||
}
|
}
|
||||||
e.Use(LoggingMiddleware)
|
e.Use(LoggingMiddleware, ClassifyInjector)
|
||||||
|
|
||||||
// set up the rate limiter from the configuration
|
// set up the rate limiter from the configuration
|
||||||
rateLimiter := AmSetupRateLimiter()
|
rateLimiter := AmSetupRateLimiter()
|
||||||
@@ -257,6 +269,8 @@ func main() {
|
|||||||
closer = email.SetupMailSender()
|
closer = email.SetupMailSender()
|
||||||
defer closer()
|
defer closer()
|
||||||
htmlcheck.SetupDicts()
|
htmlcheck.SetupDicts()
|
||||||
|
closer = agentclass.AmInitAgentClass(CONTEXTVAL_NAME)
|
||||||
|
defer closer()
|
||||||
closer = ui.SetupUILayer()
|
closer = ui.SetupUILayer()
|
||||||
defer closer()
|
defer closer()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user