From 630d129bb5369fda0b678a55da59531d86e92fdb Mon Sep 17 00:00:00 2001 From: Amy Gale Ruth Bowersox Date: Thu, 12 Nov 2026 14:25:11 -0700 Subject: [PATCH] introduce databse connection pooling and harden RowSet operations --- config/config.go | 16 ++++++++++++++++ config/default.yaml | 5 +++++ database/base.go | 17 ++++++++++++++++- database/category.go | 3 ++- database/community.go | 12 ++++++++---- database/conference.go | 20 ++++++++++++++++---- database/post.go | 8 ++++++-- database/services.go | 18 ++++++++++++++---- database/topic.go | 16 ++++++++++++---- database/user.go | 3 ++- 10 files changed, 97 insertions(+), 21 deletions(-) diff --git a/config/config.go b/config/config.go index a9205f3..d5abefa 100644 --- a/config/config.go +++ b/config/config.go @@ -121,6 +121,12 @@ type AmConfig struct { User string `yaml:"user"` Password string `yaml:"password"` DatabaseName string `yaml:"databaseName"` + Pooling struct { + MaxOpen int `yaml:"maxOpen"` + MaxIdle int `yaml:"maxIdle"` + MaxIdleTime int `yaml:"maxIdleTime"` + MaxLifetime int `yaml:"maxLifetime"` + } `yaml:"pooling"` } `yaml:"database"` Defaults struct { Language string `yaml:"language"` @@ -391,6 +397,16 @@ func SetupConfig() { GlobalConfig.baseDir = "" } + // Error-check values. + if GlobalConfig.Database.Pooling.MaxOpen < 0 { + panic("maximum database connections must not be less than zero") + } else if GlobalConfig.Database.Pooling.MaxOpen == 0 { + log.Warn("WARNING: database connections unlimited - server may crash under heavy load") + } + if GlobalConfig.Database.Pooling.MaxIdle < 0 { + panic("maximum idle database connections must not be less than zero") + } + // Compute additional values. if CommandLine.Debug { GlobalComputedConfig.DebugMode = true diff --git a/config/default.yaml b/config/default.yaml index 2c4260e..cd26565 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -46,6 +46,11 @@ database: user: "amsdb" password: "x00yes2k" databaseName: "amsterdam" + pooling: + maxOpen: 100 + maxIdle: 10 + maxIdleTime: -1 + maxLifetime: -1 defaults: language: "en-US" timezone: "America/Denver" diff --git a/database/base.go b/database/base.go index 605fb9d..6ef4adb 100644 --- a/database/base.go +++ b/database/base.go @@ -20,6 +20,7 @@ import ( "regexp" "slices" "strings" + "time" "git.erbosoft.com/amy/amsterdam/config" "github.com/go-sql-driver/mysql" @@ -218,13 +219,27 @@ func SetupDb() (string, func(), error) { } db, err := sqlx.Connect(config.GlobalComputedConfig.DatabaseDriver, buildMysqlDSN(false)) if err == nil { + // Configure the connection pool options. + db.SetMaxOpenConns(config.GlobalConfig.Database.Pooling.MaxOpen) + db.SetMaxIdleConns(config.GlobalConfig.Database.Pooling.MaxIdle) + if config.GlobalConfig.Database.Pooling.MaxIdleTime > 0 { + db.SetConnMaxIdleTime(time.Duration(config.GlobalConfig.Database.Pooling.MaxIdleTime) * time.Second) + } + if config.GlobalConfig.Database.Pooling.MaxLifetime > 0 { + db.SetConnMaxLifetime(time.Duration(config.GlobalConfig.Database.Pooling.MaxLifetime) * time.Second) + } + + // Retrieve the database globals. amdb = db g, err := AmGlobals(context.Background()) if err == nil { + // Check database version against expected version. if g.Version != version { log.Warnf("!! database version %s does not match prepared version %s", g.Version, version) version = g.Version } + + // Setup the database-dependent caches and background tasks. setupAdCache() setupUserCache() setupContactsCache() @@ -236,7 +251,7 @@ func SetupDb() (string, func(), error) { log.Infof("SetupDb(): database version %s", version) } } - slices.Reverse(exitfns) + slices.Reverse(exitfns) // exit funcs run in reverse order return version, func() { for _, f := range exitfns { f() diff --git a/database/category.go b/database/category.go index 9aec7bd..8586b23 100644 --- a/database/category.go +++ b/database/category.go @@ -243,6 +243,7 @@ func AmSearchCategories(ctx context.Context, oper int, term string, offset int, if err != nil { return nil, total, err } + defer rs.Close() rc := make([]*Category, 0, min(max, 1000)) for rs.Next() { var catid int32 @@ -256,5 +257,5 @@ func AmSearchCategories(ctx context.Context, oper int, term string, offset int, log.Errorf("AmSearchCategoris scan error: %v", err) } } - return rc, total, nil + return rc, total, rs.Err() } diff --git a/database/community.go b/database/community.go index de064be..48b129a 100644 --- a/database/community.go +++ b/database/community.go @@ -314,6 +314,7 @@ func (c *Community) ListMembers(ctx context.Context, field, oper int, term strin if err != nil { return nil, total, err } + defer rs.Close() rc := make([]*User, 0, min(maxCount, 10000)) for rs.Next() { var uid int32 @@ -324,7 +325,7 @@ func (c *Community) ListMembers(ctx context.Context, field, oper int, term strin } } } - return rc, total, nil + return rc, total, rs.Err() } /* SetMembership sets a user's membership status within the community. @@ -1054,6 +1055,7 @@ func AmGetCommunitiesForCategory(ctx context.Context, catid int32, offset, maxCo if err != nil { return nil, total, err } + defer rs.Close() rc := make([]*Community, 0, min(maxCount, 10000)) for rs.Next() { var commid int32 @@ -1064,7 +1066,7 @@ func AmGetCommunitiesForCategory(ctx context.Context, catid int32, offset, maxCo } } } - return rc, total, nil + return rc, total, rs.Err() } /* AmSearchCommunities searches for communities matching certain criteria. @@ -1131,6 +1133,7 @@ func AmSearchCommunities(ctx context.Context, field, oper int, term string, offs if err != nil { return nil, total, err } + defer rs.Close() rc := make([]*Community, 0, min(maxCount, 10000)) for rs.Next() { var commid int32 @@ -1141,7 +1144,7 @@ func AmSearchCommunities(ctx context.Context, field, oper int, term string, offs } } } - return rc, total, nil + return rc, total, rs.Err() } /* AmSearchCommunityMembers searches for members of a community matching certain criteria. @@ -1216,6 +1219,7 @@ func AmSearchCommunityMembers(ctx context.Context, c *Community, field, oper int if err != nil { return nil, total, err } + defer rs.Close() rc := make([]*User, 0, min(maxCount, 10000)) for rs.Next() { var uid int32 @@ -1230,5 +1234,5 @@ func AmSearchCommunityMembers(ctx context.Context, c *Community, field, oper int log.Errorf("AmSearchCommunityMembers scan error: %v", err) } } - return rc, total, nil + return rc, total, rs.Err() } diff --git a/database/conference.go b/database/conference.go index 128ccfd..fc766af 100644 --- a/database/conference.go +++ b/database/conference.go @@ -158,6 +158,7 @@ func (c *Conference) Aliases(ctx context.Context, commid int32) ([]string, error if err != nil { return nil, err } + defer rs.Close() rc := make([]string, 0, 5) for rs.Next() { var a string @@ -167,7 +168,7 @@ func (c *Conference) Aliases(ctx context.Context, commid int32) ([]string, error log.Errorf("Aliases scan error: %v", err) } } - return rc, nil + return rc, rs.Err() } // AddAlias adds an alias to the conference. @@ -290,6 +291,7 @@ func (c *Conference) ContainedBy(ctx context.Context) ([]*Community, error) { if err != nil { return nil, err } + defer rs.Close() rc := make([]*Community, 0, 1) for rs.Next() { var cid int32 @@ -303,7 +305,7 @@ func (c *Conference) ContainedBy(ctx context.Context) ([]*Community, error) { return nil, err } } - return rc, nil + return rc, rs.Err() } // Members returns all the members of this conference, with their granted user levels. @@ -614,12 +616,16 @@ func (c *Conference) Fixseen(ctx context.Context, u *User) error { if err != nil { return err } + defer rs.Close() work := make([]fixseenData, 0, count) for rs.Next() { var d fixseenData err = rs.Scan(&(d.topicid), &(d.topmessage), &(d.insert)) work = append(work, d) } + if rs.Err() != nil { + return rs.Err() + } // Adjust each topic in turn. for _, d := range work { @@ -727,6 +733,7 @@ func (c *Conference) GetActivity(ctx context.Context, reportType int) ([]Activit if err != nil { return nil, err } + defer rs.Close() rc := make([]ActivityReport, 0) for rs.Next() { var cur ActivityReport @@ -736,7 +743,7 @@ func (c *Conference) GetActivity(ctx context.Context, reportType int) ([]Activit } rc = append(rc, cur) } - return rc, nil + return rc, rs.Err() } // Active user selection types. @@ -777,6 +784,7 @@ func (c *Conference) GetActiveUserEMailAddrs(ctx context.Context, userSelect, da if err != nil { return nil, err } + defer rs.Close() var stopPoint *time.Time = nil if dayLimit > 0 { mynow := time.Now().UTC() @@ -796,7 +804,7 @@ func (c *Conference) GetActiveUserEMailAddrs(ctx context.Context, userSelect, da } rc = append(rc, addy) } - return rc, nil + return rc, rs.Err() } // Stats retrieves the number of topics and posts in this conference. @@ -1108,6 +1116,7 @@ func AmListConferences(ctx context.Context, cid int32, showHidden bool) ([]*Conf if err != nil { return nil, err } + defer rs.Close() rc := make([]*ConferenceSummary, 0) for rs.Next() { cs := new(ConferenceSummary) @@ -1117,6 +1126,9 @@ func AmListConferences(ctx context.Context, cid int32, showHidden bool) ([]*Conf return nil, err } } + if rs.Err() != nil { + return nil, rs.Err() + } for i := range rc { err := amdb.GetContext(ctx, &(rc[i].Alias), "SELECT alias FROM confalias WHERE commid = ? AND confid = ?", cid, rc[i].ConfId) if err != nil { diff --git a/database/post.go b/database/post.go index 5c4c1ab..cfe83e8 100644 --- a/database/post.go +++ b/database/post.go @@ -639,6 +639,7 @@ func AmGetPublishedPosts(ctx context.Context) ([]*PostHeader, []*Community, erro if err != nil { return nil, nil, err } + defer rs.Close() // Extract post IDs to an array. cids := make([]int32, gv.FrontPagePosts) pids := make([]int64, gv.FrontPagePosts) @@ -649,7 +650,9 @@ func AmGetPublishedPosts(ctx context.Context) ([]*PostHeader, []*Community, erro } i++ } - if i == 0 { // no published posts, short-circuit response + if rs.Err() != nil { + return nil, nil, rs.Err() + } else if i == 0 { // no published posts, short-circuit response return make([]*PostHeader, 0), make([]*Community, 0), nil } if i < int(gv.FrontPagePosts) { @@ -885,6 +888,7 @@ func AmSearchPosts(ctx context.Context, searchTerms string, u *User, offset, max log.Errorf("AmSearchPosts query 2 error %v", err) return nil, count, err } + defer rs.Close() rc := make([]PostSearchResult, max) i := 0 for rs.Next() { @@ -930,5 +934,5 @@ func AmSearchPosts(ctx context.Context, searchTerms string, u *User, offset, max if i < max { rc = rc[:i] // slice off any empty entries at the end } - return rc, count, nil + return rc, count, rs.Err() } diff --git a/database/services.go b/database/services.go index 9d13454..d881ecb 100644 --- a/database/services.go +++ b/database/services.go @@ -180,6 +180,7 @@ func AmGetCommunityServices(ctx context.Context, cid int32) ([]*ServiceDef, erro if err != nil { return nil, err } + defer rs.Close() dom := serviceRoot.byName[AM_DOMAIN_COMMUNITY] a := make([]*ServiceDef, 0, len(dom.Services)) for rs.Next() { @@ -188,8 +189,12 @@ func AmGetCommunityServices(ctx context.Context, cid int32) ([]*ServiceDef, erro a = append(a, dom.byIndex[ndx]) } } - servicesCache.Add(cid, a) - rc = a + if rs.Err() == nil { + servicesCache.Add(cid, a) + rc = a + } else { + return nil, rs.Err() + } } return rc.([]*ServiceDef), nil } @@ -212,6 +217,7 @@ func AmGetCommunityServicesTx(ctx context.Context, tx *sqlx.Tx, cid int32) ([]*S if err != nil { return nil, err } + defer rs.Close() dom := serviceRoot.byName[AM_DOMAIN_COMMUNITY] a := make([]*ServiceDef, 0, len(dom.Services)) for rs.Next() { @@ -220,8 +226,12 @@ func AmGetCommunityServicesTx(ctx context.Context, tx *sqlx.Tx, cid int32) ([]*S a = append(a, dom.byIndex[ndx]) } } - servicesCache.Add(cid, a) - rc = a + if rs.Err() == nil { + servicesCache.Add(cid, a) + rc = a + } else { + return nil, rs.Err() + } } return rc.([]*ServiceDef), nil } diff --git a/database/topic.go b/database/topic.go index a32deae..60ad7af 100644 --- a/database/topic.go +++ b/database/topic.go @@ -211,6 +211,7 @@ func (t *Topic) GetBozos(ctx context.Context, u *User) ([]*TopicBozo, error) { if err != nil { return nil, err } + defer rs.Close() rc := make([]*TopicBozo, 0) for rs.Next() { tb := new(TopicBozo) @@ -220,7 +221,7 @@ func (t *Topic) GetBozos(ctx context.Context, u *User) ([]*TopicBozo, error) { } rc = append(rc, tb) } - return rc, nil + return rc, rs.Err() } // IsSubscribed returns true if the given user is subscribed to receive E-mails of topic posts. @@ -258,6 +259,7 @@ func (t *Topic) GetSubscribers(ctx context.Context) ([]int32, error) { if err != nil { return nil, err } + defer rs.Close() rc := make([]int32, 0) for rs.Next() { var tmp int32 @@ -268,6 +270,9 @@ func (t *Topic) GetSubscribers(ctx context.Context) ([]int32, error) { break } } + if err == nil && rs.Err() != nil { + err = rs.Err() + } return rc, err } @@ -297,6 +302,7 @@ func (t *Topic) GetActivity(ctx context.Context, reportType int) ([]*ActivityRep if err != nil { return nil, err } + defer rs.Close() rc := make([]*ActivityReport, 0) for rs.Next() { cur := new(ActivityReport) @@ -306,7 +312,7 @@ func (t *Topic) GetActivity(ctx context.Context, reportType int) ([]*ActivityRep } rc = append(rc, cur) } - return rc, nil + return rc, rs.Err() } /* GetActiveUserEMailAddrs gets the E-mail addresses of each user that's active in the topic, omitting those that have opted out of mass E-mails. @@ -341,6 +347,7 @@ func (t *Topic) GetActiveUserEMailAddrs(ctx context.Context, userSelect, dayLimi if err != nil { return nil, err } + defer rs.Close() var stopPoint *time.Time = nil if dayLimit > 0 { mynow := time.Now().UTC() @@ -360,7 +367,7 @@ func (t *Topic) GetActiveUserEMailAddrs(ctx context.Context, userSelect, dayLimi } rc = append(rc, addy) } - return rc, nil + return rc, rs.Err() } // backgroundPurgeTopic removes all posts from a topic that's been deleted. @@ -698,6 +705,7 @@ func AmListTopics(ctx context.Context, confid int32, uid int32, viewOption int, } return nil, err } + defer rs.Close() rc := make([]*TopicSummary, 0) for rs.Next() { rec := new(TopicSummary) @@ -708,7 +716,7 @@ func AmListTopics(ctx context.Context, confid int32, uid int32, viewOption int, rc = append(rc, rec) } } - return rc, nil + return rc, rs.Err() } /* AmNewTopic creates a new topic. diff --git a/database/user.go b/database/user.go index cc15104..179677f 100644 --- a/database/user.go +++ b/database/user.go @@ -961,6 +961,7 @@ func AmSearchUsers(ctx context.Context, field, oper int, term string, offset, ma if err != nil { return nil, total, err } + defer rs.Close() rc := make([]*User, 0, min(maxCount, 10000)) for rs.Next() { var uid int32 @@ -975,5 +976,5 @@ func AmSearchUsers(ctx context.Context, field, oper int, term string, offset, ma log.Errorf("AmSearchUsers scan error: %v", err) } } - return rc, total, nil + return rc, total, rs.Err() }