introduce databse connection pooling and harden RowSet operations

This commit is contained in:
amy
2026-11-12 14:25:11 -07:00
parent ec50f048a1
commit 630d129bb5
10 changed files with 97 additions and 21 deletions
+16
View File
@@ -121,6 +121,12 @@ type AmConfig struct {
User string `yaml:"user"`
Password string `yaml:"password"`
DatabaseName string `yaml:"databaseName"`
Pooling struct {
MaxOpen int `yaml:"maxOpen"`
MaxIdle int `yaml:"maxIdle"`
MaxIdleTime int `yaml:"maxIdleTime"`
MaxLifetime int `yaml:"maxLifetime"`
} `yaml:"pooling"`
} `yaml:"database"`
Defaults struct {
Language string `yaml:"language"`
@@ -391,6 +397,16 @@ func SetupConfig() {
GlobalConfig.baseDir = ""
}
// Error-check values.
if GlobalConfig.Database.Pooling.MaxOpen < 0 {
panic("maximum database connections must not be less than zero")
} else if GlobalConfig.Database.Pooling.MaxOpen == 0 {
log.Warn("WARNING: database connections unlimited - server may crash under heavy load")
}
if GlobalConfig.Database.Pooling.MaxIdle < 0 {
panic("maximum idle database connections must not be less than zero")
}
// Compute additional values.
if CommandLine.Debug {
GlobalComputedConfig.DebugMode = true
+5
View File
@@ -46,6 +46,11 @@ database:
user: "amsdb"
password: "x00yes2k"
databaseName: "amsterdam"
pooling:
maxOpen: 100
maxIdle: 10
maxIdleTime: -1
maxLifetime: -1
defaults:
language: "en-US"
timezone: "America/Denver"
+16 -1
View File
@@ -20,6 +20,7 @@ import (
"regexp"
"slices"
"strings"
"time"
"git.erbosoft.com/amy/amsterdam/config"
"github.com/go-sql-driver/mysql"
@@ -218,13 +219,27 @@ func SetupDb() (string, func(), error) {
}
db, err := sqlx.Connect(config.GlobalComputedConfig.DatabaseDriver, buildMysqlDSN(false))
if err == nil {
// Configure the connection pool options.
db.SetMaxOpenConns(config.GlobalConfig.Database.Pooling.MaxOpen)
db.SetMaxIdleConns(config.GlobalConfig.Database.Pooling.MaxIdle)
if config.GlobalConfig.Database.Pooling.MaxIdleTime > 0 {
db.SetConnMaxIdleTime(time.Duration(config.GlobalConfig.Database.Pooling.MaxIdleTime) * time.Second)
}
if config.GlobalConfig.Database.Pooling.MaxLifetime > 0 {
db.SetConnMaxLifetime(time.Duration(config.GlobalConfig.Database.Pooling.MaxLifetime) * time.Second)
}
// Retrieve the database globals.
amdb = db
g, err := AmGlobals(context.Background())
if err == nil {
// Check database version against expected version.
if g.Version != version {
log.Warnf("!! database version %s does not match prepared version %s", g.Version, version)
version = g.Version
}
// Setup the database-dependent caches and background tasks.
setupAdCache()
setupUserCache()
setupContactsCache()
@@ -236,7 +251,7 @@ func SetupDb() (string, func(), error) {
log.Infof("SetupDb(): database version %s", version)
}
}
slices.Reverse(exitfns)
slices.Reverse(exitfns) // exit funcs run in reverse order
return version, func() {
for _, f := range exitfns {
f()
+2 -1
View File
@@ -243,6 +243,7 @@ func AmSearchCategories(ctx context.Context, oper int, term string, offset int,
if err != nil {
return nil, total, err
}
defer rs.Close()
rc := make([]*Category, 0, min(max, 1000))
for rs.Next() {
var catid int32
@@ -256,5 +257,5 @@ func AmSearchCategories(ctx context.Context, oper int, term string, offset int,
log.Errorf("AmSearchCategoris scan error: %v", err)
}
}
return rc, total, nil
return rc, total, rs.Err()
}
+8 -4
View File
@@ -314,6 +314,7 @@ func (c *Community) ListMembers(ctx context.Context, field, oper int, term strin
if err != nil {
return nil, total, err
}
defer rs.Close()
rc := make([]*User, 0, min(maxCount, 10000))
for rs.Next() {
var uid int32
@@ -324,7 +325,7 @@ func (c *Community) ListMembers(ctx context.Context, field, oper int, term strin
}
}
}
return rc, total, nil
return rc, total, rs.Err()
}
/* SetMembership sets a user's membership status within the community.
@@ -1054,6 +1055,7 @@ func AmGetCommunitiesForCategory(ctx context.Context, catid int32, offset, maxCo
if err != nil {
return nil, total, err
}
defer rs.Close()
rc := make([]*Community, 0, min(maxCount, 10000))
for rs.Next() {
var commid int32
@@ -1064,7 +1066,7 @@ func AmGetCommunitiesForCategory(ctx context.Context, catid int32, offset, maxCo
}
}
}
return rc, total, nil
return rc, total, rs.Err()
}
/* AmSearchCommunities searches for communities matching certain criteria.
@@ -1131,6 +1133,7 @@ func AmSearchCommunities(ctx context.Context, field, oper int, term string, offs
if err != nil {
return nil, total, err
}
defer rs.Close()
rc := make([]*Community, 0, min(maxCount, 10000))
for rs.Next() {
var commid int32
@@ -1141,7 +1144,7 @@ func AmSearchCommunities(ctx context.Context, field, oper int, term string, offs
}
}
}
return rc, total, nil
return rc, total, rs.Err()
}
/* AmSearchCommunityMembers searches for members of a community matching certain criteria.
@@ -1216,6 +1219,7 @@ func AmSearchCommunityMembers(ctx context.Context, c *Community, field, oper int
if err != nil {
return nil, total, err
}
defer rs.Close()
rc := make([]*User, 0, min(maxCount, 10000))
for rs.Next() {
var uid int32
@@ -1230,5 +1234,5 @@ func AmSearchCommunityMembers(ctx context.Context, c *Community, field, oper int
log.Errorf("AmSearchCommunityMembers scan error: %v", err)
}
}
return rc, total, nil
return rc, total, rs.Err()
}
+16 -4
View File
@@ -158,6 +158,7 @@ func (c *Conference) Aliases(ctx context.Context, commid int32) ([]string, error
if err != nil {
return nil, err
}
defer rs.Close()
rc := make([]string, 0, 5)
for rs.Next() {
var a string
@@ -167,7 +168,7 @@ func (c *Conference) Aliases(ctx context.Context, commid int32) ([]string, error
log.Errorf("Aliases scan error: %v", err)
}
}
return rc, nil
return rc, rs.Err()
}
// AddAlias adds an alias to the conference.
@@ -290,6 +291,7 @@ func (c *Conference) ContainedBy(ctx context.Context) ([]*Community, error) {
if err != nil {
return nil, err
}
defer rs.Close()
rc := make([]*Community, 0, 1)
for rs.Next() {
var cid int32
@@ -303,7 +305,7 @@ func (c *Conference) ContainedBy(ctx context.Context) ([]*Community, error) {
return nil, err
}
}
return rc, nil
return rc, rs.Err()
}
// Members returns all the members of this conference, with their granted user levels.
@@ -614,12 +616,16 @@ func (c *Conference) Fixseen(ctx context.Context, u *User) error {
if err != nil {
return err
}
defer rs.Close()
work := make([]fixseenData, 0, count)
for rs.Next() {
var d fixseenData
err = rs.Scan(&(d.topicid), &(d.topmessage), &(d.insert))
work = append(work, d)
}
if rs.Err() != nil {
return rs.Err()
}
// Adjust each topic in turn.
for _, d := range work {
@@ -727,6 +733,7 @@ func (c *Conference) GetActivity(ctx context.Context, reportType int) ([]Activit
if err != nil {
return nil, err
}
defer rs.Close()
rc := make([]ActivityReport, 0)
for rs.Next() {
var cur ActivityReport
@@ -736,7 +743,7 @@ func (c *Conference) GetActivity(ctx context.Context, reportType int) ([]Activit
}
rc = append(rc, cur)
}
return rc, nil
return rc, rs.Err()
}
// Active user selection types.
@@ -777,6 +784,7 @@ func (c *Conference) GetActiveUserEMailAddrs(ctx context.Context, userSelect, da
if err != nil {
return nil, err
}
defer rs.Close()
var stopPoint *time.Time = nil
if dayLimit > 0 {
mynow := time.Now().UTC()
@@ -796,7 +804,7 @@ func (c *Conference) GetActiveUserEMailAddrs(ctx context.Context, userSelect, da
}
rc = append(rc, addy)
}
return rc, nil
return rc, rs.Err()
}
// Stats retrieves the number of topics and posts in this conference.
@@ -1108,6 +1116,7 @@ func AmListConferences(ctx context.Context, cid int32, showHidden bool) ([]*Conf
if err != nil {
return nil, err
}
defer rs.Close()
rc := make([]*ConferenceSummary, 0)
for rs.Next() {
cs := new(ConferenceSummary)
@@ -1117,6 +1126,9 @@ func AmListConferences(ctx context.Context, cid int32, showHidden bool) ([]*Conf
return nil, err
}
}
if rs.Err() != nil {
return nil, rs.Err()
}
for i := range rc {
err := amdb.GetContext(ctx, &(rc[i].Alias), "SELECT alias FROM confalias WHERE commid = ? AND confid = ?", cid, rc[i].ConfId)
if err != nil {
+6 -2
View File
@@ -639,6 +639,7 @@ func AmGetPublishedPosts(ctx context.Context) ([]*PostHeader, []*Community, erro
if err != nil {
return nil, nil, err
}
defer rs.Close()
// Extract post IDs to an array.
cids := make([]int32, gv.FrontPagePosts)
pids := make([]int64, gv.FrontPagePosts)
@@ -649,7 +650,9 @@ func AmGetPublishedPosts(ctx context.Context) ([]*PostHeader, []*Community, erro
}
i++
}
if i == 0 { // no published posts, short-circuit response
if rs.Err() != nil {
return nil, nil, rs.Err()
} else if i == 0 { // no published posts, short-circuit response
return make([]*PostHeader, 0), make([]*Community, 0), nil
}
if i < int(gv.FrontPagePosts) {
@@ -885,6 +888,7 @@ func AmSearchPosts(ctx context.Context, searchTerms string, u *User, offset, max
log.Errorf("AmSearchPosts query 2 error %v", err)
return nil, count, err
}
defer rs.Close()
rc := make([]PostSearchResult, max)
i := 0
for rs.Next() {
@@ -930,5 +934,5 @@ func AmSearchPosts(ctx context.Context, searchTerms string, u *User, offset, max
if i < max {
rc = rc[:i] // slice off any empty entries at the end
}
return rc, count, nil
return rc, count, rs.Err()
}
+10
View File
@@ -180,6 +180,7 @@ func AmGetCommunityServices(ctx context.Context, cid int32) ([]*ServiceDef, erro
if err != nil {
return nil, err
}
defer rs.Close()
dom := serviceRoot.byName[AM_DOMAIN_COMMUNITY]
a := make([]*ServiceDef, 0, len(dom.Services))
for rs.Next() {
@@ -188,8 +189,12 @@ func AmGetCommunityServices(ctx context.Context, cid int32) ([]*ServiceDef, erro
a = append(a, dom.byIndex[ndx])
}
}
if rs.Err() == nil {
servicesCache.Add(cid, a)
rc = a
} else {
return nil, rs.Err()
}
}
return rc.([]*ServiceDef), nil
}
@@ -212,6 +217,7 @@ func AmGetCommunityServicesTx(ctx context.Context, tx *sqlx.Tx, cid int32) ([]*S
if err != nil {
return nil, err
}
defer rs.Close()
dom := serviceRoot.byName[AM_DOMAIN_COMMUNITY]
a := make([]*ServiceDef, 0, len(dom.Services))
for rs.Next() {
@@ -220,8 +226,12 @@ func AmGetCommunityServicesTx(ctx context.Context, tx *sqlx.Tx, cid int32) ([]*S
a = append(a, dom.byIndex[ndx])
}
}
if rs.Err() == nil {
servicesCache.Add(cid, a)
rc = a
} else {
return nil, rs.Err()
}
}
return rc.([]*ServiceDef), nil
}
+12 -4
View File
@@ -211,6 +211,7 @@ func (t *Topic) GetBozos(ctx context.Context, u *User) ([]*TopicBozo, error) {
if err != nil {
return nil, err
}
defer rs.Close()
rc := make([]*TopicBozo, 0)
for rs.Next() {
tb := new(TopicBozo)
@@ -220,7 +221,7 @@ func (t *Topic) GetBozos(ctx context.Context, u *User) ([]*TopicBozo, error) {
}
rc = append(rc, tb)
}
return rc, nil
return rc, rs.Err()
}
// IsSubscribed returns true if the given user is subscribed to receive E-mails of topic posts.
@@ -258,6 +259,7 @@ func (t *Topic) GetSubscribers(ctx context.Context) ([]int32, error) {
if err != nil {
return nil, err
}
defer rs.Close()
rc := make([]int32, 0)
for rs.Next() {
var tmp int32
@@ -268,6 +270,9 @@ func (t *Topic) GetSubscribers(ctx context.Context) ([]int32, error) {
break
}
}
if err == nil && rs.Err() != nil {
err = rs.Err()
}
return rc, err
}
@@ -297,6 +302,7 @@ func (t *Topic) GetActivity(ctx context.Context, reportType int) ([]*ActivityRep
if err != nil {
return nil, err
}
defer rs.Close()
rc := make([]*ActivityReport, 0)
for rs.Next() {
cur := new(ActivityReport)
@@ -306,7 +312,7 @@ func (t *Topic) GetActivity(ctx context.Context, reportType int) ([]*ActivityRep
}
rc = append(rc, cur)
}
return rc, nil
return rc, rs.Err()
}
/* GetActiveUserEMailAddrs gets the E-mail addresses of each user that's active in the topic, omitting those that have opted out of mass E-mails.
@@ -341,6 +347,7 @@ func (t *Topic) GetActiveUserEMailAddrs(ctx context.Context, userSelect, dayLimi
if err != nil {
return nil, err
}
defer rs.Close()
var stopPoint *time.Time = nil
if dayLimit > 0 {
mynow := time.Now().UTC()
@@ -360,7 +367,7 @@ func (t *Topic) GetActiveUserEMailAddrs(ctx context.Context, userSelect, dayLimi
}
rc = append(rc, addy)
}
return rc, nil
return rc, rs.Err()
}
// backgroundPurgeTopic removes all posts from a topic that's been deleted.
@@ -698,6 +705,7 @@ func AmListTopics(ctx context.Context, confid int32, uid int32, viewOption int,
}
return nil, err
}
defer rs.Close()
rc := make([]*TopicSummary, 0)
for rs.Next() {
rec := new(TopicSummary)
@@ -708,7 +716,7 @@ func AmListTopics(ctx context.Context, confid int32, uid int32, viewOption int,
rc = append(rc, rec)
}
}
return rc, nil
return rc, rs.Err()
}
/* AmNewTopic creates a new topic.
+2 -1
View File
@@ -961,6 +961,7 @@ func AmSearchUsers(ctx context.Context, field, oper int, term string, offset, ma
if err != nil {
return nil, total, err
}
defer rs.Close()
rc := make([]*User, 0, min(maxCount, 10000))
for rs.Next() {
var uid int32
@@ -975,5 +976,5 @@ func AmSearchUsers(ctx context.Context, field, oper int, term string, offset, ma
log.Errorf("AmSearchUsers scan error: %v", err)
}
}
return rc, total, nil
return rc, total, rs.Err()
}