introduce databse connection pooling and harden RowSet operations
This commit is contained in:
@@ -121,6 +121,12 @@ type AmConfig struct {
|
||||
User string `yaml:"user"`
|
||||
Password string `yaml:"password"`
|
||||
DatabaseName string `yaml:"databaseName"`
|
||||
Pooling struct {
|
||||
MaxOpen int `yaml:"maxOpen"`
|
||||
MaxIdle int `yaml:"maxIdle"`
|
||||
MaxIdleTime int `yaml:"maxIdleTime"`
|
||||
MaxLifetime int `yaml:"maxLifetime"`
|
||||
} `yaml:"pooling"`
|
||||
} `yaml:"database"`
|
||||
Defaults struct {
|
||||
Language string `yaml:"language"`
|
||||
@@ -391,6 +397,16 @@ func SetupConfig() {
|
||||
GlobalConfig.baseDir = ""
|
||||
}
|
||||
|
||||
// Error-check values.
|
||||
if GlobalConfig.Database.Pooling.MaxOpen < 0 {
|
||||
panic("maximum database connections must not be less than zero")
|
||||
} else if GlobalConfig.Database.Pooling.MaxOpen == 0 {
|
||||
log.Warn("WARNING: database connections unlimited - server may crash under heavy load")
|
||||
}
|
||||
if GlobalConfig.Database.Pooling.MaxIdle < 0 {
|
||||
panic("maximum idle database connections must not be less than zero")
|
||||
}
|
||||
|
||||
// Compute additional values.
|
||||
if CommandLine.Debug {
|
||||
GlobalComputedConfig.DebugMode = true
|
||||
|
||||
@@ -46,6 +46,11 @@ database:
|
||||
user: "amsdb"
|
||||
password: "x00yes2k"
|
||||
databaseName: "amsterdam"
|
||||
pooling:
|
||||
maxOpen: 100
|
||||
maxIdle: 10
|
||||
maxIdleTime: -1
|
||||
maxLifetime: -1
|
||||
defaults:
|
||||
language: "en-US"
|
||||
timezone: "America/Denver"
|
||||
|
||||
+16
-1
@@ -20,6 +20,7 @@ import (
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.erbosoft.com/amy/amsterdam/config"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
@@ -218,13 +219,27 @@ func SetupDb() (string, func(), error) {
|
||||
}
|
||||
db, err := sqlx.Connect(config.GlobalComputedConfig.DatabaseDriver, buildMysqlDSN(false))
|
||||
if err == nil {
|
||||
// Configure the connection pool options.
|
||||
db.SetMaxOpenConns(config.GlobalConfig.Database.Pooling.MaxOpen)
|
||||
db.SetMaxIdleConns(config.GlobalConfig.Database.Pooling.MaxIdle)
|
||||
if config.GlobalConfig.Database.Pooling.MaxIdleTime > 0 {
|
||||
db.SetConnMaxIdleTime(time.Duration(config.GlobalConfig.Database.Pooling.MaxIdleTime) * time.Second)
|
||||
}
|
||||
if config.GlobalConfig.Database.Pooling.MaxLifetime > 0 {
|
||||
db.SetConnMaxLifetime(time.Duration(config.GlobalConfig.Database.Pooling.MaxLifetime) * time.Second)
|
||||
}
|
||||
|
||||
// Retrieve the database globals.
|
||||
amdb = db
|
||||
g, err := AmGlobals(context.Background())
|
||||
if err == nil {
|
||||
// Check database version against expected version.
|
||||
if g.Version != version {
|
||||
log.Warnf("!! database version %s does not match prepared version %s", g.Version, version)
|
||||
version = g.Version
|
||||
}
|
||||
|
||||
// Setup the database-dependent caches and background tasks.
|
||||
setupAdCache()
|
||||
setupUserCache()
|
||||
setupContactsCache()
|
||||
@@ -236,7 +251,7 @@ func SetupDb() (string, func(), error) {
|
||||
log.Infof("SetupDb(): database version %s", version)
|
||||
}
|
||||
}
|
||||
slices.Reverse(exitfns)
|
||||
slices.Reverse(exitfns) // exit funcs run in reverse order
|
||||
return version, func() {
|
||||
for _, f := range exitfns {
|
||||
f()
|
||||
|
||||
@@ -243,6 +243,7 @@ func AmSearchCategories(ctx context.Context, oper int, term string, offset int,
|
||||
if err != nil {
|
||||
return nil, total, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*Category, 0, min(max, 1000))
|
||||
for rs.Next() {
|
||||
var catid int32
|
||||
@@ -256,5 +257,5 @@ func AmSearchCategories(ctx context.Context, oper int, term string, offset int,
|
||||
log.Errorf("AmSearchCategoris scan error: %v", err)
|
||||
}
|
||||
}
|
||||
return rc, total, nil
|
||||
return rc, total, rs.Err()
|
||||
}
|
||||
|
||||
@@ -314,6 +314,7 @@ func (c *Community) ListMembers(ctx context.Context, field, oper int, term strin
|
||||
if err != nil {
|
||||
return nil, total, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*User, 0, min(maxCount, 10000))
|
||||
for rs.Next() {
|
||||
var uid int32
|
||||
@@ -324,7 +325,7 @@ func (c *Community) ListMembers(ctx context.Context, field, oper int, term strin
|
||||
}
|
||||
}
|
||||
}
|
||||
return rc, total, nil
|
||||
return rc, total, rs.Err()
|
||||
}
|
||||
|
||||
/* SetMembership sets a user's membership status within the community.
|
||||
@@ -1054,6 +1055,7 @@ func AmGetCommunitiesForCategory(ctx context.Context, catid int32, offset, maxCo
|
||||
if err != nil {
|
||||
return nil, total, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*Community, 0, min(maxCount, 10000))
|
||||
for rs.Next() {
|
||||
var commid int32
|
||||
@@ -1064,7 +1066,7 @@ func AmGetCommunitiesForCategory(ctx context.Context, catid int32, offset, maxCo
|
||||
}
|
||||
}
|
||||
}
|
||||
return rc, total, nil
|
||||
return rc, total, rs.Err()
|
||||
}
|
||||
|
||||
/* AmSearchCommunities searches for communities matching certain criteria.
|
||||
@@ -1131,6 +1133,7 @@ func AmSearchCommunities(ctx context.Context, field, oper int, term string, offs
|
||||
if err != nil {
|
||||
return nil, total, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*Community, 0, min(maxCount, 10000))
|
||||
for rs.Next() {
|
||||
var commid int32
|
||||
@@ -1141,7 +1144,7 @@ func AmSearchCommunities(ctx context.Context, field, oper int, term string, offs
|
||||
}
|
||||
}
|
||||
}
|
||||
return rc, total, nil
|
||||
return rc, total, rs.Err()
|
||||
}
|
||||
|
||||
/* AmSearchCommunityMembers searches for members of a community matching certain criteria.
|
||||
@@ -1216,6 +1219,7 @@ func AmSearchCommunityMembers(ctx context.Context, c *Community, field, oper int
|
||||
if err != nil {
|
||||
return nil, total, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*User, 0, min(maxCount, 10000))
|
||||
for rs.Next() {
|
||||
var uid int32
|
||||
@@ -1230,5 +1234,5 @@ func AmSearchCommunityMembers(ctx context.Context, c *Community, field, oper int
|
||||
log.Errorf("AmSearchCommunityMembers scan error: %v", err)
|
||||
}
|
||||
}
|
||||
return rc, total, nil
|
||||
return rc, total, rs.Err()
|
||||
}
|
||||
|
||||
+16
-4
@@ -158,6 +158,7 @@ func (c *Conference) Aliases(ctx context.Context, commid int32) ([]string, error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]string, 0, 5)
|
||||
for rs.Next() {
|
||||
var a string
|
||||
@@ -167,7 +168,7 @@ func (c *Conference) Aliases(ctx context.Context, commid int32) ([]string, error
|
||||
log.Errorf("Aliases scan error: %v", err)
|
||||
}
|
||||
}
|
||||
return rc, nil
|
||||
return rc, rs.Err()
|
||||
}
|
||||
|
||||
// AddAlias adds an alias to the conference.
|
||||
@@ -290,6 +291,7 @@ func (c *Conference) ContainedBy(ctx context.Context) ([]*Community, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*Community, 0, 1)
|
||||
for rs.Next() {
|
||||
var cid int32
|
||||
@@ -303,7 +305,7 @@ func (c *Conference) ContainedBy(ctx context.Context) ([]*Community, error) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return rc, nil
|
||||
return rc, rs.Err()
|
||||
}
|
||||
|
||||
// Members returns all the members of this conference, with their granted user levels.
|
||||
@@ -614,12 +616,16 @@ func (c *Conference) Fixseen(ctx context.Context, u *User) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rs.Close()
|
||||
work := make([]fixseenData, 0, count)
|
||||
for rs.Next() {
|
||||
var d fixseenData
|
||||
err = rs.Scan(&(d.topicid), &(d.topmessage), &(d.insert))
|
||||
work = append(work, d)
|
||||
}
|
||||
if rs.Err() != nil {
|
||||
return rs.Err()
|
||||
}
|
||||
|
||||
// Adjust each topic in turn.
|
||||
for _, d := range work {
|
||||
@@ -727,6 +733,7 @@ func (c *Conference) GetActivity(ctx context.Context, reportType int) ([]Activit
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]ActivityReport, 0)
|
||||
for rs.Next() {
|
||||
var cur ActivityReport
|
||||
@@ -736,7 +743,7 @@ func (c *Conference) GetActivity(ctx context.Context, reportType int) ([]Activit
|
||||
}
|
||||
rc = append(rc, cur)
|
||||
}
|
||||
return rc, nil
|
||||
return rc, rs.Err()
|
||||
}
|
||||
|
||||
// Active user selection types.
|
||||
@@ -777,6 +784,7 @@ func (c *Conference) GetActiveUserEMailAddrs(ctx context.Context, userSelect, da
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
var stopPoint *time.Time = nil
|
||||
if dayLimit > 0 {
|
||||
mynow := time.Now().UTC()
|
||||
@@ -796,7 +804,7 @@ func (c *Conference) GetActiveUserEMailAddrs(ctx context.Context, userSelect, da
|
||||
}
|
||||
rc = append(rc, addy)
|
||||
}
|
||||
return rc, nil
|
||||
return rc, rs.Err()
|
||||
}
|
||||
|
||||
// Stats retrieves the number of topics and posts in this conference.
|
||||
@@ -1108,6 +1116,7 @@ func AmListConferences(ctx context.Context, cid int32, showHidden bool) ([]*Conf
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*ConferenceSummary, 0)
|
||||
for rs.Next() {
|
||||
cs := new(ConferenceSummary)
|
||||
@@ -1117,6 +1126,9 @@ func AmListConferences(ctx context.Context, cid int32, showHidden bool) ([]*Conf
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if rs.Err() != nil {
|
||||
return nil, rs.Err()
|
||||
}
|
||||
for i := range rc {
|
||||
err := amdb.GetContext(ctx, &(rc[i].Alias), "SELECT alias FROM confalias WHERE commid = ? AND confid = ?", cid, rc[i].ConfId)
|
||||
if err != nil {
|
||||
|
||||
+6
-2
@@ -639,6 +639,7 @@ func AmGetPublishedPosts(ctx context.Context) ([]*PostHeader, []*Community, erro
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
// Extract post IDs to an array.
|
||||
cids := make([]int32, gv.FrontPagePosts)
|
||||
pids := make([]int64, gv.FrontPagePosts)
|
||||
@@ -649,7 +650,9 @@ func AmGetPublishedPosts(ctx context.Context) ([]*PostHeader, []*Community, erro
|
||||
}
|
||||
i++
|
||||
}
|
||||
if i == 0 { // no published posts, short-circuit response
|
||||
if rs.Err() != nil {
|
||||
return nil, nil, rs.Err()
|
||||
} else if i == 0 { // no published posts, short-circuit response
|
||||
return make([]*PostHeader, 0), make([]*Community, 0), nil
|
||||
}
|
||||
if i < int(gv.FrontPagePosts) {
|
||||
@@ -885,6 +888,7 @@ func AmSearchPosts(ctx context.Context, searchTerms string, u *User, offset, max
|
||||
log.Errorf("AmSearchPosts query 2 error %v", err)
|
||||
return nil, count, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]PostSearchResult, max)
|
||||
i := 0
|
||||
for rs.Next() {
|
||||
@@ -930,5 +934,5 @@ func AmSearchPosts(ctx context.Context, searchTerms string, u *User, offset, max
|
||||
if i < max {
|
||||
rc = rc[:i] // slice off any empty entries at the end
|
||||
}
|
||||
return rc, count, nil
|
||||
return rc, count, rs.Err()
|
||||
}
|
||||
|
||||
@@ -180,6 +180,7 @@ func AmGetCommunityServices(ctx context.Context, cid int32) ([]*ServiceDef, erro
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
dom := serviceRoot.byName[AM_DOMAIN_COMMUNITY]
|
||||
a := make([]*ServiceDef, 0, len(dom.Services))
|
||||
for rs.Next() {
|
||||
@@ -188,8 +189,12 @@ func AmGetCommunityServices(ctx context.Context, cid int32) ([]*ServiceDef, erro
|
||||
a = append(a, dom.byIndex[ndx])
|
||||
}
|
||||
}
|
||||
if rs.Err() == nil {
|
||||
servicesCache.Add(cid, a)
|
||||
rc = a
|
||||
} else {
|
||||
return nil, rs.Err()
|
||||
}
|
||||
}
|
||||
return rc.([]*ServiceDef), nil
|
||||
}
|
||||
@@ -212,6 +217,7 @@ func AmGetCommunityServicesTx(ctx context.Context, tx *sqlx.Tx, cid int32) ([]*S
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
dom := serviceRoot.byName[AM_DOMAIN_COMMUNITY]
|
||||
a := make([]*ServiceDef, 0, len(dom.Services))
|
||||
for rs.Next() {
|
||||
@@ -220,8 +226,12 @@ func AmGetCommunityServicesTx(ctx context.Context, tx *sqlx.Tx, cid int32) ([]*S
|
||||
a = append(a, dom.byIndex[ndx])
|
||||
}
|
||||
}
|
||||
if rs.Err() == nil {
|
||||
servicesCache.Add(cid, a)
|
||||
rc = a
|
||||
} else {
|
||||
return nil, rs.Err()
|
||||
}
|
||||
}
|
||||
return rc.([]*ServiceDef), nil
|
||||
}
|
||||
|
||||
+12
-4
@@ -211,6 +211,7 @@ func (t *Topic) GetBozos(ctx context.Context, u *User) ([]*TopicBozo, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*TopicBozo, 0)
|
||||
for rs.Next() {
|
||||
tb := new(TopicBozo)
|
||||
@@ -220,7 +221,7 @@ func (t *Topic) GetBozos(ctx context.Context, u *User) ([]*TopicBozo, error) {
|
||||
}
|
||||
rc = append(rc, tb)
|
||||
}
|
||||
return rc, nil
|
||||
return rc, rs.Err()
|
||||
}
|
||||
|
||||
// IsSubscribed returns true if the given user is subscribed to receive E-mails of topic posts.
|
||||
@@ -258,6 +259,7 @@ func (t *Topic) GetSubscribers(ctx context.Context) ([]int32, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]int32, 0)
|
||||
for rs.Next() {
|
||||
var tmp int32
|
||||
@@ -268,6 +270,9 @@ func (t *Topic) GetSubscribers(ctx context.Context) ([]int32, error) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err == nil && rs.Err() != nil {
|
||||
err = rs.Err()
|
||||
}
|
||||
return rc, err
|
||||
}
|
||||
|
||||
@@ -297,6 +302,7 @@ func (t *Topic) GetActivity(ctx context.Context, reportType int) ([]*ActivityRep
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*ActivityReport, 0)
|
||||
for rs.Next() {
|
||||
cur := new(ActivityReport)
|
||||
@@ -306,7 +312,7 @@ func (t *Topic) GetActivity(ctx context.Context, reportType int) ([]*ActivityRep
|
||||
}
|
||||
rc = append(rc, cur)
|
||||
}
|
||||
return rc, nil
|
||||
return rc, rs.Err()
|
||||
}
|
||||
|
||||
/* GetActiveUserEMailAddrs gets the E-mail addresses of each user that's active in the topic, omitting those that have opted out of mass E-mails.
|
||||
@@ -341,6 +347,7 @@ func (t *Topic) GetActiveUserEMailAddrs(ctx context.Context, userSelect, dayLimi
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
var stopPoint *time.Time = nil
|
||||
if dayLimit > 0 {
|
||||
mynow := time.Now().UTC()
|
||||
@@ -360,7 +367,7 @@ func (t *Topic) GetActiveUserEMailAddrs(ctx context.Context, userSelect, dayLimi
|
||||
}
|
||||
rc = append(rc, addy)
|
||||
}
|
||||
return rc, nil
|
||||
return rc, rs.Err()
|
||||
}
|
||||
|
||||
// backgroundPurgeTopic removes all posts from a topic that's been deleted.
|
||||
@@ -698,6 +705,7 @@ func AmListTopics(ctx context.Context, confid int32, uid int32, viewOption int,
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*TopicSummary, 0)
|
||||
for rs.Next() {
|
||||
rec := new(TopicSummary)
|
||||
@@ -708,7 +716,7 @@ func AmListTopics(ctx context.Context, confid int32, uid int32, viewOption int,
|
||||
rc = append(rc, rec)
|
||||
}
|
||||
}
|
||||
return rc, nil
|
||||
return rc, rs.Err()
|
||||
}
|
||||
|
||||
/* AmNewTopic creates a new topic.
|
||||
|
||||
+2
-1
@@ -961,6 +961,7 @@ func AmSearchUsers(ctx context.Context, field, oper int, term string, offset, ma
|
||||
if err != nil {
|
||||
return nil, total, err
|
||||
}
|
||||
defer rs.Close()
|
||||
rc := make([]*User, 0, min(maxCount, 10000))
|
||||
for rs.Next() {
|
||||
var uid int32
|
||||
@@ -975,5 +976,5 @@ func AmSearchUsers(ctx context.Context, field, oper int, term string, offset, ma
|
||||
log.Errorf("AmSearchUsers scan error: %v", err)
|
||||
}
|
||||
}
|
||||
return rc, total, nil
|
||||
return rc, total, rs.Err()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user