From d42c8319816ba7d9e062f45a9f22515416c55b2d Mon Sep 17 00:00:00 2001 From: Pavel Shevaev Date: Wed, 26 Oct 2022 17:28:42 +0300 Subject: [PATCH] First commit --- db.go | 358 ++++++++++++++++++++++++++++++++++ db_test.go | 551 +++++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 12 ++ go.sum | 38 ++++ logger.go | 64 +++++++ 5 files changed, 1023 insertions(+) create mode 100644 db.go create mode 100644 db_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 logger.go diff --git a/db.go b/db.go new file mode 100644 index 0000000..7916b18 --- /dev/null +++ b/db.go @@ -0,0 +1,358 @@ +package db + +import ( + "database/sql" + "reflect" + + "git.bit5.ru/backend/colog" + "git.bit5.ru/backend/dbr" + "git.bit5.ru/backend/errors" + "git.bit5.ru/backend/mysql" + "git.bit5.ru/backend/res_tracker" +) + +const ( + ChunkSizeForIN = 50000 +) + +type Settings struct { + Host, Port, User, Pass, Name, Prefix, Params string + Driver string + LogLevel int + Weight uint32 +} + +func (s *Settings) ConnStr() string { + return s.User + ":" + s.Pass + "@tcp(" + s.Host + ":" + s.Port + ")/" + s.Name + s.Params +} + +type DBC struct { + Logger *colog.CoLog + s Settings + _con *dbr.Connection //lazy one, should be accessed via con() method + _sess *dbr.Session //lazy one, should be accessed via sess() method + trx *dbr.Tx + trxRefs int + commitTry int +} + +func GetDBC(logger *colog.CoLog, s Settings) *DBC { + + logger = logger.Clone().AddPrefix("[" + s.Prefix + "] ") + + dbc := &DBC{Logger: logger, s: s, _con: nil, _sess: nil} + + return dbc +} + +func (dbc *DBC) con() *dbr.Connection { + if dbc._con == nil { + driver := dbc.s.Driver + if len(driver) == 0 { + driver = "mysql" + } + //NOTE: sql.Open(..) doesn't happen to return an error + sqlDb, _ := sql.Open(driver, dbc.s.ConnStr()) + + dbc._con = dbr.NewConnection(sqlDb, nil) + + res_tracker.Track(dbc) + } + return dbc._con +} + +func (dbc *DBC) sess() *dbr.Session { + if dbc._sess == nil { + dbc._sess = dbc.con().NewSession(&EventReceiver{logger: dbc.Logger, s: dbc.s}) + } + return dbc._sess +} + +func (dbc *DBC) Open() { + dbc.con() +} + +func (dbc *DBC) IsOpen() bool { + return dbc._con != nil +} + +func (dbc *DBC) Close() error { + if dbc._con != nil { + res_tracker.Untrack(dbc) + + dbc.Rollback() + err := dbc._con.Db.Close() + + dbc._con = nil + dbc._sess = nil + + return err + } else { + return nil + } +} + +//NOTE: for low level stuff +func (dbc *DBC) DB() *sql.DB { + return dbc.con().Db +} + +func (dbc *DBC) Transaction(txFunc func(dbc *DBC) error) (err error) { + err = dbc.Begin() + if err != nil { + return + } + defer func() { + if p := recover(); p != nil { + dbc.Rollback() + panic(p) // re-throw panic after Rollback + } else if err != nil { + dbc.Rollback() // err is non-nil; don't change it + } else { + err = dbc.Commit() // err is nil; if Commit returns error update err + } + }() + err = txFunc(dbc) + return err +} + +func (dbc *DBC) Begin() error { + //check if we are already in a transaction + if dbc.trx != nil { + dbc.trxRefs++ + return nil + } + trx, err := dbc.sess().Begin() + if err == nil { + dbc.trx = trx + dbc.trxRefs = 1 + dbc.commitTry = 0 + } + return err +} + +func (dbc *DBC) Rollback() error { + if dbc.trxRefs > 0 { + dbc.trxRefs-- + if dbc.trxRefs == 0 { + if err := dbc.trx.Rollback(); err != nil { + return err + } + dbc.trx = nil + } + } + return nil +} + +func (dbc *DBC) RollbackOnDefer() { + if dbc.commitTry == 0 { + dbc.Rollback() + } else { + dbc.commitTry-- + } +} + +func (dbc *DBC) Commit() error { + dbc.commitTry++ + if dbc.trxRefs > 0 { + dbc.trxRefs-- + if dbc.trxRefs == 0 { + if err := dbc.trx.Commit(); err != nil { + return err + } + dbc.trx = nil + } + } + return nil +} + +// Update creates a new UpdateBuilder for the given table +func (dbc *DBC) Update(table string) *dbr.UpdateBuilder { + if dbc.trx == nil { + return &dbr.UpdateBuilder{ + Session: dbc.sess(), + Runner: dbc.con().Db, + Table: table, + } + } + + return &dbr.UpdateBuilder{ + Session: dbc.trx.Session, + Runner: dbc.trx.Tx, + Table: table, + } +} + +// UpdateBySQL creates a new UpdateBuilder for the given SQL string and arguments +func (dbc *DBC) UpdateBySQL(sql string, args ...interface{}) *dbr.UpdateBuilder { + if dbc.trx == nil { + return &dbr.UpdateBuilder{ + Session: dbc.sess(), + Runner: dbc.con().Db, + RawFullSql: sql, + RawArguments: args, + } + } + + return &dbr.UpdateBuilder{ + Session: dbc.trx.Session, + Runner: dbc.trx.Tx, + RawFullSql: sql, + RawArguments: args, + } +} + +// DeleteFrom creates a new DeleteBuilder for the given table +func (dbc *DBC) DeleteFrom(from string) *dbr.DeleteBuilder { + if dbc.trx == nil { + return &dbr.DeleteBuilder{ + Session: dbc.sess(), + Runner: dbc.con().Db, + From: from, + } + } + + return &dbr.DeleteBuilder{ + Session: dbc.trx.Session, + Runner: dbc.trx.Tx, + From: from, + } +} + +// Select creates a new SelectBuilder that select that given columns +func (dbc *DBC) Select(cols ...string) *dbr.SelectBuilder { + if dbc.trx == nil { + return &dbr.SelectBuilder{ + Session: dbc.sess(), + Runner: dbc.con().Db, + Columns: cols, + } + } + + return &dbr.SelectBuilder{ + Session: dbc.trx.Session, + Runner: dbc.trx.Tx, + Columns: cols, + } +} + +// SelectBySQL creates a new SelectBuilder for the given SQL string and arguments +func (dbc *DBC) SelectBySQL(sql string, args ...interface{}) *dbr.SelectBuilder { + if dbc.trx == nil { + return &dbr.SelectBuilder{ + Session: dbc.sess(), + Runner: dbc.con().Db, + RawFullSql: sql, + RawArguments: args, + } + } + + return &dbr.SelectBuilder{ + Session: dbc.trx.Session, + Runner: dbc.trx.Tx, + RawFullSql: sql, + RawArguments: args, + } +} + +//Note: Creates a new slice of dbr.SelectBuilder for the given SQL string +//Supported chunking only for first IN-list +func (dbc *DBC) SelectBySQLWithChunkedIN(sql string, chunkSize int, args ...interface{}) []*dbr.SelectBuilder { + var builders []*dbr.SelectBuilder + listsForIN := make(map[int]reflect.Value) + chunkedListsForIN := make(map[int][]reflect.Value) + + for i, arg := range args { + valueOfDest := reflect.ValueOf(arg) + kindOfDest := valueOfDest.Kind() + if kindOfDest == reflect.Slice || kindOfDest == reflect.Array { + //Note: i is index of arg + listsForIN[i] = valueOfDest + } + } + + if len(listsForIN) == 0 { + builder := dbc.SelectBySQL(sql, args) + builders = append(builders, builder) + return builders + } + + for index, listForIN := range listsForIN { + var chunks []reflect.Value + valuesAmount := listForIN.Len() + fullChunksAmount := valuesAmount / chunkSize + modulo := valuesAmount % chunkSize + + for i := 0; i < fullChunksAmount; i++ { + chunkStartIndex := i * chunkSize + chunkEndIndex := chunkStartIndex + chunkSize + chunkValues := listForIN.Slice(chunkStartIndex, chunkEndIndex) + chunks = append(chunks, chunkValues) + } + + if modulo > 0 { + chunkStartIndex := fullChunksAmount * chunkSize + chunkEndIndex := chunkStartIndex + modulo + chunkValues := listForIN.Slice(chunkStartIndex, chunkEndIndex) + chunks = append(chunks, chunkValues) + } + + chunkedListsForIN[index] = chunks + } + + //TODO: Supported only first IN-list, because the several IN-lists can generate too many sql queries(chunks amount of 1-st IN-list * chunks amount of 2-d IN-list * etc.) + for argIndex, argChunks := range chunkedListsForIN { + for c := 0; c < len(argChunks); c++ { + argChunk := argChunks[c] + var sqlArgs = make([]interface{}, len(args)) + + //NOTE: Fill args for a separate sql query + for i := 0; i < len(args); i++ { + if i != argIndex { + sqlArgs[i] = args[i] + } else { + sqlArgs[i] = argChunk.Interface() + } + } + + builder := dbc.SelectBySQL(sql, sqlArgs...) + builders = append(builders, builder) + } + } + + return builders +} + +// InsertInto instantiates a InsertBuilder for the given table +func (dbc *DBC) InsertInto(into string) *dbr.InsertBuilder { + if dbc.trx == nil { + return &dbr.InsertBuilder{ + Session: dbc.sess(), + Runner: dbc.con().Db, + Into: into, + } + } + + return &dbr.InsertBuilder{ + Session: dbc.trx.Session, + Runner: dbc.trx.Tx, + Into: into, + } +} + +func IsDuplicateRecordError(err error) bool { + var myerr *mysql.MySQLError + if errors.As(err, &myerr) && myerr.Number == 1062 { + return true + } + return false +} + +func IsNotFoundError(err error) bool { + return errors.Is(err, dbr.ErrNotFound) +} + +func LastInsertIdU32(res sql.Result) uint32 { + lastId, _ := res.LastInsertId() + return uint32(lastId) +} diff --git a/db_test.go b/db_test.go new file mode 100644 index 0000000..2caa406 --- /dev/null +++ b/db_test.go @@ -0,0 +1,551 @@ +package db_test + +import ( + "testing" + + "git.bit5.ru/backend/db" + "git.bit5.ru/backend/errors" + + "game/autogen" + "game/dbmeta" + "game/dbshrd" + "game/env" + "game/tests" + "game/util" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func getDBC() *db.DBC { + g := tests.Globs() + dbc := db.GetDBC(g.Logger, g.Settings.DB_MAIN) + return dbc +} + +func TestDefaultClientCharsetAndCollation(t *testing.T) { + dbc := getDBC() + defer dbc.Close() + + var result = make(map[string]string) + + characterSets, err := dbc.DB().Query("show variables where Variable_name in ('character_set_client', 'character_set_connection', 'character_set_results');") + assert.Nil(t, err) + + for characterSets.Next() { + + var variableName string + var value string + + characterSets.Scan(&variableName, &value) + + result[variableName] = value + } + + collations, err := dbc.DB().Query("show variables where Variable_name = 'collation_connection';") + assert.Nil(t, err) + + for collations.Next() { + + var variableName string + var value string + + collations.Scan(&variableName, &value) + + result[variableName] = value + } + + assert.Equal(t, "utf8", result["character_set_client"]) + assert.Equal(t, "utf8", result["character_set_connection"]) + assert.Equal(t, "utf8", result["character_set_results"]) + assert.Equal(t, "utf8_general_ci", result["collation_connection"]) +} + +func TestClientCharsetAndCollation(t *testing.T) { + g := tests.Globs() + + DSNWithLatinCollation := g.Settings.DB_MAIN + DSNWithLatinCollation.Params = "?collation=latin1_swedish_ci" + + var resultsLatin1 = make(map[string]string) + + dbLatin1 := db.GetDBC(g.Logger, DSNWithLatinCollation) + defer dbLatin1.Close() + + characterSets, err := dbLatin1.DB().Query("show variables where Variable_name in ('character_set_client', 'character_set_connection', 'character_set_results');") + assert.Nil(t, err) + + for characterSets.Next() { + var variableName string + var value string + + characterSets.Scan(&variableName, &value) + + resultsLatin1[variableName] = value + } + + collations, err := dbLatin1.DB().Query("show variables where Variable_name = 'collation_connection';") + assert.Nil(t, err) + + for collations.Next() { + + var variableName string + var value string + + collations.Scan(&variableName, &value) + + resultsLatin1[variableName] = value + } + + assert.Equal(t, "latin1", resultsLatin1["character_set_client"]) + assert.Equal(t, "latin1", resultsLatin1["character_set_connection"]) + assert.Equal(t, "latin1", resultsLatin1["character_set_results"]) + assert.Equal(t, "latin1_swedish_ci", resultsLatin1["collation_connection"]) + +} + +func TestCloseConn(t *testing.T) { + dbc := getDBC() + defer dbc.Close() + + var res int + err := dbc.SelectBySQL("SELECT 1").LoadValue(&res) + assert.Nil(t, err) + assert.EqualValues(t, 1, res) + assert.True(t, dbc.IsOpen()) + + dbc.Close() + assert.False(t, dbc.IsOpen()) + //connection is automatically restored + err = dbc.SelectBySQL("SELECT 1").LoadValue(&res) + assert.True(t, dbc.IsOpen()) + assert.Nil(t, err) + assert.EqualValues(t, 1, res) +} + +func TestDoubleCloseConnIsOk(t *testing.T) { + dbc := getDBC() + defer dbc.Close() + + var res int + err := dbc.SelectBySQL("SELECT 1").LoadValue(&res) + assert.Nil(t, err) + assert.EqualValues(t, 1, res) + assert.True(t, dbc.IsOpen()) + + dbc.Close() + assert.False(t, dbc.IsOpen()) + //connection is automatically restored + err = dbc.SelectBySQL("SELECT 1").LoadValue(&res) + assert.True(t, dbc.IsOpen()) + assert.Nil(t, err) + assert.EqualValues(t, 1, res) + + dbc.Close() + assert.False(t, dbc.IsOpen()) + //connection is automatically restored + err = dbc.SelectBySQL("SELECT 1").LoadValue(&res) + assert.True(t, dbc.IsOpen()) + assert.Nil(t, err) + assert.EqualValues(t, 1, res) +} + +func createFooTable(t *testing.T) { + dbc := getDBC() + defer dbc.Close() + + _, err := dbc.DB().Exec("DROP TABLE IF EXISTS foo") + assert.Nil(t, err) + _, err = dbc.DB().Exec("CREATE TABLE IF NOT EXISTS foo(id int not null)") + assert.Nil(t, err) +} + +func TestTransactionCommit(t *testing.T) { + createFooTable(t) + + dbc := getDBC() + defer dbc.Close() + + assert.Nil(t, dbc.Begin()) + dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(1),(2)").Exec() + + //let's try a fresh connection + dbc1 := getDBC() + defer dbc1.Close() + assert.EqualValues(t, 0, countFoos(t, dbc1)) + + assert.EqualValues(t, 2, countFoos(t, dbc)) + dbc.Commit() + assert.EqualValues(t, 2, countFoos(t, dbc)) + + //let's try a fresh connection + dbc2 := getDBC() + defer dbc2.Close() + assert.EqualValues(t, 2, countFoos(t, dbc2)) +} + +func TestTransactionCommitNestedAllOk(t *testing.T) { + createFooTable(t) + + dbc := getDBC() + defer dbc.Close() + //begin 1 + assert.Nil(t, dbc.Begin()) + dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(1),(2)").Exec() + + //let's try a fresh connection + dbc1 := getDBC() + defer dbc1.Close() + assert.EqualValues(t, 0, countFoos(t, dbc1)) + + //begin 2 + assert.Nil(t, dbc.Begin()) + dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(3)").Exec() + //commit 2 + dbc.Commit() + assert.EqualValues(t, 3, countFoos(t, dbc)) + + //let's try a fresh connection + dbc2 := getDBC() + defer dbc2.Close() + assert.EqualValues(t, 0, countFoos(t, dbc2)) + + //commit 1 + dbc.Commit() + assert.EqualValues(t, 3, countFoos(t, dbc)) + + //let's try a fresh connection + db3 := getDBC() + defer db3.Close() + assert.EqualValues(t, 3, countFoos(t, db3)) +} + +func TestTransactionCommitNestedRollback(t *testing.T) { + createFooTable(t) + + dbc := getDBC() + defer dbc.Close() + //begin 1 + assert.Nil(t, dbc.Begin()) + dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(1),(2)").Exec() + + //let's try a fresh connection + dbc1 := getDBC() + defer dbc1.Close() + assert.EqualValues(t, 0, countFoos(t, dbc1)) + + //begin 2 + assert.Nil(t, dbc.Begin()) + dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(3)").Exec() + //rollback 2 + dbc.Rollback() + //rollback above doesn't have an effect since we are in the + //nested transaction + assert.EqualValues(t, 3, countFoos(t, dbc)) + + //let's try a fresh connection + dbc2 := getDBC() + defer dbc2.Close() + assert.EqualValues(t, 0, countFoos(t, dbc2)) + + //rollback 1 + dbc.Rollback() + assert.EqualValues(t, 0, countFoos(t, dbc)) + + //let's try a fresh connection + db3 := getDBC() + defer db3.Close() + assert.EqualValues(t, 0, countFoos(t, db3)) +} + +func TestTransactionRollbackOnDeferAllOK(t *testing.T) { + createFooTable(t) + + dbc := getDBC() + defer dbc.Close() + + fn := func() { + dbc.Begin() + defer dbc.RollbackOnDefer() + dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(1),(2)").Exec() + assert.EqualValues(t, 2, countFoos(t, dbc)) + + fnNested := func() { + dbc.Begin() + defer dbc.RollbackOnDefer() + dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(3)").Exec() + assert.EqualValues(t, 3, countFoos(t, dbc)) + dbc.Commit() + } + + fnNested() + + dbc.Commit() + } + + fn() + dbc1 := getDBC() + defer dbc1.Close() + assert.EqualValues(t, 3, countFoos(t, dbc1)) +} + +func TestTransactionRollbackOnDefer(t *testing.T) { + createFooTable(t) + + dbc := getDBC() + defer dbc.Close() + + fn := func() { + dbc.Begin() + defer dbc.RollbackOnDefer() + dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(1),(2)").Exec() + assert.EqualValues(t, 2, countFoos(t, dbc)) + + fnNested := func() { + dbc.Begin() + defer dbc.RollbackOnDefer() + dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(3)").Exec() + assert.EqualValues(t, 3, countFoos(t, dbc)) + //commit is missing for some reason, emulating error + //dbc.Commit() + } + + fnNested() + + //commit is missing for some reason, emulating error + //dbc.Commit() + } + + fn() + + dbc1 := getDBC() + defer dbc1.Close() + assert.EqualValues(t, 0, countFoos(t, dbc1)) +} + +func TestTransaction(t *testing.T) { + createFooTable(t) + + dbc := getDBC() + defer dbc.Close() + + assert.Nil(t, dbc.Transaction(func(dbs *db.DBC) error { + dbs.UpdateBySQL("INSERT INTO foo(id) VALUES(1),(2)").Exec() + + //let's try a fresh connection + dbc1 := getDBC() + defer dbc1.Close() + assert.EqualValues(t, 0, countFoos(t, dbc1)) + + assert.EqualValues(t, 2, countFoos(t, dbc)) + return nil + })) + + //let's try a fresh connection + dbc2 := getDBC() + defer dbc2.Close() + assert.EqualValues(t, 2, countFoos(t, dbc2)) +} + +func TestTransactionRollbackOnError(t *testing.T) { + createFooTable(t) + + dbc := getDBC() + defer dbc.Close() + + err := dbc.Transaction(func(dbs *db.DBC) error { + dbs.UpdateBySQL("INSERT INTO foo(id) VALUES(1),(2)").Exec() + + assert.EqualValues(t, 2, countFoos(t, dbc)) + return errors.New("Opps") + }) + assert.EqualValues(t, "Opps", err.Error()) + + //let's try a fresh connection + dbc2 := getDBC() + defer dbc2.Close() + assert.EqualValues(t, 0, countFoos(t, dbc2)) +} + +func TestTransactionRollbackOnPanic(t *testing.T) { + createFooTable(t) + + dbc := getDBC() + defer dbc.Close() + + defer func() { + if r := recover(); r != nil { + str := r.(string) + assert.EqualValues(t, str, "Ooops") + + //let's try a fresh connection + dbc2 := getDBC() + defer dbc2.Close() + assert.EqualValues(t, 0, countFoos(t, dbc2)) + + } + }() + + dbc.Transaction(func(dbs *db.DBC) error { + dbs.UpdateBySQL("INSERT INTO foo(id) VALUES(1),(2)").Exec() + + assert.EqualValues(t, 2, countFoos(t, dbc)) + panic("Ooops") + return nil + }) +} + +func countFoos(t *testing.T, dbc *db.DBC) int { + var res int + err := dbc.SelectBySQL("SELECT COUNT(id) FROM foo").LoadValue(&res) + assert.Nil(t, err) + return res +} + +func TestTransactionRollback(t *testing.T) { + createFooTable(t) + + dbc := getDBC() + defer dbc.Close() + assert.Nil(t, dbc.Begin()) + { + _, err := dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(1),(2)").Exec() + assert.Nil(t, err) + } + + var res int + { + err := dbc.SelectBySQL("SELECT COUNT(id) FROM foo").LoadValue(&res) + assert.Nil(t, err) + assert.EqualValues(t, 2, res) + } + dbc.Rollback() + + { + err := dbc.SelectBySQL("SELECT COUNT(id) FROM foo").LoadValue(&res) + assert.Nil(t, err) + assert.EqualValues(t, 0, res) + } + + //let's try a fresh connection + dbc2 := getDBC() + defer dbc2.Close() + { + err := dbc2.SelectBySQL("SELECT COUNT(id) FROM foo").LoadValue(&res) + assert.Nil(t, err) + assert.EqualValues(t, 0, res) + } +} + +func TestTransactionNestedRollback(t *testing.T) { + createFooTable(t) + + dbc := getDBC() + defer dbc.Close() + //begin 1 + assert.Nil(t, dbc.Begin()) + { + _, err := dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(1)").Exec() + assert.Nil(t, err) + } + + var res int + { + err := dbc.SelectBySQL("SELECT COUNT(id) FROM foo").LoadValue(&res) + assert.Nil(t, err) + assert.EqualValues(t, 1, res) + } + + //begin 2 + assert.Nil(t, dbc.Begin()) + { + _, err := dbc.UpdateBySQL("INSERT INTO foo(id) VALUES(1)").Exec() + assert.Nil(t, err) + } + + //no real rollback happens here since we are in a 'bigger' transaction + //rollback 2 + dbc.Rollback() + { + err := dbc.SelectBySQL("SELECT COUNT(id) FROM foo").LoadValue(&res) + assert.Nil(t, err) + assert.EqualValues(t, 2, res) + } + + //rollback 1 + dbc.Rollback() + + { + err := dbc.SelectBySQL("SELECT COUNT(id) FROM foo").LoadValue(&res) + assert.Nil(t, err) + assert.EqualValues(t, 0, res) + } + +} + +func makePlayers(t *testing.T, env *env.Env, amount int) []uint32 { + var players []uint32 + mainDb := env.MainDb() + + for i := 0; i < amount; i++ { + shardPlayer, err := dbshrd.CreateShardPlayer(env.Settings.DB_SHARDS, mainDb, 1) + require.NoError(t, err) + players = append(players, shardPlayer.Id) + } + + //Note: There is only one test shard db, so the shard id = 1 for all players + shardDb, err := dbshrd.GetShardDb(env.Logger, env.Settings.DB_SHARDS, 1) + require.NoError(t, err) + require.NotNil(t, shardDb) + defer shardDb.Close() + + for i := 0; i < len(players); i++ { + player := autogen.NewDataPlayer() + player.Id = players[i] + err = dbmeta.SaveRow(shardDb, player) + require.NoError(t, err) + } + + return players +} + +func TestSelectBySQLWithChunkedIN(t *testing.T) { + env := tests.NewEnvCleanStorage() + defer env.Close() + + //Note: There is only one test shard db, so the shard id = 1 for all players + shardDb, err := dbshrd.GetShardDb(env.Logger, env.Settings.DB_SHARDS, 1) + require.NoError(t, err) + require.NotNil(t, shardDb) + defer shardDb.Close() + + var playersAmount = 100 + var playerIds []uint32 + playerIds = makePlayers(t, env, playersAmount) + + for chunkSizeForIN := 1; chunkSizeForIN <= 101; chunkSizeForIN++ { + var totalIds []uint32 + fullChunksAmount := playersAmount / chunkSizeForIN + modulo := playersAmount % chunkSizeForIN + sql := "SELECT id FROM player WHERE 1 = ? AND id IN ? AND 2 = ?" + builders := shardDb.SelectBySQLWithChunkedIN(sql, chunkSizeForIN, 1, playerIds, 2) + + require.Len(t, builders, fullChunksAmount+util.BoolToInt(modulo > 0)) + + for queryIndex, builder := range builders { + var queryIds []uint32 + _, err := builder.LoadValues(&queryIds) + require.NoError(t, err) + require.NotNil(t, queryIds) + totalIds = append(totalIds, queryIds...) + + if queryIndex < fullChunksAmount { + require.Len(t, queryIds, chunkSizeForIN) + } else { + require.Len(t, queryIds, modulo) + } + } + + require.Len(t, totalIds, playersAmount) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..cdb143c --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module git.bit5.ru/backend/db + +go 1.13 + +require ( + git.bit5.ru/backend/colog v1.0.0 + git.bit5.ru/backend/dbr v1.0.0 + git.bit5.ru/backend/errors v1.0.0 + git.bit5.ru/backend/mysql v1.0.0 + git.bit5.ru/backend/res_tracker v1.1.1 + github.com/stretchr/testify v1.8.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7390c06 --- /dev/null +++ b/go.sum @@ -0,0 +1,38 @@ +git.bit5.ru/backend/colog v1.0.0 h1:FLykTQuRfnhWYQ+dZhoaOfkxjHNEfWp4AwP/KD+ob+M= +git.bit5.ru/backend/colog v1.0.0/go.mod h1:fiOrMQ7SPBD5Pn/Tb7rG1phsDQbZaA7S0vtdnlM6BK4= +git.bit5.ru/backend/dbr v1.0.0 h1:sDxNkjYuyVovFP+Tx/TE0a17zLY7Sfn6qKEIb6ZIJYM= +git.bit5.ru/backend/dbr v1.0.0/go.mod h1:NP1BTND68eshI4Mv6SmSMh7COuy0fTiiHnUCT7ufB+o= +git.bit5.ru/backend/errors v1.0.0 h1:WWJ0sly44q1HQjN01X75ZAGKZwwY5Ml+XVDXMjCkToA= +git.bit5.ru/backend/errors v1.0.0/go.mod h1:75faRwsnpM0Se00/Bh7fysWQXV8oMjNJFQ6f7+r9k3Y= +git.bit5.ru/backend/mysql v1.0.0 h1:NKxTr5p7vti/qrig6zZtWaDeiLRRgPkMB0Nz7AGGlIY= +git.bit5.ru/backend/mysql v1.0.0/go.mod h1:iK3dzIpv9YAGN8Fy6zi1XGUX8R5nl9XQ9NJbwYM7y0w= +git.bit5.ru/backend/res_tracker v1.1.1 h1:MVGJe8G4TFMoSZdV55BxEmEJr10gjLl13Izw6J+02n4= +git.bit5.ru/backend/res_tracker v1.1.1/go.mod h1:ffjnItxqkGc6rxOK9XgrQDirGhmIBwoqibmyLJ4TZtQ= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/logger.go b/logger.go new file mode 100644 index 0000000..1c45d2b --- /dev/null +++ b/logger.go @@ -0,0 +1,64 @@ +package db + +import ( + "strings" + + "git.bit5.ru/backend/colog" +) + +// EventReceiver logs db stuff if neccessary +type EventReceiver struct { + logger *colog.CoLog + s Settings +} + +// Event receives a simple notification when various events occur +func (n *EventReceiver) Event(eventName string) { + if n.logger != nil { + n.logger.Output(colog.LDebug, 5, eventName) + } +} + +// EventKv receives a notification when various events occur along with +// optional key/value data +func (n *EventReceiver) EventKv(eventName string, kvs map[string]string) { + if n.logger != nil { + n.logger.Output(colog.LDebug, 5, eventName) + } +} + +// EventErr receives a notification of an error if one occurs +func (n *EventReceiver) EventErr(eventName string, err error) error { return err } + +// EventErrKv receives a notification of an error if one occurs along with +// optional key/value data +func (n *EventReceiver) EventErrKv(eventName string, err error, kvs map[string]string) error { + return err +} + +// Timing receives the time an event took to happen +func (n *EventReceiver) Timing(eventName string, nanoseconds int64) { +} + +// TimingKv receives the time an event took to happen along with optional key/value data +func (n *EventReceiver) TimingKv(eventName string, nanoseconds int64, kvs map[string]string) { + if n.logger != nil { + sql := kvs["sql"] + sp := strings.Index(sql, " ") + if sp != -1 { + query := sql[:sp] + if n.s.LogLevel > 1 { + n.logger.Output(colog.LDebug, 6, sql) + } else { + if len(sql) > 50 { + sql = sql[:50] + "..." + } + if query == "SELECT" && n.s.LogLevel > 0 { + n.logger.Output(colog.LDebug, 6, sql) + } else { + n.logger.Output(colog.LDebug, 6, sql) + } + } + } + } +}