From aa0194dbeccdb9e79d5775f0a8903c3cdbb4e753 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 25 Apr 2023 20:36:57 +0900 Subject: [PATCH 01/53] Drop Go 1.13-17 support (#1420) Start v1.8 development --- .github/workflows/test.yml | 11 +++-------- README.md | 4 ++-- go.mod | 2 +- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d45ed0fa9..cd474767b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,11 +27,6 @@ jobs: # Older production releases '1.19', '1.18', - '1.17', - '1.16', - '1.15', - '1.14', - '1.13', ] mysql = [ '8.0', @@ -47,7 +42,7 @@ jobs: includes = [] # Go versions compatibility check for v in go[1:]: - includes.append({'os': 'ubuntu-latest', 'go': v, 'mysql': mysql[0]}) + includes.append({'os': 'ubuntu-latest', 'go': v, 'mysql': mysql[0]}) matrix = { # OS vs MySQL versions @@ -69,10 +64,10 @@ jobs: matrix: ${{ fromJSON(needs.list.outputs.matrix) }} steps: - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} - - uses: shogo82148/actions-setup-mysql@v1.15.0 + - uses: shogo82148/actions-setup-mysql@v1.16.0 with: mysql-version: ${{ matrix.mysql }} user: ${{ env.MYSQL_TEST_USER }} diff --git a/README.md b/README.md index 3b5d229aa..5a242e9d7 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.13 or higher. We aim to support the 3 latest versions of Go. - * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) + * Go 1.18 or higher. We aim to support the 3 latest versions of Go. + * MySQL (5.6+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- diff --git a/go.mod b/go.mod index 251110478..77bbb8dbf 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/go-sql-driver/mysql -go 1.13 +go 1.18 From cffc85ce9efe406a98c1d82749a237cc0338a8b2 Mon Sep 17 00:00:00 2001 From: Evil Puncker Date: Tue, 25 Apr 2023 19:10:42 -0300 Subject: [PATCH 02/53] Reduced allocation on connection.go (#1421) reduces allocations when there is only one param because current calculation is off by 2 --- connection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connection.go b/connection.go index 947a883e3..0aeef207b 100644 --- a/connection.go +++ b/connection.go @@ -68,7 +68,7 @@ func (mc *mysqlConn) handleParams() (err error) { default: if cmdSet.Len() == 0 { // Heuristic: 29 chars for each other key=value to reduce reallocations - cmdSet.Grow(4 + len(param) + 1 + len(val) + 30*(len(mc.cfg.Params)-1)) + cmdSet.Grow(4 + len(param) + 3 + len(val) + 30*(len(mc.cfg.Params)-1)) cmdSet.WriteString("SET ") } else { cmdSet.WriteString(", ") From fbfb3f6a34bd0d4e73e1569831e054ec36b38ce9 Mon Sep 17 00:00:00 2001 From: jypelle <52546084+jypelle@users.noreply.github.com> Date: Mon, 1 May 2023 17:52:55 +0200 Subject: [PATCH 03/53] Adding DeregisterDialContext (#1422) Co-authored-by: jypelle --- AUTHORS | 1 + driver.go | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/AUTHORS b/AUTHORS index fb1478c3b..ea9b96789 100644 --- a/AUTHORS +++ b/AUTHORS @@ -47,6 +47,7 @@ INADA Naoki Jacek Szwec James Harr Janek Vedock +Jean-Yves Pellé Jeff Hodges Jeffrey Charles Jerome Meyer diff --git a/driver.go b/driver.go index ad7aec215..8b0c3ec0a 100644 --- a/driver.go +++ b/driver.go @@ -55,6 +55,17 @@ func RegisterDialContext(net string, dial DialContextFunc) { dials[net] = dial } +// DeregisterDialContext removes the custom dial function registered with the given net. +func DeregisterDialContext(net string) { + dialsLock.Lock() + defer dialsLock.Unlock() + if dials != nil { + if _, ok := dials[net]; ok { + delete(dials, net) + } + } +} + // RegisterDial registers a custom dial function. It can then be used by the // network address mynet(addr), where mynet is the registered new network. // addr is passed as a parameter to the dial function. From 191a7c4c519ef60cf3e8656fde8728eee9194308 Mon Sep 17 00:00:00 2001 From: frozenbonito Date: Thu, 4 May 2023 23:30:22 +0900 Subject: [PATCH 04/53] Make logger configurable per Connector (#1408) --- AUTHORS | 1 + auth.go | 2 +- connection.go | 16 ++++++++-------- connection_test.go | 1 + connector.go | 2 +- driver_test.go | 2 +- dsn.go | 6 ++++++ dsn_test.go | 34 +++++++++++++++++----------------- errors.go | 12 +++++++++--- errors_test.go | 6 +++--- packets.go | 26 +++++++++++++------------- packets_test.go | 1 + statement.go | 4 ++-- 13 files changed, 64 insertions(+), 49 deletions(-) diff --git a/AUTHORS b/AUTHORS index ea9b96789..129ca665a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -96,6 +96,7 @@ Stan Putrya Stanley Gunawan Steven Hartland Tan Jinhua <312841925 at qq.com> +Tetsuro Aoki Thomas Wodarek Tim Ruffles Tom Jenkinson diff --git a/auth.go b/auth.go index 1ff203e57..b591e7b8a 100644 --- a/auth.go +++ b/auth.go @@ -291,7 +291,7 @@ func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { return enc, err default: - errLog.Print("unknown auth plugin:", plugin) + mc.cfg.Logger.Print("unknown auth plugin:", plugin) return nil, ErrUnknownPlugin } } diff --git a/connection.go b/connection.go index 0aeef207b..a7da9e7e2 100644 --- a/connection.go +++ b/connection.go @@ -105,7 +105,7 @@ func (mc *mysqlConn) Begin() (driver.Tx, error) { func (mc *mysqlConn) begin(readOnly bool) (driver.Tx, error) { if mc.closed.Load() { - errLog.Print(ErrInvalidConn) + mc.cfg.Logger.Print(ErrInvalidConn) return nil, driver.ErrBadConn } var q string @@ -147,7 +147,7 @@ func (mc *mysqlConn) cleanup() { return } if err := mc.netConn.Close(); err != nil { - errLog.Print(err) + mc.cfg.Logger.Print(err) } } @@ -163,14 +163,14 @@ func (mc *mysqlConn) error() error { func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { if mc.closed.Load() { - errLog.Print(ErrInvalidConn) + mc.cfg.Logger.Print(ErrInvalidConn) return nil, driver.ErrBadConn } // Send command err := mc.writeCommandPacketStr(comStmtPrepare, query) if err != nil { // STMT_PREPARE is safe to retry. So we can return ErrBadConn here. - errLog.Print(err) + mc.cfg.Logger.Print(err) return nil, driver.ErrBadConn } @@ -204,7 +204,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin buf, err := mc.buf.takeCompleteBuffer() if err != nil { // can not take the buffer. Something must be wrong with the connection - errLog.Print(err) + mc.cfg.Logger.Print(err) return "", ErrInvalidConn } buf = buf[:0] @@ -296,7 +296,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, error) { if mc.closed.Load() { - errLog.Print(ErrInvalidConn) + mc.cfg.Logger.Print(ErrInvalidConn) return nil, driver.ErrBadConn } if len(args) != 0 { @@ -357,7 +357,7 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) { if mc.closed.Load() { - errLog.Print(ErrInvalidConn) + mc.cfg.Logger.Print(ErrInvalidConn) return nil, driver.ErrBadConn } if len(args) != 0 { @@ -451,7 +451,7 @@ func (mc *mysqlConn) finish() { // Ping implements driver.Pinger interface func (mc *mysqlConn) Ping(ctx context.Context) (err error) { if mc.closed.Load() { - errLog.Print(ErrInvalidConn) + mc.cfg.Logger.Print(ErrInvalidConn) return driver.ErrBadConn } diff --git a/connection_test.go b/connection_test.go index b6764a2f6..98c985ae1 100644 --- a/connection_test.go +++ b/connection_test.go @@ -179,6 +179,7 @@ func TestPingErrInvalidConn(t *testing.T) { buf: newBuffer(nc), maxAllowedPacket: defaultMaxAllowedPacket, closech: make(chan struct{}), + cfg: NewConfig(), } err := ms.Ping(context.Background()) diff --git a/connector.go b/connector.go index d567b4e4f..a5c988e13 100644 --- a/connector.go +++ b/connector.go @@ -92,7 +92,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { authResp, err := mc.auth(authData, plugin) if err != nil { // try the default auth plugin, if using the requested plugin failed - errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) + c.cfg.Logger.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) plugin = defaultAuthPlugin authResp, err = mc.auth(authData, plugin) if err != nil { diff --git a/driver_test.go b/driver_test.go index a1c776728..1741a13ef 100644 --- a/driver_test.go +++ b/driver_test.go @@ -1995,7 +1995,7 @@ func TestInsertRetrieveEscapedData(t *testing.T) { func TestUnixSocketAuthFail(t *testing.T) { runTests(t, dsn, func(dbt *DBTest) { // Save the current logger so we can restore it. - oldLogger := errLog + oldLogger := defaultLogger // Set a new logger so we can capture its output. buffer := bytes.NewBuffer(make([]byte, 0, 64)) diff --git a/dsn.go b/dsn.go index 4b71aaab0..ded459c94 100644 --- a/dsn.go +++ b/dsn.go @@ -50,6 +50,7 @@ type Config struct { Timeout time.Duration // Dial timeout ReadTimeout time.Duration // I/O read timeout WriteTimeout time.Duration // I/O write timeout + Logger Logger // Logger AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE AllowCleartextPasswords bool // Allows the cleartext client side plugin @@ -71,6 +72,7 @@ func NewConfig() *Config { Collation: defaultCollation, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, + Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, } @@ -153,6 +155,10 @@ func (cfg *Config) normalize() error { } } + if cfg.Logger == nil { + cfg.Logger = defaultLogger + } + return nil } diff --git a/dsn_test.go b/dsn_test.go index 41a6a29fa..cb97d557e 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -22,55 +22,55 @@ var testDSNs = []struct { out *Config }{{ "username:password@protocol(address)/dbname?param=value", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true}, }, { "user@unix(/path/to/socket)/dbname?charset=utf8", - &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"}, }, { "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, CheckConnLiveness: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, Logger: defaultLogger, AllowAllFiles: true, AllowOldPasswords: true, CheckConnLiveness: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, }, { "user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0&allowFallbackToPlaintext=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowFallbackToPlaintext: true, AllowNativePasswords: false, CheckConnLiveness: false}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, Logger: defaultLogger, AllowFallbackToPlaintext: true, AllowNativePasswords: false, CheckConnLiveness: false}, }, { "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", - &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "@/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:p@/ssword@/", - &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "unix/?arg=%2Fsome%2Fpath.ext", - &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(127.0.0.1)/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(de:ad:be:ef::ca:fe)/dbname", - &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, } diff --git a/errors.go b/errors.go index ff9a8f088..5680b6c05 100644 --- a/errors.go +++ b/errors.go @@ -37,20 +37,26 @@ var ( errBadConnNoWrite = errors.New("bad connection") ) -var errLog = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime|log.Lshortfile)) +var defaultLogger = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime|log.Lshortfile)) // Logger is used to log critical error messages. type Logger interface { Print(v ...interface{}) } -// SetLogger is used to set the logger for critical errors. +// NopLogger is a nop implementation of the Logger interface. +type NopLogger struct{} + +// Print implements Logger interface. +func (nl *NopLogger) Print(_ ...interface{}) {} + +// SetLogger is used to set the default logger for critical errors. // The initial logger is os.Stderr. func SetLogger(logger Logger) error { if logger == nil { return errors.New("logger is nil") } - errLog = logger + defaultLogger = logger return nil } diff --git a/errors_test.go b/errors_test.go index 43213f98e..53d634454 100644 --- a/errors_test.go +++ b/errors_test.go @@ -16,9 +16,9 @@ import ( ) func TestErrorsSetLogger(t *testing.T) { - previous := errLog + previous := defaultLogger defer func() { - errLog = previous + defaultLogger = previous }() // set up logger @@ -28,7 +28,7 @@ func TestErrorsSetLogger(t *testing.T) { // print SetLogger(logger) - errLog.Print("test") + defaultLogger.Print("test") // check result if actual := buffer.String(); actual != expected { diff --git a/packets.go b/packets.go index ee05c95a8..8fd67997b 100644 --- a/packets.go +++ b/packets.go @@ -34,7 +34,7 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { if cerr := mc.canceled.Value(); cerr != nil { return nil, cerr } - errLog.Print(err) + mc.cfg.Logger.Print(err) mc.Close() return nil, ErrInvalidConn } @@ -56,7 +56,7 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { if pktLen == 0 { // there was no previous packet if prevData == nil { - errLog.Print(ErrMalformPkt) + mc.cfg.Logger.Print(ErrMalformPkt) mc.Close() return nil, ErrInvalidConn } @@ -70,7 +70,7 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { if cerr := mc.canceled.Value(); cerr != nil { return nil, cerr } - errLog.Print(err) + mc.cfg.Logger.Print(err) mc.Close() return nil, ErrInvalidConn } @@ -119,7 +119,7 @@ func (mc *mysqlConn) writePacket(data []byte) error { } } if err != nil { - errLog.Print("closing bad idle connection: ", err) + mc.cfg.Logger.Print("closing bad idle connection: ", err) mc.Close() return driver.ErrBadConn } @@ -161,7 +161,7 @@ func (mc *mysqlConn) writePacket(data []byte) error { // Handle error if err == nil { // n != len(data) mc.cleanup() - errLog.Print(ErrMalformPkt) + mc.cfg.Logger.Print(ErrMalformPkt) } else { if cerr := mc.canceled.Value(); cerr != nil { return cerr @@ -171,7 +171,7 @@ func (mc *mysqlConn) writePacket(data []byte) error { return errBadConnNoWrite } mc.cleanup() - errLog.Print(err) + mc.cfg.Logger.Print(err) } return ErrInvalidConn } @@ -322,7 +322,7 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string data, err := mc.buf.takeSmallBuffer(pktLen + 4) if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) + mc.cfg.Logger.Print(err) return errBadConnNoWrite } @@ -404,7 +404,7 @@ func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error { data, err := mc.buf.takeSmallBuffer(pktLen) if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) + mc.cfg.Logger.Print(err) return errBadConnNoWrite } @@ -424,7 +424,7 @@ func (mc *mysqlConn) writeCommandPacket(command byte) error { data, err := mc.buf.takeSmallBuffer(4 + 1) if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) + mc.cfg.Logger.Print(err) return errBadConnNoWrite } @@ -443,7 +443,7 @@ func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { data, err := mc.buf.takeBuffer(pktLen + 4) if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) + mc.cfg.Logger.Print(err) return errBadConnNoWrite } @@ -464,7 +464,7 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { data, err := mc.buf.takeSmallBuffer(4 + 1 + 4) if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) + mc.cfg.Logger.Print(err) return errBadConnNoWrite } @@ -938,7 +938,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { } if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) + mc.cfg.Logger.Print(err) return errBadConnNoWrite } @@ -1137,7 +1137,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if valuesCap != cap(paramValues) { data = append(data[:pos], paramValues...) if err = mc.buf.store(data); err != nil { - errLog.Print(err) + mc.cfg.Logger.Print(err) return errBadConnNoWrite } } diff --git a/packets_test.go b/packets_test.go index b61e4dbf7..cacec1c68 100644 --- a/packets_test.go +++ b/packets_test.go @@ -265,6 +265,7 @@ func TestReadPacketFail(t *testing.T) { mc := &mysqlConn{ buf: newBuffer(conn), closech: make(chan struct{}), + cfg: NewConfig(), } // illegal empty (stand-alone) packet diff --git a/statement.go b/statement.go index 10ece8bd6..d8b3975a5 100644 --- a/statement.go +++ b/statement.go @@ -51,7 +51,7 @@ func (stmt *mysqlStmt) CheckNamedValue(nv *driver.NamedValue) (err error) { func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { if stmt.mc.closed.Load() { - errLog.Print(ErrInvalidConn) + stmt.mc.cfg.Logger.Print(ErrInvalidConn) return nil, driver.ErrBadConn } // Send command @@ -99,7 +99,7 @@ func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) { if stmt.mc.closed.Load() { - errLog.Print(ErrInvalidConn) + stmt.mc.cfg.Logger.Print(ErrInvalidConn) return nil, driver.ErrBadConn } // Send command From 0b40aee005dafcce42833210b672c1f0930008aa Mon Sep 17 00:00:00 2001 From: wayyoungboy <35394786+wayyoungboy@users.noreply.github.com> Date: Sat, 6 May 2023 17:07:41 +0800 Subject: [PATCH 05/53] Avoid panic in TestRowsColumnTypes (#1426) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * optimized the execution flow of the TestRowsColumnTypes unit test * Update driver_test.go --------- Co-authored-by: 渠磊 Co-authored-by: Inada Naoki --- driver_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/driver_test.go b/driver_test.go index 1741a13ef..d24488a82 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2945,7 +2945,10 @@ func TestRowsColumnTypes(t *testing.T) { continue } } - + // Avoid panic caused by nil scantype. + if t.Failed() { + return + } values := make([]interface{}, len(tt)) for i := range values { values[i] = reflect.New(types[i]).Interface() From 736b6faabe4947c9a0a7fef6407839dc72114011 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 7 May 2023 20:24:09 +0900 Subject: [PATCH 06/53] Stop `ColumnTypeScanType()` from returning `sql.RawBytes` (#1424) ColumnTypeScanType() returns []byte, string, or sql.NullString. It returned sql.RawBytes but it was dangoerous. Fixes #1423 --- driver_test.go | 67 +++++++++++++++++++++++++------------------------- fields.go | 48 +++++++++++++++++++++--------------- 2 files changed, 62 insertions(+), 53 deletions(-) diff --git a/driver_test.go b/driver_test.go index d24488a82..50c617274 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2778,13 +2778,18 @@ func TestRowsColumnTypes(t *testing.T) { nd1 := sql.NullTime{Time: time.Date(2006, 01, 02, 0, 0, 0, 0, time.UTC), Valid: true} nd2 := sql.NullTime{Time: time.Date(2006, 03, 04, 0, 0, 0, 0, time.UTC), Valid: true} ndNULL := sql.NullTime{Time: time.Time{}, Valid: false} - rbNULL := sql.RawBytes(nil) - rb0 := sql.RawBytes("0") - rb42 := sql.RawBytes("42") - rbTest := sql.RawBytes("Test") - rb0pad4 := sql.RawBytes("0\x00\x00\x00") // BINARY right-pads values with 0x00 - rbx0 := sql.RawBytes("\x00") - rbx42 := sql.RawBytes("\x42") + bNULL := []byte(nil) + nsNULL := sql.NullString{String: "", Valid: false} + // Helper function to build NullString from string literal. + ns := func(s string) sql.NullString { return sql.NullString{String: s, Valid: true} } + ns0 := ns("0") + b0 := []byte("0") + b42 := []byte("42") + nsTest := ns("Test") + bTest := []byte("Test") + b0pad4 := []byte("0\x00\x00\x00") // BINARY right-pads values with 0x00 + bx0 := []byte("\x00") + bx42 := []byte("\x42") var columns = []struct { name string @@ -2797,7 +2802,7 @@ func TestRowsColumnTypes(t *testing.T) { valuesIn [3]string valuesOut [3]interface{} }{ - {"bit8null", "BIT(8)", "BIT", scanTypeRawBytes, true, 0, 0, [3]string{"0x0", "NULL", "0x42"}, [3]interface{}{rbx0, rbNULL, rbx42}}, + {"bit8null", "BIT(8)", "BIT", scanTypeBytes, true, 0, 0, [3]string{"0x0", "NULL", "0x42"}, [3]interface{}{bx0, bNULL, bx42}}, {"boolnull", "BOOL", "TINYINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "true", "0"}, [3]interface{}{niNULL, ni1, ni0}}, {"bool", "BOOL NOT NULL", "TINYINT", scanTypeInt8, false, 0, 0, [3]string{"1", "0", "FALSE"}, [3]interface{}{int8(1), int8(0), int8(0)}}, {"intnull", "INTEGER", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, @@ -2817,24 +2822,24 @@ func TestRowsColumnTypes(t *testing.T) { {"float74null", "FLOAT(7,4)", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, 4, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, {"double", "DOUBLE NOT NULL", "DOUBLE", scanTypeFloat64, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float64(0), float64(42), float64(13.37)}}, {"doublenull", "DOUBLE", "DOUBLE", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, - {"decimal1", "DECIMAL(10,6) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 10, 6, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), sql.RawBytes("13.370000"), sql.RawBytes("1234.123456")}}, - {"decimal1null", "DECIMAL(10,6)", "DECIMAL", scanTypeRawBytes, true, 10, 6, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), rbNULL, sql.RawBytes("1234.123456")}}, - {"decimal2", "DECIMAL(8,4) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 8, 4, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), sql.RawBytes("13.3700"), sql.RawBytes("1234.1235")}}, - {"decimal2null", "DECIMAL(8,4)", "DECIMAL", scanTypeRawBytes, true, 8, 4, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), rbNULL, sql.RawBytes("1234.1235")}}, - {"decimal3", "DECIMAL(5,0) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 5, 0, [3]string{"0", "13.37", "-12345.123456"}, [3]interface{}{rb0, sql.RawBytes("13"), sql.RawBytes("-12345")}}, - {"decimal3null", "DECIMAL(5,0)", "DECIMAL", scanTypeRawBytes, true, 5, 0, [3]string{"0", "NULL", "-12345.123456"}, [3]interface{}{rb0, rbNULL, sql.RawBytes("-12345")}}, - {"char25null", "CHAR(25)", "CHAR", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"binary4null", "BINARY(4)", "BINARY", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0pad4, rbNULL, rbTest}}, - {"varbinary42", "VARBINARY(42) NOT NULL", "VARBINARY", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"tinyblobnull", "TINYBLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"tinytextnull", "TINYTEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"blobnull", "BLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"textnull", "TEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"mediumblob", "MEDIUMBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"longblob", "LONGBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"decimal1", "DECIMAL(10,6) NOT NULL", "DECIMAL", scanTypeString, false, 10, 6, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{"0.000000", "13.370000", "1234.123456"}}, + {"decimal1null", "DECIMAL(10,6)", "DECIMAL", scanTypeNullString, true, 10, 6, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{ns("0.000000"), nsNULL, ns("1234.123456")}}, + {"decimal2", "DECIMAL(8,4) NOT NULL", "DECIMAL", scanTypeString, false, 8, 4, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{"0.0000", "13.3700", "1234.1235"}}, + {"decimal2null", "DECIMAL(8,4)", "DECIMAL", scanTypeNullString, true, 8, 4, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{ns("0.0000"), nsNULL, ns("1234.1235")}}, + {"decimal3", "DECIMAL(5,0) NOT NULL", "DECIMAL", scanTypeString, false, 5, 0, [3]string{"0", "13.37", "-12345.123456"}, [3]interface{}{"0", "13", "-12345"}}, + {"decimal3null", "DECIMAL(5,0)", "DECIMAL", scanTypeNullString, true, 5, 0, [3]string{"0", "NULL", "-12345.123456"}, [3]interface{}{ns0, nsNULL, ns("-12345")}}, + {"char25null", "CHAR(25)", "CHAR", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{ns0, nsNULL, nsTest}}, + {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{"0", "Test", "42"}}, + {"binary4null", "BINARY(4)", "BINARY", scanTypeBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{b0pad4, bNULL, bTest}}, + {"varbinary42", "VARBINARY(42) NOT NULL", "VARBINARY", scanTypeBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{b0, bTest, b42}}, + {"tinyblobnull", "TINYBLOB", "BLOB", scanTypeBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{b0, bNULL, bTest}}, + {"tinytextnull", "TINYTEXT", "TEXT", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{ns0, nsNULL, nsTest}}, + {"blobnull", "BLOB", "BLOB", scanTypeBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{b0, bNULL, bTest}}, + {"textnull", "TEXT", "TEXT", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{ns0, nsNULL, nsTest}}, + {"mediumblob", "MEDIUMBLOB NOT NULL", "BLOB", scanTypeBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{b0, bTest, b42}}, + {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{"0", "Test", "42"}}, + {"longblob", "LONGBLOB NOT NULL", "BLOB", scanTypeBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{b0, bTest, b42}}, + {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{"0", "Test", "42"}}, {"datetime", "DATETIME", "DATETIME", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt0, nt0}}, {"datetime2", "DATETIME(2)", "DATETIME", scanTypeNullTime, true, 2, 2, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt2}}, {"datetime6", "DATETIME(6)", "DATETIME", scanTypeNullTime, true, 6, 6, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt6}}, @@ -2959,14 +2964,10 @@ func TestRowsColumnTypes(t *testing.T) { if err != nil { t.Fatalf("failed to scan values in %v", err) } - for j := range values { - value := reflect.ValueOf(values[j]).Elem().Interface() + for j, value := range values { + value := reflect.ValueOf(value).Elem().Interface() if !reflect.DeepEqual(value, columns[j].valuesOut[i]) { - if columns[j].scanType == scanTypeRawBytes { - t.Errorf("row %d, column %d: %v != %v", i, j, string(value.(sql.RawBytes)), string(columns[j].valuesOut[i].(sql.RawBytes))) - } else { - t.Errorf("row %d, column %d: %v != %v", i, j, value, columns[j].valuesOut[i]) - } + t.Errorf("row %d, column %d: %v != %v", i, j, value, columns[j].valuesOut[i]) } } i++ diff --git a/fields.go b/fields.go index e0654a83d..18c23e0cb 100644 --- a/fields.go +++ b/fields.go @@ -110,21 +110,23 @@ func (mf *mysqlField) typeDatabaseName() string { } var ( - scanTypeFloat32 = reflect.TypeOf(float32(0)) - scanTypeFloat64 = reflect.TypeOf(float64(0)) - scanTypeInt8 = reflect.TypeOf(int8(0)) - scanTypeInt16 = reflect.TypeOf(int16(0)) - scanTypeInt32 = reflect.TypeOf(int32(0)) - scanTypeInt64 = reflect.TypeOf(int64(0)) - scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) - scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) - scanTypeNullTime = reflect.TypeOf(sql.NullTime{}) - scanTypeUint8 = reflect.TypeOf(uint8(0)) - scanTypeUint16 = reflect.TypeOf(uint16(0)) - scanTypeUint32 = reflect.TypeOf(uint32(0)) - scanTypeUint64 = reflect.TypeOf(uint64(0)) - scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) - scanTypeUnknown = reflect.TypeOf(new(interface{})) + scanTypeFloat32 = reflect.TypeOf(float32(0)) + scanTypeFloat64 = reflect.TypeOf(float64(0)) + scanTypeInt8 = reflect.TypeOf(int8(0)) + scanTypeInt16 = reflect.TypeOf(int16(0)) + scanTypeInt32 = reflect.TypeOf(int32(0)) + scanTypeInt64 = reflect.TypeOf(int64(0)) + scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) + scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) + scanTypeNullTime = reflect.TypeOf(sql.NullTime{}) + scanTypeUint8 = reflect.TypeOf(uint8(0)) + scanTypeUint16 = reflect.TypeOf(uint16(0)) + scanTypeUint32 = reflect.TypeOf(uint32(0)) + scanTypeUint64 = reflect.TypeOf(uint64(0)) + scanTypeString = reflect.TypeOf("") + scanTypeNullString = reflect.TypeOf(sql.NullString{}) + scanTypeBytes = reflect.TypeOf([]byte{}) + scanTypeUnknown = reflect.TypeOf(new(interface{})) ) type mysqlField struct { @@ -187,12 +189,18 @@ func (mf *mysqlField) scanType() reflect.Type { } return scanTypeNullFloat + case fieldTypeBit, fieldTypeTinyBLOB, fieldTypeMediumBLOB, fieldTypeLongBLOB, + fieldTypeBLOB, fieldTypeVarString, fieldTypeString, fieldTypeGeometry: + if mf.charSet == 63 /* binary */ { + return scanTypeBytes + } + fallthrough case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, - fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, - fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, - fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON, - fieldTypeTime: - return scanTypeRawBytes + fieldTypeEnum, fieldTypeSet, fieldTypeJSON, fieldTypeTime: + if mf.flags&flagNotNULL != 0 { + return scanTypeString + } + return scanTypeNullString case fieldTypeDate, fieldTypeNewDate, fieldTypeTimestamp, fieldTypeDateTime: From 081308f66228fdc51224614d1cf414c918cc1596 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 7 May 2023 20:25:21 +0900 Subject: [PATCH 07/53] Add benchmark to receive massive rows. (#1415) --- benchmark_test.go | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/benchmark_test.go b/benchmark_test.go index 97ed781f8..fc70df60d 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -372,3 +372,59 @@ func BenchmarkQueryRawBytes(b *testing.B) { }) } } + +// BenchmarkReceiveMassiveRows measures performance of receiving large number of rows. +func BenchmarkReceiveMassiveRows(b *testing.B) { + // Setup -- prepare 10000 rows. + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val TEXT)") + defer db.Close() + + sval := strings.Repeat("x", 50) + stmt, err := db.Prepare(`INSERT INTO foo (id, val) VALUES (?, ?)` + strings.Repeat(",(?,?)", 99)) + if err != nil { + b.Errorf("failed to prepare query: %v", err) + return + } + for i := 0; i < 10000; i += 100 { + args := make([]any, 200) + for j := 0; j < 100; j++ { + args[j*2] = i + j + args[j*2+1] = sval + } + _, err := stmt.Exec(args...) + if err != nil { + b.Error(err) + return + } + } + stmt.Close() + + // Use b.Run() to skip expensive setup. + b.Run("query", func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + rows, err := db.Query(`SELECT id, val FROM foo`) + if err != nil { + b.Errorf("failed to select: %v", err) + return + } + for rows.Next() { + var i int + var s sql.RawBytes + err = rows.Scan(&i, &s) + if err != nil { + b.Errorf("failed to scan: %v", err) + _ = rows.Close() + return + } + } + if err = rows.Err(); err != nil { + b.Errorf("failed to read rows: %v", err) + } + _ = rows.Close() + } + }) +} From a841e816042356288f94f7c5a586d83040cb63ea Mon Sep 17 00:00:00 2001 From: Evan Elias Date: Wed, 17 May 2023 14:28:03 -0400 Subject: [PATCH 08/53] Fix ColumnType.DatabaseTypeName for mediumint unsigned (#1428) --- AUTHORS | 1 + README.md | 2 +- driver_test.go | 1 + fields.go | 3 +++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 129ca665a..24dc43652 100644 --- a/AUTHORS +++ b/AUTHORS @@ -33,6 +33,7 @@ Dave Protasowski DisposaBoy Egor Smolyakov Erwan Martin +Evan Elias Evan Shaw Frederick Mayle Gustavo Kristic diff --git a/README.md b/README.md index 5a242e9d7..ddb5cefc7 100644 --- a/README.md +++ b/README.md @@ -465,7 +465,7 @@ user:password@/ The connection pool is managed by Go's database/sql package. For details on how to configure the size of the pool and how long connections stay in the pool see `*DB.SetMaxOpenConns`, `*DB.SetMaxIdleConns`, and `*DB.SetConnMaxLifetime` in the [database/sql documentation](https://golang.org/pkg/database/sql/). The read, write, and dial timeouts for each individual connection are configured with the DSN parameters [`readTimeout`](#readtimeout), [`writeTimeout`](#writetimeout), and [`timeout`](#timeout), respectively. ## `ColumnType` Support -This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported. All Unsigned database type names will be returned `UNSIGNED ` with `INT`, `TINYINT`, `SMALLINT`, `BIGINT`. +This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported. All Unsigned database type names will be returned `UNSIGNED ` with `INT`, `TINYINT`, `SMALLINT`, `MEDIUMINT`, `BIGINT`. ## `context.Context` Support Go 1.8 added `database/sql` support for `context.Context`. This driver supports query timeouts and cancellation via contexts. diff --git a/driver_test.go b/driver_test.go index 50c617274..118c0d7ba 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2816,6 +2816,7 @@ func TestRowsColumnTypes(t *testing.T) { {"tinyuint", "TINYINT UNSIGNED NOT NULL", "UNSIGNED TINYINT", scanTypeUint8, false, 0, 0, [3]string{"0", "255", "42"}, [3]interface{}{uint8(0), uint8(255), uint8(42)}}, {"smalluint", "SMALLINT UNSIGNED NOT NULL", "UNSIGNED SMALLINT", scanTypeUint16, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint16(0), uint16(65535), uint16(42)}}, {"biguint", "BIGINT UNSIGNED NOT NULL", "UNSIGNED BIGINT", scanTypeUint64, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint64(0), uint64(65535), uint64(42)}}, + {"mediumuint", "MEDIUMINT UNSIGNED NOT NULL", "UNSIGNED MEDIUMINT", scanTypeUint32, false, 0, 0, [3]string{"0", "16777215", "42"}, [3]interface{}{uint32(0), uint32(16777215), uint32(42)}}, {"uint13", "INT(13) UNSIGNED NOT NULL", "UNSIGNED INT", scanTypeUint32, false, 0, 0, [3]string{"0", "1337", "42"}, [3]interface{}{uint32(0), uint32(1337), uint32(42)}}, {"float", "FLOAT NOT NULL", "FLOAT", scanTypeFloat32, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float32(0), float32(42), float32(13.37)}}, {"floatnull", "FLOAT", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, diff --git a/fields.go b/fields.go index 18c23e0cb..ae709363f 100644 --- a/fields.go +++ b/fields.go @@ -37,6 +37,9 @@ func (mf *mysqlField) typeDatabaseName() string { case fieldTypeGeometry: return "GEOMETRY" case fieldTypeInt24: + if mf.flags&flagUnsigned != 0 { + return "UNSIGNED MEDIUMINT" + } return "MEDIUMINT" case fieldTypeJSON: return "JSON" From 72e78ee26806a26405ee462c4cf82406f094a143 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 May 2023 23:04:35 +0900 Subject: [PATCH 09/53] README: Update multistatement (#1431) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ddb5cefc7..ad7ca718e 100644 --- a/README.md +++ b/README.md @@ -295,9 +295,9 @@ Valid Values: true, false Default: false ``` -Allow multiple statements in one query. While this allows batch queries, it also greatly increases the risk of SQL injections. Only the result of the first query is returned, all other results are silently discarded. +Allow multiple statements in one query. This can be used to bach multiple queries. Use [Rows.NextResultSet()](https://pkg.go.dev/database/sql#Rows.NextResultSet) to get result of the second and subsequent queries. -When `multiStatements` is used, `?` parameters must only be used in the first statement. +When `multiStatements` is used, `?` parameters must only be used in the first statement. [interpolateParams](#interpolateparams) can be used to avoid this limitation unless prepared statement is used explicitly. ##### `parseTime` From 924f8336da7226f4cd4bfac575d394ffa20aacb4 Mon Sep 17 00:00:00 2001 From: Daemonxiao <35677990+Daemonxiao@users.noreply.github.com> Date: Wed, 24 May 2023 00:44:19 +0800 Subject: [PATCH 10/53] Send connection attributes (#1389) Co-authored-by: Inada Naoki --- .github/workflows/test.yml | 1 + README.md | 9 ++++++++ connection.go | 1 + connector.go | 46 ++++++++++++++++++++++++++++++++++++- connector_test.go | 9 +++++--- const.go | 12 ++++++++++ driver.go | 11 ++++----- driver_test.go | 47 ++++++++++++++++++++++++++++++++++++++ dsn.go | 40 ++++++++++++++++++-------------- packets.go | 13 +++++++++++ packets_test.go | 7 +++++- utils.go | 5 ++++ 12 files changed, 173 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cd474767b..b2ab5e82a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,6 +79,7 @@ jobs: ; TestConcurrent fails if max_connections is too large max_connections=50 local_infile=1 + performance_schema=on - name: setup database run: | mysql --user 'root' --host '127.0.0.1' -e 'create database gotest;' diff --git a/README.md b/README.md index ad7ca718e..5935afd0c 100644 --- a/README.md +++ b/README.md @@ -393,6 +393,15 @@ Default: 0 I/O write timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. +##### `connectionAttributes` + +``` +Type: comma-delimited string of user-defined "key:value" pairs +Valid Values: (:,:,...) +Default: none +``` + +[Connection attributes](https://dev.mysql.com/doc/refman/8.0/en/performance-schema-connection-attribute-tables.html) are key-value pairs that application programs can pass to the server at connect time. ##### System Variables diff --git a/connection.go b/connection.go index a7da9e7e2..67cea1fcb 100644 --- a/connection.go +++ b/connection.go @@ -27,6 +27,7 @@ type mysqlConn struct { affectedRows uint64 insertId uint64 cfg *Config + connector *connector maxAllowedPacket int maxWriteSize int writeTimeout time.Duration diff --git a/connector.go b/connector.go index a5c988e13..6acf3dd50 100644 --- a/connector.go +++ b/connector.go @@ -11,11 +11,54 @@ package mysql import ( "context" "database/sql/driver" + "fmt" "net" + "os" + "strconv" + "strings" ) type connector struct { - cfg *Config // immutable private copy. + cfg *Config // immutable private copy. + encodedAttributes string // Encoded connection attributes. +} + +func encodeConnectionAttributes(textAttributes string) string { + connAttrsBuf := make([]byte, 0, 251) + + // default connection attributes + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrClientName) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrClientNameValue) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrOS) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrOSValue) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrPlatform) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrPlatformValue) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrPid) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, strconv.Itoa(os.Getpid())) + + // user-defined connection attributes + for _, connAttr := range strings.Split(textAttributes, ",") { + attr := strings.SplitN(connAttr, ":", 2) + if len(attr) != 2 { + continue + } + for _, v := range attr { + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, v) + } + } + + return string(connAttrsBuf) +} + +func newConnector(cfg *Config) (*connector, error) { + encodedAttributes := encodeConnectionAttributes(cfg.ConnectionAttributes) + if len(encodedAttributes) > 250 { + return nil, fmt.Errorf("connection attributes are longer than 250 bytes: %dbytes (%q)", len(encodedAttributes), cfg.ConnectionAttributes) + } + return &connector{ + cfg: cfg, + encodedAttributes: encodedAttributes, + }, nil } // Connect implements driver.Connector interface. @@ -29,6 +72,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { maxWriteSize: maxPacketSize - 1, closech: make(chan struct{}), cfg: c.cfg, + connector: c, } mc.parseTime = mc.cfg.ParseTime diff --git a/connector_test.go b/connector_test.go index 976903c5b..bedb44ce2 100644 --- a/connector_test.go +++ b/connector_test.go @@ -8,13 +8,16 @@ import ( ) func TestConnectorReturnsTimeout(t *testing.T) { - connector := &connector{&Config{ + connector, err := newConnector(&Config{ Net: "tcp", Addr: "1.1.1.1:1234", Timeout: 10 * time.Millisecond, - }} + }) + if err != nil { + t.Fatal(err) + } - _, err := connector.Connect(context.Background()) + _, err = connector.Connect(context.Background()) if err == nil { t.Fatal("error expected") } diff --git a/const.go b/const.go index 64e2bced6..0f2621a6f 100644 --- a/const.go +++ b/const.go @@ -8,12 +8,24 @@ package mysql +import "runtime" + const ( defaultAuthPlugin = "mysql_native_password" defaultMaxAllowedPacket = 64 << 20 // 64 MiB. See https://github.com/go-sql-driver/mysql/issues/1355 minProtocolVersion = 10 maxPacketSize = 1<<24 - 1 timeFormat = "2006-01-02 15:04:05.999999" + + // Connection attributes + // See https://dev.mysql.com/doc/refman/8.0/en/performance-schema-connection-attribute-tables.html#performance-schema-connection-attributes-available + connAttrClientName = "_client_name" + connAttrClientNameValue = "Go-MySQL-Driver" + connAttrOS = "_os" + connAttrOSValue = runtime.GOOS + connAttrPlatform = "_platform" + connAttrPlatformValue = runtime.GOARCH + connAttrPid = "_pid" ) // MySQL constants documentation: diff --git a/driver.go b/driver.go index 8b0c3ec0a..c19e04207 100644 --- a/driver.go +++ b/driver.go @@ -85,8 +85,9 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { if err != nil { return nil, err } - c := &connector{ - cfg: cfg, + c, err := newConnector(cfg) + if err != nil { + return nil, err } return c.Connect(context.Background()) } @@ -103,7 +104,7 @@ func NewConnector(cfg *Config) (driver.Connector, error) { if err := cfg.normalize(); err != nil { return nil, err } - return &connector{cfg: cfg}, nil + return newConnector(cfg) } // OpenConnector implements driver.DriverContext. @@ -112,7 +113,5 @@ func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { if err != nil { return nil, err } - return &connector{ - cfg: cfg, - }, nil + return newConnector(cfg) } diff --git a/driver_test.go b/driver_test.go index 118c0d7ba..7c25aa905 100644 --- a/driver_test.go +++ b/driver_test.go @@ -3214,3 +3214,50 @@ func TestConnectorTimeoutsWatchCancel(t *testing.T) { t.Errorf("connection not closed") } } + +func TestConnectionAttributes(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + attr1 := "attr1" + value1 := "value1" + attr2 := "foo" + value2 := "boo" + dsn += fmt.Sprintf("&connectionAttributes=%s:%s,%s:%s", attr1, value1, attr2, value2) + + var db *sql.DB + if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { + db, err = sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + } + + dbt := &DBTest{t, db} + + var attrValue string + queryString := "SELECT ATTR_VALUE FROM performance_schema.session_account_connect_attrs WHERE PROCESSLIST_ID = CONNECTION_ID() and ATTR_NAME = ?" + rows := dbt.mustQuery(queryString, connAttrClientName) + if rows.Next() { + rows.Scan(&attrValue) + if attrValue != connAttrClientNameValue { + dbt.Errorf("expected %q, got %q", connAttrClientNameValue, attrValue) + } + } else { + dbt.Errorf("no data") + } + rows.Close() + + rows = dbt.mustQuery(queryString, attr2) + if rows.Next() { + rows.Scan(&attrValue) + if attrValue != value2 { + dbt.Errorf("expected %q, got %q", value2, attrValue) + } + } else { + dbt.Errorf("no data") + } + rows.Close() +} diff --git a/dsn.go b/dsn.go index ded459c94..7c788517c 100644 --- a/dsn.go +++ b/dsn.go @@ -34,23 +34,24 @@ var ( // If a new Config is created instead of being parsed from a DSN string, // the NewConfig function should be used, which sets default values. type Config struct { - User string // Username - Passwd string // Password (requires User) - Net string // Network type - Addr string // Network address (requires Net) - DBName string // Database name - Params map[string]string // Connection parameters - Collation string // Connection collation - Loc *time.Location // Location for time.Time values - MaxAllowedPacket int // Max packet size allowed - ServerPubKey string // Server public key name - pubKey *rsa.PublicKey // Server public key - TLSConfig string // TLS configuration name - TLS *tls.Config // TLS configuration, its priority is higher than TLSConfig - Timeout time.Duration // Dial timeout - ReadTimeout time.Duration // I/O read timeout - WriteTimeout time.Duration // I/O write timeout - Logger Logger // Logger + User string // Username + Passwd string // Password (requires User) + Net string // Network type + Addr string // Network address (requires Net) + DBName string // Database name + Params map[string]string // Connection parameters + ConnectionAttributes string // Connection Attributes, comma-delimited string of user-defined "key:value" pairs + Collation string // Connection collation + Loc *time.Location // Location for time.Time values + MaxAllowedPacket int // Max packet size allowed + ServerPubKey string // Server public key name + pubKey *rsa.PublicKey // Server public key + TLSConfig string // TLS configuration name + TLS *tls.Config // TLS configuration, its priority is higher than TLSConfig + Timeout time.Duration // Dial timeout + ReadTimeout time.Duration // I/O read timeout + WriteTimeout time.Duration // I/O write timeout + Logger Logger // Logger AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE AllowCleartextPasswords bool // Allows the cleartext client side plugin @@ -560,6 +561,11 @@ func parseDSNParams(cfg *Config, params string) (err error) { if err != nil { return } + + // Connection attributes + case "connectionAttributes": + cfg.ConnectionAttributes = value + default: // lazy init if cfg.Params == nil { diff --git a/packets.go b/packets.go index 8fd67997b..d6a11fd21 100644 --- a/packets.go +++ b/packets.go @@ -285,6 +285,7 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string clientLocalFiles | clientPluginAuth | clientMultiResults | + clientConnectAttrs | mc.flags&clientLongFlag if mc.cfg.ClientFoundRows { @@ -318,6 +319,13 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string pktLen += n + 1 } + // 1 byte to store length of all key-values + // NOTE: Actually, this is length encoded integer. + // But we support only len(connAttrBuf) < 251 for now because takeSmallBuffer + // doesn't support buffer size more than 4096 bytes. + // TODO(methane): Rewrite buffer management. + pktLen += 1 + len(mc.connector.encodedAttributes) + // Calculate packet length and get buffer with that size data, err := mc.buf.takeSmallBuffer(pktLen + 4) if err != nil { @@ -394,6 +402,11 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string data[pos] = 0x00 pos++ + // Connection Attributes + data[pos] = byte(len(mc.connector.encodedAttributes)) + pos++ + pos += copy(data[pos:], []byte(mc.connector.encodedAttributes)) + // Send Auth packet return mc.writePacket(data[:pos]) } diff --git a/packets_test.go b/packets_test.go index cacec1c68..f429087e9 100644 --- a/packets_test.go +++ b/packets_test.go @@ -96,9 +96,14 @@ var _ net.Conn = new(mockConn) func newRWMockConn(sequence uint8) (*mockConn, *mysqlConn) { conn := new(mockConn) + connector, err := newConnector(NewConfig()) + if err != nil { + panic(err) + } mc := &mysqlConn{ buf: newBuffer(conn), - cfg: NewConfig(), + cfg: connector.cfg, + connector: connector, netConn: conn, closech: make(chan struct{}), maxAllowedPacket: defaultMaxAllowedPacket, diff --git a/utils.go b/utils.go index 15dbd8d16..753ebd65c 100644 --- a/utils.go +++ b/utils.go @@ -616,6 +616,11 @@ func appendLengthEncodedInteger(b []byte, n uint64) []byte { byte(n>>32), byte(n>>40), byte(n>>48), byte(n>>56)) } +func appendLengthEncodedString(b []byte, s string) []byte { + b = appendLengthEncodedInteger(b, uint64(len(s))) + return append(b, s...) +} + // reserveBuffer checks cap(buf) and expand buffer to len(buf) + appendSize. // If cap(buf) is not enough, reallocate new buffer. func reserveBuffer(buf []byte, appendSize int) []byte { From d3e4fe64aaa1e99a19f711233dc682f2114ffbfd Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 23:49:33 +0900 Subject: [PATCH 11/53] Use PathEscape for dbname in DSN. (#1432) Support for slashes in database names via url escape codes. On the other hand, '%' in DSN is now treated as percent-encoding. Co-authored-by: Brian Hendriks --- AUTHORS | 2 ++ README.md | 6 +++++ dsn.go | 8 +++++-- dsn_test.go | 66 ++++++++++++++++++++++++++++++----------------------- 4 files changed, 51 insertions(+), 31 deletions(-) diff --git a/AUTHORS b/AUTHORS index 24dc43652..7e4fac5a1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -110,6 +110,7 @@ Xuehong Chan Zhenye Xie Zhixin Wen Ziheng Lyu +Brian Hendriks # Organizations @@ -127,3 +128,4 @@ Percona LLC Pivotal Inc. Stripe Inc. Zendesk Inc. +Dolthub Inc. diff --git a/README.md b/README.md index 5935afd0c..156aaa965 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,12 @@ This has the same effect as an empty DSN string: ``` +`dbname` is escaped by [PathEscape()]()https://pkg.go.dev/net/url#PathEscape) since v1.8.0. If your database name is `dbname/withslash`, it becomes: + +``` +/dbname%2Fwithslash +``` + Alternatively, [Config.FormatDSN](https://godoc.org/github.com/go-sql-driver/mysql#Config.FormatDSN) can be used to create a DSN string by filling a struct. #### Password diff --git a/dsn.go b/dsn.go index 7c788517c..3a6537e6c 100644 --- a/dsn.go +++ b/dsn.go @@ -203,7 +203,7 @@ func (cfg *Config) FormatDSN() string { // /dbname buf.WriteByte('/') - buf.WriteString(cfg.DBName) + buf.WriteString(url.PathEscape(cfg.DBName)) // [?param1=value1&...¶mN=valueN] hasParam := false @@ -365,7 +365,11 @@ func ParseDSN(dsn string) (cfg *Config, err error) { break } } - cfg.DBName = dsn[i+1 : j] + + dbname := dsn[i+1 : j] + if cfg.DBName, err = url.PathUnescape(dbname); err != nil { + return nil, fmt.Errorf("invalid dbname %q: %w", dbname, err) + } break } diff --git a/dsn_test.go b/dsn_test.go index cb97d557e..8b623df01 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -50,6 +50,9 @@ var testDSNs = []struct { }, { "/dbname", &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, +}, { + "/dbname%2Fwithslash", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname/withslash", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "@/", &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, @@ -76,17 +79,20 @@ var testDSNs = []struct { func TestDSNParser(t *testing.T) { for i, tst := range testDSNs { - cfg, err := ParseDSN(tst.in) - if err != nil { - t.Error(err.Error()) - } + t.Run(tst.in, func(t *testing.T) { + cfg, err := ParseDSN(tst.in) + if err != nil { + t.Error(err.Error()) + return + } - // pointer not static - cfg.TLS = nil + // pointer not static + cfg.TLS = nil - if !reflect.DeepEqual(cfg, tst.out) { - t.Errorf("%d. ParseDSN(%q) mismatch:\ngot %+v\nwant %+v", i, tst.in, cfg, tst.out) - } + if !reflect.DeepEqual(cfg, tst.out) { + t.Errorf("%d. ParseDSN(%q) mismatch:\ngot %+v\nwant %+v", i, tst.in, cfg, tst.out) + } + }) } } @@ -113,27 +119,29 @@ func TestDSNParserInvalid(t *testing.T) { func TestDSNReformat(t *testing.T) { for i, tst := range testDSNs { - dsn1 := tst.in - cfg1, err := ParseDSN(dsn1) - if err != nil { - t.Error(err.Error()) - continue - } - cfg1.TLS = nil // pointer not static - res1 := fmt.Sprintf("%+v", cfg1) - - dsn2 := cfg1.FormatDSN() - cfg2, err := ParseDSN(dsn2) - if err != nil { - t.Error(err.Error()) - continue - } - cfg2.TLS = nil // pointer not static - res2 := fmt.Sprintf("%+v", cfg2) + t.Run(tst.in, func(t *testing.T) { + dsn1 := tst.in + cfg1, err := ParseDSN(dsn1) + if err != nil { + t.Error(err.Error()) + return + } + cfg1.TLS = nil // pointer not static + res1 := fmt.Sprintf("%+v", cfg1) - if res1 != res2 { - t.Errorf("%d. %q does not match %q", i, res2, res1) - } + dsn2 := cfg1.FormatDSN() + cfg2, err := ParseDSN(dsn2) + if err != nil { + t.Error(err.Error()) + return + } + cfg2.TLS = nil // pointer not static + res2 := fmt.Sprintf("%+v", cfg2) + + if res1 != res2 { + t.Errorf("%d. %q does not match %q", i, res2, res1) + } + }) } } From 7b4d7eb08bc4e705373ad835b2384df28676fb2f Mon Sep 17 00:00:00 2001 From: uji <49834542+uji@users.noreply.github.com> Date: Fri, 26 May 2023 11:32:30 +0900 Subject: [PATCH 12/53] all: replace ioutil pkg to new package (#1438) --- auth.go | 2 +- driver_test.go | 3 +-- utils.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/auth.go b/auth.go index b591e7b8a..e758e6d00 100644 --- a/auth.go +++ b/auth.go @@ -33,7 +33,7 @@ var ( // Note: The provided rsa.PublicKey instance is exclusively owned by the driver // after registering it and may not be modified. // -// data, err := ioutil.ReadFile("mykey.pem") +// data, err := os.ReadFile("mykey.pem") // if err != nil { // log.Fatal(err) // } diff --git a/driver_test.go b/driver_test.go index 7c25aa905..abf91a486 100644 --- a/driver_test.go +++ b/driver_test.go @@ -17,7 +17,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "log" "math" "net" @@ -1245,7 +1244,7 @@ func TestLoadData(t *testing.T) { dbt.mustExec("CREATE TABLE test (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL) CHARACTER SET utf8") // Local File - file, err := ioutil.TempFile("", "gotest") + file, err := os.CreateTemp("", "gotest") defer os.Remove(file.Name()) if err != nil { dbt.Fatal(err) diff --git a/utils.go b/utils.go index 753ebd65c..a24197b93 100644 --- a/utils.go +++ b/utils.go @@ -36,7 +36,7 @@ var ( // registering it. // // rootCertPool := x509.NewCertPool() -// pem, err := ioutil.ReadFile("/path/ca-cert.pem") +// pem, err := os.ReadFile("/path/ca-cert.pem") // if err != nil { // log.Fatal(err) // } From 7b22099c7ea60190ef92f953ee62263a1808bd4b Mon Sep 17 00:00:00 2001 From: guangwu Date: Fri, 26 May 2023 12:52:11 +0800 Subject: [PATCH 13/53] code optimization (#1439) --- driver.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/driver.go b/driver.go index c19e04207..0ed8fa1c5 100644 --- a/driver.go +++ b/driver.go @@ -60,9 +60,7 @@ func DeregisterDialContext(net string) { dialsLock.Lock() defer dialsLock.Unlock() if dials != nil { - if _, ok := dials[net]; ok { - delete(dials, net) - } + delete(dials, net) } } From 99976f4f587dd1a26900f6dd91ca96f6e3e2f724 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 28 May 2023 01:43:28 +0900 Subject: [PATCH 14/53] Use `SET NAMES charset COLLATE collation`. (#1437) --- README.md | 10 ++++++---- connection.go | 9 +++++++-- dsn.go | 5 ++--- dsn_test.go | 34 +++++++++++++++++----------------- packets.go | 11 +++++++---- 5 files changed, 39 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 156aaa965..d747a7446 100644 --- a/README.md +++ b/README.md @@ -202,8 +202,7 @@ Default: none Sets the charset used for client-server interaction (`"SET NAMES "`). If multiple charsets are set (separated by a comma), the following charset is used if setting the charset failes. This enables for example support for `utf8mb4` ([introduced in MySQL 5.5.3](http://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)) with fallback to `utf8` for older servers (`charset=utf8mb4,utf8`). -Usage of the `charset` parameter is discouraged because it issues additional queries to the server. -Unless you need the fallback behavior, please use `collation` instead. +See also [Unicode Support](#unicode-support). ##### `checkConnLiveness` @@ -232,6 +231,7 @@ The default collation (`utf8mb4_general_ci`) is supported from MySQL 5.5. You s Collations for charset "ucs2", "utf16", "utf16le", and "utf32" can not be used ([ref](https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset)). +See also [Unicode Support](#unicode-support). ##### `clientFoundRows` @@ -511,9 +511,11 @@ However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` v ### Unicode support Since version 1.5 Go-MySQL-Driver automatically uses the collation ` utf8mb4_general_ci` by default. -Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. +Other charsets / collations can be set using the [`charset`](#charset) or [`collation`](#collation) DSN parameter. -Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. +- When only the `charset` is specified, the `SET NAMES ` query is sent and the server's default collation is used. +- When both the `charset` and `collation` are specified, the `SET NAMES COLLATE ` query is sent. +- When only the `collation` is specified, the collation is specified in the protocol handshake and the `SET NAMES` query is not sent. This can save one roundtrip, but note that the server may ignore the specified collation silently and use the server's default charset/collation instead. See http://dev.mysql.com/doc/refman/8.0/en/charset-unicode.html for more details on MySQL's Unicode support. diff --git a/connection.go b/connection.go index 67cea1fcb..14a972b40 100644 --- a/connection.go +++ b/connection.go @@ -49,14 +49,19 @@ type mysqlConn struct { // Handles parameters set in DSN after the connection is established func (mc *mysqlConn) handleParams() (err error) { var cmdSet strings.Builder + for param, val := range mc.cfg.Params { switch param { // Charset: character_set_connection, character_set_client, character_set_results case "charset": charsets := strings.Split(val, ",") - for i := range charsets { + for _, cs := range charsets { // ignore errors here - a charset may not exist - err = mc.exec("SET NAMES " + charsets[i]) + if mc.cfg.Collation != "" { + err = mc.exec("SET NAMES " + cs + " COLLATE " + mc.cfg.Collation) + } else { + err = mc.exec("SET NAMES " + cs) + } if err == nil { break } diff --git a/dsn.go b/dsn.go index 3a6537e6c..693aa4e5a 100644 --- a/dsn.go +++ b/dsn.go @@ -70,7 +70,6 @@ type Config struct { // NewConfig creates a new Config and sets default values. func NewConfig() *Config { return &Config{ - Collation: defaultCollation, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, @@ -100,7 +99,7 @@ func (cfg *Config) Clone() *Config { } func (cfg *Config) normalize() error { - if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { + if cfg.InterpolateParams && cfg.Collation != "" && unsafeCollations[cfg.Collation] { return errInvalidDSNUnsafeCollation } @@ -237,7 +236,7 @@ func (cfg *Config) FormatDSN() string { writeDSNParam(&buf, &hasParam, "clientFoundRows", "true") } - if col := cfg.Collation; col != defaultCollation && len(col) > 0 { + if col := cfg.Collation; col != "" { writeDSNParam(&buf, &hasParam, "collation", col) } diff --git a/dsn_test.go b/dsn_test.go index 8b623df01..a729d0ef8 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -22,58 +22,58 @@ var testDSNs = []struct { out *Config }{{ "username:password@protocol(address)/dbname?param=value", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true}, }, { "user@unix(/path/to/socket)/dbname?charset=utf8", - &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"}, }, { "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, Logger: defaultLogger, AllowAllFiles: true, AllowOldPasswords: true, CheckConnLiveness: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, }, { "user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0&allowFallbackToPlaintext=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, Logger: defaultLogger, AllowFallbackToPlaintext: true, AllowNativePasswords: false, CheckConnLiveness: false}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Loc: time.UTC, MaxAllowedPacket: 0, Logger: defaultLogger, AllowFallbackToPlaintext: true, AllowNativePasswords: false, CheckConnLiveness: false}, }, { "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", - &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/dbname%2Fwithslash", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname/withslash", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname/withslash", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "@/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:p@/ssword@/", - &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "unix/?arg=%2Fsome%2Fpath.ext", - &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(127.0.0.1)/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(de:ad:be:ef::ca:fe)/dbname", - &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, } diff --git a/packets.go b/packets.go index d6a11fd21..c10072c94 100644 --- a/packets.go +++ b/packets.go @@ -14,7 +14,6 @@ import ( "database/sql/driver" "encoding/binary" "encoding/json" - "errors" "fmt" "io" "math" @@ -346,14 +345,18 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string data[10] = 0x00 data[11] = 0x00 - // Charset [1 byte] + // Collation ID [1 byte] + cname := mc.cfg.Collation + if cname == "" { + cname = defaultCollation + } var found bool - data[12], found = collations[mc.cfg.Collation] + data[12], found = collations[cname] if !found { // Note possibility for false negatives: // could be triggered although the collation is valid if the // collations map does not contain entries the server supports. - return errors.New("unknown collation") + return fmt.Errorf("unknown collation: %q", cname) } // Filler [23 bytes] (all 0x00) From f43effaa7c9271606b37b04a6235e5f7ed37c3e0 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 28 May 2023 02:07:46 +0900 Subject: [PATCH 15/53] Reduce map lookup in ColumnTypeDatabaseTypeName. (#1436) --- collations.go | 2 +- fields.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/collations.go b/collations.go index 295bfbe52..1cdf97b67 100644 --- a/collations.go +++ b/collations.go @@ -9,7 +9,7 @@ package mysql const defaultCollation = "utf8mb4_general_ci" -const binaryCollation = "binary" +const binaryCollationID = 63 // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: diff --git a/fields.go b/fields.go index ae709363f..30f31cbfb 100644 --- a/fields.go +++ b/fields.go @@ -18,7 +18,7 @@ func (mf *mysqlField) typeDatabaseName() string { case fieldTypeBit: return "BIT" case fieldTypeBLOB: - if mf.charSet != collations[binaryCollation] { + if mf.charSet != binaryCollationID { return "TEXT" } return "BLOB" @@ -49,7 +49,7 @@ func (mf *mysqlField) typeDatabaseName() string { } return "INT" case fieldTypeLongBLOB: - if mf.charSet != collations[binaryCollation] { + if mf.charSet != binaryCollationID { return "LONGTEXT" } return "LONGBLOB" @@ -59,7 +59,7 @@ func (mf *mysqlField) typeDatabaseName() string { } return "BIGINT" case fieldTypeMediumBLOB: - if mf.charSet != collations[binaryCollation] { + if mf.charSet != binaryCollationID { return "MEDIUMTEXT" } return "MEDIUMBLOB" @@ -77,7 +77,7 @@ func (mf *mysqlField) typeDatabaseName() string { } return "SMALLINT" case fieldTypeString: - if mf.charSet == collations[binaryCollation] { + if mf.charSet == binaryCollationID { return "BINARY" } return "CHAR" @@ -91,17 +91,17 @@ func (mf *mysqlField) typeDatabaseName() string { } return "TINYINT" case fieldTypeTinyBLOB: - if mf.charSet != collations[binaryCollation] { + if mf.charSet != binaryCollationID { return "TINYTEXT" } return "TINYBLOB" case fieldTypeVarChar: - if mf.charSet == collations[binaryCollation] { + if mf.charSet == binaryCollationID { return "VARBINARY" } return "VARCHAR" case fieldTypeVarString: - if mf.charSet == collations[binaryCollation] { + if mf.charSet == binaryCollationID { return "VARBINARY" } return "VARCHAR" @@ -194,7 +194,7 @@ func (mf *mysqlField) scanType() reflect.Type { case fieldTypeBit, fieldTypeTinyBLOB, fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, fieldTypeVarString, fieldTypeString, fieldTypeGeometry: - if mf.charSet == 63 /* binary */ { + if mf.charSet == binaryCollationID { return scanTypeBytes } fallthrough From 397e2f5323e1c03bc4513d6c9ab345dfd47108cd Mon Sep 17 00:00:00 2001 From: Matthew Herrmann <47012945+mherr-google@users.noreply.github.com> Date: Mon, 29 May 2023 13:33:49 +1000 Subject: [PATCH 16/53] Exec() now provides access to status of multiple statements. (#1309) It now reports the last inserted ID and affected row count for all statements, not just the last one. This is useful to execute batches of statements such as UPDATE with minimal roundtrips. Co-authored-by: Inada Naoki --- README.md | 16 +++++++ auth.go | 6 +-- connection.go | 29 ++++++------- driver_test.go | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ infile.go | 8 ++-- packets.go | 77 ++++++++++++++++++++++++++++------ result.go | 37 ++++++++++++++-- rows.go | 7 +++- statement.go | 17 ++++---- 9 files changed, 259 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index d747a7446..4eade6853 100644 --- a/README.md +++ b/README.md @@ -305,6 +305,22 @@ Allow multiple statements in one query. This can be used to bach multiple querie When `multiStatements` is used, `?` parameters must only be used in the first statement. [interpolateParams](#interpolateparams) can be used to avoid this limitation unless prepared statement is used explicitly. +It's possible to access the last inserted ID and number of affected rows for multiple statements by using `sql.Conn.Raw()` and the `mysql.Result`. For example: + +```go +conn, _ := db.Conn(ctx) +conn.Raw(func(conn interface{}) error { + ex := conn.(driver.Execer) + res, err := ex.Exec(` + UPDATE point SET x = 1 WHERE y = 2; + UPDATE point SET x = 2 WHERE y = 3; + `, nil) + // Both slices have 2 elements. + log.Print(res.(mysql.Result).AllRowsAffected()) + log.Print(res.(mysql.Result).AllLastInsertIds()) +}) +``` + ##### `parseTime` ``` diff --git a/auth.go b/auth.go index e758e6d00..f6b157a12 100644 --- a/auth.go +++ b/auth.go @@ -346,7 +346,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { case 1: switch authData[0] { case cachingSha2PasswordFastAuthSuccess: - if err = mc.readResultOK(); err == nil { + if err = mc.resultUnchanged().readResultOK(); err == nil { return nil // auth successful } @@ -397,7 +397,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { return err } } - return mc.readResultOK() + return mc.resultUnchanged().readResultOK() default: return ErrMalformPkt @@ -426,7 +426,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { if err != nil { return err } - return mc.readResultOK() + return mc.resultUnchanged().readResultOK() } default: diff --git a/connection.go b/connection.go index 14a972b40..631a1dc24 100644 --- a/connection.go +++ b/connection.go @@ -23,9 +23,8 @@ import ( type mysqlConn struct { buf buffer netConn net.Conn - rawConn net.Conn // underlying connection when netConn is TLS connection. - affectedRows uint64 - insertId uint64 + rawConn net.Conn // underlying connection when netConn is TLS connection. + result mysqlResult // managed by clearResult() and handleOkPacket(). cfg *Config connector *connector maxAllowedPacket int @@ -155,6 +154,7 @@ func (mc *mysqlConn) cleanup() { if err := mc.netConn.Close(); err != nil { mc.cfg.Logger.Print(err) } + mc.clearResult() } func (mc *mysqlConn) error() error { @@ -316,28 +316,25 @@ func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, err } query = prepared } - mc.affectedRows = 0 - mc.insertId = 0 err := mc.exec(query) if err == nil { - return &mysqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, err + copied := mc.result + return &copied, err } return nil, mc.markBadConn(err) } // Internal function to execute commands func (mc *mysqlConn) exec(query string) error { + handleOk := mc.clearResult() // Send command if err := mc.writeCommandPacketStr(comQuery, query); err != nil { return mc.markBadConn(err) } // Read Result - resLen, err := mc.readResultSetHeaderPacket() + resLen, err := handleOk.readResultSetHeaderPacket() if err != nil { return err } @@ -354,7 +351,7 @@ func (mc *mysqlConn) exec(query string) error { } } - return mc.discardResults() + return handleOk.discardResults() } func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, error) { @@ -362,6 +359,8 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro } func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) { + handleOk := mc.clearResult() + if mc.closed.Load() { mc.cfg.Logger.Print(ErrInvalidConn) return nil, driver.ErrBadConn @@ -382,7 +381,7 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) if err == nil { // Read Result var resLen int - resLen, err = mc.readResultSetHeaderPacket() + resLen, err = handleOk.readResultSetHeaderPacket() if err == nil { rows := new(textRows) rows.mc = mc @@ -410,12 +409,13 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) // The returned byte slice is only valid until the next read func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { // Send command + handleOk := mc.clearResult() if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil { return nil, err } // Read Result - resLen, err := mc.readResultSetHeaderPacket() + resLen, err := handleOk.readResultSetHeaderPacket() if err == nil { rows := new(textRows) rows.mc = mc @@ -466,11 +466,12 @@ func (mc *mysqlConn) Ping(ctx context.Context) (err error) { } defer mc.finish() + handleOk := mc.clearResult() if err = mc.writeCommandPacket(comPing); err != nil { return mc.markBadConn(err) } - return mc.readResultOK() + return handleOk.readResultOK() } // BeginTx implements driver.ConnBeginTx interface diff --git a/driver_test.go b/driver_test.go index abf91a486..cd94c434e 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2154,11 +2154,51 @@ func TestRejectReadOnly(t *testing.T) { } func TestPing(t *testing.T) { + ctx := context.Background() runTests(t, dsn, func(dbt *DBTest) { if err := dbt.db.Ping(); err != nil { dbt.fail("Ping", "Ping", err) } }) + + runTests(t, dsn, func(dbt *DBTest) { + conn, err := dbt.db.Conn(ctx) + if err != nil { + dbt.fail("db", "Conn", err) + } + + // Check that affectedRows and insertIds are cleared after each call. + conn.Raw(func(conn interface{}) error { + c := conn.(*mysqlConn) + + // Issue a query that sets affectedRows and insertIds. + q, err := c.Query(`SELECT 1`, nil) + if err != nil { + dbt.fail("Conn", "Query", err) + } + if got, want := c.result.affectedRows, []int64{0}; !reflect.DeepEqual(got, want) { + dbt.Fatalf("bad affectedRows: got %v, want=%v", got, want) + } + if got, want := c.result.insertIds, []int64{0}; !reflect.DeepEqual(got, want) { + dbt.Fatalf("bad insertIds: got %v, want=%v", got, want) + } + q.Close() + + // Verify that Ping() clears both fields. + for i := 0; i < 2; i++ { + if err := c.Ping(ctx); err != nil { + dbt.fail("Pinger", "Ping", err) + } + if got, want := c.result.affectedRows, []int64(nil); !reflect.DeepEqual(got, want) { + t.Errorf("bad affectedRows: got %v, want=%v", got, want) + } + if got, want := c.result.insertIds, []int64(nil); !reflect.DeepEqual(got, want) { + t.Errorf("bad affectedRows: got %v, want=%v", got, want) + } + } + return nil + }) + }) } // See Issue #799 @@ -2378,6 +2418,42 @@ func TestMultiResultSetNoSelect(t *testing.T) { }) } +func TestExecMultipleResults(t *testing.T) { + ctx := context.Background() + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + dbt.mustExec(` + CREATE TABLE test ( + id INT NOT NULL AUTO_INCREMENT, + value VARCHAR(255), + PRIMARY KEY (id) + )`) + conn, err := dbt.db.Conn(ctx) + if err != nil { + t.Fatalf("failed to connect: %v", err) + } + conn.Raw(func(conn interface{}) error { + ex := conn.(driver.Execer) + res, err := ex.Exec(` + INSERT INTO test (value) VALUES ('a'), ('b'); + INSERT INTO test (value) VALUES ('c'), ('d'), ('e'); + `, nil) + if err != nil { + t.Fatalf("insert statements failed: %v", err) + } + mres := res.(Result) + if got, want := mres.AllRowsAffected(), []int64{2, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("bad AllRowsAffected: got %v, want=%v", got, want) + } + // For INSERTs containing multiple rows, LAST_INSERT_ID() returns the + // first inserted ID, not the last. + if got, want := mres.AllLastInsertIds(), []int64{1, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("bad AllLastInsertIds: got %v, want %v", got, want) + } + return nil + }) + }) +} + // tests if rows are set in a proper state if some results were ignored before // calling rows.NextResultSet. func TestSkipResults(t *testing.T) { @@ -2399,6 +2475,42 @@ func TestSkipResults(t *testing.T) { }) } +func TestQueryMultipleResults(t *testing.T) { + ctx := context.Background() + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + dbt.mustExec(` + CREATE TABLE test ( + id INT NOT NULL AUTO_INCREMENT, + value VARCHAR(255), + PRIMARY KEY (id) + )`) + conn, err := dbt.db.Conn(ctx) + if err != nil { + t.Fatalf("failed to connect: %v", err) + } + conn.Raw(func(conn interface{}) error { + qr := conn.(driver.Queryer) + + c := conn.(*mysqlConn) + + // Demonstrate that repeated queries reset the affectedRows + for i := 0; i < 2; i++ { + _, err := qr.Query(` + INSERT INTO test (value) VALUES ('a'), ('b'); + INSERT INTO test (value) VALUES ('c'), ('d'), ('e'); + `, nil) + if err != nil { + t.Fatalf("insert statements failed: %v", err) + } + if got, want := c.result.affectedRows, []int64{2, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("bad affectedRows: got %v, want=%v", got, want) + } + } + return nil + }) + }) +} + func TestPingContext(t *testing.T) { runTests(t, dsn, func(dbt *DBTest) { ctx, cancel := context.WithCancel(context.Background()) diff --git a/infile.go b/infile.go index 3279dcffd..cfd41914e 100644 --- a/infile.go +++ b/infile.go @@ -93,7 +93,7 @@ func deferredClose(err *error, closer io.Closer) { const defaultPacketSize = 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP -func (mc *mysqlConn) handleInFileRequest(name string) (err error) { +func (mc *okHandler) handleInFileRequest(name string) (err error) { var rdr io.Reader var data []byte packetSize := defaultPacketSize @@ -154,7 +154,7 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { for err == nil { n, err = rdr.Read(data[4:]) if n > 0 { - if ioErr := mc.writePacket(data[:4+n]); ioErr != nil { + if ioErr := mc.conn().writePacket(data[:4+n]); ioErr != nil { return ioErr } } @@ -168,7 +168,7 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { if data == nil { data = make([]byte, 4) } - if ioErr := mc.writePacket(data[:4]); ioErr != nil { + if ioErr := mc.conn().writePacket(data[:4]); ioErr != nil { return ioErr } @@ -177,6 +177,6 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { return mc.readResultOK() } - mc.readPacket() + mc.conn().readPacket() return err } diff --git a/packets.go b/packets.go index c10072c94..1a7f2c376 100644 --- a/packets.go +++ b/packets.go @@ -511,7 +511,9 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { switch data[0] { case iOK: - return nil, "", mc.handleOkPacket(data) + // resultUnchanged, since auth happens before any queries or + // commands have been executed. + return nil, "", mc.resultUnchanged().handleOkPacket(data) case iAuthMoreData: return data[1:], "", err @@ -535,8 +537,8 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { } // Returns error if Packet is not an 'Result OK'-Packet -func (mc *mysqlConn) readResultOK() error { - data, err := mc.readPacket() +func (mc *okHandler) readResultOK() error { + data, err := mc.conn().readPacket() if err != nil { return err } @@ -544,13 +546,17 @@ func (mc *mysqlConn) readResultOK() error { if data[0] == iOK { return mc.handleOkPacket(data) } - return mc.handleErrorPacket(data) + return mc.conn().handleErrorPacket(data) } // Result Set Header Packet // http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-ProtocolText::Resultset -func (mc *mysqlConn) readResultSetHeaderPacket() (int, error) { - data, err := mc.readPacket() +func (mc *okHandler) readResultSetHeaderPacket() (int, error) { + // handleOkPacket replaces both values; other cases leave the values unchanged. + mc.result.affectedRows = append(mc.result.affectedRows, 0) + mc.result.insertIds = append(mc.result.insertIds, 0) + + data, err := mc.conn().readPacket() if err == nil { switch data[0] { @@ -558,7 +564,7 @@ func (mc *mysqlConn) readResultSetHeaderPacket() (int, error) { return 0, mc.handleOkPacket(data) case iERR: - return 0, mc.handleErrorPacket(data) + return 0, mc.conn().handleErrorPacket(data) case iLocalInFile: return 0, mc.handleInFileRequest(string(data[1:])) @@ -623,18 +629,61 @@ func readStatus(b []byte) statusFlag { return statusFlag(b[0]) | statusFlag(b[1])<<8 } +// Returns an instance of okHandler for codepaths where mysqlConn.result doesn't +// need to be cleared first (e.g. during authentication, or while additional +// resultsets are being fetched.) +func (mc *mysqlConn) resultUnchanged() *okHandler { + return (*okHandler)(mc) +} + +// okHandler represents the state of the connection when mysqlConn.result has +// been prepared for processing of OK packets. +// +// To correctly populate mysqlConn.result (updated by handleOkPacket()), all +// callpaths must either: +// +// 1. first clear it using clearResult(), or +// 2. confirm that they don't need to (by calling resultUnchanged()). +// +// Both return an instance of type *okHandler. +type okHandler mysqlConn + +// Exposees the underlying type's methods. +func (mc *okHandler) conn() *mysqlConn { + return (*mysqlConn)(mc) +} + +// clearResult clears the connection's stored affectedRows and insertIds +// fields. +// +// It returns a handler that can process OK responses. +func (mc *mysqlConn) clearResult() *okHandler { + mc.result = mysqlResult{} + return (*okHandler)(mc) +} + // Ok Packet // http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet -func (mc *mysqlConn) handleOkPacket(data []byte) error { +func (mc *okHandler) handleOkPacket(data []byte) error { var n, m int + var affectedRows, insertId uint64 // 0x00 [1 byte] // Affected rows [Length Coded Binary] - mc.affectedRows, _, n = readLengthEncodedInteger(data[1:]) + affectedRows, _, n = readLengthEncodedInteger(data[1:]) // Insert id [Length Coded Binary] - mc.insertId, _, m = readLengthEncodedInteger(data[1+n:]) + insertId, _, m = readLengthEncodedInteger(data[1+n:]) + + // Update for the current statement result (only used by + // readResultSetHeaderPacket). + if len(mc.result.affectedRows) > 0 { + mc.result.affectedRows[len(mc.result.affectedRows)-1] = int64(affectedRows) + } + if len(mc.result.insertIds) > 0 { + mc.result.insertIds[len(mc.result.insertIds)-1] = int64(insertId) + } // server_status [2 bytes] mc.status = readStatus(data[1+n+m : 1+n+m+2]) @@ -1165,7 +1214,9 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { return mc.writePacket(data) } -func (mc *mysqlConn) discardResults() error { +// For each remaining resultset in the stream, discards its rows and updates +// mc.affectedRows and mc.insertIds. +func (mc *okHandler) discardResults() error { for mc.status&statusMoreResultsExists != 0 { resLen, err := mc.readResultSetHeaderPacket() if err != nil { @@ -1173,11 +1224,11 @@ func (mc *mysqlConn) discardResults() error { } if resLen > 0 { // columns - if err := mc.readUntilEOF(); err != nil { + if err := mc.conn().readUntilEOF(); err != nil { return err } // rows - if err := mc.readUntilEOF(); err != nil { + if err := mc.conn().readUntilEOF(); err != nil { return err } } diff --git a/result.go b/result.go index c6438d034..36a432e81 100644 --- a/result.go +++ b/result.go @@ -8,15 +8,44 @@ package mysql +import "database/sql/driver" + +// Result exposes data not available through *connection.Result. +// +// This is accessible by executing statements using sql.Conn.Raw() and +// downcasting the returned result: +// +// res, err := rawConn.Exec(...) +// res.(mysql.Result).AllRowsAffected() +// +type Result interface { + driver.Result + // AllRowsAffected returns a slice containing the affected rows for each + // executed statement. + AllRowsAffected() []int64 + // AllLastInsertIds returns a slice containing the last inserted ID for each + // executed statement. + AllLastInsertIds() []int64 +} + type mysqlResult struct { - affectedRows int64 - insertId int64 + // One entry in both slices is created for every executed statement result. + affectedRows []int64 + insertIds []int64 } func (res *mysqlResult) LastInsertId() (int64, error) { - return res.insertId, nil + return res.insertIds[len(res.insertIds)-1], nil } func (res *mysqlResult) RowsAffected() (int64, error) { - return res.affectedRows, nil + return res.affectedRows[len(res.affectedRows)-1], nil +} + +func (res *mysqlResult) AllLastInsertIds() []int64 { + return append([]int64{}, res.insertIds...) // defensive copy +} + +func (res *mysqlResult) AllRowsAffected() []int64 { + return append([]int64{}, res.affectedRows...) // defensive copy } diff --git a/rows.go b/rows.go index 888bdb5f0..63d0ed2d5 100644 --- a/rows.go +++ b/rows.go @@ -123,7 +123,8 @@ func (rows *mysqlRows) Close() (err error) { err = mc.readUntilEOF() } if err == nil { - if err = mc.discardResults(); err != nil { + handleOk := mc.clearResult() + if err = handleOk.discardResults(); err != nil { return err } } @@ -160,7 +161,9 @@ func (rows *mysqlRows) nextResultSet() (int, error) { return 0, io.EOF } rows.rs = resultSet{} - return rows.mc.readResultSetHeaderPacket() + // rows.mc.affectedRows and rows.mc.insertIds accumulate on each call to + // nextResultSet. + return rows.mc.resultUnchanged().readResultSetHeaderPacket() } func (rows *mysqlRows) nextNotEmptyResultSet() (int, error) { diff --git a/statement.go b/statement.go index d8b3975a5..31e7799c4 100644 --- a/statement.go +++ b/statement.go @@ -61,12 +61,10 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { } mc := stmt.mc - - mc.affectedRows = 0 - mc.insertId = 0 + handleOk := stmt.mc.clearResult() // Read Result - resLen, err := mc.readResultSetHeaderPacket() + resLen, err := handleOk.readResultSetHeaderPacket() if err != nil { return nil, err } @@ -83,14 +81,12 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { } } - if err := mc.discardResults(); err != nil { + if err := handleOk.discardResults(); err != nil { return nil, err } - return &mysqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, nil + copied := mc.result + return &copied, nil } func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { @@ -111,7 +107,8 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) { mc := stmt.mc // Read Result - resLen, err := mc.readResultSetHeaderPacket() + handleOk := stmt.mc.clearResult() + resLen, err := handleOk.readResultSetHeaderPacket() if err != nil { return nil, err } From 8365b948403b6a9d0724518c2f722e09d4561794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Mengu=C3=A9?= Date: Fri, 2 Jun 2023 17:28:47 +0200 Subject: [PATCH 17/53] doc: add link to NewConnector from FormatDSN (#1442) Advise to use NewConnector instead of FormatDSN because roundtripping is known to not work well. See https://github.com/go-sql-driver/mysql/issues/1410#issuecomment-1510866931 --- dsn.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dsn.go b/dsn.go index 693aa4e5a..380ca9570 100644 --- a/dsn.go +++ b/dsn.go @@ -177,6 +177,8 @@ func writeDSNParam(buf *bytes.Buffer, hasParam *bool, name, value string) { // FormatDSN formats the given Config into a DSN string which can be passed to // the driver. +// +// Note: use [NewConnector] and [database/sql.OpenDB] to open a connection from a [*Config]. func (cfg *Config) FormatDSN() string { var buf bytes.Buffer From 65ed3c5d4007ad7ea74c33e78b953b82a9ed80ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Mengu=C3=A9?= Date: Fri, 2 Jun 2023 17:30:26 +0200 Subject: [PATCH 18/53] Add fuzz test for FormatDSN (#1444) Run (go 1.18+): go test -fuzz FuzzFormatDSN Note: invalid host:addr values are currently ignored as they are known to break (ParseDSN doesn't strictly check address format). --- dsn_fuzz_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 dsn_fuzz_test.go diff --git a/dsn_fuzz_test.go b/dsn_fuzz_test.go new file mode 100644 index 000000000..04c56ad45 --- /dev/null +++ b/dsn_fuzz_test.go @@ -0,0 +1,47 @@ +//go:build go1.18 +// +build go1.18 + +package mysql + +import ( + "net" + "testing" +) + +func FuzzFormatDSN(f *testing.F) { + for _, test := range testDSNs { // See dsn_test.go + f.Add(test.in) + } + + f.Fuzz(func(t *testing.T, dsn1 string) { + // Do not waste resources + if len(dsn1) > 1000 { + t.Skip("ignore: too long") + } + + cfg1, err := ParseDSN(dsn1) + if err != nil { + t.Skipf("invalid DSN: %v", err) + } + + dsn2 := cfg1.FormatDSN() + if dsn2 == dsn1 { + return + } + + // Skip known cases of bad config that are not strictly checked by ParseDSN + if _, _, err := net.SplitHostPort(cfg1.Addr); err != nil { + t.Skipf("invalid addr %q: %v", cfg1.Addr, err) + } + + cfg2, err := ParseDSN(dsn2) + if err != nil { + t.Fatalf("%q rewritten as %q: %v", dsn1, dsn2, err) + } + + dsn3 := cfg2.FormatDSN() + if dsn3 != dsn2 { + t.Errorf("%q rewritten as %q", dsn2, dsn3) + } + }) +} From cf948e4a9df2e97c1b0e3d068a52b5e2d53485a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Mengu=C3=A9?= Date: Tue, 13 Jun 2023 06:24:06 +0200 Subject: [PATCH 19/53] TestDSNReformat: add more roundtrip checks (#1443) Add more roundtrip checks for ParseDSN/FormatDSN. --- dsn_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dsn_test.go b/dsn_test.go index a729d0ef8..be50102de 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -130,6 +130,11 @@ func TestDSNReformat(t *testing.T) { res1 := fmt.Sprintf("%+v", cfg1) dsn2 := cfg1.FormatDSN() + if dsn2 != dsn1 { + // Just log + t.Logf("%d. %q reformated as %q", i, dsn1, dsn2) + } + cfg2, err := ParseDSN(dsn2) if err != nil { t.Error(err.Error()) @@ -141,6 +146,11 @@ func TestDSNReformat(t *testing.T) { if res1 != res2 { t.Errorf("%d. %q does not match %q", i, res2, res1) } + + dsn3 := cfg2.FormatDSN() + if dsn3 != dsn2 { + t.Errorf("%d. %q does not match %q", i, dsn2, dsn3) + } }) } } From 943264b76442d87ceea460ae7745208c8143f098 Mon Sep 17 00:00:00 2001 From: Achille Date: Mon, 12 Jun 2023 23:39:30 -0700 Subject: [PATCH 20/53] ignore errors returned by SetKeepAlive (#1448) Signed-off-by: Achille Roussel --- connector.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/connector.go b/connector.go index 6acf3dd50..7e0b16734 100644 --- a/connector.go +++ b/connector.go @@ -100,10 +100,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { // Enable TCP Keepalives on TCP connections if tc, ok := mc.netConn.(*net.TCPConn); ok { if err := tc.SetKeepAlive(true); err != nil { - // Don't send COM_QUIT before handshake. - mc.netConn.Close() - mc.netConn = nil - return nil, err + c.cfg.Logger.Print(err) } } From 564dee9b80ffc1e406b8b91e2215d29919730ae2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 16 Jun 2023 10:33:09 +0900 Subject: [PATCH 21/53] CI: use staticcheck (#1449) --- .github/workflows/test.yml | 8 ++++++++ auth.go | 2 +- driver_test.go | 11 +++++++---- errors.go | 2 +- infile.go | 4 ++-- nulltime.go | 2 +- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2ab5e82a..3122c0e17 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,6 +11,14 @@ env: MYSQL_TEST_CONCURRENT: 1 jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dominikh/staticcheck-action@v1.3.0 + with: + version: "2023.1.3" + list: runs-on: ubuntu-latest outputs: diff --git a/auth.go b/auth.go index f6b157a12..d2ab0103d 100644 --- a/auth.go +++ b/auth.go @@ -382,7 +382,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { // parse public key block, rest := pem.Decode(data[1:]) if block == nil { - return fmt.Errorf("No Pem data found, data: %s", rest) + return fmt.Errorf("no pem data found, data: %s", rest) } pkix, err := x509.ParsePKIXPublicKey(block.Bytes) if err != nil { diff --git a/driver_test.go b/driver_test.go index cd94c434e..c937b8416 100644 --- a/driver_test.go +++ b/driver_test.go @@ -346,8 +346,8 @@ func TestMultiQuery(t *testing.T) { rows := dbt.mustQuery("SELECT value FROM test WHERE id=1;") if rows.Next() { rows.Scan(&out) - if 5 != out { - dbt.Errorf("5 != %d", out) + if out != 5 { + dbt.Errorf("expected 5, got %d", out) } if rows.Next() { @@ -1293,7 +1293,7 @@ func TestLoadData(t *testing.T) { _, err = dbt.db.Exec("LOAD DATA LOCAL INFILE 'Reader::doesnotexist' INTO TABLE test") if err == nil { dbt.Fatal("load non-existent Reader didn't fail") - } else if err.Error() != "Reader 'doesnotexist' is not registered" { + } else if err.Error() != "reader 'doesnotexist' is not registered" { dbt.Fatal(err.Error()) } }) @@ -1401,6 +1401,7 @@ func TestReuseClosedConnection(t *testing.T) { if err != nil { t.Fatalf("error preparing statement: %s", err.Error()) } + //lint:ignore SA1019 this is a test _, err = stmt.Exec(nil) if err != nil { t.Fatalf("error executing statement: %s", err.Error()) @@ -1415,6 +1416,7 @@ func TestReuseClosedConnection(t *testing.T) { t.Errorf("panic after reusing a closed connection: %v", err) } }() + //lint:ignore SA1019 this is a test _, err = stmt.Exec(nil) if err != nil && err != driver.ErrBadConn { t.Errorf("unexpected error '%s', expected '%s'", @@ -2432,6 +2434,7 @@ func TestExecMultipleResults(t *testing.T) { t.Fatalf("failed to connect: %v", err) } conn.Raw(func(conn interface{}) error { + //lint:ignore SA1019 this is a test ex := conn.(driver.Execer) res, err := ex.Exec(` INSERT INTO test (value) VALUES ('a'), ('b'); @@ -2489,8 +2492,8 @@ func TestQueryMultipleResults(t *testing.T) { t.Fatalf("failed to connect: %v", err) } conn.Raw(func(conn interface{}) error { + //lint:ignore SA1019 this is a test qr := conn.(driver.Queryer) - c := conn.(*mysqlConn) // Demonstrate that repeated queries reset the affectedRows diff --git a/errors.go b/errors.go index 5680b6c05..a9a3060c9 100644 --- a/errors.go +++ b/errors.go @@ -21,7 +21,7 @@ var ( ErrMalformPkt = errors.New("malformed packet") ErrNoTLS = errors.New("TLS requested but server does not support TLS") ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN") - ErrNativePassword = errors.New("this user requires mysql native password authentication.") + ErrNativePassword = errors.New("this user requires mysql native password authentication") ErrOldPassword = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") ErrUnknownPlugin = errors.New("this authentication plugin is not supported") ErrOldProtocol = errors.New("MySQL server does not support required protocol 41+") diff --git a/infile.go b/infile.go index cfd41914e..0c8af9f11 100644 --- a/infile.go +++ b/infile.go @@ -116,10 +116,10 @@ func (mc *okHandler) handleInFileRequest(name string) (err error) { defer deferredClose(&err, cl) } } else { - err = fmt.Errorf("Reader '%s' is ", name) + err = fmt.Errorf("reader '%s' is ", name) } } else { - err = fmt.Errorf("Reader '%s' is not registered", name) + err = fmt.Errorf("reader '%s' is not registered", name) } } else { // File name = strings.Trim(name, `"`) diff --git a/nulltime.go b/nulltime.go index 36c8a42c5..7d381d5c2 100644 --- a/nulltime.go +++ b/nulltime.go @@ -59,7 +59,7 @@ func (nt *NullTime) Scan(value interface{}) (err error) { } nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) + return fmt.Errorf("can't convert %T to time.Time", value) } // Value implements the driver Valuer interface. From 5d4a83127cf18cadc447807c320666de5367cc4d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 3 Jul 2023 15:50:22 +0900 Subject: [PATCH 22/53] Parse numbers on text protocol too (#1452) --- driver_test.go | 87 ++++++++++++++++++++++++++++++++++---------------- packets.go | 39 +++++++++++++++++----- 2 files changed, 90 insertions(+), 36 deletions(-) diff --git a/driver_test.go b/driver_test.go index c937b8416..2748870b7 100644 --- a/driver_test.go +++ b/driver_test.go @@ -148,29 +148,18 @@ func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { defer db2.Close() } - dsn3 := dsn + "&multiStatements=true" - var db3 *sql.DB - if _, err := ParseDSN(dsn3); err != errInvalidDSNUnsafeCollation { - db3, err = sql.Open("mysql", dsn3) - if err != nil { - t.Fatalf("error connecting: %s", err.Error()) - } - defer db3.Close() - } - - dbt := &DBTest{t, db} - dbt2 := &DBTest{t, db2} - dbt3 := &DBTest{t, db3} for _, test := range tests { - test(dbt) - dbt.db.Exec("DROP TABLE IF EXISTS test") + t.Run("default", func(t *testing.T) { + dbt := &DBTest{t, db} + test(dbt) + dbt.db.Exec("DROP TABLE IF EXISTS test") + }) if db2 != nil { - test(dbt2) - dbt2.db.Exec("DROP TABLE IF EXISTS test") - } - if db3 != nil { - test(dbt3) - dbt3.db.Exec("DROP TABLE IF EXISTS test") + t.Run("interpolateParams", func(t *testing.T) { + dbt2 := &DBTest{t, db2} + test(dbt2) + dbt2.db.Exec("DROP TABLE IF EXISTS test") + }) } } } @@ -316,6 +305,48 @@ func TestCRUD(t *testing.T) { }) } +// TestNumbers test that selecting numeric columns. +// Both of textRows and binaryRows should return same type and value. +func TestNumbersToAny(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE `test` (id INT PRIMARY KEY, b BOOL, i8 TINYINT, " + + "i16 SMALLINT, i32 INT, i64 BIGINT, f32 FLOAT, f64 DOUBLE)") + dbt.mustExec("INSERT INTO `test` VALUES (1, true, 127, 32767, 2147483647, 9223372036854775807, 1.25, 2.5)") + + // Use binaryRows for intarpolateParams=false and textRows for intarpolateParams=true. + rows := dbt.mustQuery("SELECT b, i8, i16, i32, i64, f32, f64 FROM `test` WHERE id=?", 1) + if !rows.Next() { + dbt.Fatal("no data") + } + var b, i8, i16, i32, i64, f32, f64 any + err := rows.Scan(&b, &i8, &i16, &i32, &i64, &f32, &f64) + if err != nil { + dbt.Fatal(err) + } + if b.(int64) != 1 { + dbt.Errorf("b != 1") + } + if i8.(int64) != 127 { + dbt.Errorf("i8 != 127") + } + if i16.(int64) != 32767 { + dbt.Errorf("i16 != 32767") + } + if i32.(int64) != 2147483647 { + dbt.Errorf("i32 != 2147483647") + } + if i64.(int64) != 9223372036854775807 { + dbt.Errorf("i64 != 9223372036854775807") + } + if f32.(float32) != 1.25 { + dbt.Errorf("f32 != 1.25") + } + if f64.(float64) != 2.5 { + dbt.Errorf("f64 != 2.5") + } + }) +} + func TestMultiQuery(t *testing.T) { runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { // Create Table @@ -1808,13 +1839,13 @@ func TestConcurrent(t *testing.T) { } runTests(t, dsn, func(dbt *DBTest) { - var version string - if err := dbt.db.QueryRow("SELECT @@version").Scan(&version); err != nil { - dbt.Fatalf("%s", err.Error()) - } - if strings.Contains(strings.ToLower(version), "mariadb") { - t.Skip(`TODO: "fix commands out of sync. Did you run multiple statements at once?" on MariaDB`) - } + // var version string + // if err := dbt.db.QueryRow("SELECT @@version").Scan(&version); err != nil { + // dbt.Fatal(err) + // } + // if strings.Contains(strings.ToLower(version), "mariadb") { + // t.Skip(`TODO: "fix commands out of sync. Did you run multiple statements at once?" on MariaDB`) + // } var max int err := dbt.db.QueryRow("SELECT @@max_connections").Scan(&max) diff --git a/packets.go b/packets.go index 1a7f2c376..66635c55b 100644 --- a/packets.go +++ b/packets.go @@ -17,6 +17,7 @@ import ( "fmt" "io" "math" + "strconv" "time" ) @@ -834,7 +835,8 @@ func (rows *textRows) readRow(dest []driver.Value) error { for i := range dest { // Read bytes and convert to string - dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) + var buf []byte + buf, isNull, n, err = readLengthEncodedString(data[pos:]) pos += n if err != nil { @@ -846,19 +848,40 @@ func (rows *textRows) readRow(dest []driver.Value) error { continue } - if !mc.parseTime { - continue - } - - // Parse time field switch rows.rs.columns[i].fieldType { case fieldTypeTimestamp, fieldTypeDateTime, fieldTypeDate, fieldTypeNewDate: - if dest[i], err = parseDateTime(dest[i].([]byte), mc.cfg.Loc); err != nil { - return err + if mc.parseTime { + dest[i], err = parseDateTime(buf, mc.cfg.Loc) + } else { + dest[i] = buf + } + + case fieldTypeTiny, fieldTypeShort, fieldTypeInt24, fieldTypeYear, fieldTypeLong: + dest[i], err = strconv.ParseInt(string(buf), 10, 32) + + case fieldTypeLongLong: + if rows.rs.columns[i].flags&flagUnsigned != 0 { + dest[i], err = strconv.ParseUint(string(buf), 10, 64) + } else { + dest[i], err = strconv.ParseInt(string(buf), 10, 64) } + + case fieldTypeFloat: + var d float64 + d, err = strconv.ParseFloat(string(buf), 32) + dest[i] = float32(d) + + case fieldTypeDouble: + dest[i], err = strconv.ParseFloat(string(buf), 64) + + default: + dest[i] = buf + } + if err != nil { + return err } } From 0b18dac46f7f10d00411ab6fb10b8d6e4522c2d9 Mon Sep 17 00:00:00 2001 From: Daemonxiao <35677990+Daemonxiao@users.noreply.github.com> Date: Thu, 13 Jul 2023 16:52:35 +0800 Subject: [PATCH 23/53] Add Daemonxiao to AUTHORS (#1459) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 7e4fac5a1..29e08b0ca 100644 --- a/AUTHORS +++ b/AUTHORS @@ -26,6 +26,7 @@ Carlos Nieto Chris Kirkland Chris Moos Craig Wilson +Daemonxiao <735462752 at qq.com> Daniel Montoya Daniel Nichter Daniël van Eeden From 2c81c69ebe815b611383d18002074e073bed745a Mon Sep 17 00:00:00 2001 From: i7a7467 <61368544+i7a7467@users.noreply.github.com> Date: Thu, 3 Aug 2023 15:51:54 +0900 Subject: [PATCH 24/53] update docs link about load data local (#1468) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4eade6853..6ef19966c 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ Default: false ``` `allowAllFiles=true` disables the file allowlist for `LOAD DATA LOCAL INFILE` and allows *all* files. -[*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html) +[*Might be insecure!*](https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-local) ##### `allowCleartextPasswords` @@ -509,7 +509,7 @@ For this feature you need direct access to the package. Therefore you must chang import "github.com/go-sql-driver/mysql" ``` -Files must be explicitly allowed by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the allowlist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)). +Files must be explicitly allowed by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the allowlist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-local)). To use a `io.Reader` a handler function must be registered with `mysql.RegisterReaderHandler(name, handler)` which returns a `io.Reader` or `io.ReadCloser`. The Reader is available with the filepath `Reader::` then. Choose different names for different handlers and `DeregisterReaderHandler` when you don't need it anymore. From e503d8d2c01d622d312e4b044fc2c19948d4663f Mon Sep 17 00:00:00 2001 From: Netzer7 <58796038+Netzer7@users.noreply.github.com> Date: Mon, 7 Aug 2023 00:34:14 -0700 Subject: [PATCH 25/53] Update README.md (#1464) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ef19966c..18fcc0276 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ Valid Values: Default: none ``` -Sets the charset used for client-server interaction (`"SET NAMES "`). If multiple charsets are set (separated by a comma), the following charset is used if setting the charset failes. This enables for example support for `utf8mb4` ([introduced in MySQL 5.5.3](http://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)) with fallback to `utf8` for older servers (`charset=utf8mb4,utf8`). +Sets the charset used for client-server interaction (`"SET NAMES "`). If multiple charsets are set (separated by a comma), the following charset is used if setting the charset fails. This enables for example support for `utf8mb4` ([introduced in MySQL 5.5.3](http://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)) with fallback to `utf8` for older servers (`charset=utf8mb4,utf8`). See also [Unicode Support](#unicode-support). From 7cf548287682c36ebce3b7966f2693d58094bd5a Mon Sep 17 00:00:00 2001 From: ICHINOSE Shogo Date: Wed, 9 Aug 2023 20:35:39 +0900 Subject: [PATCH 26/53] add Go 1.21 and MySQL 8.1 to the build matrix (#1472) * add Go 1.21 and MySQL 8.1 to the build matrix * bump shogo82148/actions-setup-mysql v1.21.0 --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3122c0e17..b25c9e389 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,12 +31,14 @@ jobs: import os go = [ # Keep the most recent production release at the top - '1.20', + '1.21', # Older production releases + '1.20', '1.19', '1.18', ] mysql = [ + '8.1', '8.0', '5.7', '5.6', @@ -75,7 +77,7 @@ jobs: - uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} - - uses: shogo82148/actions-setup-mysql@v1.16.0 + - uses: shogo82148/actions-setup-mysql@v1.21.0 with: mysql-version: ${{ matrix.mysql }} user: ${{ env.MYSQL_TEST_USER }} From 43e9bef05581335f84d246aba6211af1b5133aae Mon Sep 17 00:00:00 2001 From: Pyry Kontio Date: Sat, 2 Sep 2023 03:35:23 +0900 Subject: [PATCH 27/53] Improve DSN docstsrings (#1475) --- dsn.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dsn.go b/dsn.go index 380ca9570..f5b184e3f 100644 --- a/dsn.go +++ b/dsn.go @@ -36,8 +36,8 @@ var ( type Config struct { User string // Username Passwd string // Password (requires User) - Net string // Network type - Addr string // Network address (requires Net) + Net string // Network (e.g. "tcp", "tcp6", "unix". default: "tcp") + Addr string // Address (default: "127.0.0.1:3306" for "tcp" and "/tmp/mysql.sock" for "unix") DBName string // Database name Params map[string]string // Connection parameters ConnectionAttributes string // Connection Attributes, comma-delimited string of user-defined "key:value" pairs From 78e0387dba9f2894f3ee6004b98c49b9b11bf367 Mon Sep 17 00:00:00 2001 From: ShenFeng312 <49786112+ShenFeng312@users.noreply.github.com> Date: Wed, 20 Sep 2023 11:55:24 +0800 Subject: [PATCH 28/53] packet: remove length check (#1481) Fix #1478 --- packets.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packets.go b/packets.go index 66635c55b..4e27004aa 100644 --- a/packets.go +++ b/packets.go @@ -572,12 +572,9 @@ func (mc *okHandler) readResultSetHeaderPacket() (int, error) { } // column count - num, _, n := readLengthEncodedInteger(data) - if n-len(data) == 0 { - return int(num), nil - } - - return 0, ErrMalformPkt + num, _, _ := readLengthEncodedInteger(data) + // ignore remaining data in the packet. see #1478. + return int(num), nil } return 0, err } From 22e750b046938b5c13375da56a5f85ae9ce10e0b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Sep 2023 20:16:32 +0900 Subject: [PATCH 29/53] README: fix markup error (#1480) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 18fcc0276..9257c1fd2 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ This has the same effect as an empty DSN string: ``` -`dbname` is escaped by [PathEscape()]()https://pkg.go.dev/net/url#PathEscape) since v1.8.0. If your database name is `dbname/withslash`, it becomes: +`dbname` is escaped by [PathEscape()](https://pkg.go.dev/net/url#PathEscape) since v1.8.0. If your database name is `dbname/withslash`, it becomes: ``` /dbname%2Fwithslash From 19171b59bf90e6bf7a5bdf979e5e24a84b328b8a Mon Sep 17 00:00:00 2001 From: Oliver Bone Date: Sat, 30 Sep 2023 20:33:48 +0100 Subject: [PATCH 30/53] Close connection on ErrPktSync and ErrPktSyncMul (#1473) An `ErrPktSync` or `ErrPktSyncMul` error always means that a packet header has been read, but since the sequence ID was not correct then the packet payload has not been read. This results in the connection being left in a broken state, since any future operations will always result in a "busy buffer" error. Keeping such connections alive leads to them being repeatedly returned to the pool in this state, which can in turn result in a large number of failures due to these "busy buffer" errors. This commit fixes this problem by simply closing the connection before returning either `ErrPktSync` or `ErrPktSyncMul`. This ensures that the connection won't be returned to the pool, preventing it from causing any further errors. --- AUTHORS | 1 + packets.go | 1 + packets_test.go | 52 ++++++++++++++++++++++++++----------------------- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/AUTHORS b/AUTHORS index 29e08b0ca..dec27daca 100644 --- a/AUTHORS +++ b/AUTHORS @@ -77,6 +77,7 @@ Maciej Zimnoch Michael Woolnough Nathanial Murphy Nicola Peduzzi +Oliver Bone Olivier Mengué oscarzhao Paul Bonser diff --git a/packets.go b/packets.go index 4e27004aa..0994d41a3 100644 --- a/packets.go +++ b/packets.go @@ -44,6 +44,7 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { // check packet sync [8 bit] if data[3] != mc.sequence { + mc.Close() if data[3] > mc.sequence { return nil, ErrPktSyncMul } diff --git a/packets_test.go b/packets_test.go index f429087e9..56c455188 100644 --- a/packets_test.go +++ b/packets_test.go @@ -133,30 +133,34 @@ func TestReadPacketSingleByte(t *testing.T) { } func TestReadPacketWrongSequenceID(t *testing.T) { - conn := new(mockConn) - mc := &mysqlConn{ - buf: newBuffer(conn), - } - - // too low sequence id - conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} - conn.maxReads = 1 - mc.sequence = 1 - _, err := mc.readPacket() - if err != ErrPktSync { - t.Errorf("expected ErrPktSync, got %v", err) - } - - // reset - conn.reads = 0 - mc.sequence = 0 - mc.buf = newBuffer(conn) - - // too high sequence id - conn.data = []byte{0x01, 0x00, 0x00, 0x42, 0xff} - _, err = mc.readPacket() - if err != ErrPktSyncMul { - t.Errorf("expected ErrPktSyncMul, got %v", err) + for _, testCase := range []struct { + ClientSequenceID byte + ServerSequenceID byte + ExpectedErr error + }{ + { + ClientSequenceID: 1, + ServerSequenceID: 0, + ExpectedErr: ErrPktSync, + }, + { + ClientSequenceID: 0, + ServerSequenceID: 0x42, + ExpectedErr: ErrPktSyncMul, + }, + } { + conn, mc := newRWMockConn(testCase.ClientSequenceID) + + conn.data = []byte{0x01, 0x00, 0x00, testCase.ServerSequenceID, 0xff} + _, err := mc.readPacket() + if err != testCase.ExpectedErr { + t.Errorf("expected %v, got %v", testCase.ExpectedErr, err) + } + + // connection should not be returned to the pool in this state + if mc.IsValid() { + t.Errorf("expected IsValid() to be false") + } } } From e5a2abc9cca895ca44570b171ff1f2f976d5921d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 4 Oct 2023 21:24:11 +0300 Subject: [PATCH 31/53] Spelling, grammar, and link fixes (#1485) --- CHANGELOG.md | 6 +++--- README.md | 2 +- auth.go | 4 ++-- driver_test.go | 10 +++++----- dsn_test.go | 2 +- packets.go | 6 +++--- packets_test.go | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5166e4adb..213215c8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -162,7 +162,7 @@ New Features: - Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249) - Support for returning table alias on Columns() (#289, #359, #382) - - Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318, #490) + - Placeholder interpolation, can be activated with the DSN parameter `interpolateParams=true` (#309, #318, #490) - Support for uint64 parameters with high bit set (#332, #345) - Cleartext authentication plugin support (#327) - Exported ParseDSN function and the Config struct (#403, #419, #429) @@ -206,7 +206,7 @@ Changes: - Also exported the MySQLWarning type - mysqlConn.Close returns the first error encountered instead of ignoring all errors - writePacket() automatically writes the packet size to the header - - readPacket() uses an iterative approach instead of the recursive approach to merge splitted packets + - readPacket() uses an iterative approach instead of the recursive approach to merge split packets New Features: @@ -254,7 +254,7 @@ Bugfixes: - Fixed MySQL 4.1 support: MySQL 4.1 sends packets with lengths which differ from the specification - Convert to DB timezone when inserting `time.Time` - - Splitted packets (more than 16MB) are now merged correctly + - Split packets (more than 16MB) are now merged correctly - Fixed false positive `io.EOF` errors when the data was fully read - Avoid panics on reuse of closed connections - Fixed empty string producing false nil values diff --git a/README.md b/README.md index 9257c1fd2..fff8969f3 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ Passwords can consist of any character. Escaping is **not** necessary. #### Protocol See [net.Dial](https://golang.org/pkg/net/#Dial) for more information which networks are available. -In general you should use an Unix domain socket if available and TCP otherwise for best performance. +In general you should use a Unix domain socket if available and TCP otherwise for best performance. #### Address For TCP and UDP networks, addresses have the form `host[:port]`. diff --git a/auth.go b/auth.go index d2ab0103d..bab282bd2 100644 --- a/auth.go +++ b/auth.go @@ -338,7 +338,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { switch plugin { - // https://insidemysql.com/preparing-your-community-connector-for-mysql-8-part-2-sha256/ + // https://dev.mysql.com/blog-archive/preparing-your-community-connector-for-mysql-8-part-2-sha256/ case "caching_sha2_password": switch len(authData) { case 0: @@ -376,7 +376,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { } if data[0] != iAuthMoreData { - return fmt.Errorf("unexpect resp from server for caching_sha2_password perform full authentication") + return fmt.Errorf("unexpected resp from server for caching_sha2_password, perform full authentication") } // parse public key diff --git a/driver_test.go b/driver_test.go index 2748870b7..dd3d73141 100644 --- a/driver_test.go +++ b/driver_test.go @@ -1198,7 +1198,7 @@ func TestLongData(t *testing.T) { dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(inS), len(out)) } if rows.Next() { - dbt.Error("LONGBLOB: unexpexted row") + dbt.Error("LONGBLOB: unexpected row") } } else { dbt.Fatalf("LONGBLOB: no data") @@ -1217,7 +1217,7 @@ func TestLongData(t *testing.T) { dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(in), len(out)) } if rows.Next() { - dbt.Error("LONGBLOB: unexpexted row") + dbt.Error("LONGBLOB: unexpected row") } } else { if err = rows.Err(); err != nil { @@ -1293,7 +1293,7 @@ func TestLoadData(t *testing.T) { dbt.Fatalf("unexpected row count: got %d, want 0", count) } - // Then fille File with data and try to load it + // Then fill File with data and try to load it file.WriteString("1\ta string\n2\ta string containing a \\t\n3\ta string containing a \\n\n4\ta string containing both \\t\\n\n") file.Close() dbt.mustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE %q INTO TABLE test", file.Name())) @@ -1899,7 +1899,7 @@ func TestConcurrent(t *testing.T) { }(i) } - // wait until all conections are open + // wait until all connections are open wg.Wait() if fatalError != "" { @@ -1948,7 +1948,7 @@ func TestCustomDial(t *testing.T) { t.Skipf("MySQL server not running on %s", netAddr) } - // our custom dial function which justs wraps net.Dial here + // our custom dial function which just wraps net.Dial here RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { var d net.Dialer return d.DialContext(ctx, prot, addr) diff --git a/dsn_test.go b/dsn_test.go index be50102de..8a6a0c10e 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -132,7 +132,7 @@ func TestDSNReformat(t *testing.T) { dsn2 := cfg1.FormatDSN() if dsn2 != dsn1 { // Just log - t.Logf("%d. %q reformated as %q", i, dsn1, dsn2) + t.Logf("%d. %q reformatted as %q", i, dsn1, dsn2) } cfg2, err := ParseDSN(dsn2) diff --git a/packets.go b/packets.go index 0994d41a3..a1aaf20ee 100644 --- a/packets.go +++ b/packets.go @@ -240,7 +240,7 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro // reserved (all [00]) [10 bytes] pos += 1 + 2 + 2 + 1 + 10 - // second part of the password cipher [mininum 13 bytes], + // second part of the password cipher [minimum 13 bytes], // where len=MAX(13, length of auth-plugin-data - 8) // // The web documentation is ambiguous about the length. However, @@ -538,7 +538,7 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { } } -// Returns error if Packet is not an 'Result OK'-Packet +// Returns error if Packet is not a 'Result OK'-Packet func (mc *okHandler) readResultOK() error { data, err := mc.conn().readPacket() if err != nil { @@ -647,7 +647,7 @@ func (mc *mysqlConn) resultUnchanged() *okHandler { // Both return an instance of type *okHandler. type okHandler mysqlConn -// Exposees the underlying type's methods. +// Exposes the underlying type's methods. func (mc *okHandler) conn() *mysqlConn { return (*mysqlConn)(mc) } diff --git a/packets_test.go b/packets_test.go index 56c455188..e86ec5848 100644 --- a/packets_test.go +++ b/packets_test.go @@ -188,7 +188,7 @@ func TestReadPacketSplit(t *testing.T) { data[4] = 0x11 data[maxPacketSize+3] = 0x22 - // 2nd packet has payload length 0 and squence id 1 + // 2nd packet has payload length 0 and sequence id 1 // 00 00 00 01 data[pkt2ofs+3] = 0x01 @@ -220,7 +220,7 @@ func TestReadPacketSplit(t *testing.T) { data[pkt2ofs+4] = 0x33 data[pkt2ofs+maxPacketSize+3] = 0x44 - // 3rd packet has payload length 0 and squence id 2 + // 3rd packet has payload length 0 and sequence id 2 // 00 00 00 02 data[pkt3ofs+3] = 0x02 From 37980127edfb00edd1ba2eb397a33fdea2828828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Thu, 5 Oct 2023 11:44:35 +0300 Subject: [PATCH 32/53] use strings.Cut (#1486) --- connector.go | 9 ++++----- dsn.go | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/connector.go b/connector.go index 7e0b16734..ba3be71e7 100644 --- a/connector.go +++ b/connector.go @@ -38,13 +38,12 @@ func encodeConnectionAttributes(textAttributes string) string { // user-defined connection attributes for _, connAttr := range strings.Split(textAttributes, ",") { - attr := strings.SplitN(connAttr, ":", 2) - if len(attr) != 2 { + k, v, found := strings.Cut(connAttr, ":") + if !found { continue } - for _, v := range attr { - connAttrsBuf = appendLengthEncodedString(connAttrsBuf, v) - } + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, k) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, v) } return string(connAttrsBuf) diff --git a/dsn.go b/dsn.go index f5b184e3f..50c7ec413 100644 --- a/dsn.go +++ b/dsn.go @@ -390,13 +390,13 @@ func ParseDSN(dsn string) (cfg *Config, err error) { // Values must be url.QueryEscape'ed func parseDSNParams(cfg *Config, params string) (err error) { for _, v := range strings.Split(params, "&") { - param := strings.SplitN(v, "=", 2) - if len(param) != 2 { + key, value, found := strings.Cut(v, "=") + if !found { continue } // cfg params - switch value := param[1]; param[0] { + switch key { // Disable INFILE allowlist / enable all files case "allowAllFiles": var isBool bool @@ -577,7 +577,7 @@ func parseDSNParams(cfg *Config, params string) (err error) { cfg.Params = make(map[string]string) } - if cfg.Params[param[0]], err = url.QueryUnescape(value); err != nil { + if cfg.Params[key], err = url.QueryUnescape(value); err != nil { return } } From 5f74bcbcf0550e74cf0ac0170e5dd9f87683a355 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 9 Oct 2023 18:44:08 +0900 Subject: [PATCH 33/53] move stale connection check to ResetSession() (#1496) When ResetSession was added, it was called when the connection is put into the pool. Thet is why we had only set `mc.reset` flag on ResetSession(). In Go 1.15, this behavior was changed. (golang/go@971f8a2) ResetSession is called when the connection is checked out from the pool. So we can call checkConnLiveness() directly from ResetSession. --- connection.go | 27 +++++++++++++++++++++++++-- packets.go | 28 ---------------------------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/connection.go b/connection.go index 631a1dc24..660b2b0e0 100644 --- a/connection.go +++ b/connection.go @@ -34,7 +34,6 @@ type mysqlConn struct { status statusFlag sequence uint8 parseTime bool - reset bool // set when the Go SQL package calls ResetSession // for context support (Go 1.8+) watching bool @@ -646,7 +645,31 @@ func (mc *mysqlConn) ResetSession(ctx context.Context) error { if mc.closed.Load() { return driver.ErrBadConn } - mc.reset = true + + // Perform a stale connection check. We only perform this check for + // the first query on a connection that has been checked out of the + // connection pool: a fresh connection from the pool is more likely + // to be stale, and it has not performed any previous writes that + // could cause data corruption, so it's safe to return ErrBadConn + // if the check fails. + if mc.cfg.CheckConnLiveness { + conn := mc.netConn + if mc.rawConn != nil { + conn = mc.rawConn + } + var err error + if mc.cfg.ReadTimeout != 0 { + err = conn.SetReadDeadline(time.Now().Add(mc.cfg.ReadTimeout)) + } + if err == nil { + err = connCheck(conn) + } + if err != nil { + mc.cfg.Logger.Print("closing bad idle connection: ", err) + return driver.ErrBadConn + } + } + return nil } diff --git a/packets.go b/packets.go index a1aaf20ee..0127232ee 100644 --- a/packets.go +++ b/packets.go @@ -98,34 +98,6 @@ func (mc *mysqlConn) writePacket(data []byte) error { return ErrPktTooLarge } - // Perform a stale connection check. We only perform this check for - // the first query on a connection that has been checked out of the - // connection pool: a fresh connection from the pool is more likely - // to be stale, and it has not performed any previous writes that - // could cause data corruption, so it's safe to return ErrBadConn - // if the check fails. - if mc.reset { - mc.reset = false - conn := mc.netConn - if mc.rawConn != nil { - conn = mc.rawConn - } - var err error - if mc.cfg.CheckConnLiveness { - if mc.cfg.ReadTimeout != 0 { - err = conn.SetReadDeadline(time.Now().Add(mc.cfg.ReadTimeout)) - } - if err == nil { - err = connCheck(conn) - } - } - if err != nil { - mc.cfg.Logger.Print("closing bad idle connection: ", err) - mc.Close() - return driver.ErrBadConn - } - } - for { var size int if pktLen >= maxPacketSize { From 9c633df1f62eadfdc840840a0f229ea59cc15c33 Mon Sep 17 00:00:00 2001 From: ICHINOSE Shogo Date: Tue, 10 Oct 2023 17:39:46 +0900 Subject: [PATCH 34/53] fix race condition of TestConcurrent (#1490) * fix race condition of TestConcurrent * run tests with the '-race' option --- .github/workflows/test.yml | 2 +- driver_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b25c9e389..8e1cb9bc3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -96,7 +96,7 @@ jobs: - name: test run: | - go test -v '-covermode=count' '-coverprofile=coverage.out' + go test -v '-race' '-covermode=atomic' '-coverprofile=coverage.out' - name: Send coverage uses: shogo82148/actions-goveralls@v1 diff --git a/driver_test.go b/driver_test.go index dd3d73141..74f15c2d2 100644 --- a/driver_test.go +++ b/driver_test.go @@ -1872,7 +1872,6 @@ func TestConcurrent(t *testing.T) { defer wg.Done() tx, err := dbt.db.Begin() - atomic.AddInt32(&remaining, -1) if err != nil { if err.Error() != "Error 1040: Too many connections" { @@ -1882,7 +1881,7 @@ func TestConcurrent(t *testing.T) { } // keep the connection busy until all connections are open - for remaining > 0 { + for atomic.AddInt32(&remaining, -1) > 0 { if _, err = tx.Exec("DO 1"); err != nil { fatalf("error on conn %d: %s", id, err.Error()) return From 278a0b9e6b34ccc52aa213681836a79336714d34 Mon Sep 17 00:00:00 2001 From: ICHINOSE Shogo Date: Tue, 10 Oct 2023 17:48:58 +0900 Subject: [PATCH 35/53] mark fail, mustExec and mustQuery as test helpers (#1488) --- driver_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/driver_test.go b/driver_test.go index 74f15c2d2..f46d38df6 100644 --- a/driver_test.go +++ b/driver_test.go @@ -165,6 +165,7 @@ func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { } func (dbt *DBTest) fail(method, query string, err error) { + dbt.Helper() if len(query) > 300 { query = "[query too large to print]" } @@ -172,6 +173,7 @@ func (dbt *DBTest) fail(method, query string, err error) { } func (dbt *DBTest) mustExec(query string, args ...interface{}) (res sql.Result) { + dbt.Helper() res, err := dbt.db.Exec(query, args...) if err != nil { dbt.fail("exec", query, err) @@ -180,6 +182,7 @@ func (dbt *DBTest) mustExec(query string, args ...interface{}) (res sql.Result) } func (dbt *DBTest) mustQuery(query string, args ...interface{}) (rows *sql.Rows) { + dbt.Helper() rows, err := dbt.db.Query(query, args...) if err != nil { dbt.fail("query", query, err) From 1e6b8d7df47928193f2b1a04b5f7f06907187508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Mengu=C3=A9?= Date: Thu, 19 Oct 2023 07:33:23 +0200 Subject: [PATCH 36/53] Remove obsolete fuzz.go (#1498) fuzz.go (added in #1097) uses gofuzz. But #1444 added a better fuzzer that uses Go builtin fuzzing. Closes #1445. --- fuzz.go | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 fuzz.go diff --git a/fuzz.go b/fuzz.go deleted file mode 100644 index 3a4ec25a9..000000000 --- a/fuzz.go +++ /dev/null @@ -1,25 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package. -// -// Copyright 2020 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -//go:build gofuzz -// +build gofuzz - -package mysql - -import ( - "database/sql" -) - -func Fuzz(data []byte) int { - db, err := sql.Open("mysql", string(data)) - if err != nil { - return 0 - } - db.Close() - return 1 -} From 62c29ce0b1b8f84567de97ca0d32cebd53f05aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Mengu=C3=A9?= Date: Tue, 24 Oct 2023 10:05:53 +0200 Subject: [PATCH 37/53] Allow to change (or disable) the default driver name for registration (#1499) A link variable now allows to change or disable the name of the driver that is automatically registered with database/sql: Change driver name: go build "-ldflags=-X github.com/go-sql-driver/mysql.driverName=custom" Disable driver registration (set driverName to empty string): go build "-ldflags=-X github.com/go-sql-driver/mysql.driverName=" In the same way, a variable overridable at link time is also provided to override the driver name used in the test suite. This allows to run our test suite on another driver. go test "-ldflags=-X github.com/go-sql-driver/mysql.driverNameTest=custom" driverName is propagated to driverNameTest unless driverNameTest is explicitely defined. --- benchmark_test.go | 8 ++++---- driver.go | 8 +++++++- driver_test.go | 28 +++++++++++++++++++--------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/benchmark_test.go b/benchmark_test.go index fc70df60d..a4ecc0a63 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -48,7 +48,7 @@ func (tb *TB) checkStmt(stmt *sql.Stmt, err error) *sql.Stmt { func initDB(b *testing.B, queries ...string) *sql.DB { tb := (*TB)(b) - db := tb.checkDB(sql.Open("mysql", dsn)) + db := tb.checkDB(sql.Open(driverNameTest, dsn)) for _, query := range queries { if _, err := db.Exec(query); err != nil { b.Fatalf("error on %q: %v", query, err) @@ -105,7 +105,7 @@ func BenchmarkExec(b *testing.B) { tb := (*TB)(b) b.StopTimer() b.ReportAllocs() - db := tb.checkDB(sql.Open("mysql", dsn)) + db := tb.checkDB(sql.Open(driverNameTest, dsn)) db.SetMaxIdleConns(concurrencyLevel) defer db.Close() @@ -151,7 +151,7 @@ func BenchmarkRoundtripTxt(b *testing.B) { sampleString := string(sample) b.ReportAllocs() tb := (*TB)(b) - db := tb.checkDB(sql.Open("mysql", dsn)) + db := tb.checkDB(sql.Open(driverNameTest, dsn)) defer db.Close() b.StartTimer() var result string @@ -184,7 +184,7 @@ func BenchmarkRoundtripBin(b *testing.B) { sample, min, max := initRoundtripBenchmarks() b.ReportAllocs() tb := (*TB)(b) - db := tb.checkDB(sql.Open("mysql", dsn)) + db := tb.checkDB(sql.Open(driverNameTest, dsn)) defer db.Close() stmt := tb.checkStmt(db.Prepare("SELECT ?")) defer stmt.Close() diff --git a/driver.go b/driver.go index 0ed8fa1c5..45528b920 100644 --- a/driver.go +++ b/driver.go @@ -90,8 +90,14 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { return c.Connect(context.Background()) } +// This variable can be replaced with -ldflags like below: +// go build "-ldflags=-X github.com/go-sql-driver/mysql.driverName=custom" +var driverName = "mysql" + func init() { - sql.Register("mysql", &MySQLDriver{}) + if driverName != "" { + sql.Register(driverName, &MySQLDriver{}) + } } // NewConnector returns new driver.Connector. diff --git a/driver_test.go b/driver_test.go index f46d38df6..13e07e753 100644 --- a/driver_test.go +++ b/driver_test.go @@ -31,6 +31,16 @@ import ( "time" ) +// This variable can be replaced with -ldflags like below: +// go test "-ldflags=-X github.com/go-sql-driver/mysql.driverNameTest=custom" +var driverNameTest string + +func init() { + if driverNameTest == "" { + driverNameTest = driverName + } +} + // Ensure that all the driver interfaces are implemented var ( _ driver.Rows = &binaryRows{} @@ -111,7 +121,7 @@ func runTestsWithMultiStatement(t *testing.T, dsn string, tests ...func(dbt *DBT dsn += "&multiStatements=true" var db *sql.DB if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { - db, err = sql.Open("mysql", dsn) + db, err = sql.Open(driverNameTest, dsn) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -130,7 +140,7 @@ func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { t.Skipf("MySQL server not running on %s", netAddr) } - db, err := sql.Open("mysql", dsn) + db, err := sql.Open(driverNameTest, dsn) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -141,7 +151,7 @@ func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { dsn2 := dsn + "&interpolateParams=true" var db2 *sql.DB if _, err := ParseDSN(dsn2); err != errInvalidDSNUnsafeCollation { - db2, err = sql.Open("mysql", dsn2) + db2, err = sql.Open(driverNameTest, dsn2) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -1917,7 +1927,7 @@ func testDialError(t *testing.T, dialErr error, expectErr error) { return nil, dialErr }) - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + db, err := sql.Open(driverNameTest, fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -1956,7 +1966,7 @@ func TestCustomDial(t *testing.T) { return d.DialContext(ctx, prot, addr) }) - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + db, err := sql.Open(driverNameTest, fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -2054,7 +2064,7 @@ func TestUnixSocketAuthFail(t *testing.T) { } t.Logf("socket: %s", socket) badDSN := fmt.Sprintf("%s:%s@unix(%s)/%s?timeout=30s", user, badPass, socket, dbname) - db, err := sql.Open("mysql", badDSN) + db, err := sql.Open(driverNameTest, badDSN) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -2243,7 +2253,7 @@ func TestEmptyPassword(t *testing.T) { } dsn := fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, "", netAddr, dbname) - db, err := sql.Open("mysql", dsn) + db, err := sql.Open(driverNameTest, dsn) if err == nil { defer db.Close() err = db.Ping() @@ -3210,7 +3220,7 @@ func TestConnectorObeysDialTimeouts(t *testing.T) { return d.DialContext(ctx, prot, addr) }) - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + db, err := sql.Open(driverNameTest, fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -3375,7 +3385,7 @@ func TestConnectionAttributes(t *testing.T) { var db *sql.DB if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { - db, err = sql.Open("mysql", dsn) + db, err = sql.Open(driverNameTest, dsn) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } From c175348d98a9a245462ade75c6fde69424eb6fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Mengu=C3=A9?= Date: Tue, 24 Oct 2023 10:08:26 +0200 Subject: [PATCH 38/53] testing: expose testing.TB in DBTest instead of full *testing.T (#1500) Reduce the methods exposed by DBTest to the subset of testing.T exposed in the testing.TB interface. --- driver_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver_test.go b/driver_test.go index 13e07e753..f256011a7 100644 --- a/driver_test.go +++ b/driver_test.go @@ -92,7 +92,7 @@ func init() { } type DBTest struct { - *testing.T + testing.TB db *sql.DB } From 18b74e415dc148b486af13faa300fdefe26e484f Mon Sep 17 00:00:00 2001 From: Vaibhav Panvalkar <42548559+panvalkar1994@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:27:05 +0530 Subject: [PATCH 39/53] symbol removed from installation command (#1510) Co-authored-by: panvalkar1994 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fff8969f3..ac79890a7 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac ## Installation Simple install the package to your [$GOPATH](https://github.com/golang/go/wiki/GOPATH "GOPATH") with the [go tool](https://golang.org/cmd/go/ "go command") from shell: ```bash -$ go get -u github.com/go-sql-driver/mysql +go get -u github.com/go-sql-driver/mysql ``` Make sure [Git is installed](https://git-scm.com/downloads) on your machine and in your system's `PATH`. From b2e2ccbf16565d9706a2ffe77aafb21fb545a8d5 Mon Sep 17 00:00:00 2001 From: Xiang Zhang Date: Tue, 14 Nov 2023 19:17:17 +0800 Subject: [PATCH 40/53] QueryUnescape DSN ConnectionAttribute value (#1470) --- AUTHORS | 2 ++ driver_test.go | 18 +++++++++++++++--- dsn.go | 6 +++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index dec27daca..c84293100 100644 --- a/AUTHORS +++ b/AUTHORS @@ -109,6 +109,7 @@ Xiangyu Hu Xiaobing Jiang Xiuming Chen Xuehong Chan +Zhang Xiang Zhenye Xie Zhixin Wen Ziheng Lyu @@ -127,6 +128,7 @@ InfoSum Ltd. Keybase Inc. Multiplay Ltd. Percona LLC +PingCAP Inc. Pivotal Inc. Stripe Inc. Zendesk Inc. diff --git a/driver_test.go b/driver_test.go index f256011a7..8c02f6d1c 100644 --- a/driver_test.go +++ b/driver_test.go @@ -3379,9 +3379,10 @@ func TestConnectionAttributes(t *testing.T) { attr1 := "attr1" value1 := "value1" - attr2 := "foo" - value2 := "boo" - dsn += fmt.Sprintf("&connectionAttributes=%s:%s,%s:%s", attr1, value1, attr2, value2) + attr2 := "fo/o" + value2 := "bo/o" + dsn += "&connectionAttributes=" + url.QueryEscape(fmt.Sprintf("%s:%s,%s:%s", attr1, value1, attr2, value2)) + var db *sql.DB if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { @@ -3407,6 +3408,17 @@ func TestConnectionAttributes(t *testing.T) { } rows.Close() + rows = dbt.mustQuery(queryString, attr1) + if rows.Next() { + rows.Scan(&attrValue) + if attrValue != value1 { + dbt.Errorf("expected %q, got %q", value1, attrValue) + } + } else { + dbt.Errorf("no data") + } + rows.Close() + rows = dbt.mustQuery(queryString, attr2) if rows.Next() { rows.Scan(&attrValue) diff --git a/dsn.go b/dsn.go index 50c7ec413..ef0608636 100644 --- a/dsn.go +++ b/dsn.go @@ -569,7 +569,11 @@ func parseDSNParams(cfg *Config, params string) (err error) { // Connection attributes case "connectionAttributes": - cfg.ConnectionAttributes = value + connectionAttributes, err := url.QueryUnescape(value) + if err != nil { + return fmt.Errorf("invalid connectionAttributes value: %v", err) + } + cfg.ConnectionAttributes = connectionAttributes default: // lazy init From a4c260b40eeb51bd823d8b04d0e0e8d072e56adf Mon Sep 17 00:00:00 2001 From: Aidan <97376271+keeplearning20221@users.noreply.github.com> Date: Wed, 15 Nov 2023 18:40:52 +0800 Subject: [PATCH 41/53] fix hangup when error in multi resultsets (#1462) Fix #1361 Co-authored-by: Inada Naoki --- AUTHORS | 1 + driver_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ rows.go | 8 +++++++- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index c84293100..c7e159603 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,6 +13,7 @@ Aaron Hopkins Achille Roussel +Aidan Alex Snast Alexey Palazhchenko Andrew Reid diff --git a/driver_test.go b/driver_test.go index 8c02f6d1c..ab780f04c 100644 --- a/driver_test.go +++ b/driver_test.go @@ -3430,3 +3430,44 @@ func TestConnectionAttributes(t *testing.T) { } rows.Close() } + +func TestErrorInMultiResult(t *testing.T) { + // https://github.com/go-sql-driver/mysql/issues/1361 + var db *sql.DB + if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { + db, err = sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + } + + dbt := &DBTest{t, db} + query := ` +CREATE PROCEDURE test_proc1() +BEGIN + SELECT 1,2; + SELECT 3,4; + SIGNAL SQLSTATE '10000' SET MESSAGE_TEXT = "some error", MYSQL_ERRNO = 10000; +END; +` + runCallCommand(dbt, query, "test_proc1") +} + +func runCallCommand(dbt *DBTest, query, name string) { + dbt.mustExec(fmt.Sprintf("DROP PROCEDURE IF EXISTS %s", name)) + dbt.mustExec(query) + defer dbt.mustExec("DROP PROCEDURE " + name) + rows, err := dbt.db.Query(fmt.Sprintf("CALL %s", name)) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + } + for rows.NextResultSet() { + for rows.Next() { + } + } +} diff --git a/rows.go b/rows.go index 63d0ed2d5..81fa6062c 100644 --- a/rows.go +++ b/rows.go @@ -163,7 +163,13 @@ func (rows *mysqlRows) nextResultSet() (int, error) { rows.rs = resultSet{} // rows.mc.affectedRows and rows.mc.insertIds accumulate on each call to // nextResultSet. - return rows.mc.resultUnchanged().readResultSetHeaderPacket() + resLen, err := rows.mc.resultUnchanged().readResultSetHeaderPacket() + if err != nil { + // Clean up about multi-results flag + rows.rs.done = true + rows.mc.status = rows.mc.status & (^statusMoreResultsExists) + } + return resLen, err } func (rows *mysqlRows) nextNotEmptyResultSet() (int, error) { From 98d72897bab37633105da6dce698ce074fd19995 Mon Sep 17 00:00:00 2001 From: Jason Ng Date: Thu, 23 Nov 2023 21:01:24 +0800 Subject: [PATCH 42/53] Add default connection attribute '_server_host' (#1506) The `_server_host` connection attribute is supported in MariaDB (Connector/C) https://mariadb.com/kb/en/mysql_optionsv/#connection-attribute-options --- AUTHORS | 2 ++ connector.go | 21 +++++++------- connector_test.go | 7 ++--- const.go | 1 + driver.go | 9 ++---- driver_test.go | 71 ++++++++++++++++++++++++----------------------- packets.go | 16 +++++------ packets_test.go | 5 +--- 8 files changed, 64 insertions(+), 68 deletions(-) diff --git a/AUTHORS b/AUTHORS index c7e159603..2caa7d706 100644 --- a/AUTHORS +++ b/AUTHORS @@ -50,6 +50,7 @@ INADA Naoki Jacek Szwec James Harr Janek Vedock +Jason Ng Jean-Yves Pellé Jeff Hodges Jeffrey Charles @@ -131,6 +132,7 @@ Multiplay Ltd. Percona LLC PingCAP Inc. Pivotal Inc. +Shattered Silicon Ltd. Stripe Inc. Zendesk Inc. Dolthub Inc. diff --git a/connector.go b/connector.go index ba3be71e7..3cef7963f 100644 --- a/connector.go +++ b/connector.go @@ -11,7 +11,6 @@ package mysql import ( "context" "database/sql/driver" - "fmt" "net" "os" "strconv" @@ -23,8 +22,8 @@ type connector struct { encodedAttributes string // Encoded connection attributes. } -func encodeConnectionAttributes(textAttributes string) string { - connAttrsBuf := make([]byte, 0, 251) +func encodeConnectionAttributes(cfg *Config) string { + connAttrsBuf := make([]byte, 0) // default connection attributes connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrClientName) @@ -35,9 +34,14 @@ func encodeConnectionAttributes(textAttributes string) string { connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrPlatformValue) connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrPid) connAttrsBuf = appendLengthEncodedString(connAttrsBuf, strconv.Itoa(os.Getpid())) + serverHost, _, _ := net.SplitHostPort(cfg.Addr) + if serverHost != "" { + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrServerHost) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, serverHost) + } // user-defined connection attributes - for _, connAttr := range strings.Split(textAttributes, ",") { + for _, connAttr := range strings.Split(cfg.ConnectionAttributes, ",") { k, v, found := strings.Cut(connAttr, ":") if !found { continue @@ -49,15 +53,12 @@ func encodeConnectionAttributes(textAttributes string) string { return string(connAttrsBuf) } -func newConnector(cfg *Config) (*connector, error) { - encodedAttributes := encodeConnectionAttributes(cfg.ConnectionAttributes) - if len(encodedAttributes) > 250 { - return nil, fmt.Errorf("connection attributes are longer than 250 bytes: %dbytes (%q)", len(encodedAttributes), cfg.ConnectionAttributes) - } +func newConnector(cfg *Config) *connector { + encodedAttributes := encodeConnectionAttributes(cfg) return &connector{ cfg: cfg, encodedAttributes: encodedAttributes, - }, nil + } } // Connect implements driver.Connector interface. diff --git a/connector_test.go b/connector_test.go index bedb44ce2..82d8c5989 100644 --- a/connector_test.go +++ b/connector_test.go @@ -8,16 +8,13 @@ import ( ) func TestConnectorReturnsTimeout(t *testing.T) { - connector, err := newConnector(&Config{ + connector := newConnector(&Config{ Net: "tcp", Addr: "1.1.1.1:1234", Timeout: 10 * time.Millisecond, }) - if err != nil { - t.Fatal(err) - } - _, err = connector.Connect(context.Background()) + _, err := connector.Connect(context.Background()) if err == nil { t.Fatal("error expected") } diff --git a/const.go b/const.go index 0f2621a6f..22526e031 100644 --- a/const.go +++ b/const.go @@ -26,6 +26,7 @@ const ( connAttrPlatform = "_platform" connAttrPlatformValue = runtime.GOARCH connAttrPid = "_pid" + connAttrServerHost = "_server_host" ) // MySQL constants documentation: diff --git a/driver.go b/driver.go index 45528b920..105316b81 100644 --- a/driver.go +++ b/driver.go @@ -83,10 +83,7 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { if err != nil { return nil, err } - c, err := newConnector(cfg) - if err != nil { - return nil, err - } + c := newConnector(cfg) return c.Connect(context.Background()) } @@ -108,7 +105,7 @@ func NewConnector(cfg *Config) (driver.Connector, error) { if err := cfg.normalize(); err != nil { return nil, err } - return newConnector(cfg) + return newConnector(cfg), nil } // OpenConnector implements driver.DriverContext. @@ -117,5 +114,5 @@ func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { if err != nil { return nil, err } - return newConnector(cfg) + return newConnector(cfg), nil } diff --git a/driver_test.go b/driver_test.go index ab780f04c..efbff1792 100644 --- a/driver_test.go +++ b/driver_test.go @@ -24,6 +24,7 @@ import ( "os" "reflect" "runtime" + "strconv" "strings" "sync" "sync/atomic" @@ -3377,12 +3378,30 @@ func TestConnectionAttributes(t *testing.T) { t.Skipf("MySQL server not running on %s", netAddr) } - attr1 := "attr1" - value1 := "value1" - attr2 := "fo/o" - value2 := "bo/o" - dsn += "&connectionAttributes=" + url.QueryEscape(fmt.Sprintf("%s:%s,%s:%s", attr1, value1, attr2, value2)) + defaultAttrs := []string{ + connAttrClientName, + connAttrOS, + connAttrPlatform, + connAttrPid, + connAttrServerHost, + } + host, _, _ := net.SplitHostPort(addr) + defaultAttrValues := []string{ + connAttrClientNameValue, + connAttrOSValue, + connAttrPlatformValue, + strconv.Itoa(os.Getpid()), + host, + } + + customAttrs := []string{"attr1", "fo/o"} + customAttrValues := []string{"value1", "bo/o"} + customAttrStrs := make([]string, len(customAttrs)) + for i := range customAttrs { + customAttrStrs[i] = fmt.Sprintf("%s:%s", customAttrs[i], customAttrValues[i]) + } + dsn += "&connectionAttributes=" + url.QueryEscape(strings.Join(customAttrStrs, ",")) var db *sql.DB if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { @@ -3395,40 +3414,24 @@ func TestConnectionAttributes(t *testing.T) { dbt := &DBTest{t, db} - var attrValue string - queryString := "SELECT ATTR_VALUE FROM performance_schema.session_account_connect_attrs WHERE PROCESSLIST_ID = CONNECTION_ID() and ATTR_NAME = ?" - rows := dbt.mustQuery(queryString, connAttrClientName) - if rows.Next() { - rows.Scan(&attrValue) - if attrValue != connAttrClientNameValue { - dbt.Errorf("expected %q, got %q", connAttrClientNameValue, attrValue) - } - } else { - dbt.Errorf("no data") - } - rows.Close() + queryString := "SELECT ATTR_NAME, ATTR_VALUE FROM performance_schema.session_account_connect_attrs WHERE PROCESSLIST_ID = CONNECTION_ID()" + rows := dbt.mustQuery(queryString) + defer rows.Close() - rows = dbt.mustQuery(queryString, attr1) - if rows.Next() { - rows.Scan(&attrValue) - if attrValue != value1 { - dbt.Errorf("expected %q, got %q", value1, attrValue) - } - } else { - dbt.Errorf("no data") + rowsMap := make(map[string]string) + for rows.Next() { + var attrName, attrValue string + rows.Scan(&attrName, &attrValue) + rowsMap[attrName] = attrValue } - rows.Close() - rows = dbt.mustQuery(queryString, attr2) - if rows.Next() { - rows.Scan(&attrValue) - if attrValue != value2 { - dbt.Errorf("expected %q, got %q", value2, attrValue) + connAttrs := append(append([]string{}, defaultAttrs...), customAttrs...) + expectedAttrValues := append(append([]string{}, defaultAttrValues...), customAttrValues...) + for i := range connAttrs { + if gotValue := rowsMap[connAttrs[i]]; gotValue != expectedAttrValues[i] { + dbt.Errorf("expected %q, got %q", expectedAttrValues[i], gotValue) } - } else { - dbt.Errorf("no data") } - rows.Close() } func TestErrorInMultiResult(t *testing.T) { diff --git a/packets.go b/packets.go index 0127232ee..49e6bb058 100644 --- a/packets.go +++ b/packets.go @@ -292,15 +292,14 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string pktLen += n + 1 } - // 1 byte to store length of all key-values - // NOTE: Actually, this is length encoded integer. - // But we support only len(connAttrBuf) < 251 for now because takeSmallBuffer - // doesn't support buffer size more than 4096 bytes. - // TODO(methane): Rewrite buffer management. - pktLen += 1 + len(mc.connector.encodedAttributes) + // encode length of the connection attributes + var connAttrsLEIBuf [9]byte + connAttrsLen := len(mc.connector.encodedAttributes) + connAttrsLEI := appendLengthEncodedInteger(connAttrsLEIBuf[:0], uint64(connAttrsLen)) + pktLen += len(connAttrsLEI) + len(mc.connector.encodedAttributes) // Calculate packet length and get buffer with that size - data, err := mc.buf.takeSmallBuffer(pktLen + 4) + data, err := mc.buf.takeBuffer(pktLen + 4) if err != nil { // cannot take the buffer. Something must be wrong with the connection mc.cfg.Logger.Print(err) @@ -380,8 +379,7 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string pos++ // Connection Attributes - data[pos] = byte(len(mc.connector.encodedAttributes)) - pos++ + pos += copy(data[pos:], connAttrsLEI) pos += copy(data[pos:], []byte(mc.connector.encodedAttributes)) // Send Auth packet diff --git a/packets_test.go b/packets_test.go index e86ec5848..fa4683eab 100644 --- a/packets_test.go +++ b/packets_test.go @@ -96,10 +96,7 @@ var _ net.Conn = new(mockConn) func newRWMockConn(sequence uint8) (*mockConn, *mysqlConn) { conn := new(mockConn) - connector, err := newConnector(NewConfig()) - if err != nil { - panic(err) - } + connector := newConnector(NewConfig()) mc := &mysqlConn{ buf: newBuffer(conn), cfg: connector.cfg, From d9f43839450e9361c16685ea24f0bce0da1935b7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 12 Dec 2023 14:21:53 +0900 Subject: [PATCH 43/53] fix fragile test (#1522) --- driver_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/driver_test.go b/driver_test.go index efbff1792..87892a09a 100644 --- a/driver_test.go +++ b/driver_test.go @@ -128,6 +128,8 @@ func runTestsWithMultiStatement(t *testing.T, dsn string, tests ...func(dbt *DBT } defer db.Close() } + // Previous test may be skipped without dropping the test table + db.Exec("DROP TABLE IF EXISTS test") dbt := &DBTest{t, db} for _, test := range tests { @@ -147,6 +149,7 @@ func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { } defer db.Close() + // Previous test may be skipped without dropping the test table db.Exec("DROP TABLE IF EXISTS test") dsn2 := dsn + "&interpolateParams=true" From fc589cbaba22032382488393c72b9b3b5366917c Mon Sep 17 00:00:00 2001 From: Gusted Date: Tue, 12 Dec 2023 10:26:35 +0100 Subject: [PATCH 44/53] Add client_ed25519 authentication (#1518) Implements the necessary client code for [ed25519 authentication](https://mariadb.com/kb/en/authentication-plugin-ed25519/). This patch uses filippo.io/edwards25519 to implement the crypto bits. The standard library `crypto/ed25519` cannot be used as MariaDB chose a scheme that is simply not compatible with what the standard library provides. --- AUTHORS | 1 + auth.go | 47 ++++++++++++++++++++++++++++++++++++++++++++++ auth_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++ driver_test.go | 10 +++++----- go.mod | 2 ++ go.sum | 2 ++ 6 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 go.sum diff --git a/AUTHORS b/AUTHORS index 2caa7d706..954e7ac7a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -39,6 +39,7 @@ Evan Elias Evan Shaw Frederick Mayle Gustavo Kristic +Gusted Hajime Nakagami Hanno Braun Henri Yandell diff --git a/auth.go b/auth.go index bab282bd2..658259b24 100644 --- a/auth.go +++ b/auth.go @@ -13,10 +13,13 @@ import ( "crypto/rsa" "crypto/sha1" "crypto/sha256" + "crypto/sha512" "crypto/x509" "encoding/pem" "fmt" "sync" + + "filippo.io/edwards25519" ) // server pub keys registry @@ -225,6 +228,44 @@ func encryptPassword(password string, seed []byte, pub *rsa.PublicKey) ([]byte, return rsa.EncryptOAEP(sha1, rand.Reader, pub, plain, nil) } +// authEd25519 does ed25519 authentication used by MariaDB. +func authEd25519(scramble []byte, password string) ([]byte, error) { + // Derived from https://github.com/MariaDB/server/blob/d8e6bb00888b1f82c031938f4c8ac5d97f6874c3/plugin/auth_ed25519/ref10/sign.c + // Code style is from https://cs.opensource.google/go/go/+/refs/tags/go1.21.5:src/crypto/ed25519/ed25519.go;l=207 + h := sha512.Sum512([]byte(password)) + + s, err := edwards25519.NewScalar().SetBytesWithClamping(h[:32]) + if err != nil { + return nil, err + } + A := (&edwards25519.Point{}).ScalarBaseMult(s) + + mh := sha512.New() + mh.Write(h[32:]) + mh.Write(scramble) + messageDigest := mh.Sum(nil) + r, err := edwards25519.NewScalar().SetUniformBytes(messageDigest) + if err != nil { + return nil, err + } + + R := (&edwards25519.Point{}).ScalarBaseMult(r) + + kh := sha512.New() + kh.Write(R.Bytes()) + kh.Write(A.Bytes()) + kh.Write(scramble) + hramDigest := kh.Sum(nil) + k, err := edwards25519.NewScalar().SetUniformBytes(hramDigest) + if err != nil { + return nil, err + } + + S := k.MultiplyAdd(k, s, r) + + return append(R.Bytes(), S.Bytes()...), nil +} + func (mc *mysqlConn) sendEncryptedPassword(seed []byte, pub *rsa.PublicKey) error { enc, err := encryptPassword(mc.cfg.Passwd, seed, pub) if err != nil { @@ -290,6 +331,12 @@ func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey) return enc, err + case "client_ed25519": + if len(authData) != 32 { + return nil, ErrMalformPkt + } + return authEd25519(authData, mc.cfg.Passwd) + default: mc.cfg.Logger.Print("unknown auth plugin:", plugin) return nil, ErrUnknownPlugin diff --git a/auth_test.go b/auth_test.go index 3ce0ea6e0..8caed1fff 100644 --- a/auth_test.go +++ b/auth_test.go @@ -1328,3 +1328,54 @@ func TestAuthSwitchSHA256PasswordSecure(t *testing.T) { t.Errorf("got unexpected data: %v", conn.written) } } + +// Derived from https://github.com/MariaDB/server/blob/6b2287fff23fbdc362499501c562f01d0d2db52e/plugin/auth_ed25519/ed25519-t.c +func TestEd25519Auth(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "foobar" + + authData := []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + plugin := "client_ed25519" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{ + 232, 61, 201, 63, 67, 63, 51, 53, 86, 73, 238, 35, 170, 117, 146, + 214, 26, 17, 35, 9, 8, 132, 245, 141, 48, 99, 66, 58, 36, 228, 48, + 84, 115, 254, 187, 168, 88, 162, 249, 57, 35, 85, 79, 238, 167, 106, + 68, 117, 56, 135, 171, 47, 20, 14, 133, 79, 15, 229, 124, 160, 176, + 100, 138, 14, + } + if writtenAuthRespLen != 64 { + t.Fatalf("expected 64 bytes from client, got %d", writtenAuthRespLen) + } + if !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("auth response did not match expected value:\n%v\n%v", writtenAuthResp, expectedAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} diff --git a/driver_test.go b/driver_test.go index 87892a09a..97fd5a17a 100644 --- a/driver_test.go +++ b/driver_test.go @@ -165,14 +165,14 @@ func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { for _, test := range tests { t.Run("default", func(t *testing.T) { dbt := &DBTest{t, db} + defer dbt.db.Exec("DROP TABLE IF EXISTS test") test(dbt) - dbt.db.Exec("DROP TABLE IF EXISTS test") }) if db2 != nil { t.Run("interpolateParams", func(t *testing.T) { dbt2 := &DBTest{t, db2} + defer dbt2.db.Exec("DROP TABLE IF EXISTS test") test(dbt2) - dbt2.db.Exec("DROP TABLE IF EXISTS test") }) } } @@ -3181,14 +3181,14 @@ func TestRawBytesAreNotModified(t *testing.T) { rows, err := dbt.db.QueryContext(ctx, `SELECT id, value FROM test`) if err != nil { - t.Fatal(err) + dbt.Fatal(err) } var b int var raw sql.RawBytes for rows.Next() { if err := rows.Scan(&b, &raw); err != nil { - t.Fatal(err) + dbt.Fatal(err) } before := string(raw) @@ -3198,7 +3198,7 @@ func TestRawBytesAreNotModified(t *testing.T) { after := string(raw) if before != after { - t.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) + dbt.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) } } rows.Close() diff --git a/go.mod b/go.mod index 77bbb8dbf..4629714c0 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/go-sql-driver/mysql go 1.18 + +require filippo.io/edwards25519 v1.1.0 diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..359ca94b4 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= From 2cdf62442f2edb873d1270897d994fc83b78f118 Mon Sep 17 00:00:00 2001 From: ICHINOSE Shogo Date: Wed, 13 Dec 2023 15:21:30 +0900 Subject: [PATCH 45/53] Fix sql.RawBytes corruption issue (#1523) --- driver_test.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/driver_test.go b/driver_test.go index 97fd5a17a..d7359085d 100644 --- a/driver_test.go +++ b/driver_test.go @@ -3183,25 +3183,26 @@ func TestRawBytesAreNotModified(t *testing.T) { if err != nil { dbt.Fatal(err) } + defer rows.Close() var b int var raw sql.RawBytes - for rows.Next() { - if err := rows.Scan(&b, &raw); err != nil { - dbt.Fatal(err) - } + if !rows.Next() { + dbt.Fatal("expected at least one row") + } + if err := rows.Scan(&b, &raw); err != nil { + dbt.Fatal(err) + } - before := string(raw) - // Ensure cancelling the query does not corrupt the contents of `raw` - cancel() - time.Sleep(time.Microsecond * 100) - after := string(raw) + before := string(raw) + // Ensure cancelling the query does not corrupt the contents of `raw` + cancel() + time.Sleep(time.Microsecond * 100) + after := string(raw) - if before != after { - dbt.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) - } + if before != after { + dbt.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) } - rows.Close() }() } }) From d4517c5d905ccd3cc1e750f592edfa88d774d908 Mon Sep 17 00:00:00 2001 From: jennifersp <44716627+jennifersp@users.noreply.github.com> Date: Wed, 13 Dec 2023 00:50:21 -0800 Subject: [PATCH 46/53] Support ENUM and SET type in DatabaseTypeName() (#1520) --- AUTHORS | 5 +++-- driver_test.go | 2 ++ fields.go | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 954e7ac7a..0ada02d86 100644 --- a/AUTHORS +++ b/AUTHORS @@ -21,6 +21,7 @@ Animesh Ray Arne Hormann Ariel Mashraki Asta Xie +Brian Hendriks Bulat Gaifullin Caine Jette Carlos Nieto @@ -55,6 +56,7 @@ Jason Ng Jean-Yves Pellé Jeff Hodges Jeffrey Charles +Jennifer Purevsuren Jerome Meyer Jiajia Zhong Jian Zhen @@ -116,13 +118,13 @@ Zhang Xiang Zhenye Xie Zhixin Wen Ziheng Lyu -Brian Hendriks # Organizations Barracuda Networks, Inc. Counting Ltd. DigitalOcean Inc. +Dolthub Inc. dyves labs AG Facebook Inc. GitHub Inc. @@ -136,4 +138,3 @@ Pivotal Inc. Shattered Silicon Ltd. Stripe Inc. Zendesk Inc. -Dolthub Inc. diff --git a/driver_test.go b/driver_test.go index d7359085d..8ec1be412 100644 --- a/driver_test.go +++ b/driver_test.go @@ -3007,6 +3007,8 @@ func TestRowsColumnTypes(t *testing.T) { {"datetime6", "DATETIME(6)", "DATETIME", scanTypeNullTime, true, 6, 6, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt6}}, {"date", "DATE", "DATE", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02'", "NULL", "'2006-03-04'"}, [3]interface{}{nd1, ndNULL, nd2}}, {"year", "YEAR NOT NULL", "YEAR", scanTypeUint16, false, 0, 0, [3]string{"2006", "2000", "1994"}, [3]interface{}{uint16(2006), uint16(2000), uint16(1994)}}, + {"enum", "ENUM('', 'v1', 'v2')", "ENUM", scanTypeNullString, true, 0, 0, [3]string{"''", "'v1'", "'v2'"}, [3]interface{}{ns(""), ns("v1"), ns("v2")}}, + {"set", "set('', 'v1', 'v2')", "SET", scanTypeNullString, true, 0, 0, [3]string{"''", "'v1'", "'v1,v2'"}, [3]interface{}{ns(""), ns("v1"), ns("v1,v2")}}, } schema := "" diff --git a/fields.go b/fields.go index 30f31cbfb..2a397b245 100644 --- a/fields.go +++ b/fields.go @@ -77,6 +77,11 @@ func (mf *mysqlField) typeDatabaseName() string { } return "SMALLINT" case fieldTypeString: + if mf.flags&flagEnum != 0 { + return "ENUM" + } else if mf.flags&flagSet != 0 { + return "SET" + } if mf.charSet == binaryCollationID { return "BINARY" } From 0004702b931d3429afb3e16df444ed80be24d1f4 Mon Sep 17 00:00:00 2001 From: ICHINOSE Shogo Date: Wed, 13 Dec 2023 20:25:41 +0900 Subject: [PATCH 47/53] Parallelize test (#1525) * Refactor test cleanup in driver_test.go * parallelize TestEmptyQuery and TestCRUD * parallelize TestNumbersToAny * parallelize TestInt * parallelize TestFloat32 * parallelize TestFloat64 * parallelize TestFloat64Placeholder * parallelize TestString * parallelize TestRawBytes * parallelize TestRawMessage * parallelize TestValuer * parallelize TestValuerWithValidation * parallelize TestTimestampMicros * parallelize TestNULL * parallelize TestUint64 * parallelize TestLongData * parallelize TestContextCancelExec * parallelize TestPingContext * parallelize TestContextCancelQuery * parallelize TestContextCancelQueryRow * Revert "parallelize TestLongData" This reverts commit a360be7a110bb6372bed8cf7bc467e3c2dae3c66. * parallelize TestContextCancelPrepare * parallelize TestContextCancelStmtExec * parallelize TestContextCancelStmtQuery * parallelize TestContextCancelBegin * parallelize TestContextBeginIsolationLevel * parallelize TestContextBeginReadOnly * parallelize TestValuerWithValueReceiverGivenNilValue * parallelize TestRawBytesAreNotModified * parallelize TestFoundRows * parallelize TestRowsClose * parallelize TestCloseStmtBeforeRows * parallelize TestStmtMultiRows * Revert "parallelize TestRawBytesAreNotModified" This reverts commit 91622f05d44481dd9867eeaaf382da239afe3925. * parallelize TestStaleConnectionChecks * parallelize TestFailingCharset * parallelize TestColumnsWithAlias * parallelize TestRawBytesResultExceedsBuffer * parallelize TestUnixSocketAuthFail * parallelize TestSkipResults * Add parallel flag to go test command * Revert "parallelize TestUnixSocketAuthFail" This reverts commit b3df7bd130a21294a45c3733f1d2541b15582111. --- .github/workflows/test.yml | 2 +- conncheck_test.go | 2 +- driver_test.go | 332 ++++++++++++++++++++++--------------- 3 files changed, 198 insertions(+), 138 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8e1cb9bc3..aae421196 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -96,7 +96,7 @@ jobs: - name: test run: | - go test -v '-race' '-covermode=atomic' '-coverprofile=coverage.out' + go test -v '-race' '-covermode=atomic' '-coverprofile=coverage.out' -parallel 10 - name: Send coverage uses: shogo82148/actions-goveralls@v1 diff --git a/conncheck_test.go b/conncheck_test.go index f7e025680..6b60cb7d6 100644 --- a/conncheck_test.go +++ b/conncheck_test.go @@ -17,7 +17,7 @@ import ( ) func TestStaleConnectionChecks(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { dbt.mustExec("SET @@SESSION.wait_timeout = 2") if err := dbt.db.Ping(); err != nil { diff --git a/driver_test.go b/driver_test.go index 8ec1be412..6bdb78c78 100644 --- a/driver_test.go +++ b/driver_test.go @@ -11,6 +11,7 @@ package mysql import ( "bytes" "context" + "crypto/rand" "crypto/tls" "database/sql" "database/sql/driver" @@ -149,8 +150,9 @@ func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { } defer db.Close() - // Previous test may be skipped without dropping the test table - db.Exec("DROP TABLE IF EXISTS test") + cleanup := func() { + db.Exec("DROP TABLE IF EXISTS test") + } dsn2 := dsn + "&interpolateParams=true" var db2 *sql.DB @@ -163,21 +165,80 @@ func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { } for _, test := range tests { + test := test t.Run("default", func(t *testing.T) { dbt := &DBTest{t, db} - defer dbt.db.Exec("DROP TABLE IF EXISTS test") + t.Cleanup(cleanup) test(dbt) }) if db2 != nil { t.Run("interpolateParams", func(t *testing.T) { dbt2 := &DBTest{t, db2} - defer dbt2.db.Exec("DROP TABLE IF EXISTS test") + t.Cleanup(cleanup) test(dbt2) }) } } } +// runTestsParallel runs the tests in parallel with a separate database connection for each test. +func runTestsParallel(t *testing.T, dsn string, tests ...func(dbt *DBTest, tableName string)) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + newTableName := func(t *testing.T) string { + t.Helper() + var buf [8]byte + if _, err := rand.Read(buf[:]); err != nil { + t.Fatal(err) + } + return fmt.Sprintf("test_%x", buf[:]) + } + + t.Parallel() + for _, test := range tests { + test := test + + t.Run("default", func(t *testing.T) { + t.Parallel() + + tableName := newTableName(t) + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + t.Cleanup(func() { + db.Exec("DROP TABLE IF EXISTS " + tableName) + db.Close() + }) + + dbt := &DBTest{t, db} + test(dbt, tableName) + }) + + dsn2 := dsn + "&interpolateParams=true" + if _, err := ParseDSN(dsn2); err == errInvalidDSNUnsafeCollation { + t.Run("interpolateParams", func(t *testing.T) { + t.Parallel() + + tableName := newTableName(t) + db, err := sql.Open("mysql", dsn2) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + t.Cleanup(func() { + db.Exec("DROP TABLE IF EXISTS " + tableName) + db.Close() + }) + + dbt := &DBTest{t, db} + test(dbt, tableName) + }) + } + } +} + func (dbt *DBTest) fail(method, query string, err error) { dbt.Helper() if len(query) > 300 { @@ -216,7 +277,7 @@ func maybeSkip(t *testing.T, err error, skipErrno uint16) { } func TestEmptyQuery(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { // just a comment, no query rows := dbt.mustQuery("--") defer rows.Close() @@ -228,20 +289,20 @@ func TestEmptyQuery(t *testing.T) { } func TestCRUD(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { // Create Table - dbt.mustExec("CREATE TABLE test (value BOOL)") + dbt.mustExec("CREATE TABLE " + tbl + " (value BOOL)") // Test for unexpected data var out bool - rows := dbt.mustQuery("SELECT * FROM test") + rows := dbt.mustQuery("SELECT * FROM " + tbl) if rows.Next() { dbt.Error("unexpected data in empty table") } rows.Close() // Create Data - res := dbt.mustExec("INSERT INTO test VALUES (1)") + res := dbt.mustExec("INSERT INTO " + tbl + " VALUES (1)") count, err := res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -259,7 +320,7 @@ func TestCRUD(t *testing.T) { } // Read - rows = dbt.mustQuery("SELECT value FROM test") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if true != out { @@ -275,7 +336,7 @@ func TestCRUD(t *testing.T) { rows.Close() // Update - res = dbt.mustExec("UPDATE test SET value = ? WHERE value = ?", false, true) + res = dbt.mustExec("UPDATE "+tbl+" SET value = ? WHERE value = ?", false, true) count, err = res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -285,7 +346,7 @@ func TestCRUD(t *testing.T) { } // Check Update - rows = dbt.mustQuery("SELECT value FROM test") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if false != out { @@ -301,7 +362,7 @@ func TestCRUD(t *testing.T) { rows.Close() // Delete - res = dbt.mustExec("DELETE FROM test WHERE value = ?", false) + res = dbt.mustExec("DELETE FROM "+tbl+" WHERE value = ?", false) count, err = res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -311,7 +372,7 @@ func TestCRUD(t *testing.T) { } // Check for unexpected rows - res = dbt.mustExec("DELETE FROM test") + res = dbt.mustExec("DELETE FROM " + tbl) count, err = res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -325,13 +386,13 @@ func TestCRUD(t *testing.T) { // TestNumbers test that selecting numeric columns. // Both of textRows and binaryRows should return same type and value. func TestNumbersToAny(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE `test` (id INT PRIMARY KEY, b BOOL, i8 TINYINT, " + + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (id INT PRIMARY KEY, b BOOL, i8 TINYINT, " + "i16 SMALLINT, i32 INT, i64 BIGINT, f32 FLOAT, f64 DOUBLE)") - dbt.mustExec("INSERT INTO `test` VALUES (1, true, 127, 32767, 2147483647, 9223372036854775807, 1.25, 2.5)") + dbt.mustExec("INSERT INTO " + tbl + " VALUES (1, true, 127, 32767, 2147483647, 9223372036854775807, 1.25, 2.5)") // Use binaryRows for intarpolateParams=false and textRows for intarpolateParams=true. - rows := dbt.mustQuery("SELECT b, i8, i16, i32, i64, f32, f64 FROM `test` WHERE id=?", 1) + rows := dbt.mustQuery("SELECT b, i8, i16, i32, i64, f32, f64 FROM "+tbl+" WHERE id=?", 1) if !rows.Next() { dbt.Fatal("no data") } @@ -410,7 +471,7 @@ func TestMultiQuery(t *testing.T) { } func TestInt(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { types := [5]string{"TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT"} in := int64(42) var out int64 @@ -418,11 +479,11 @@ func TestInt(t *testing.T) { // SIGNED for _, v := range types { - dbt.mustExec("CREATE TABLE test (value " + v + ")") + dbt.mustExec("CREATE TABLE " + tbl + " (value " + v + ")") - dbt.mustExec("INSERT INTO test VALUES (?)", in) + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM test") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if in != out { @@ -433,16 +494,16 @@ func TestInt(t *testing.T) { } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } // UNSIGNED ZEROFILL for _, v := range types { - dbt.mustExec("CREATE TABLE test (value " + v + " ZEROFILL)") + dbt.mustExec("CREATE TABLE " + tbl + " (value " + v + " ZEROFILL)") - dbt.mustExec("INSERT INTO test VALUES (?)", in) + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM test") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if in != out { @@ -453,21 +514,21 @@ func TestInt(t *testing.T) { } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } }) } func TestFloat32(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { types := [2]string{"FLOAT", "DOUBLE"} in := float32(42.23) var out float32 var rows *sql.Rows for _, v := range types { - dbt.mustExec("CREATE TABLE test (value " + v + ")") - dbt.mustExec("INSERT INTO test VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM test") + dbt.mustExec("CREATE TABLE " + tbl + " (value " + v + ")") + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if in != out { @@ -477,21 +538,21 @@ func TestFloat32(t *testing.T) { dbt.Errorf("%s: no data", v) } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } }) } func TestFloat64(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { types := [2]string{"FLOAT", "DOUBLE"} var expected float64 = 42.23 var out float64 var rows *sql.Rows for _, v := range types { - dbt.mustExec("CREATE TABLE test (value " + v + ")") - dbt.mustExec("INSERT INTO test VALUES (42.23)") - rows = dbt.mustQuery("SELECT value FROM test") + dbt.mustExec("CREATE TABLE " + tbl + " (value " + v + ")") + dbt.mustExec("INSERT INTO " + tbl + " VALUES (42.23)") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if expected != out { @@ -501,21 +562,21 @@ func TestFloat64(t *testing.T) { dbt.Errorf("%s: no data", v) } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } }) } func TestFloat64Placeholder(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { types := [2]string{"FLOAT", "DOUBLE"} var expected float64 = 42.23 var out float64 var rows *sql.Rows for _, v := range types { - dbt.mustExec("CREATE TABLE test (id int, value " + v + ")") - dbt.mustExec("INSERT INTO test VALUES (1, 42.23)") - rows = dbt.mustQuery("SELECT value FROM test WHERE id = ?", 1) + dbt.mustExec("CREATE TABLE " + tbl + " (id int, value " + v + ")") + dbt.mustExec("INSERT INTO " + tbl + " VALUES (1, 42.23)") + rows = dbt.mustQuery("SELECT value FROM "+tbl+" WHERE id = ?", 1) if rows.Next() { rows.Scan(&out) if expected != out { @@ -525,24 +586,24 @@ func TestFloat64Placeholder(t *testing.T) { dbt.Errorf("%s: no data", v) } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } }) } func TestString(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { types := [6]string{"CHAR(255)", "VARCHAR(255)", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"} in := "κόσμε üöäßñóùéàâÿœ'îë Árvíztűrő いろはにほへとちりぬるを イロハニホヘト דג סקרן чащах น่าฟังเอย" var out string var rows *sql.Rows for _, v := range types { - dbt.mustExec("CREATE TABLE test (value " + v + ") CHARACTER SET utf8") + dbt.mustExec("CREATE TABLE " + tbl + " (value " + v + ") CHARACTER SET utf8") - dbt.mustExec("INSERT INTO test VALUES (?)", in) + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM test") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if in != out { @@ -553,11 +614,11 @@ func TestString(t *testing.T) { } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } // BLOB - dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + dbt.mustExec("CREATE TABLE " + tbl + " (id int, value BLOB) CHARACTER SET utf8") id := 2 in = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, " + @@ -568,9 +629,9 @@ func TestString(t *testing.T) { "sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, " + "sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. " + "Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." - dbt.mustExec("INSERT INTO test VALUES (?, ?)", id, in) + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?, ?)", id, in) - err := dbt.db.QueryRow("SELECT value FROM test WHERE id = ?", id).Scan(&out) + err := dbt.db.QueryRow("SELECT value FROM "+tbl+" WHERE id = ?", id).Scan(&out) if err != nil { dbt.Fatalf("Error on BLOB-Query: %s", err.Error()) } else if out != in { @@ -580,7 +641,7 @@ func TestString(t *testing.T) { } func TestRawBytes(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { v1 := []byte("aaa") v2 := []byte("bbb") rows := dbt.mustQuery("SELECT ?, ?", v1, v2) @@ -609,7 +670,7 @@ func TestRawBytes(t *testing.T) { } func TestRawMessage(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { v1 := json.RawMessage("{}") v2 := json.RawMessage("[]") rows := dbt.mustQuery("SELECT ?, ?", v1, v2) @@ -640,14 +701,14 @@ func (tv testValuer) Value() (driver.Value, error) { } func TestValuer(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { in := testValuer{"a_value"} var out string var rows *sql.Rows - dbt.mustExec("CREATE TABLE test (value VARCHAR(255)) CHARACTER SET utf8") - dbt.mustExec("INSERT INTO test VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM test") + dbt.mustExec("CREATE TABLE " + tbl + " (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if in.value != out { @@ -657,8 +718,6 @@ func TestValuer(t *testing.T) { dbt.Errorf("Valuer: no data") } rows.Close() - - dbt.mustExec("DROP TABLE IF EXISTS test") }) } @@ -675,15 +734,15 @@ func (tv testValuerWithValidation) Value() (driver.Value, error) { } func TestValuerWithValidation(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { in := testValuerWithValidation{"a_value"} var out string var rows *sql.Rows - dbt.mustExec("CREATE TABLE testValuer (value VARCHAR(255)) CHARACTER SET utf8") - dbt.mustExec("INSERT INTO testValuer VALUES (?)", in) + dbt.mustExec("CREATE TABLE " + tbl + " (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM testValuer") + rows = dbt.mustQuery("SELECT value FROM " + tbl) defer rows.Close() if rows.Next() { @@ -695,19 +754,17 @@ func TestValuerWithValidation(t *testing.T) { dbt.Errorf("Valuer: no data") } - if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", testValuerWithValidation{""}); err == nil { + if _, err := dbt.db.Exec("INSERT INTO "+tbl+" VALUES (?)", testValuerWithValidation{""}); err == nil { dbt.Errorf("Failed to check valuer error") } - if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", nil); err != nil { + if _, err := dbt.db.Exec("INSERT INTO "+tbl+" VALUES (?)", nil); err != nil { dbt.Errorf("Failed to check nil") } - if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", map[string]bool{}); err == nil { + if _, err := dbt.db.Exec("INSERT INTO "+tbl+" VALUES (?)", map[string]bool{}); err == nil { dbt.Errorf("Failed to check not valuer") } - - dbt.mustExec("DROP TABLE IF EXISTS testValuer") }) } @@ -941,7 +998,7 @@ func TestTimestampMicros(t *testing.T) { f0 := format[:19] f1 := format[:21] f6 := format[:26] - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { // check if microseconds are supported. // Do not use timestamp(x) for that check - before 5.5.6, x would mean display width // and not precision. @@ -956,7 +1013,7 @@ func TestTimestampMicros(t *testing.T) { return } _, err := dbt.db.Exec(` - CREATE TABLE test ( + CREATE TABLE ` + tbl + ` ( value0 TIMESTAMP NOT NULL DEFAULT '` + f0 + `', value1 TIMESTAMP(1) NOT NULL DEFAULT '` + f1 + `', value6 TIMESTAMP(6) NOT NULL DEFAULT '` + f6 + `' @@ -965,10 +1022,10 @@ func TestTimestampMicros(t *testing.T) { if err != nil { dbt.Error(err) } - defer dbt.mustExec("DROP TABLE IF EXISTS test") - dbt.mustExec("INSERT INTO test SET value0=?, value1=?, value6=?", f0, f1, f6) + defer dbt.mustExec("DROP TABLE IF EXISTS " + tbl) + dbt.mustExec("INSERT INTO "+tbl+" SET value0=?, value1=?, value6=?", f0, f1, f6) var res0, res1, res6 string - rows := dbt.mustQuery("SELECT * FROM test") + rows := dbt.mustQuery("SELECT * FROM " + tbl) defer rows.Close() if !rows.Next() { dbt.Errorf("test contained no selectable values") @@ -990,7 +1047,7 @@ func TestTimestampMicros(t *testing.T) { } func TestNULL(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { nullStmt, err := dbt.db.Prepare("SELECT NULL") if err != nil { dbt.Fatal(err) @@ -1122,12 +1179,12 @@ func TestNULL(t *testing.T) { } // Insert NULL - dbt.mustExec("CREATE TABLE test (dummmy1 int, value int, dummy2 int)") + dbt.mustExec("CREATE TABLE " + tbl + " (dummmy1 int, value int, dummy2 int)") - dbt.mustExec("INSERT INTO test VALUES (?, ?, ?)", 1, nil, 2) + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?, ?, ?)", 1, nil, 2) var out interface{} - rows := dbt.mustQuery("SELECT * FROM test") + rows := dbt.mustQuery("SELECT * FROM " + tbl) defer rows.Close() if rows.Next() { rows.Scan(&out) @@ -1151,7 +1208,7 @@ func TestUint64(t *testing.T) { shigh = int64(uhigh) stop = ^shigh ) - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { stmt, err := dbt.db.Prepare(`SELECT ?, ?, ? ,?, ?, ?, ?, ?`) if err != nil { dbt.Fatal(err) @@ -1347,12 +1404,12 @@ func TestLoadData(t *testing.T) { }) } -func TestFoundRows(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") - dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") +func TestFoundRows1(t *testing.T) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO " + tbl + " (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") - res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + res := dbt.mustExec("UPDATE " + tbl + " SET data = 1 WHERE id = 0") count, err := res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -1360,7 +1417,7 @@ func TestFoundRows(t *testing.T) { if count != 2 { dbt.Fatalf("Expected 2 affected rows, got %d", count) } - res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + res = dbt.mustExec("UPDATE " + tbl + " SET data = 1 WHERE id = 1") count, err = res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -1369,11 +1426,14 @@ func TestFoundRows(t *testing.T) { dbt.Fatalf("Expected 2 affected rows, got %d", count) } }) - runTests(t, dsn+"&clientFoundRows=true", func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") - dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") +} + +func TestFoundRows2(t *testing.T) { + runTestsParallel(t, dsn+"&clientFoundRows=true", func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO " + tbl + " (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") - res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + res := dbt.mustExec("UPDATE " + tbl + " SET data = 1 WHERE id = 0") count, err := res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -1381,7 +1441,7 @@ func TestFoundRows(t *testing.T) { if count != 2 { dbt.Fatalf("Expected 2 matched rows, got %d", count) } - res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + res = dbt.mustExec("UPDATE " + tbl + " SET data = 1 WHERE id = 1") count, err = res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -1507,7 +1567,7 @@ func TestCharset(t *testing.T) { } func TestFailingCharset(t *testing.T) { - runTests(t, dsn+"&charset=none", func(dbt *DBTest) { + runTestsParallel(t, dsn+"&charset=none", func(dbt *DBTest, _ string) { // run query to really establish connection... _, err := dbt.db.Exec("SELECT 1") if err == nil { @@ -1556,7 +1616,7 @@ func TestCollation(t *testing.T) { } func TestColumnsWithAlias(t *testing.T) { - runTests(t, dsn+"&columnsWithAlias=true", func(dbt *DBTest) { + runTestsParallel(t, dsn+"&columnsWithAlias=true", func(dbt *DBTest, _ string) { rows := dbt.mustQuery("SELECT 1 AS A") defer rows.Close() cols, _ := rows.Columns() @@ -1580,7 +1640,7 @@ func TestColumnsWithAlias(t *testing.T) { } func TestRawBytesResultExceedsBuffer(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { // defaultBufSize from buffer.go expected := strings.Repeat("abc", defaultBufSize) @@ -1639,7 +1699,7 @@ func TestTimezoneConversion(t *testing.T) { // Special cases func TestRowsClose(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { rows, err := dbt.db.Query("SELECT 1") if err != nil { dbt.Fatal(err) @@ -1664,7 +1724,7 @@ func TestRowsClose(t *testing.T) { // dangling statements // http://code.google.com/p/go/issues/detail?id=3865 func TestCloseStmtBeforeRows(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { stmt, err := dbt.db.Prepare("SELECT 1") if err != nil { dbt.Fatal(err) @@ -1705,7 +1765,7 @@ func TestCloseStmtBeforeRows(t *testing.T) { // It is valid to have multiple Rows for the same Stmt // http://code.google.com/p/go/issues/detail?id=3734 func TestStmtMultiRows(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { stmt, err := dbt.db.Prepare("SELECT 1 UNION SELECT 0") if err != nil { dbt.Fatal(err) @@ -2507,7 +2567,7 @@ func TestExecMultipleResults(t *testing.T) { // tests if rows are set in a proper state if some results were ignored before // calling rows.NextResultSet. func TestSkipResults(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { rows := dbt.mustQuery("SELECT 1, 2") defer rows.Close() @@ -2562,7 +2622,7 @@ func TestQueryMultipleResults(t *testing.T) { } func TestPingContext(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { ctx, cancel := context.WithCancel(context.Background()) cancel() if err := dbt.db.PingContext(ctx); err != context.Canceled { @@ -2572,8 +2632,8 @@ func TestPingContext(t *testing.T) { } func TestContextCancelExec(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) // Delay execution for just a bit until db.ExecContext has begun. @@ -2581,7 +2641,7 @@ func TestContextCancelExec(t *testing.T) { // This query will be canceled. startTime := time.Now() - if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO "+tbl+" VALUES (SLEEP(1))"); err != context.Canceled { dbt.Errorf("expected context.Canceled, got %v", err) } if d := time.Since(startTime); d > 500*time.Millisecond { @@ -2593,7 +2653,7 @@ func TestContextCancelExec(t *testing.T) { // Check how many times the query is executed. var v int - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { // TODO: need to kill the query, and v should be 0. @@ -2601,14 +2661,14 @@ func TestContextCancelExec(t *testing.T) { } // Context is already canceled, so error should come before execution. - if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (1)"); err == nil { + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO "+tbl+" VALUES (1)"); err == nil { dbt.Error("expected error") } else if err.Error() != "context canceled" { dbt.Fatalf("unexpected error: %s", err) } // The second insert query will fail, so the table has no changes. - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { @@ -2618,8 +2678,8 @@ func TestContextCancelExec(t *testing.T) { } func TestContextCancelQuery(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) // Delay execution for just a bit until db.ExecContext has begun. @@ -2627,7 +2687,7 @@ func TestContextCancelQuery(t *testing.T) { // This query will be canceled. startTime := time.Now() - if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO "+tbl+" VALUES (SLEEP(1))"); err != context.Canceled { dbt.Errorf("expected context.Canceled, got %v", err) } if d := time.Since(startTime); d > 500*time.Millisecond { @@ -2639,7 +2699,7 @@ func TestContextCancelQuery(t *testing.T) { // Check how many times the query is executed. var v int - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { // TODO: need to kill the query, and v should be 0. @@ -2647,12 +2707,12 @@ func TestContextCancelQuery(t *testing.T) { } // Context is already canceled, so error should come before execution. - if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (1)"); err != context.Canceled { + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO "+tbl+" VALUES (1)"); err != context.Canceled { dbt.Errorf("expected context.Canceled, got %v", err) } // The second insert query will fail, so the table has no changes. - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { @@ -2662,12 +2722,12 @@ func TestContextCancelQuery(t *testing.T) { } func TestContextCancelQueryRow(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") - dbt.mustExec("INSERT INTO test VALUES (1), (2), (3)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") + dbt.mustExec("INSERT INTO " + tbl + " VALUES (1), (2), (3)") ctx, cancel := context.WithCancel(context.Background()) - rows, err := dbt.db.QueryContext(ctx, "SELECT v FROM test") + rows, err := dbt.db.QueryContext(ctx, "SELECT v FROM "+tbl) if err != nil { dbt.Fatalf("%s", err.Error()) } @@ -2695,7 +2755,7 @@ func TestContextCancelQueryRow(t *testing.T) { } func TestContextCancelPrepare(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { ctx, cancel := context.WithCancel(context.Background()) cancel() if _, err := dbt.db.PrepareContext(ctx, "SELECT 1"); err != context.Canceled { @@ -2705,10 +2765,10 @@ func TestContextCancelPrepare(t *testing.T) { } func TestContextCancelStmtExec(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) - stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO "+tbl+" VALUES (SLEEP(1))") if err != nil { dbt.Fatalf("unexpected error: %v", err) } @@ -2730,7 +2790,7 @@ func TestContextCancelStmtExec(t *testing.T) { // Check how many times the query is executed. var v int - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { // TODO: need to kill the query, and v should be 0. @@ -2740,10 +2800,10 @@ func TestContextCancelStmtExec(t *testing.T) { } func TestContextCancelStmtQuery(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) - stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO "+tbl+" VALUES (SLEEP(1))") if err != nil { dbt.Fatalf("unexpected error: %v", err) } @@ -2765,7 +2825,7 @@ func TestContextCancelStmtQuery(t *testing.T) { // Check how many times the query is executed. var v int - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { // TODO: need to kill the query, and v should be 0. @@ -2779,8 +2839,8 @@ func TestContextCancelBegin(t *testing.T) { t.Skip(`FIXME: it sometime fails with "expected driver.ErrBadConn, got sql: connection is already closed" on windows and macOS`) } - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) conn, err := dbt.db.Conn(ctx) if err != nil { @@ -2797,7 +2857,7 @@ func TestContextCancelBegin(t *testing.T) { // This query will be canceled. startTime := time.Now() - if _, err := tx.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + if _, err := tx.ExecContext(ctx, "INSERT INTO "+tbl+" VALUES (SLEEP(1))"); err != context.Canceled { dbt.Errorf("expected context.Canceled, got %v", err) } if d := time.Since(startTime); d > 500*time.Millisecond { @@ -2835,8 +2895,8 @@ func TestContextCancelBegin(t *testing.T) { } func TestContextBeginIsolationLevel(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2854,13 +2914,13 @@ func TestContextBeginIsolationLevel(t *testing.T) { dbt.Fatal(err) } - _, err = tx1.ExecContext(ctx, "INSERT INTO test VALUES (1)") + _, err = tx1.ExecContext(ctx, "INSERT INTO "+tbl+" VALUES (1)") if err != nil { dbt.Fatal(err) } var v int - row := tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + row := tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+tbl) if err := row.Scan(&v); err != nil { dbt.Fatal(err) } @@ -2874,7 +2934,7 @@ func TestContextBeginIsolationLevel(t *testing.T) { dbt.Fatal(err) } - row = tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + row = tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+tbl) if err := row.Scan(&v); err != nil { dbt.Fatal(err) } @@ -2887,8 +2947,8 @@ func TestContextBeginIsolationLevel(t *testing.T) { } func TestContextBeginReadOnly(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2903,14 +2963,14 @@ func TestContextBeginReadOnly(t *testing.T) { } // INSERT queries fail in a READ ONLY transaction. - _, err = tx.ExecContext(ctx, "INSERT INTO test VALUES (1)") + _, err = tx.ExecContext(ctx, "INSERT INTO "+tbl+" VALUES (1)") if _, ok := err.(*MySQLError); !ok { dbt.Errorf("expected MySQLError, got %v", err) } // SELECT queries can be executed. var v int - row := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + row := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+tbl) if err := row.Scan(&v); err != nil { dbt.Fatal(err) } @@ -3147,9 +3207,9 @@ func TestRowsColumnTypes(t *testing.T) { } func TestValuerWithValueReceiverGivenNilValue(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (value VARCHAR(255))") - dbt.db.Exec("INSERT INTO test VALUES (?)", (*testValuer)(nil)) + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (value VARCHAR(255))") + dbt.db.Exec("INSERT INTO "+tbl+" VALUES (?)", (*testValuer)(nil)) // This test will panic on the INSERT if ConvertValue() does not check for typed nil before calling Value() }) } From c48c0e7da17e8fc06133e431ce7c10e7a3e94f06 Mon Sep 17 00:00:00 2001 From: shi yuhang <52435083+shiyuhang0@users.noreply.github.com> Date: Fri, 5 Jan 2024 16:47:16 +0800 Subject: [PATCH 48/53] Fix unsigned int overflow (#1530) --- driver_test.go | 15 +++++++++------ packets.go | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/driver_test.go b/driver_test.go index 6bdb78c78..5934caab6 100644 --- a/driver_test.go +++ b/driver_test.go @@ -388,16 +388,16 @@ func TestCRUD(t *testing.T) { func TestNumbersToAny(t *testing.T) { runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { dbt.mustExec("CREATE TABLE " + tbl + " (id INT PRIMARY KEY, b BOOL, i8 TINYINT, " + - "i16 SMALLINT, i32 INT, i64 BIGINT, f32 FLOAT, f64 DOUBLE)") - dbt.mustExec("INSERT INTO " + tbl + " VALUES (1, true, 127, 32767, 2147483647, 9223372036854775807, 1.25, 2.5)") + "i16 SMALLINT, i32 INT, i64 BIGINT, f32 FLOAT, f64 DOUBLE, iu32 INT UNSIGNED)") + dbt.mustExec("INSERT INTO " + tbl + " VALUES (1, true, 127, 32767, 2147483647, 9223372036854775807, 1.25, 2.5, 4294967295)") - // Use binaryRows for intarpolateParams=false and textRows for intarpolateParams=true. - rows := dbt.mustQuery("SELECT b, i8, i16, i32, i64, f32, f64 FROM "+tbl+" WHERE id=?", 1) + // Use binaryRows for interpolateParams=false and textRows for interpolateParams=true. + rows := dbt.mustQuery("SELECT b, i8, i16, i32, i64, f32, f64, iu32 FROM "+tbl+" WHERE id=?", 1) if !rows.Next() { dbt.Fatal("no data") } - var b, i8, i16, i32, i64, f32, f64 any - err := rows.Scan(&b, &i8, &i16, &i32, &i64, &f32, &f64) + var b, i8, i16, i32, i64, f32, f64, iu32 any + err := rows.Scan(&b, &i8, &i16, &i32, &i64, &f32, &f64, &iu32) if err != nil { dbt.Fatal(err) } @@ -422,6 +422,9 @@ func TestNumbersToAny(t *testing.T) { if f64.(float64) != 2.5 { dbt.Errorf("f64 != 2.5") } + if iu32.(int64) != 4294967295 { + dbt.Errorf("iu32 != 4294967295") + } }) } diff --git a/packets.go b/packets.go index 49e6bb058..94b46b10f 100644 --- a/packets.go +++ b/packets.go @@ -828,7 +828,7 @@ func (rows *textRows) readRow(dest []driver.Value) error { } case fieldTypeTiny, fieldTypeShort, fieldTypeInt24, fieldTypeYear, fieldTypeLong: - dest[i], err = strconv.ParseInt(string(buf), 10, 32) + dest[i], err = strconv.ParseInt(string(buf), 10, 64) case fieldTypeLongLong: if rows.rs.columns[i].flags&flagUnsigned != 0 { From 743e263bab87912dfb61789f36c21d9685887c76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paulius=20Lo=C5=BEys?= <42966213+PauliusLozys@users.noreply.github.com> Date: Wed, 31 Jan 2024 11:34:24 +0200 Subject: [PATCH 49/53] Introduce `timeTruncate` parameter for `time.Time` arguments (#1541) Co-authored-by: Inada Naoki --- AUTHORS | 1 + README.md | 9 ++++++ connection.go | 2 +- dsn.go | 12 +++++++ dsn_test.go | 3 ++ packets.go | 2 +- utils.go | 6 +++- utils_test.go | 89 ++++++++++++++++++++++++++++++++++++++------------- 8 files changed, 98 insertions(+), 26 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0ada02d86..63ee516e5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -86,6 +86,7 @@ Oliver Bone Olivier Mengué oscarzhao Paul Bonser +Paulius Lozys Peter Schultz Phil Porada Rebecca Chin diff --git a/README.md b/README.md index ac79890a7..018e1dd7c 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,15 @@ Note that this sets the location for time.Time values but does not change MySQL' Please keep in mind, that param values must be [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`. +##### `timeTruncate` + +``` +Type: duration +Default: 0 +``` + +[Truncate time values](https://pkg.go.dev/time#Duration.Truncate) to the specified duration. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. + ##### `maxAllowedPacket` ``` Type: decimal number diff --git a/connection.go b/connection.go index 660b2b0e0..99eb8a808 100644 --- a/connection.go +++ b/connection.go @@ -251,7 +251,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin buf = append(buf, "'0000-00-00'"...) } else { buf = append(buf, '\'') - buf, err = appendDateTime(buf, v.In(mc.cfg.Loc)) + buf, err = appendDateTime(buf, v.In(mc.cfg.Loc), mc.cfg.TimeTruncate) if err != nil { return "", err } diff --git a/dsn.go b/dsn.go index ef0608636..ce5d85ff0 100644 --- a/dsn.go +++ b/dsn.go @@ -48,6 +48,7 @@ type Config struct { pubKey *rsa.PublicKey // Server public key TLSConfig string // TLS configuration name TLS *tls.Config // TLS configuration, its priority is higher than TLSConfig + TimeTruncate time.Duration // Truncate time.Time values to the specified duration Timeout time.Duration // Dial timeout ReadTimeout time.Duration // I/O read timeout WriteTimeout time.Duration // I/O write timeout @@ -262,6 +263,10 @@ func (cfg *Config) FormatDSN() string { writeDSNParam(&buf, &hasParam, "parseTime", "true") } + if cfg.TimeTruncate > 0 { + writeDSNParam(&buf, &hasParam, "timeTruncate", cfg.TimeTruncate.String()) + } + if cfg.ReadTimeout > 0 { writeDSNParam(&buf, &hasParam, "readTimeout", cfg.ReadTimeout.String()) } @@ -502,6 +507,13 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // time.Time truncation + case "timeTruncate": + cfg.TimeTruncate, err = time.ParseDuration(value) + if err != nil { + return + } + // I/O read Timeout case "readTimeout": cfg.ReadTimeout, err = time.ParseDuration(value) diff --git a/dsn_test.go b/dsn_test.go index 8a6a0c10e..75cbda700 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -74,6 +74,9 @@ var testDSNs = []struct { }, { "tcp(de:ad:be:ef::ca:fe)/dbname", &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, +}, { + "user:password@/dbname?loc=UTC&timeout=30s&parseTime=true&timeTruncate=1h", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Loc: time.UTC, Timeout: 30 * time.Second, ParseTime: true, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, TimeTruncate: time.Hour}, }, } diff --git a/packets.go b/packets.go index 94b46b10f..e5a6e4727 100644 --- a/packets.go +++ b/packets.go @@ -1172,7 +1172,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if v.IsZero() { b = append(b, "0000-00-00"...) } else { - b, err = appendDateTime(b, v.In(mc.cfg.Loc)) + b, err = appendDateTime(b, v.In(mc.cfg.Loc), mc.cfg.TimeTruncate) if err != nil { return err } diff --git a/utils.go b/utils.go index a24197b93..cda24fe74 100644 --- a/utils.go +++ b/utils.go @@ -265,7 +265,11 @@ func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Va return nil, fmt.Errorf("invalid DATETIME packet length %d", num) } -func appendDateTime(buf []byte, t time.Time) ([]byte, error) { +func appendDateTime(buf []byte, t time.Time, timeTruncate time.Duration) ([]byte, error) { + if timeTruncate > 0 { + t = t.Truncate(timeTruncate) + } + year, month, day := t.Date() hour, min, sec := t.Clock() nsec := t.Nanosecond() diff --git a/utils_test.go b/utils_test.go index 4e5fc3cb7..80aebddff 100644 --- a/utils_test.go +++ b/utils_test.go @@ -237,8 +237,10 @@ func TestIsolationLevelMapping(t *testing.T) { func TestAppendDateTime(t *testing.T) { tests := []struct { - t time.Time - str string + t time.Time + str string + timeTruncate time.Duration + expectedErr bool }{ { t: time.Date(1234, 5, 6, 0, 0, 0, 0, time.UTC), @@ -276,34 +278,75 @@ func TestAppendDateTime(t *testing.T) { t: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), str: "0001-01-01", }, + // Truncated time + { + t: time.Date(1234, 5, 6, 0, 0, 0, 0, time.UTC), + str: "1234-05-06", + timeTruncate: time.Second, + }, + { + t: time.Date(4567, 12, 31, 12, 0, 0, 0, time.UTC), + str: "4567-12-31 12:00:00", + timeTruncate: time.Minute, + }, + { + t: time.Date(2020, 5, 30, 12, 34, 0, 0, time.UTC), + str: "2020-05-30 12:34:00", + timeTruncate: 0, + }, + { + t: time.Date(2020, 5, 30, 12, 34, 56, 0, time.UTC), + str: "2020-05-30 12:34:56", + timeTruncate: time.Second, + }, + { + t: time.Date(2020, 5, 30, 22, 33, 44, 123000000, time.UTC), + str: "2020-05-30 22:33:44", + timeTruncate: time.Second, + }, + { + t: time.Date(2020, 5, 30, 22, 33, 44, 123456000, time.UTC), + str: "2020-05-30 22:33:44.123", + timeTruncate: time.Millisecond, + }, + { + t: time.Date(2020, 5, 30, 22, 33, 44, 123456789, time.UTC), + str: "2020-05-30 22:33:44", + timeTruncate: time.Second, + }, + { + t: time.Date(9999, 12, 31, 23, 59, 59, 999999999, time.UTC), + str: "9999-12-31 23:59:59.999999999", + timeTruncate: 0, + }, + { + t: time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC), + str: "0001-01-01", + timeTruncate: 365 * 24 * time.Hour, + }, + // year out of range + { + t: time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC), + expectedErr: true, + }, + { + t: time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC), + expectedErr: true, + }, } for _, v := range tests { buf := make([]byte, 0, 32) - buf, _ = appendDateTime(buf, v.t) + buf, err := appendDateTime(buf, v.t, v.timeTruncate) + if err != nil { + if !v.expectedErr { + t.Errorf("appendDateTime(%v) returned an errror: %v", v.t, err) + } + continue + } if str := string(buf); str != v.str { t.Errorf("appendDateTime(%v), have: %s, want: %s", v.t, str, v.str) } } - - // year out of range - { - v := time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC) - buf := make([]byte, 0, 32) - _, err := appendDateTime(buf, v) - if err == nil { - t.Error("want an error") - return - } - } - { - v := time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC) - buf := make([]byte, 0, 32) - _, err := appendDateTime(buf, v) - if err == nil { - t.Error("want an error") - return - } - } } func TestParseDateTime(t *testing.T) { From f019727e4706bf9c4f60579382f6e72b94bd0305 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 5 Feb 2024 16:57:21 +0800 Subject: [PATCH 50/53] add TiDB support in README.md (#1333) Signed-off-by: crazycs520 Co-authored-by: Inada Naoki --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 018e1dd7c..9d0d806ef 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,16 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.18 or higher. We aim to support the 3 latest versions of Go. - * MySQL (5.6+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) + +* Go 1.19 or higher. We aim to support the 3 latest versions of Go. +* MySQL (5.7+) and MariaDB (10.3+) are supported. +* [TiDB](https://github.com/pingcap/tidb) is supported by PingCAP. + * Do not ask questions about TiDB in our issue tracker or forum. + * [Document](https://docs.pingcap.com/tidb/v6.1/dev-guide-sample-application-golang) + * [Forum](https://ask.pingcap.com/) +* go-mysql would work with Percona Server, Google CloudSQL or Sphinx (2.2.3+). + * Maintainers won't support them. Do not expect issues are investigated and resolved by maintainers. + * Investigate issues yourself and please send a pull request to fix it. --------------------------------------- From 097fe6e3ad83bbd7c84debe810aec4c4a533bcaa Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 5 Feb 2024 20:29:00 +0900 Subject: [PATCH 51/53] Update workflows (#1547) --- .github/workflows/codeql.yml | 8 ++++---- .github/workflows/test.yml | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d9d29a8b7..83a3d6ee8 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -24,18 +24,18 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} queries: +security-and-quality - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aae421196..f5a115802 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,10 +14,10 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dominikh/staticcheck-action@v1.3.0 with: - version: "2023.1.3" + version: "2023.1.6" list: runs-on: ubuntu-latest @@ -73,11 +73,11 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.list.outputs.matrix) }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - - uses: shogo82148/actions-setup-mysql@v1.21.0 + - uses: shogo82148/actions-setup-mysql@v1 with: mysql-version: ${{ matrix.mysql }} user: ${{ env.MYSQL_TEST_USER }} From 6964272ffd13a41ad66383cd2ea738fded75ad06 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Mar 2024 00:32:18 +0900 Subject: [PATCH 52/53] Make TimeTruncate functional option (#1552) --- connection.go | 2 +- dsn.go | 47 ++++++++++++++++++++++++++++++++++++++++------- dsn_test.go | 2 +- packets.go | 2 +- result.go | 5 ++--- 5 files changed, 45 insertions(+), 13 deletions(-) diff --git a/connection.go b/connection.go index 99eb8a808..c170114fe 100644 --- a/connection.go +++ b/connection.go @@ -251,7 +251,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin buf = append(buf, "'0000-00-00'"...) } else { buf = append(buf, '\'') - buf, err = appendDateTime(buf, v.In(mc.cfg.Loc), mc.cfg.TimeTruncate) + buf, err = appendDateTime(buf, v.In(mc.cfg.Loc), mc.cfg.timeTruncate) if err != nil { return "", err } diff --git a/dsn.go b/dsn.go index ce5d85ff0..d0fbf3bd9 100644 --- a/dsn.go +++ b/dsn.go @@ -34,6 +34,8 @@ var ( // If a new Config is created instead of being parsed from a DSN string, // the NewConfig function should be used, which sets default values. type Config struct { + // non boolean fields + User string // Username Passwd string // Password (requires User) Net string // Network (e.g. "tcp", "tcp6", "unix". default: "tcp") @@ -45,15 +47,15 @@ type Config struct { Loc *time.Location // Location for time.Time values MaxAllowedPacket int // Max packet size allowed ServerPubKey string // Server public key name - pubKey *rsa.PublicKey // Server public key TLSConfig string // TLS configuration name TLS *tls.Config // TLS configuration, its priority is higher than TLSConfig - TimeTruncate time.Duration // Truncate time.Time values to the specified duration Timeout time.Duration // Dial timeout ReadTimeout time.Duration // I/O read timeout WriteTimeout time.Duration // I/O write timeout Logger Logger // Logger + // boolean fields + AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE AllowCleartextPasswords bool // Allows the cleartext client side plugin AllowFallbackToPlaintext bool // Allows fallback to unencrypted connection if server does not support TLS @@ -66,17 +68,48 @@ type Config struct { MultiStatements bool // Allow multiple statements in one query ParseTime bool // Parse time values to time.Time RejectReadOnly bool // Reject read-only connections + + // unexported fields. new options should be come here + + pubKey *rsa.PublicKey // Server public key + timeTruncate time.Duration // Truncate time.Time values to the specified duration } +// Functional Options Pattern +// https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis +type Option func(*Config) error + // NewConfig creates a new Config and sets default values. func NewConfig() *Config { - return &Config{ + cfg := &Config{ Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, } + + return cfg +} + +// Apply applies the given options to the Config object. +func (c *Config) Apply(opts ...Option) error { + for _, opt := range opts { + err := opt(c) + if err != nil { + return err + } + } + return nil +} + +// TimeTruncate sets the time duration to truncate time.Time values in +// query parameters. +func TimeTruncate(d time.Duration) Option { + return func(cfg *Config) error { + cfg.timeTruncate = d + return nil + } } func (cfg *Config) Clone() *Config { @@ -263,8 +296,8 @@ func (cfg *Config) FormatDSN() string { writeDSNParam(&buf, &hasParam, "parseTime", "true") } - if cfg.TimeTruncate > 0 { - writeDSNParam(&buf, &hasParam, "timeTruncate", cfg.TimeTruncate.String()) + if cfg.timeTruncate > 0 { + writeDSNParam(&buf, &hasParam, "timeTruncate", cfg.timeTruncate.String()) } if cfg.ReadTimeout > 0 { @@ -509,9 +542,9 @@ func parseDSNParams(cfg *Config, params string) (err error) { // time.Time truncation case "timeTruncate": - cfg.TimeTruncate, err = time.ParseDuration(value) + cfg.timeTruncate, err = time.ParseDuration(value) if err != nil { - return + return fmt.Errorf("invalid timeTruncate value: %v, error: %w", value, err) } // I/O read Timeout diff --git a/dsn_test.go b/dsn_test.go index 75cbda700..dd8cd935c 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -76,7 +76,7 @@ var testDSNs = []struct { &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:password@/dbname?loc=UTC&timeout=30s&parseTime=true&timeTruncate=1h", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Loc: time.UTC, Timeout: 30 * time.Second, ParseTime: true, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, TimeTruncate: time.Hour}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Loc: time.UTC, Timeout: 30 * time.Second, ParseTime: true, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, timeTruncate: time.Hour}, }, } diff --git a/packets.go b/packets.go index e5a6e4727..3d6e5308c 100644 --- a/packets.go +++ b/packets.go @@ -1172,7 +1172,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if v.IsZero() { b = append(b, "0000-00-00"...) } else { - b, err = appendDateTime(b, v.In(mc.cfg.Loc), mc.cfg.TimeTruncate) + b, err = appendDateTime(b, v.In(mc.cfg.Loc), mc.cfg.timeTruncate) if err != nil { return err } diff --git a/result.go b/result.go index 36a432e81..d51631468 100644 --- a/result.go +++ b/result.go @@ -15,9 +15,8 @@ import "database/sql/driver" // This is accessible by executing statements using sql.Conn.Raw() and // downcasting the returned result: // -// res, err := rawConn.Exec(...) -// res.(mysql.Result).AllRowsAffected() -// +// res, err := rawConn.Exec(...) +// res.(mysql.Result).AllRowsAffected() type Result interface { driver.Result // AllRowsAffected returns a slice containing the affected rows for each From 33b7747a9144946e50399904d3f27ecc0f96c2b6 Mon Sep 17 00:00:00 2001 From: "Alessandro (Ale) Segala" <43508+ItalyPaleAle@users.noreply.github.com> Date: Sat, 9 Mar 2024 07:57:08 +0100 Subject: [PATCH 53/53] Add BeforeConnect callback to configuration object (#1469) This can be used to alter the connection options for each connection, right before it's established Co-authored-by: Inada Naoki --- AUTHORS | 1 + connector.go | 12 +++++++++++- driver_test.go | 34 ++++++++++++++++++++++++++++++++++ dsn.go | 14 ++++++++++++-- 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 63ee516e5..4021b96cc 100644 --- a/AUTHORS +++ b/AUTHORS @@ -132,6 +132,7 @@ GitHub Inc. Google Inc. InfoSum Ltd. Keybase Inc. +Microsoft Corp. Multiplay Ltd. Percona LLC PingCAP Inc. diff --git a/connector.go b/connector.go index 3cef7963f..a0ee62839 100644 --- a/connector.go +++ b/connector.go @@ -66,12 +66,22 @@ func newConnector(cfg *Config) *connector { func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { var err error + // Invoke beforeConnect if present, with a copy of the configuration + cfg := c.cfg + if c.cfg.beforeConnect != nil { + cfg = c.cfg.Clone() + err = c.cfg.beforeConnect(ctx, cfg) + if err != nil { + return nil, err + } + } + // New mysqlConn mc := &mysqlConn{ maxAllowedPacket: maxPacketSize, maxWriteSize: maxPacketSize - 1, closech: make(chan struct{}), - cfg: c.cfg, + cfg: cfg, connector: c, } mc.parseTime = mc.cfg.ParseTime diff --git a/driver_test.go b/driver_test.go index 5934caab6..001957244 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2044,6 +2044,40 @@ func TestCustomDial(t *testing.T) { } } +func TestBeforeConnect(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + // dbname is set in the BeforeConnect handle + cfg, err := ParseDSN(fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, pass, netAddr, "_")) + if err != nil { + t.Fatalf("error parsing DSN: %v", err) + } + + cfg.Apply(BeforeConnect(func(ctx context.Context, c *Config) error { + c.DBName = dbname + return nil + })) + + connector, err := NewConnector(cfg) + if err != nil { + t.Fatalf("error creating connector: %v", err) + } + + db := sql.OpenDB(connector) + defer db.Close() + + var connectedDb string + err = db.QueryRow("SELECT DATABASE();").Scan(&connectedDb) + if err != nil { + t.Fatalf("error executing query: %v", err) + } + if connectedDb != dbname { + t.Fatalf("expected to connect to DB %s, but connected to %s instead", dbname, connectedDb) + } +} + func TestSQLInjection(t *testing.T) { createTest := func(arg string) func(dbt *DBTest) { return func(dbt *DBTest) { diff --git a/dsn.go b/dsn.go index d0fbf3bd9..65f5a0242 100644 --- a/dsn.go +++ b/dsn.go @@ -10,6 +10,7 @@ package mysql import ( "bytes" + "context" "crypto/rsa" "crypto/tls" "errors" @@ -71,8 +72,9 @@ type Config struct { // unexported fields. new options should be come here - pubKey *rsa.PublicKey // Server public key - timeTruncate time.Duration // Truncate time.Time values to the specified duration + beforeConnect func(context.Context, *Config) error // Invoked before a connection is established + pubKey *rsa.PublicKey // Server public key + timeTruncate time.Duration // Truncate time.Time values to the specified duration } // Functional Options Pattern @@ -112,6 +114,14 @@ func TimeTruncate(d time.Duration) Option { } } +// BeforeConnect sets the function to be invoked before a connection is established. +func BeforeConnect(fn func(context.Context, *Config) error) Option { + return func(cfg *Config) error { + cfg.beforeConnect = fn + return nil + } +} + func (cfg *Config) Clone() *Config { cp := *cfg if cp.TLS != nil {