From 0ab6f80f9929384a8cf6cfc299b43233534eb705 Mon Sep 17 00:00:00 2001 From: avivklas Date: Mon, 3 Jun 2019 13:44:43 +0300 Subject: [PATCH 01/10] added PortalSuspended message --- pgproto3/frontend.go | 9 ++++++--- pgproto3/portal_suspended.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 pgproto3/portal_suspended.go diff --git a/pgproto3/frontend.go b/pgproto3/frontend.go index 4c05fcc3..be2c01cd 100644 --- a/pgproto3/frontend.go +++ b/pgproto3/frontend.go @@ -23,7 +23,7 @@ type Frontend struct { copyInResponse CopyInResponse copyOutResponse CopyOutResponse copyDone CopyDone - copyFail CopyFail + copyFail CopyFail dataRow DataRow emptyQueryResponse EmptyQueryResponse errorResponse ErrorResponse @@ -36,6 +36,7 @@ type Frontend struct { parseComplete ParseComplete readyForQuery ReadyForQuery rowDescription RowDescription + portalSuspended PortalSuspended bodyLen int msgType byte @@ -76,8 +77,8 @@ func (b *Frontend) Receive() (BackendMessage, error) { msg = &b.notificationResponse case 'c': msg = &b.copyDone - case 'f': - msg = &b.copyFail + case 'f': + msg = &b.copyFail case 'C': msg = &b.commandComplete case 'd': @@ -112,6 +113,8 @@ func (b *Frontend) Receive() (BackendMessage, error) { msg = &b.copyBothResponse case 'Z': msg = &b.readyForQuery + case 's': + msg = &b.portalSuspended default: return nil, errors.Errorf("unknown message type: %c", b.msgType) } diff --git a/pgproto3/portal_suspended.go b/pgproto3/portal_suspended.go new file mode 100644 index 00000000..dc81b027 --- /dev/null +++ b/pgproto3/portal_suspended.go @@ -0,0 +1,29 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type PortalSuspended struct{} + +func (*PortalSuspended) Backend() {} + +func (dst *PortalSuspended) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "PortalSuspended", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +func (src *PortalSuspended) Encode(dst []byte) []byte { + return append(dst, 's', 0, 0, 0, 4) +} + +func (src *PortalSuspended) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "PortalSuspended", + }) +} From d678216f468d1fe4dc28649feacd4b30a176769e Mon Sep 17 00:00:00 2001 From: David Hudson Date: Fri, 7 Jun 2019 15:08:36 +0100 Subject: [PATCH 02/10] pgtype: Fix -0 for numeric types Due to the special case of when the digits string was longer than 1 but only contained the negative sign and a 0, it was incorrectly stripping the 0 and attempting to parse "-" as a number. The solution is to check an extra position along to make sure a trailing 0 is not immediately preceeded by a negetive sign. Fixes #543 --- pgtype/numeric.go | 2 +- pgtype/numeric_array_test.go | 15 +++++++++++++++ pgtype/numeric_test.go | 3 +++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pgtype/numeric.go b/pgtype/numeric.go index fb63df75..e14d02e4 100644 --- a/pgtype/numeric.go +++ b/pgtype/numeric.go @@ -321,7 +321,7 @@ func parseNumericString(str string) (n *big.Int, exp int32, err error) { if len(parts) > 1 { exp = int32(-len(parts[1])) } else { - for len(digits) > 1 && digits[len(digits)-1] == '0' { + for len(digits) > 1 && digits[len(digits)-1] == '0' && digits[len(digits)-2] != '-' { digits = digits[:len(digits)-1] exp++ } diff --git a/pgtype/numeric_array_test.go b/pgtype/numeric_array_test.go index 22ee1bc4..28aa67d9 100644 --- a/pgtype/numeric_array_test.go +++ b/pgtype/numeric_array_test.go @@ -1,6 +1,7 @@ package pgtype_test import ( + "math" "math/big" "reflect" "testing" @@ -65,6 +66,13 @@ func TestNumericArraySet(t *testing.T) { Dimensions: []pgtype.ArrayDimension{{LowerBound: 1, Length: 1}}, Status: pgtype.Present}, }, + { + source: []float32{float32(math.Copysign(0, -1))}, + result: pgtype.NumericArray{ + Elements: []pgtype.Numeric{{Int: big.NewInt(0), Status: pgtype.Present}}, + Dimensions: []pgtype.ArrayDimension{{LowerBound: 1, Length: 1}}, + Status: pgtype.Present}, + }, { source: []float64{1}, result: pgtype.NumericArray{ @@ -72,6 +80,13 @@ func TestNumericArraySet(t *testing.T) { Dimensions: []pgtype.ArrayDimension{{LowerBound: 1, Length: 1}}, Status: pgtype.Present}, }, + { + source: []float64{math.Copysign(0, -1)}, + result: pgtype.NumericArray{ + Elements: []pgtype.Numeric{{Int: big.NewInt(0), Status: pgtype.Present}}, + Dimensions: []pgtype.ArrayDimension{{LowerBound: 1, Length: 1}}, + Status: pgtype.Present}, + }, { source: (([]float32)(nil)), result: pgtype.NumericArray{Status: pgtype.Null}, diff --git a/pgtype/numeric_test.go b/pgtype/numeric_test.go index 9d7d83d6..a5f70c9e 100644 --- a/pgtype/numeric_test.go +++ b/pgtype/numeric_test.go @@ -1,6 +1,7 @@ package pgtype_test import ( + "math" "math/big" "math/rand" "reflect" @@ -188,7 +189,9 @@ func TestNumericSet(t *testing.T) { result *pgtype.Numeric }{ {source: float32(1), result: &pgtype.Numeric{Int: big.NewInt(1), Status: pgtype.Present}}, + {source: float32(math.Copysign(0, -1)), result: &pgtype.Numeric{Int: big.NewInt(0), Status: pgtype.Present}}, {source: float64(1), result: &pgtype.Numeric{Int: big.NewInt(1), Status: pgtype.Present}}, + {source: float64(math.Copysign(0, -1)), result: &pgtype.Numeric{Int: big.NewInt(0), Status: pgtype.Present}}, {source: int8(1), result: &pgtype.Numeric{Int: big.NewInt(1), Status: pgtype.Present}}, {source: int16(1), result: &pgtype.Numeric{Int: big.NewInt(1), Status: pgtype.Present}}, {source: int32(1), result: &pgtype.Numeric{Int: big.NewInt(1), Status: pgtype.Present}}, From 39b09f2c4af27a3ed19ab8def20ea094df8f65e5 Mon Sep 17 00:00:00 2001 From: jinhua luo Date: Tue, 25 Jun 2019 01:05:28 +0800 Subject: [PATCH 03/10] cast bytea to make []byte suitable for both string and binary string types --- internal/sanitize/sanitize.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sanitize/sanitize.go b/internal/sanitize/sanitize.go index 53543b89..8939d797 100644 --- a/internal/sanitize/sanitize.go +++ b/internal/sanitize/sanitize.go @@ -87,7 +87,7 @@ func QuoteString(str string) string { } func QuoteBytes(buf []byte) string { - return `'\x` + hex.EncodeToString(buf) + "'" + return `'\x` + hex.EncodeToString(buf) + "'::bytea" } type sqlLexer struct { From e07faf207d03a337183040882337aa0d7dd2e743 Mon Sep 17 00:00:00 2001 From: jinhua luo Date: Tue, 25 Jun 2019 02:12:56 +0800 Subject: [PATCH 04/10] adjust the test for the patch --- internal/sanitize/sanitize_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sanitize/sanitize_test.go b/internal/sanitize/sanitize_test.go index 9597840e..f4337253 100644 --- a/internal/sanitize/sanitize_test.go +++ b/internal/sanitize/sanitize_test.go @@ -108,7 +108,7 @@ func TestQuerySanitize(t *testing.T) { { query: sanitize.Query{Parts: []sanitize.Part{"select ", 1}}, args: []interface{}{[]byte{0, 1, 2, 3, 255}}, - expected: `select '\x00010203ff'`, + expected: `select '\x00010203ff'::bytea`, }, { query: sanitize.Query{Parts: []sanitize.Part{"select ", 1}}, From 134d3e8d7e9de0443bc9e5094667cdb4275f71e7 Mon Sep 17 00:00:00 2001 From: Nick Jones Date: Tue, 25 Jun 2019 12:37:52 +1000 Subject: [PATCH 05/10] Read OIDs for composite types on connection init. This used to be done, but pulled in tables which slowed down connections on databases with a large number of tables; see https://github.com/jackc/pgx/issues/140. This change includes composite types but excludes tables by joining against [pg_class](https://www.postgresql.org/docs/11/catalog-pg-class.html) in which `relkind` is `'c'` for the former and `'r'` for the latter. Fixes https://github.com/jackc/pgx/issues/420. --- conn.go | 4 +++- pgmock/pgmock.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/conn.go b/conn.go index cb24748c..0cf6c167 100644 --- a/conn.go +++ b/conn.go @@ -404,9 +404,11 @@ func initPostgresql(c *Conn) (*pgtype.ConnInfo, error) { from pg_type t left join pg_type base_type on t.typelem=base_type.oid left join pg_namespace nsp on t.typnamespace=nsp.oid +left join pg_class cls on t.typrelid=cls.oid where ( - t.typtype in('b', 'p', 'r', 'e') + t.typtype in('b', 'p', 'r', 'e', 'c') and (base_type.oid is null or base_type.typtype in('b', 'p', 'r')) + and (cls.oid is null or cls.relkind='c') )` ) diff --git a/pgmock/pgmock.go b/pgmock/pgmock.go index 4d15f7b8..d4ab0d13 100644 --- a/pgmock/pgmock.go +++ b/pgmock/pgmock.go @@ -211,9 +211,11 @@ func PgxInitSteps() []Step { from pg_type t left join pg_type base_type on t.typelem=base_type.oid left join pg_namespace nsp on t.typnamespace=nsp.oid +left join pg_class cls on t.typrelid=cls.oid where ( - t.typtype in('b', 'p', 'r', 'e') + t.typtype in('b', 'p', 'r', 'e', 'c') and (base_type.oid is null or base_type.typtype in('b', 'p', 'r')) + and (cls.oid is null or cls.relkind='c') )`, }), ExpectMessage(&pgproto3.Describe{ From c474426c1143f927738bfa077ecf10d888cc0d97 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Tue, 25 Jun 2019 21:40:32 -0700 Subject: [PATCH 06/10] Log error message on rows-close error --- query.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query.go b/query.go index 27969be9..14df853f 100644 --- a/query.go +++ b/query.go @@ -84,7 +84,7 @@ func (rows *Rows) Close() { rows.conn.log(LogLevelInfo, "Query", map[string]interface{}{"sql": rows.sql, "args": logQueryArgs(rows.args), "time": endTime.Sub(rows.startTime), "rowCount": rows.rowCount}) } } else if rows.conn.shouldLog(LogLevelError) { - rows.conn.log(LogLevelError, "Query", map[string]interface{}{"sql": rows.sql, "args": logQueryArgs(rows.args)}) + rows.conn.log(LogLevelError, "Query", map[string]interface{}{"sql": rows.sql, "args": logQueryArgs(rows.args), "err": rows.err}) } if rows.batch != nil && rows.err != nil { From c5be74ca4e2f5f26b00bd6f02c98f43df6b8407b Mon Sep 17 00:00:00 2001 From: jinhua luo Date: Thu, 27 Jun 2019 13:16:35 +0800 Subject: [PATCH 07/10] send simple query if no args no need to parse and sanitize the sql string when no args. --- query.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/query.go b/query.go index 14df853f..5c6cbf7f 100644 --- a/query.go +++ b/query.go @@ -522,6 +522,10 @@ func (c *Conn) readUntilRowDescription() ([]FieldDescription, error) { } func (c *Conn) sanitizeAndSendSimpleQuery(sql string, args ...interface{}) (err error) { + if len(args) == 0 { + return c.sendSimpleQuery(sql) + } + if c.RuntimeParams["standard_conforming_strings"] != "on" { return errors.New("simple protocol queries must be run with standard_conforming_strings=on") } From a1d6202434aa40c3624688f6c2cacbc27eef5472 Mon Sep 17 00:00:00 2001 From: Jack Christensen Date: Sat, 29 Jun 2019 11:19:29 -0500 Subject: [PATCH 08/10] Release 3.5.0 --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27c7d9df..f22d8d29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +# 3.5.0 (June 29, 2019) + +## Features + +* Protocol support for PortalSuspended message (avivklas) +* Read OIDs for composite types on connection init (Nick Jones) + +## Fixes + +* Hstore can have empty keys (Josh Leverette) +* Fix -0 value for numeric type (David Hudson) +* Log error message on rows-close error (Euan Kemp) + +## Changes + +* Explicitly cast binary string to bytea in simple protocol (jinhua luo) +* Skip parse and sanitize simple query when no arguments (jinhua luo) + # 3.4.0 (May 3, 2019) ## Features From 7c5d801f058d3a2c41650f9643f2873a10d98964 Mon Sep 17 00:00:00 2001 From: Jack Christensen Date: Sat, 29 Jun 2019 14:13:10 -0500 Subject: [PATCH 09/10] Add v4 prerelease notice --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index d6499ba4..b7051f65 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ if err != nil { } ``` +## v4 Coming Soon + +This is the current stable v3 version. v4 is currently is in prelease status. Consider using [v4](https://github.com/jackc/pgx/tree/v4) for new development or test upgrading existing applications. + ## Features pgx supports many additional features beyond what is available through database/sql. From 8ba5485db6ebfa824458df98db6df29cbe128332 Mon Sep 17 00:00:00 2001 From: Nicholas Wilson Date: Thu, 4 Jul 2019 10:21:32 +0100 Subject: [PATCH 10/10] Use zap.Any for handling interface{} -> zap.Field conversion zap.Any falls back to zap.Reflect, but is better for this case, because it first checks for the types that zap handles specially. For example, time.Duration, or error, which zap.Reflect will just treat as untyped int64 or struct objects, but zap.Any is able to detect these types and print them properly. --- log/zapadapter/adapter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log/zapadapter/adapter.go b/log/zapadapter/adapter.go index 82263b6e..a5a377e6 100644 --- a/log/zapadapter/adapter.go +++ b/log/zapadapter/adapter.go @@ -19,7 +19,7 @@ func (pl *Logger) Log(level pgx.LogLevel, msg string, data map[string]interface{ fields := make([]zapcore.Field, len(data)) i := 0 for k, v := range data { - fields[i] = zap.Reflect(k, v) + fields[i] = zap.Any(k, v) i++ }