Skip to content

Commit 17a251f

Browse files
Decode/Unmarshal a msgpack array of arbitrary length into a struct (new directive:vartuple) (#399)
* Decode/Unmarshal a msgpack array of arbitrary length into a struct (new directive:vartuple)
1 parent a180baa commit 17a251f

File tree

9 files changed

+285
-12
lines changed

9 files changed

+285
-12
lines changed

gen/decode.go

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,12 +73,14 @@ func (d *decodeGen) assignAndCheck(name string, typ string) {
7373
}
7474

7575
func (d *decodeGen) structAsTuple(s *Struct) {
76-
nfields := len(s.Fields)
77-
7876
sz := randIdent()
7977
d.p.declare(sz, u32)
8078
d.assignAndCheck(sz, arrayHeader)
81-
d.p.arrayCheck(strconv.Itoa(nfields), sz)
79+
if s.AsVarTuple {
80+
d.p.printf("\nif %[1]s == 0 { return }", sz)
81+
} else {
82+
d.p.arrayCheck(strconv.Itoa(len(s.Fields)), sz)
83+
}
8284
for i := range s.Fields {
8385
if !d.p.ok() {
8486
return
@@ -98,6 +100,12 @@ func (d *decodeGen) structAsTuple(s *Struct) {
98100
if anField {
99101
d.p.printf("\n}") // close if statement
100102
}
103+
if s.AsVarTuple {
104+
d.p.printf("\nif %[1]s--; %[1]s == 0 { return }", sz)
105+
}
106+
}
107+
if s.AsVarTuple {
108+
d.p.printf("\nfor ; %[1]s > 0; %[1]s-- {\nif err = dc.Skip(); err != nil {\nerr = msgp.WrapError(err)\nreturn\n}\n}", sz)
101109
}
102110
}
103111

gen/elem.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -459,8 +459,9 @@ func (s *Ptr) IfZeroExpr() string { return s.Varname() + " == nil" }
459459

460460
type Struct struct {
461461
common
462-
Fields []StructField // field list
463-
AsTuple bool // write as an array instead of a map
462+
Fields []StructField // field list
463+
AsTuple bool // write as an array instead of a map
464+
AsVarTuple bool // write as an array of variable length instead of a map
464465
}
465466

466467
func (s *Struct) TypeName() string {

gen/encode.go

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -97,9 +97,7 @@ func (e *encodeGen) tuple(s *Struct) {
9797
data := msgp.AppendArrayHeader(nil, uint32(nfields))
9898
e.p.printf("\n// array header, size %d", nfields)
9999
e.Fuse(data)
100-
if len(s.Fields) == 0 {
101-
e.fuseHook()
102-
}
100+
e.fuseHook()
103101
for i := range s.Fields {
104102
if !e.p.ok() {
105103
return

gen/marshal.go

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -104,9 +104,7 @@ func (m *marshalGen) tuple(s *Struct) {
104104
data = msgp.AppendArrayHeader(data, uint32(len(s.Fields)))
105105
m.p.printf("\n// array header, size %d", len(s.Fields))
106106
m.Fuse(data)
107-
if len(s.Fields) == 0 {
108-
m.fuseHook()
109-
}
107+
m.fuseHook()
110108
for i := range s.Fields {
111109
if !m.p.ok() {
112110
return

gen/unmarshal.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,11 @@ func (u *unmarshalGen) tuple(s *Struct) {
7777
sz := randIdent()
7878
u.p.declare(sz, u32)
7979
u.assignAndCheck(sz, arrayHeader)
80-
u.p.arrayCheck(strconv.Itoa(len(s.Fields)), sz)
80+
if s.AsVarTuple {
81+
u.p.printf("\nif %[1]s == 0 {\no = bts\nreturn\n}", sz)
82+
} else {
83+
u.p.arrayCheck(strconv.Itoa(len(s.Fields)), sz)
84+
}
8185
for i := range s.Fields {
8286
if !u.p.ok() {
8387
return
@@ -101,6 +105,12 @@ func (u *unmarshalGen) tuple(s *Struct) {
101105
if anField {
102106
u.p.printf("\n}")
103107
}
108+
if s.AsVarTuple {
109+
u.p.printf("\nif %[1]s--; %[1]s == 0 {\no = bts\nreturn\n}", sz)
110+
}
111+
}
112+
if s.AsVarTuple {
113+
u.p.printf("\nfor ; %[1]s > 0; %[1]s-- {\nbts, err = msgp.Skip(bts)\nif err != nil {\nerr = msgp.WrapError(err)\nreturn\n}\n}", sz)
104114
}
105115
}
106116

helper_test.go

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package main
2+
3+
import (
4+
"os"
5+
"os/exec"
6+
"path/filepath"
7+
"strings"
8+
"testing"
9+
10+
"github.com/tinylib/msgp/gen"
11+
)
12+
13+
const showGeneratedFile = false
14+
15+
func generate(t *testing.T, content string) (string, error) {
16+
tempDir := t.TempDir()
17+
18+
mainFilename := filepath.Join(tempDir, "main.go")
19+
20+
fd, err := os.OpenFile(mainFilename, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o600)
21+
if err != nil {
22+
return "", err
23+
}
24+
defer fd.Close()
25+
26+
if _, err := fd.WriteString(content); err != nil {
27+
return "", err
28+
}
29+
30+
mode := gen.Encode | gen.Decode | gen.Size | gen.Marshal | gen.Unmarshal | gen.Test
31+
if err := Run(mainFilename, mode, false); err != nil {
32+
return "", err
33+
}
34+
35+
if showGeneratedFile {
36+
mainGenFilename := strings.TrimSuffix(mainFilename, ".go") + "_gen.go"
37+
content, err := os.ReadFile(mainGenFilename)
38+
if err != nil {
39+
return "", err
40+
}
41+
t.Logf("generated %s content:\n%s", mainGenFilename, content)
42+
}
43+
44+
return mainFilename, nil
45+
}
46+
47+
func goExec(t *testing.T, mainFilename string, test bool) {
48+
mainGenFilename := strings.TrimSuffix(mainFilename, ".go") + "_gen.go"
49+
50+
args := []string{"run", mainFilename, mainGenFilename}
51+
if test {
52+
mainGenTestFilename := strings.TrimSuffix(mainFilename, ".go") + "_gen_test.go"
53+
args = []string{"test", mainFilename, mainGenFilename, mainGenTestFilename}
54+
}
55+
56+
output, err := exec.Command("go", args...).CombinedOutput()
57+
if err != nil {
58+
t.Fatalf("go run failed: %v, output:\n%s", err, output)
59+
}
60+
}

issue275_test.go

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package main
2+
3+
import "testing"
4+
5+
func TestIssue275Tuples(t *testing.T) {
6+
mainFilename, err := generate(t, issue275Tuples)
7+
if err != nil {
8+
t.Fatalf("generate failed: %v", err)
9+
}
10+
goExec(t, mainFilename, false) // exec go run
11+
goExec(t, mainFilename, true) // exec go test
12+
}
13+
14+
var issue275Tuples = `package main
15+
16+
import (
17+
"fmt"
18+
"os"
19+
)
20+
21+
//go:generate msgp
22+
23+
//msgp:tuple Test1
24+
type Test1 struct {
25+
Foo string
26+
}
27+
28+
//msgp:tuple Test2
29+
type Test2 struct {
30+
Foo string
31+
Bar string
32+
}
33+
34+
//msgp:tuple Test3
35+
type Test3 struct {
36+
Foo string
37+
Bar string
38+
Baz string
39+
}
40+
41+
//msgp:vartuple Test
42+
type Test struct {
43+
Foo string
44+
Bar string
45+
}
46+
47+
func main() {
48+
t1 := Test1{Foo: "Foo1"}
49+
d1, err := t1.MarshalMsg(nil)
50+
if err != nil {
51+
fmt.Println("Test1 MarshalMsg failed:", err)
52+
os.Exit(1)
53+
}
54+
55+
t2 := Test2{Foo: "Foo2", Bar: "Bar2"}
56+
d2, err := t2.MarshalMsg(nil)
57+
if err != nil {
58+
fmt.Println("Test2 MarshalMsg failed:", err)
59+
os.Exit(1)
60+
}
61+
62+
t3 := Test3{Foo: "Foo3", Bar: "Bar3", Baz: "Baz3"}
63+
d3, err := t3.MarshalMsg(nil)
64+
if err != nil {
65+
fmt.Println("Test3 MarshalMsg failed:", err)
66+
os.Exit(1)
67+
}
68+
69+
var msg Test
70+
71+
msg = Test{}
72+
if _, err := msg.UnmarshalMsg(d1); err != nil {
73+
fmt.Println("Test UnmarshalMsg from Test1 failed:", err)
74+
os.Exit(1)
75+
}
76+
if msg.Foo != "Foo1" {
77+
fmt.Println("Test UnmarshalMsg from Test1 bad msg:", msg)
78+
os.Exit(1)
79+
}
80+
81+
msg = Test{}
82+
if _, err := msg.UnmarshalMsg(d2); err != nil {
83+
fmt.Println("Test UnmarshalMsg from Test2 failed:", err)
84+
os.Exit(1)
85+
}
86+
if msg.Foo != "Foo2" || msg.Bar != "Bar2" {
87+
fmt.Println("Test UnmarshalMsg from Test2 bad msg:", msg)
88+
os.Exit(1)
89+
}
90+
91+
msg = Test{}
92+
if _, err := msg.UnmarshalMsg(d3); err != nil {
93+
fmt.Println("Test UnmarshalMsg from Test3 failed:", err)
94+
os.Exit(1)
95+
}
96+
if msg.Foo != "Foo3" || msg.Bar != "Bar3" {
97+
fmt.Println("Test UnmarshalMsg from Test3 bad msg:", msg)
98+
os.Exit(1)
99+
}
100+
101+
var msg1 Test1
102+
if _, err := msg1.UnmarshalMsg(d2); err != nil {
103+
if err.Error() != "msgp: wanted array of size 1; got 2" {
104+
fmt.Println("Test1 UnmarshalMsg from Test2 failed:", err)
105+
os.Exit(1)
106+
}
107+
}
108+
109+
var msg2 Test2
110+
if _, err := msg2.UnmarshalMsg(d2); err != nil {
111+
fmt.Println("Test2 UnmarshalMsg from Test2 failed:", err)
112+
os.Exit(1)
113+
}
114+
if msg2.Foo != "Foo2" || msg2.Bar != "Bar2" {
115+
fmt.Println("Test3 UnmarshalMsg from Test2 bad msg:", msg)
116+
os.Exit(1)
117+
}
118+
119+
var msg3 Test3
120+
if _, err := msg3.UnmarshalMsg(d2); err != nil {
121+
if err.Error() != "msgp: wanted array of size 3; got 2" {
122+
fmt.Println("Test3 UnmarshalMsg from Test2 failed:", err)
123+
os.Exit(1)
124+
}
125+
}
126+
}
127+
`

issue395_test.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package main
2+
3+
import "testing"
4+
5+
func TestIssue395TupleAllownil(t *testing.T) {
6+
mainFilename, err := generate(t, issue395TupleAllownil)
7+
if err != nil {
8+
t.Fatalf("generate failed: %v", err)
9+
}
10+
11+
goExec(t, mainFilename, false) // exec go run
12+
goExec(t, mainFilename, true) // exec go test
13+
}
14+
15+
var issue395TupleAllownil = `package main
16+
17+
import (
18+
"fmt"
19+
"os"
20+
)
21+
22+
//go:generate msgp
23+
24+
//msgp:tuple User
25+
type User struct {
26+
ID []byte ` + "`msgpack:\",allownil\"`" + `
27+
Name string
28+
Email string
29+
IsActive bool
30+
}
31+
32+
func main() {
33+
u := User{ID: nil, Name: "user"}
34+
data, err := u.MarshalMsg(nil)
35+
if err != nil {
36+
fmt.Println("User MarshalMsg failed:", err)
37+
os.Exit(1)
38+
}
39+
40+
var user User
41+
if _, err := user.UnmarshalMsg(data); err != nil {
42+
fmt.Println("User UnmarshalMsg failed:", err)
43+
os.Exit(1)
44+
}
45+
if user.ID != nil || user.Name != "user" {
46+
fmt.Println("User UnmarshalMsg bad user:", user)
47+
os.Exit(1)
48+
}
49+
}
50+
`

parse/directives.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ var directives = map[string]directive{
2626
"replace": replace,
2727
"ignore": ignore,
2828
"tuple": astuple,
29+
"vartuple": asvartuple,
2930
"compactfloats": compactfloats,
3031
"clearomitted": clearomitted,
3132
"newtime": newtime,
@@ -176,6 +177,26 @@ func astuple(text []string, f *FileSet) error {
176177
return nil
177178
}
178179

180+
//msgp:vartuple {TypeA} {TypeB}...
181+
func asvartuple(text []string, f *FileSet) error {
182+
if len(text) < 2 {
183+
return nil
184+
}
185+
for _, item := range text[1:] {
186+
name := strings.TrimSpace(item)
187+
if el, ok := f.Identities[name]; ok {
188+
if st, ok := el.(*gen.Struct); ok {
189+
st.AsTuple = true
190+
st.AsVarTuple = true
191+
infof(name)
192+
} else {
193+
warnf("%s: only structs can be tuples\n", name)
194+
}
195+
}
196+
}
197+
return nil
198+
}
199+
179200
//msgp:tag {tagname}
180201
func tag(text []string, f *FileSet) error {
181202
if len(text) != 2 {

0 commit comments

Comments
 (0)