From af8341ee071138455dc92592a5368243c873bcba Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 6 May 2022 17:15:45 +0200 Subject: [PATCH 1/4] py: improve ParseTuple{,AndKeywords} to handle 's{,*,#}' Signed-off-by: Sebastien Binet --- py/args.go | 55 ++++++++-- py/args_test.go | 277 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 326 insertions(+), 6 deletions(-) create mode 100644 py/args_test.go diff --git a/py/args.go b/py/args.go index e3bfc36b..d7e5a35b 100644 --- a/py/args.go +++ b/py/args.go @@ -465,17 +465,60 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist switch op.code { case 'O': *result = arg - case 'Z', 'z': - if _, ok := arg.(NoneType); ok { - *result = arg - break + case 'Z': + switch op.modifier { + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or None, not %s", name, i+1, arg.Type().Name) + case '#', 0: + switch arg := arg.(type) { + case String, NoneType: + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or None, not %s", name, i+1, arg.Type().Name) + } + } + *result = arg + case 'z': + switch op.modifier { + default: + switch arg := arg.(type) { + case String, NoneType: + // ok + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or None, not %s", name, i+1, arg.Type().Name) + } + case '#': + fallthrough // FIXME(sbinet): check for read-only? + case '*': + switch arg := arg.(type) { + case String, Bytes, NoneType: + // ok. + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str, bytes-like or None, not %s", name, i+1, arg.Type().Name) + } } - fallthrough - case 'U', 's': + *result = arg + case 'U': if _, ok := arg.(String); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be str, not %s", name, i+1, arg.Type().Name) } *result = arg + case 's': + switch op.modifier { + default: + if _, ok := arg.(String); !ok { + return ExceptionNewf(TypeError, "%s() argument %d must be str, not %s", name, i+1, arg.Type().Name) + } + case '#': + fallthrough // FIXME(sbinet): check for read-only? + case '*': + switch arg := arg.(type) { + case String, Bytes: + // ok. + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or bytes-like, not %s", name, i+1, arg.Type().Name) + } + } + *result = arg case 'i', 'n': if _, ok := arg.(Int); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be int, not %s", name, i+1, arg.Type().Name) diff --git a/py/args_test.go b/py/args_test.go new file mode 100644 index 00000000..2f291827 --- /dev/null +++ b/py/args_test.go @@ -0,0 +1,277 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +import ( + "fmt" + "testing" +) + +func TestParseTupleAndKeywords(t *testing.T) { + for _, tc := range []struct { + args Tuple + kwargs StringDict + format string + kwlist []string + results []Object + err error + }{ + { + args: Tuple{String("a")}, + format: "O:func", + results: []Object{String("a")}, + }, + { + args: Tuple{None}, + format: "Z:func", + results: []Object{None}, + }, + { + args: Tuple{String("a")}, + format: "Z:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "Z:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not int'"), + }, + { + args: Tuple{None}, + format: "Z*:func", // FIXME(sbinet): invalid format. + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not NoneType'"), + }, + { + args: Tuple{None}, + format: "Z#:func", + results: []Object{None}, + }, + { + args: Tuple{String("a")}, + format: "Z#:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "Z#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not int'"), + }, + { + args: Tuple{None}, + format: "z:func", + results: []Object{None}, + }, + { + args: Tuple{String("a")}, + format: "z:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "z:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not int'"), + }, + { + args: Tuple{Bytes("a")}, + format: "z:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not bytes'"), + }, + { + args: Tuple{None}, + format: "z*:func", + results: []Object{None}, + }, + { + args: Tuple{Bytes("a")}, + format: "z*:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "z*:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "z*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, bytes-like or None, not int'"), + }, + { + args: Tuple{None}, + format: "z#:func", + results: []Object{None}, + }, + { + args: Tuple{Bytes("a")}, + format: "z#:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "z#:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "z#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, bytes-like or None, not int'"), + }, + { + args: Tuple{String("a")}, + format: "s:func", + results: []Object{String("a")}, + }, + { + args: Tuple{None}, + format: "s:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not NoneType'"), + }, + { + args: Tuple{Bytes("a")}, + format: "s:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{String("a")}, + format: "s#:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Bytes("a")}, + format: "s#:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{None}, + format: "s#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or bytes-like, not NoneType'"), + }, + { + args: Tuple{String("a")}, + format: "s*:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Bytes("a")}, + format: "s*:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{None}, + format: "s*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or bytes-like, not NoneType'"), + }, + { + args: Tuple{String("a")}, + format: "U:func", + results: []Object{String("a")}, + }, + { + args: Tuple{None}, + format: "U:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not NoneType'"), + }, + { + args: Tuple{Bytes("a")}, + format: "U:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{Bytes("a")}, + format: "U*:func", // FIXME(sbinet): invalid format + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{Bytes("a")}, + format: "U#:func", // FIXME(sbinet): invalid format + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{Int(42)}, + format: "i:func", + results: []Object{Int(42)}, + }, + { + args: Tuple{None}, + format: "i:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be int, not NoneType'"), + }, + { + args: Tuple{Int(42)}, + format: "n:func", + results: []Object{Int(42)}, + }, + { + args: Tuple{None}, + format: "n:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be int, not NoneType'"), + }, + { + args: Tuple{Bool(true)}, + format: "p:func", + results: []Object{Bool(true)}, + }, + { + args: Tuple{None}, + format: "p:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bool, not NoneType'"), + }, + { + args: Tuple{Float(42)}, + format: "d:func", + results: []Object{Float(42)}, + }, + { + args: Tuple{Int(42)}, + format: "d:func", + results: []Object{Float(42)}, + }, + { + args: Tuple{None}, + format: "p:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bool, not NoneType'"), + }, + } { + t.Run(tc.format, func(t *testing.T) { + results := make([]*Object, len(tc.results)) + for i := range tc.results { + results[i] = &tc.results[i] + } + err := ParseTupleAndKeywords(tc.args, tc.kwargs, tc.format, tc.kwlist, results...) + switch { + case err != nil && tc.err != nil: + if got, want := err.Error(), tc.err.Error(); got != want { + t.Fatalf("invalid error:\ngot= %s\nwant=%s", got, want) + } + case err != nil && tc.err == nil: + t.Fatalf("could not parse tuple+kwargs: %+v", err) + case err == nil && tc.err != nil: + t.Fatalf("expected an error (got=nil): %+v", tc.err) + case err == nil && tc.err == nil: + // ok. + } + // FIXME(sbinet): check results + }) + } +} From 8ca157b84c9fdfe4726ff015fc1d6626eb95c991 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 6 May 2022 18:08:59 +0200 Subject: [PATCH 2/4] py: improve ParseTuple{,AndKeywords} to handle 'y{,*,#}' Signed-off-by: Sebastien Binet --- py/args.go | 17 +++++++++++++++++ py/args_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/py/args.go b/py/args.go index d7e5a35b..04734ef3 100644 --- a/py/args.go +++ b/py/args.go @@ -519,6 +519,23 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist } } *result = arg + case 'y': + switch op.modifier { + default: + if _, ok := arg.(Bytes); !ok { + return ExceptionNewf(TypeError, "%s() argument %d must be bytes-like, not %s", name, i+1, arg.Type().Name) + } + case '#': + fallthrough // FIXME(sbinet): check for read-only? + case '*': + switch arg := arg.(type) { + case Bytes: + // ok. + default: + return ExceptionNewf(TypeError, "%s() argument %d must be bytes-like, not %s", name, i+1, arg.Type().Name) + } + } + *result = arg case 'i', 'n': if _, ok := arg.(Int); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be int, not %s", name, i+1, arg.Type().Name) diff --git a/py/args_test.go b/py/args_test.go index 2f291827..408ad342 100644 --- a/py/args_test.go +++ b/py/args_test.go @@ -174,6 +174,57 @@ func TestParseTupleAndKeywords(t *testing.T) { results: []Object{nil}, err: fmt.Errorf("TypeError: 'func() argument 1 must be str or bytes-like, not NoneType'"), }, + { + args: Tuple{Bytes("a")}, + format: "y:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{None}, + format: "y:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not NoneType'"), + }, + { + args: Tuple{String("a")}, + format: "y:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not str'"), + }, + { + args: Tuple{Bytes("a")}, + format: "y#:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "y#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not str'"), + }, + { + args: Tuple{None}, + format: "y#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not NoneType'"), + }, + { + args: Tuple{Bytes("a")}, + format: "y*:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "y*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not str'"), + }, + { + args: Tuple{None}, + format: "y*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not NoneType'"), + }, { args: Tuple{String("a")}, format: "U:func", From 6b3e35b14353e9cff3eb89701187515b02cbe664 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Mon, 9 May 2022 15:31:40 +0200 Subject: [PATCH 3/4] py: make Exception implement __{str,repr}__ Signed-off-by: Sebastien Binet --- py/exception.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/py/exception.go b/py/exception.go index a2abbcac..2e8f91a2 100644 --- a/py/exception.go +++ b/py/exception.go @@ -360,6 +360,22 @@ func (e *Exception) M__getattr__(name string) (Object, error) { return e.Args, nil // FIXME All attributes are args! } +func (e *Exception) M__str__() (Object, error) { + msg := e.Args.(Tuple)[0] + return msg, nil +} + +func (e *Exception) M__repr__() (Object, error) { + msg := e.Args.(Tuple)[0].(String) + typ := e.Base.Name + return String(fmt.Sprintf("%s(%q)", typ, string(msg))), nil +} + // Check Interfaces -var _ error = (*Exception)(nil) -var _ error = (*ExceptionInfo)(nil) +var ( + _ error = (*ExceptionInfo)(nil) + + _ error = (*Exception)(nil) + _ I__str__ = (*Exception)(nil) + _ I__repr__ = (*Exception)(nil) +) From 189965b62e710677c39f9b84953ce1b941a3fdb1 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Mon, 9 May 2022 11:39:43 +0200 Subject: [PATCH 4/4] stdlib/binascii: first import Signed-off-by: Sebastien Binet --- stdlib/binascii/binascii.go | 158 +++++++++++++++++++++++ stdlib/binascii/binascii_test.go | 15 +++ stdlib/binascii/testdata/test.py | 54 ++++++++ stdlib/binascii/testdata/test_golden.txt | 11 ++ stdlib/stdlib.go | 1 + 5 files changed, 239 insertions(+) create mode 100644 stdlib/binascii/binascii.go create mode 100644 stdlib/binascii/binascii_test.go create mode 100644 stdlib/binascii/testdata/test.py create mode 100644 stdlib/binascii/testdata/test_golden.txt diff --git a/stdlib/binascii/binascii.go b/stdlib/binascii/binascii.go new file mode 100644 index 00000000..4a87d3ee --- /dev/null +++ b/stdlib/binascii/binascii.go @@ -0,0 +1,158 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package binascii provides the implementation of the python's 'binascii' module. +package binascii + +import ( + "encoding/base64" + "encoding/hex" + "errors" + "hash/crc32" + + "github.com/go-python/gpython/py" +) + +var ( + Incomplete = py.ExceptionType.NewType("binascii.Incomplete", "", nil, nil) + Error = py.ValueError.NewType("binascii.Error", "", nil, nil) +) + +func init() { + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "binascii", + Doc: "Conversion between binary data and ASCII", + }, + Methods: []*py.Method{ + py.MustNewMethod("a2b_base64", a2b_base64, 0, "Decode a line of base64 data."), + py.MustNewMethod("b2a_base64", b2a_base64, 0, "Base64-code line of data."), + py.MustNewMethod("a2b_hex", a2b_hex, 0, a2b_hex_doc), + py.MustNewMethod("b2a_hex", b2a_hex, 0, b2a_hex_doc), + py.MustNewMethod("crc32", crc32_, 0, "Compute CRC-32 incrementally."), + py.MustNewMethod("unhexlify", a2b_hex, 0, unhexlify_doc), + py.MustNewMethod("hexlify", b2a_hex, 0, hexlify_doc), + }, + Globals: py.StringDict{ + "Incomplete": Incomplete, + "Error": Error, + }, + }) +} + +func b2a_base64(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pydata py.Object + pynewl py.Object = py.True + ) + err := py.ParseTupleAndKeywords(args, kwargs, "y*|p:binascii.b2a_base64", []string{"data", "newline"}, &pydata, &pynewl) + if err != nil { + return nil, err + } + + var ( + buf = []byte(pydata.(py.Bytes)) + newline = bool(pynewl.(py.Bool)) + ) + + out := base64.StdEncoding.EncodeToString(buf) + if newline { + out += "\n" + } + return py.Bytes(out), nil +} + +func a2b_base64(self py.Object, args py.Tuple) (py.Object, error) { + var pydata py.Object + err := py.ParseTuple(args, "s:binascii.a2b_base64", &pydata) + if err != nil { + return nil, err + } + + out, err := base64.StdEncoding.DecodeString(string(pydata.(py.String))) + if err != nil { + return nil, py.ExceptionNewf(Error, "could not decode base64 data: %+v", err) + } + + return py.Bytes(out), nil +} + +func crc32_(self py.Object, args py.Tuple) (py.Object, error) { + var ( + pydata py.Object + pycrc py.Object = py.Int(0) + ) + + err := py.ParseTuple(args, "y*|i:binascii.crc32", &pydata, &pycrc) + if err != nil { + return nil, err + } + + crc := crc32.Update(uint32(pycrc.(py.Int)), crc32.IEEETable, []byte(pydata.(py.Bytes))) + return py.Int(crc), nil + +} + +const a2b_hex_doc = `Binary data of hexadecimal representation. + +hexstr must contain an even number of hex digits (upper or lower case). +This function is also available as "unhexlify()".` + +func a2b_hex(self py.Object, args py.Tuple) (py.Object, error) { + var ( + hexErr hex.InvalidByteError + pydata py.Object + src string + ) + err := py.ParseTuple(args, "s*:binascii.a2b_hex", &pydata) + if err != nil { + return nil, err + } + + switch v := pydata.(type) { + case py.String: + src = string(v) + case py.Bytes: + src = string(v) + } + + o, err := hex.DecodeString(src) + if err != nil { + switch { + case errors.Is(err, hex.ErrLength): + return nil, py.ExceptionNewf(Error, "Odd-length string") + case errors.As(err, &hexErr): + return nil, py.ExceptionNewf(Error, "Non-hexadecimal digit found") + default: + return nil, py.ExceptionNewf(Error, "could not decode hex data: %+v", err) + } + } + + return py.Bytes(o), nil +} + +const b2a_hex_doc = `Hexadecimal representation of binary data. + +The return value is a bytes object. This function is also +available as "hexlify()".` + +func b2a_hex(self py.Object, args py.Tuple) (py.Object, error) { + var pydata py.Object + err := py.ParseTuple(args, "y*:binascii.b2a_hex", &pydata) + if err != nil { + return nil, err + } + + o := hex.EncodeToString([]byte(pydata.(py.Bytes))) + return py.Bytes(o), nil +} + +const unhexlify_doc = `Binary data of hexadecimal representation. + +hexstr must contain an even number of hex digits (upper or lower case).` + +const hexlify_doc = `Hexadecimal representation of binary data. + +The return value is a bytes object. This function is also +available as "b2a_hex()".` diff --git a/stdlib/binascii/binascii_test.go b/stdlib/binascii/binascii_test.go new file mode 100644 index 00000000..bd40c782 --- /dev/null +++ b/stdlib/binascii/binascii_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binascii_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestBinascii(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/binascii/testdata/test.py b/stdlib/binascii/testdata/test.py new file mode 100644 index 00000000..f6227aff --- /dev/null +++ b/stdlib/binascii/testdata/test.py @@ -0,0 +1,54 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import binascii + +print("globals:") +for name in ("Error", "Incomplete"): + v = getattr(binascii, name) + print("\nbinascii.%s:\n%s" % (name,repr(v))) + +def assertEqual(x, y): + assert x == y, "got: %s, want: %s" % (repr(x), repr(y)) + +## base64 +assertEqual(binascii.b2a_base64(b'hello world!'), b'aGVsbG8gd29ybGQh\n') +assertEqual(binascii.b2a_base64(b'hello\nworld!'), b'aGVsbG8Kd29ybGQh\n') +assertEqual(binascii.b2a_base64(b'hello world!', newline=False), b'aGVsbG8gd29ybGQh') +assertEqual(binascii.b2a_base64(b'hello\nworld!', newline=False), b'aGVsbG8Kd29ybGQh') +assertEqual(binascii.a2b_base64("aGVsbG8gd29ybGQh\n"), b'hello world!') + +try: + binascii.b2a_base64("string") + print("expected an exception") +except TypeError as e: + print("expected an exception:", e) + pass + +## crc32 +assertEqual(binascii.crc32(b'hello world!'), 62177901) +assertEqual(binascii.crc32(b'hello world!', 0), 62177901) +assertEqual(binascii.crc32(b'hello world!', 42), 4055036404) + +## hex +assertEqual(binascii.b2a_hex(b'hello world!'), b'68656c6c6f20776f726c6421') +assertEqual(binascii.a2b_hex(b'68656c6c6f20776f726c6421'), b'hello world!') +assertEqual(binascii.hexlify(b'hello world!'), b'68656c6c6f20776f726c6421') +assertEqual(binascii.unhexlify(b'68656c6c6f20776f726c6421'), b'hello world!') + +try: + binascii.a2b_hex(b'123') + print("expected an exception") +except binascii.Error as e: + print("expected an exception:",e) + pass + +try: + binascii.a2b_hex(b'hell') + print("expected an exception") +except binascii.Error as e: + print("expected an exception:",e) + pass + +print("done.") diff --git a/stdlib/binascii/testdata/test_golden.txt b/stdlib/binascii/testdata/test_golden.txt new file mode 100644 index 00000000..29f03223 --- /dev/null +++ b/stdlib/binascii/testdata/test_golden.txt @@ -0,0 +1,11 @@ +globals: + +binascii.Error: + + +binascii.Incomplete: + +expected an exception: binascii.b2a_base64() argument 1 must be bytes-like, not str +expected an exception: Odd-length string +expected an exception: Non-hexadecimal digit found +done. diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index 38483df1..8182d462 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -18,6 +18,7 @@ import ( "github.com/go-python/gpython/stdlib/marshal" "github.com/go-python/gpython/vm" + _ "github.com/go-python/gpython/stdlib/binascii" _ "github.com/go-python/gpython/stdlib/builtin" _ "github.com/go-python/gpython/stdlib/math" _ "github.com/go-python/gpython/stdlib/string"