diff --git a/.gitignore b/.gitignore index a3eaead1..1981ae0a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ __pycache__ cover.out /junk /dist + +examples/embedding/embedding +builtin/testfile \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..b0d10522 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch file", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${file}" + }, + { + "name": "examples/embedding", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/examples/embedding", + "cwd": "${workspaceFolder}/examples/embedding", + "args": [ + "mylib-demo.py" + ], + }, + ] +} \ No newline at end of file diff --git a/README.md b/README.md index a5068a0f..14d9fe43 100644 --- a/README.md +++ b/README.md @@ -5,18 +5,20 @@ [![GoDoc](https://godoc.org/github.com/go-python/gpython?status.svg)](https://godoc.org/github.com/go-python/gpython) [![License](https://img.shields.io/badge/License-BSD--3-blue.svg)](https://github.com/go-python/gpython/blob/master/LICENSE) -gpython is a part re-implementation / part port of the Python 3.4 -interpreter to the Go language, "batteries not included". - -It includes: - - * runtime - using compatible byte code to python3.4 - * lexer - * parser - * compiler +gpython is a part re-implementation, part port of the Python 3.4 +interpreter in Go. Although there are many areas of improvement, +it stands as an noteworthy achievement in capability and potential. + +gpython includes: + + * lexer, parser, and compiler + * runtime and high-level convenience functions + * multi-context interpreter instancing + * easy embedding into your Go application * interactive mode (REPL) ([try online!](https://gpython.org)) -It does not include very many python modules as many of the core + +gpython does not include many python modules as many of the core modules are written in C not python. The converted modules are: * builtins @@ -27,53 +29,52 @@ modules are written in C not python. The converted modules are: ## Install -Gpython is a Go program and comes as a single binary file. - -Download the relevant binary from here: https://github.com/go-python/gpython/releases +Download directly from the [releases page](https://github.com/go-python/gpython/releases) -Or alternatively if you have Go installed use +Or if you have Go installed: - go get github.com/go-python/gpython - -and this will build the binary in `$GOPATH/bin`. You can then modify -the source and submit patches. + go install github.com/go-python/gpython ## Objectives -Gpython was written as a learning experiment to investigate how hard +gpython started as an experiment to investigate how hard porting Python to Go might be. It turns out that all those C modules -are a significant barrier to making a fully functional port. +are a significant barrier to making gpython a complete replacement +to CPython. -## Status +However, to those who want to embed a highly popular and known language +into their Go application, gpython could be a great choice over less +capable (or lesser known) alternatives. -The project works well enough to parse all the code in the python 3.4 -distribution and to compile and run python 3 programs which don't -depend on a module gpython doesn't support. +## Status -See the examples directory for some python programs which run with -gpython. +gpython currently: + - Parses all the code in the Python 3.4 distribution + - Runs Python 3 for the modules that are currently supported + - Supports concurrent multi-interpreter execution ("multi-context") Speed hasn't been a goal of the conversions however it runs pystone at -about 20% of the speed of cpython. The pi test runs quicker under -gpython as I think the Go long integer primitives are faster than the +about 20% of the speed of CPython. The [π test](https://github.com/go-python/gpython/tree/master/examples/pi_chudnovsky_bs.py) runs quicker under +gpython as the Go long integer primitives are likely faster than the Python ones. -There are many directions this project could go in. I think the most -profitable would be to re-use the -[grumpy](https://github.com/grumpyhome/grumpy) runtime (which would mean -changing the object model). This would give access to the C modules -that need to be ported and would give grumpy access to a compiler and -interpreter (gpython does support `eval` for instance). +@ncw started gpython it in 2013 and work on is sporadic. If you or someone +you know would be interested to take it futher, it would be much appreciated. + +## Getting Started -I (@ncw) haven't had much time to work on gpython (I started it in -2013 and have worked on it very sporadically) so someone who wants to -take it in the next direction would be much appreciated. +The [embedding example](https://github.com/go-python/gpython/tree/master/examples/embedding) demonstrates how to +easily embed and invoke gpython from any Go application. -## Limitations and Bugs +Importantly, gpython is able to run multiple interpreter instances simultaneously, +allowing you to embed gpython naturally into your Go application. This makes it +possible to use gpython in a server situation where complete interpreter +independence is an absolute requirement. See this in action in the [multi-context example](https://github.com/go-python/gpython/tree/master/examples/multi-context) + +If you are looking to get involved, a light and easy place to start is adding more convenience functions to [py/util.go](https://github.com/go-python/gpython/tree/master/py/util.go). See [notes.txt](https://github.com/go-python/gpython/blob/master/notes.txt) for bigger ideas. -Lots! -## Similar projects +## Other Projects of Interest * [grumpy](https://github.com/grumpyhome/grumpy) - a python to go transpiler @@ -86,5 +87,5 @@ or on the [Gophers Slack](https://gophers.slack.com/) in the `#go-python` channe ## License This is licensed under the MIT licence, however it contains code which -was ported fairly directly directly from the cpython source code under +was ported fairly directly directly from the CPython source code under the [PSF LICENSE](https://github.com/python/cpython/blob/master/LICENSE). diff --git a/ast/asdl_go.py b/ast/asdl_go.py old mode 100755 new mode 100644 diff --git a/ast/asttest.py b/ast/asttest.py old mode 100755 new mode 100644 diff --git a/builtin/builtin.go b/builtin/builtin.go index 8f4ef674..2d8d7a9b 100644 --- a/builtin/builtin.go +++ b/builtin/builtin.go @@ -162,7 +162,17 @@ func init() { "Warning": py.Warning, "ZeroDivisionError": py.ZeroDivisionError, } - py.NewModule("builtins", builtin_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "builtins", + Doc: builtin_doc, + Flags: py.ShareModule, + }, + Methods: methods, + Globals: globals, + }) + } const print_doc = `print(value, ..., sep=' ', end='\\n', file=sys.stdout, flush=False) @@ -178,18 +188,22 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje var ( sepObj py.Object = py.String(" ") endObj py.Object = py.String("\n") - file py.Object = py.MustGetModule("sys").Globals["stdout"] flush py.Object ) + sysModule, err := self.(*py.Module).Context.GetModule("sys") + if err != nil { + return nil, err + } + stdout := sysModule.Globals["stdout"] kwlist := []string{"sep", "end", "file", "flush"} - err := py.ParseTupleAndKeywords(nil, kwargs, "|ssOO:print", kwlist, &sepObj, &endObj, &file, &flush) + err = py.ParseTupleAndKeywords(nil, kwargs, "|ssOO:print", kwlist, &sepObj, &endObj, &stdout, &flush) if err != nil { return nil, err } sep := sepObj.(py.String) end := endObj.(py.String) - write, err := py.GetAttrString(file, "write") + write, err := py.GetAttrString(stdout, "write") if err != nil { return nil, err } @@ -219,7 +233,7 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje } if shouldFlush, _ := py.MakeBool(flush); shouldFlush == py.True { - fflush, err := py.GetAttrString(file, "flush") + fflush, err := py.GetAttrString(stdout, "flush") if err == nil { return py.Call(fflush, nil, nil) } @@ -449,7 +463,7 @@ func builtin___build_class__(self py.Object, args py.Tuple, kwargs py.StringDict } // fmt.Printf("Calling %v with %v and %v\n", fn.Name, fn.Globals, ns) // fmt.Printf("Code = %#v\n", fn.Code) - cell, err = py.VmRun(fn.Globals, ns, fn.Code, fn.Closure) + cell, err = fn.Context.RunCode(fn.Code, fn.Globals, ns, fn.Closure) if err != nil { return nil, err } @@ -749,9 +763,9 @@ func builtin_compile(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Ob return nil, py.ExceptionNewf(py.ValueError, "compile(): invalid optimize value") } - if dont_inherit.(py.Int) != 0 { - // PyEval_MergeCompilerFlags(&cf) - } + // if dont_inherit.(py.Int) != 0 { + // PyEval_MergeCompilerFlags(&cf) + // } // switch string(startstr.(py.String)) { // case "exec": @@ -782,7 +796,7 @@ func builtin_compile(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Ob return nil, err } // result = py.CompileStringExFlags(str, filename, start[mode], &cf, optimize) - result, err = compile.Compile(str, string(filename.(py.String)), string(startstr.(py.String)), int(supplied_flags.(py.Int)), dont_inherit.(py.Int) != 0) + result, err = compile.Compile(str, string(filename.(py.String)), py.CompileMode(startstr.(py.String)), int(supplied_flags.(py.Int)), dont_inherit.(py.Int) != 0) if err != nil { return nil, err } @@ -882,9 +896,8 @@ or ... etc. ` func isinstance(obj py.Object, classOrTuple py.Object) (py.Bool, error) { - switch classOrTuple.(type) { + switch class_tuple := classOrTuple.(type) { case py.Tuple: - var class_tuple = classOrTuple.(py.Tuple) for idx := range class_tuple { res, _ := isinstance(obj, class_tuple[idx]) if res { diff --git a/builtin/tests/builtin.py b/builtin/tests/builtin.py index e88db3a0..ae08e336 100644 --- a/builtin/tests/builtin.py +++ b/builtin/tests/builtin.py @@ -299,40 +299,43 @@ def gen2(): doc="print" ok = False try: - print("hello", sep=1) + print("hello", sep=1, end="!") except TypeError as e: - #if e.args[0] != "sep must be None or a string, not int": - # raise + if e.args[0] != "print() argument 1 must be str, not int": + raise ok = True assert ok, "TypeError not raised" try: - print("hello", sep=" ", end=1) + print("hello", sep=",", end=1) except TypeError as e: - #if e.args[0] != "end must be None or a string, not int": - # raise + if e.args[0] != "print() argument 2 must be str, not int": + raise ok = True assert ok, "TypeError not raised" try: - print("hello", sep=" ", end="\n", file=1) + print("hello", sep=",", end="!", file=1) except AttributeError as e: - #if e.args[0] != "'int' object has no attribute 'write'": - # raise + if e.args[0] != "'int' has no attribute 'write'": + raise ok = True assert ok, "AttributeError not raised" with open("testfile", "w") as f: - print("hello", "world", sep=" ", end="\n", file=f) + print("hello", "world", end="!\n", file=f, sep=", ") + print("hells", "bells", end="...", file=f) + print(" ~", "Brother ", "Foo", "bar", file=f, end="", sep="") with open("testfile", "r") as f: - assert f.read() == "hello world\n" + assert f.read() == "hello, world!\nhells bells... ~Brother Foobar" with open("testfile", "w") as f: - print(1,2,3,sep=",",end=",\n", file=f) + print(1,2,3,sep=",", flush=False, end=",\n", file=f) + print("4",5, file=f, end="!", flush=True, sep=",") with open("testfile", "r") as f: - assert f.read() == "1,2,3,\n" + assert f.read() == "1,2,3,\n4,5!" doc="round" assert round(1.1) == 1.0 diff --git a/compile/compile.go b/compile/compile.go index dc8851cf..775717a9 100644 --- a/compile/compile.go +++ b/compile/compile.go @@ -89,33 +89,36 @@ func init() { py.Compile = Compile } -// Compile(source, filename, mode, flags, dont_inherit) -> code object +// Compile(src, srcDesc, compileMode, flags, dont_inherit) -> code object // // Compile the source string (a Python module, statement or expression) -// into a code object that can be executed by exec() or eval(). -// The filename will be used for run-time error messages. -// The mode must be 'exec' to compile a module, 'single' to compile a -// single (interactive) statement, or 'eval' to compile an expression. +// into a code object that can be executed. +// +// srcDesc is used for run-time error messages and is typically a file system pathname, +// +// See py.CompileMode for compile mode options. +// // The flags argument, if present, controls which future statements influence // the compilation of the code. +// // The dont_inherit argument, if non-zero, stops the compilation inheriting // the effects of any future statements in effect in the code calling // compile; if absent or zero these statements do influence the compilation, // in addition to any features explicitly specified. -func Compile(str, filename, mode string, futureFlags int, dont_inherit bool) (py.Object, error) { +func Compile(src, srcDesc string, mode py.CompileMode, futureFlags int, dont_inherit bool) (*py.Code, error) { // Parse Ast - Ast, err := parser.ParseString(str, mode) + Ast, err := parser.ParseString(src, mode) if err != nil { return nil, err } // Make symbol table - SymTable, err := symtable.NewSymTable(Ast, filename) + SymTable, err := symtable.NewSymTable(Ast, srcDesc) if err != nil { return nil, err } c := newCompiler(nil, compilerScopeModule) - c.Filename = filename - err = c.compileAst(Ast, filename, futureFlags, dont_inherit, SymTable) + c.Filename = srcDesc + err = c.compileAst(Ast, srcDesc, futureFlags, dont_inherit, SymTable) if err != nil { return nil, err } @@ -1342,7 +1345,6 @@ func (c *compiler) NameOp(name string, ctx ast.ExprContext) { default: panic("NameOp: ctx invalid for name variable") } - break } if op == 0 { panic("NameOp: Op not set") diff --git a/compile/compile_data_test.go b/compile/compile_data_test.go index edc53720..f0ee0422 100644 --- a/compile/compile_data_test.go +++ b/compile/compile_data_test.go @@ -12,7 +12,7 @@ import ( var compileTestData = []struct { in string - mode string // exec, eval or single + mode py.CompileMode out *py.Code exceptionType *py.Type errString string diff --git a/compile/compile_test.go b/compile/compile_test.go index 72d4a3f6..aee3caac 100644 --- a/compile/compile_test.go +++ b/compile/compile_test.go @@ -163,7 +163,7 @@ func EqCode(t *testing.T, name string, a, b *py.Code) { func TestCompile(t *testing.T) { for _, test := range compileTestData { // log.Printf(">>> %s", test.in) - codeObj, err := Compile(test.in, "", test.mode, 0, true) + code, err := Compile(test.in, "", test.mode, 0, true) if err != nil { if test.exceptionType == nil { t.Errorf("%s: Got exception %v when not expecting one", test.in, err) @@ -196,17 +196,12 @@ func TestCompile(t *testing.T) { } } else { if test.out == nil { - if codeObj != nil { - t.Errorf("%s: Expecting nil *py.Code but got %T", test.in, codeObj) + if code != nil { + t.Errorf("%s: Expecting nil *py.Code but got %T", test.in, code) } } else { - code, ok := codeObj.(*py.Code) - if !ok { - t.Errorf("%s: Expecting *py.Code but got %T", test.in, codeObj) - } else { - //t.Logf("Testing %q", test.in) - EqCode(t, test.in, test.out, code) - } + //t.Logf("Testing %q", test.in) + EqCode(t, test.in, test.out, code) } } } diff --git a/compile/diffdis.py b/compile/diffdis.py old mode 100755 new mode 100644 diff --git a/compile/make_compile_test.py b/compile/make_compile_test.py old mode 100755 new mode 100644 diff --git a/examples/embedding/README.md b/examples/embedding/README.md new file mode 100644 index 00000000..15cf6f3d --- /dev/null +++ b/examples/embedding/README.md @@ -0,0 +1,102 @@ +## Embedding gpython + +This example demonstrates how embed gpython into a Go application. + + +### Why embed gpython? + +Embedding a highly capable and familiar "interpreted" language allows your users +to easily augment app behavior, configuration, and customization -- all post-deployment. + +Have you ever found an exciting software project but lose interest when you discover that you +also need to learn an esoteric language schema? In an era of limited attention span, +most people are generally turned off if they have to learn a new language in addition to learning +to use your app. + +If you consider [why use Python](https://www.stxnext.com/what-is-python-used-for/), then perhaps also +consider your users could immediately feel interested to hear that your software offers +an additional familiar dimension of value. + +Python is widespread in finance, sciences of all kinds, hobbyist programming and is often +endearingly regarded as most popular programming language for non-developers. +If your application can be driven by embedded Python, then chances are others will +feel excited and empowered that your project can be used out of the box +with positive feelings of being in familiar territory. + +### But what about the lack of python modules? + +There are only be a small number of native modules available, but don't forget you have the entire +Go standard library and *any* Go package you can name at your fingertips to expose! +This plus multi-context capability gives gpython enormous potential on how it can +serve your project. + +So basically, gpython is only off the table if you need to run python that makes heavy use of +modules that are only available in CPython. + +### Packing List + +| | | +|---------------------- | ------------------------------------------------------------------| +| `main.go` | if no args, runs in REPL mode, otherwise runs the given file | +| `lib/mylib.py` | models a library that your application would expose | +| `lib/REPL-startup.py` | invoked by `main.go` when starting REPL mode | +| `mylib-demo.py` | models a user-authored script that consumes `mylib` | +| `mylib.module.go` | Go implementation of `mylib_go` consumed by `mylib` | + + +### Invoking a Python Script + +```bash +$ cd examples/embedding/ +$ go build . +$ ./embedding mylib-demo.py +``` +``` +Welcome to a gpython embedded example, + where your wildest Go-based python dreams come true! + +========================================================== + Python 3.4 (github.com/go-python/gpython) + go1.17.6 on darwin amd64 +========================================================== + +Spring Break itinerary: + Stop 1: Miami, Florida | 7 nights + Stop 2: Mallorca, Spain | 3 nights + Stop 3: Ibiza, Spain | 14 nights + Stop 4: Monaco | 12 nights +### Made with Vacaton 1.0 by Fletch F. Fletcher + +I bet Monaco will be the best! +``` + +### REPL Mode + +```bash +$ ./embedding +``` +``` +======= Entering REPL mode, press Ctrl+D to exit ======= + +========================================================== + Python 3.4 (github.com/go-python/gpython) + go1.17.6 on darwin amd64 +========================================================== + +>>> v = Vacation("Spring Break", Stop("Florida", 3), Stop("Nice", 7)) +>>> print(str(v)) +Spring Break, 2 stop(s) +>>> v.PrintItinerary() +Spring Break itinerary: + Stop 1: Florida | 3 nights + Stop 2: Nice | 7 nights +### Made with Vacaton 1.0 by Fletch F. Fletcher +``` + +## Takeways + + - `main.go` demonstrates high-level convenience functions such as `py.RunFile()`. + - Embedding any Go `struct` only requires that it implements `py.Object`, which is a single function: + `Type() *py.Type` + - See [py/run.go](https://github.com/go-python/gpython/tree/master/py/run.go) for more about interpreter instances and `py.Context` + - There are many helper functions available for you in [py/util.go](https://github.com/go-python/gpython/tree/master/py/util.go) and your contributions are welcome! diff --git a/examples/embedding/lib/REPL-startup.py b/examples/embedding/lib/REPL-startup.py new file mode 100644 index 00000000..6b27c415 --- /dev/null +++ b/examples/embedding/lib/REPL-startup.py @@ -0,0 +1,8 @@ + + +# This file is called from main.go when in REPL mode + +# This is here to demonstrate making life easier for your users in REPL mode +# by doing pre-setup here so they don't have to import every time they start. +from mylib import * + diff --git a/examples/embedding/lib/mylib.py b/examples/embedding/lib/mylib.py new file mode 100644 index 00000000..4c279c78 --- /dev/null +++ b/examples/embedding/lib/mylib.py @@ -0,0 +1,53 @@ +import mylib_go as _go + +PY_VERSION = _go.PY_VERSION +GO_VERSION = _go.GO_VERSION + + +print(''' +========================================================== + %s + %s +========================================================== +''' % (PY_VERSION, GO_VERSION)) + + +def Stop(location, num_nights = 2): + return _go.VacationStop_new(location, num_nights) + + +class Vacation: + + def __init__(self, tripName, *stops): + self._v, self._libVers = _go.Vacation_new() + self.tripName = tripName + self.AddStops(*stops) + + def __str__(self): + return "%s, %d stop(s)" % (self.tripName, self.NumStops()) + + def NumStops(self): + return self._v.num_stops() + + def GetStop(self, stop_num): + return self._v.get_stop(stop_num) + + def AddStops(self, *stops): + self._v.add_stops(stops) + + def PrintItinerary(self): + print(self.tripName, "itinerary:") + i = 1 + while 1: + + try: + stop = self.GetStop(i) + except IndexError: + break + + print(" Stop %d: %s" % (i, str(stop))) + i += 1 + + print("### Made with %s " % self._libVers) + + \ No newline at end of file diff --git a/examples/embedding/main.go b/examples/embedding/main.go new file mode 100644 index 00000000..faa5120e --- /dev/null +++ b/examples/embedding/main.go @@ -0,0 +1,51 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + + // This initializes gpython for runtime execution and is essential. + // It defines forward-declared symbols and registers native built-in modules, such as sys and time. + _ "github.com/go-python/gpython/modules" + + // Commonly consumed gpython + "github.com/go-python/gpython/py" + "github.com/go-python/gpython/repl" + "github.com/go-python/gpython/repl/cli" +) + +func main() { + flag.Parse() + runWithFile(flag.Arg(0)) +} + +func runWithFile(pyFile string) error { + + // See type Context interface and related docs + ctx := py.NewContext(py.DefaultContextOpts()) + + var err error + if len(pyFile) == 0 { + replCtx := repl.New(ctx) + + fmt.Print("\n======= Entering REPL mode, press Ctrl+D to exit =======\n") + + _, err = py.RunFile(ctx, "lib/REPL-startup.py", py.CompileOpts{}, replCtx.Module) + if err == nil { + cli.RunREPL(replCtx) + } + + } else { + _, err = py.RunFile(ctx, pyFile, py.CompileOpts{}, nil) + } + + if err != nil { + py.TracebackDump(err) + } + + return err +} diff --git a/examples/embedding/mylib-demo.py b/examples/embedding/mylib-demo.py new file mode 100644 index 00000000..4a461023 --- /dev/null +++ b/examples/embedding/mylib-demo.py @@ -0,0 +1,15 @@ +print(''' +Welcome to a gpython embedded example, + where your wildest Go-based python dreams come true!''') + +# This is a model for a public/user-side script that you or users would maintain, +# offering an open canvas to drive app behavior, customization, or anything you can dream up. +# +# Modules you offer for consumption can also serve to document such things. +from mylib import * + +springBreak = Vacation("Spring Break", Stop("Miami, Florida", 7), Stop("Mallorca, Spain", 3)) +springBreak.AddStops(Stop("Ibiza, Spain", 14), Stop("Monaco", 12)) +springBreak.PrintItinerary() + +print("\nI bet %s will be the best!\n" % springBreak.GetStop(4).Get()[0]) diff --git a/examples/embedding/mylib.module.go b/examples/embedding/mylib.module.go new file mode 100644 index 00000000..b510277c --- /dev/null +++ b/examples/embedding/mylib.module.go @@ -0,0 +1,175 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "runtime" + + "github.com/go-python/gpython/py" +) + +// These gpython py.Object type delcarations are the bridge between gpython and embedded Go types. +var ( + PyVacationStopType = py.NewType("Stop", "") + PyVacationType = py.NewType("Vacation", "") +) + +// init is where you register your embedded module and attach methods to your embedded class types. +func init() { + + // For each of your embedded python types, attach instance methods. + // When an instance method is invoked, the "self" py.Object is the instance. + PyVacationStopType.Dict["Set"] = py.MustNewMethod("Set", VacationStop_Set, 0, "") + PyVacationStopType.Dict["Get"] = py.MustNewMethod("Get", VacationStop_Get, 0, "") + PyVacationType.Dict["add_stops"] = py.MustNewMethod("Vacation.add_stops", Vacation_add_stops, 0, "") + PyVacationType.Dict["num_stops"] = py.MustNewMethod("Vacation.num_stops", Vacation_num_stops, 0, "") + PyVacationType.Dict["get_stop"] = py.MustNewMethod("Vacation.get_stop", Vacation_get_stop, 0, "") + + // Bind methods attached at the module (global) level. + // When these are invoked, the first py.Object param (typically "self") is the bound *Module instance. + methods := []*py.Method{ + py.MustNewMethod("VacationStop_new", VacationStop_new, 0, ""), + py.MustNewMethod("Vacation_new", Vacation_new, 0, ""), + } + + // Register a ModuleImpl instance used by the gpython runtime to instantiate new py.Module when first imported. + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "mylib_go", + Doc: "Example embedded python module", + }, + Methods: methods, + Globals: py.StringDict{ + "PY_VERSION": py.String("Python 3.4 (github.com/go-python/gpython)"), + "GO_VERSION": py.String(fmt.Sprintf("%s on %s %s", runtime.Version(), runtime.GOOS, runtime.GOARCH)), + "MYLIB_VERS": py.String("Vacaton 1.0 by Fletch F. Fletcher"), + }, + }) +} + +// VacationStop is an example Go struct to embed. +type VacationStop struct { + Desc py.String + NumNights py.Int +} + +// Type comprises the py.Object interface, allowing a Go struct to be cast as a py.Object. +// Instance methods of an type are then attached to this type object +func (stop *VacationStop) Type() *py.Type { + return PyVacationStopType +} + +func (stop *VacationStop) M__str__() (py.Object, error) { + line := fmt.Sprintf(" %-16v | %2v nights", stop.Desc, stop.NumNights) + return py.String(line), nil +} + +func (stop *VacationStop) M__repr__() (py.Object, error) { + return stop.M__str__() +} + +func VacationStop_new(module py.Object, args py.Tuple) (py.Object, error) { + stop := &VacationStop{} + VacationStop_Set(stop, args) + return stop, nil +} + +// VacationStop_Set is an embedded instance method of VacationStop +func VacationStop_Set(self py.Object, args py.Tuple) (py.Object, error) { + stop := self.(*VacationStop) + + // Check out other convenience functions in py/util.go + // Also available is py.ParseTuple(args, "si", ...) + err := py.LoadTuple(args, []interface{}{&stop.Desc, &stop.NumNights}) + if err != nil { + return nil, err + } + + /* Alternative util func is ParseTuple(): + var desc, nights py.Object + err := py.ParseTuple(args, "si", &desc, &nights) + if err != nil { + return nil, err + } + stop.Desc = desc.(py.String) + stop.NumNights = desc.(py.Int) + */ + + return py.None, nil +} + +// VacationStop_Get is an embedded instance method of VacationStop +func VacationStop_Get(self py.Object, args py.Tuple) (py.Object, error) { + stop := self.(*VacationStop) + + return py.Tuple{ + stop.Desc, + stop.NumNights, + }, nil +} + +type Vacation struct { + Stops []*VacationStop + MadeBy string +} + +func (v *Vacation) Type() *py.Type { + return PyVacationType +} + +func Vacation_new(module py.Object, args py.Tuple) (py.Object, error) { + v := &Vacation{} + + // For Module-bound methods, we have easy access to the parent Module + py.LoadAttr(module, "MYLIB_VERS", &v.MadeBy) + + ret := py.Tuple{ + v, + py.String(v.MadeBy), + } + return ret, nil +} + +func Vacation_num_stops(self py.Object, args py.Tuple) (py.Object, error) { + v := self.(*Vacation) + return py.Int(len(v.Stops)), nil +} + +func Vacation_get_stop(self py.Object, args py.Tuple) (py.Object, error) { + v := self.(*Vacation) + + // Check out other convenience functions in py/util.go + // If you would like to be a contributor for gpython, improving these or adding more is a great place to start! + stopNum, err := py.GetInt(args[0]) + if err != nil { + return nil, err + } + + if stopNum < 1 || int(stopNum) > len(v.Stops) { + return nil, py.ExceptionNewf(py.IndexError, "invalid stop index") + } + + return py.Object(v.Stops[stopNum-1]), nil +} + +func Vacation_add_stops(self py.Object, args py.Tuple) (py.Object, error) { + v := self.(*Vacation) + srcStops, ok := args[0].(py.Tuple) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "expected Tuple, got %T", args[0]) + } + + for _, arg := range srcStops { + stop, ok := arg.(*VacationStop) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "expected Stop, got %T", arg) + } + + v.Stops = append(v.Stops, stop) + } + + return py.None, nil +} diff --git a/examples/multi-context/main.go b/examples/multi-context/main.go new file mode 100644 index 00000000..e9f72212 --- /dev/null +++ b/examples/multi-context/main.go @@ -0,0 +1,138 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "log" + "runtime" + "strings" + "sync" + "time" + + // This initializes gpython for runtime execution and is critical. + // It defines forward-declared symbols and registers native built-in modules, such as sys and time. + _ "github.com/go-python/gpython/modules" + + // This is the primary import for gpython. + // It contains all symbols needed to fully compile and run python. + "github.com/go-python/gpython/py" +) + +func main() { + + // The total job count implies a fixed amount of work. + // The number of workers is how many py.Context (in concurrent goroutines) to pull jobs off the queue. + // One worker does all the work serially while N number of workers will (ideally) divides up. + totalJobs := 20 + + for i := 0; i < 10; i++ { + numWorkers := i + 1 + elapsed := RunMultiPi(numWorkers, totalJobs) + fmt.Printf("=====> %2d worker(s): %v\n\n", numWorkers, elapsed) + + // Give each trial a fresh start + runtime.GC() + } + +} + +var jobScript = ` +pi = chud.pi_chudnovsky_bs(numDigits) +last_5 = pi % 100000 +print("%s: last 5 digits of %d is %d (job #%0d)" % (WORKER_ID, numDigits, last_5, jobID)) +` + +var jobSrcTemplate = ` +import pi_chudnovsky_bs as chud + +WORKER_ID = "{{WORKER_ID}}" + +print("%s ready!" % (WORKER_ID)) +` + +type worker struct { + name string + ctx py.Context + main *py.Module + job *py.Code +} + +func (w *worker) compileTemplate(pySrc string) { + pySrc = strings.Replace(pySrc, "{{WORKER_ID}}", w.name, -1) + + mainImpl := py.ModuleImpl{ + CodeSrc: pySrc, + } + + var err error + w.main, err = w.ctx.ModuleInit(&mainImpl) + if err != nil { + log.Fatal(err) + } +} + +func RunMultiPi(numWorkers, numTimes int) time.Duration { + var workersRunning sync.WaitGroup + + fmt.Printf("Starting %d worker(s) to calculate %d jobs...\n", numWorkers, numTimes) + + jobPipe := make(chan int) + go func() { + for i := 0; i < numTimes; i++ { + jobPipe <- i + 1 + } + close(jobPipe) + }() + + // Note that py.Code can be shared (accessed concurrently) since it is an inherently read-only object + jobCode, err := py.Compile(jobScript, "", py.ExecMode, 0, true) + if err != nil { + log.Fatal("jobScript failed to comple") + } + + workers := make([]worker, numWorkers) + for i := 0; i < numWorkers; i++ { + + opts := py.DefaultContextOpts() + + // Make sure our import statement will find pi_chudnovsky_bs + opts.SysPaths = append(opts.SysPaths, "..") + + workers[i] = worker{ + name: fmt.Sprintf("Worker #%d", i+1), + ctx: py.NewContext(opts), + job: jobCode, + } + + workersRunning.Add(1) + } + + startTime := time.Now() + + for i := range workers { + w := workers[i] + go func() { + + // Compiling can be concurrent since there is no associated py.Context + w.compileTemplate(jobSrcTemplate) + + for jobID := range jobPipe { + numDigits := 100000 + if jobID%2 == 0 { + numDigits *= 10 + } + py.SetAttrString(w.main.Globals, "numDigits", py.Int(numDigits)) + py.SetAttrString(w.main.Globals, "jobID", py.Int(jobID)) + w.ctx.RunCode(jobCode, w.main.Globals, w.main.Globals, nil) + } + workersRunning.Done() + }() + } + + workersRunning.Wait() + + return time.Since(startTime) +} diff --git a/examples/pystone.py b/examples/pystone.py old mode 100755 new mode 100644 diff --git a/importlib/importlib.go b/importlib/importlib.go index e1d307db..b70d330d 100644 --- a/importlib/importlib.go +++ b/importlib/importlib.go @@ -7,15 +7,18 @@ package py import ( - "log" - - "github.com/go-python/gpython/marshal" + "github.com/go-python/gpython/py" ) // Load the frozen module func init() { - _, err := marshal.LoadFrozenModule("importlib", data) - log.Fatalf("Failed to load importlib: %v", err) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "importlib", + }, + CodeBuf: data, + }) } // Auto-generated by Modules/_freeze_importlib.c diff --git a/main.go b/main.go index 8148056b..9743bc0f 100644 --- a/main.go +++ b/main.go @@ -12,22 +12,13 @@ import ( "runtime" "runtime/pprof" - _ "github.com/go-python/gpython/builtin" + "github.com/go-python/gpython/repl" "github.com/go-python/gpython/repl/cli" - //_ "github.com/go-python/gpython/importlib" - "io/ioutil" "log" "os" - "strings" - "github.com/go-python/gpython/compile" - "github.com/go-python/gpython/marshal" - _ "github.com/go-python/gpython/math" "github.com/go-python/gpython/py" - pysys "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" - "github.com/go-python/gpython/vm" ) // Globals @@ -48,33 +39,14 @@ Full options: flag.PrintDefaults() } -// Exit with the message -func fatal(message string, args ...interface{}) { - if !strings.HasSuffix(message, "\n") { - message += "\n" - } - syntaxError() - fmt.Fprintf(os.Stderr, message, args...) - os.Exit(1) -} - func main() { flag.Usage = syntaxError flag.Parse() args := flag.Args() - py.MustGetModule("sys").Globals["argv"] = pysys.MakeArgv(args) - if len(args) == 0 { - - fmt.Printf("Python 3.4.0 (%s, %s)\n", commit, date) - fmt.Printf("[Gpython %s]\n", version) - fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) - fmt.Printf("- go version: %s\n", runtime.Version()) - cli.RunREPL() - return - } - prog := args[0] - // fmt.Printf("Running %q\n", prog) + opts := py.DefaultContextOpts() + opts.SysArgs = flag.Args() + ctx := py.NewContext(opts) if *cpuprofile != "" { f, err := os.Create(*cpuprofile) @@ -88,41 +60,23 @@ func main() { defer pprof.StopCPUProfile() } - // FIXME should be using ImportModuleLevelObject() here - f, err := os.Open(prog) - if err != nil { - log.Fatalf("Failed to open %q: %v", prog, err) - } - var obj py.Object - if strings.HasSuffix(prog, ".pyc") { - obj, err = marshal.ReadPyc(f) - if err != nil { - log.Fatalf("Failed to marshal %q: %v", prog, err) - } - } else if strings.HasSuffix(prog, ".py") { - str, err := ioutil.ReadAll(f) - if err != nil { - log.Fatalf("Failed to read %q: %v", prog, err) - } - obj, err = compile.Compile(string(str), prog, "exec", 0, true) + // IF no args, enter REPL mode + if len(args) == 0 { + + fmt.Printf("Python 3.4.0 (%s, %s)\n", commit, date) + fmt.Printf("[Gpython %s]\n", version) + fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) + fmt.Printf("- go version: %s\n", runtime.Version()) + + replCtx := repl.New(ctx) + cli.RunREPL(replCtx) + + } else { + _, err := py.RunFile(ctx, args[0], py.CompileOpts{}, nil) if err != nil { - log.Fatalf("Can't compile %q: %v", prog, err) + py.TracebackDump(err) + log.Fatal(err) } - } else { - log.Fatalf("Can't execute %q", prog) - } - if err = f.Close(); err != nil { - log.Fatalf("Failed to close %q: %v", prog, err) - } - code := obj.(*py.Code) - module := py.NewModule("__main__", "", nil, nil) - module.Globals["__file__"] = py.String(prog) - res, err := vm.Run(module.Globals, module.Globals, code, nil) - if err != nil { - py.TracebackDump(err) - log.Fatal(err) } - // fmt.Printf("Return = %v\n", res) - _ = res } diff --git a/marshal/marshal.go b/marshal/marshal.go index dd039d5c..937008cc 100644 --- a/marshal/marshal.go +++ b/marshal/marshal.go @@ -6,7 +6,6 @@ package marshal import ( - "bytes" "encoding/binary" "errors" "fmt" @@ -15,7 +14,6 @@ import ( "strconv" "github.com/go-python/gpython/py" - "github.com/go-python/gpython/vm" ) const ( @@ -448,29 +446,12 @@ func ReadPyc(r io.Reader) (obj py.Object, err error) { } // FIXME do something with timestamp & length? if header.Magic>>16 != 0x0a0d { - return nil, errors.New("Bad magic in .pyc file") + return nil, errors.New("bad magic in .pyc file") } // fmt.Printf("header = %v\n", header) return ReadObject(r) } -// Unmarshals a frozen module -func LoadFrozenModule(name string, data []byte) (*py.Module, error) { - r := bytes.NewBuffer(data) - obj, err := ReadObject(r) - if err != nil { - return nil, err - } - code := obj.(*py.Code) - module := py.NewModule(name, "", nil, nil) - _, err = vm.Run(module.Globals, module.Globals, code, nil) - if err != nil { - py.TracebackDump(err) - return nil, err - } - return module, nil -} - const dump_doc = `dump(value, file[, version]) Write the value on the open file. The value must be a supported type. @@ -634,5 +615,14 @@ func init() { globals := py.StringDict{ "version": py.Int(MARSHAL_VERSION), } - py.NewModule("marshal", module_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "marshal", + Doc: module_doc, + Flags: py.ShareModule, + }, + Globals: globals, + Methods: methods, + }) } diff --git a/math/math.go b/math/math.go index 88d60fb3..0e6790fd 100644 --- a/math/math.go +++ b/math/math.go @@ -1333,9 +1333,18 @@ func init() { py.MustNewMethod("trunc", math_trunc, 0, math_trunc_doc), py.MustNewMethod("to_ulps", math_to_ulps, 0, math_to_ulps_doc), } - globals := py.StringDict{ - "pi": py.Float(math.Pi), - "e": py.Float(math.E), - } - py.NewModule("math", math_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "math", + Doc: math_doc, + Flags: py.ShareModule, + }, + Methods: methods, + Globals: py.StringDict{ + "pi": py.Float(math.Pi), + "e": py.Float(math.E), + }, + }) + } diff --git a/modules/runtime.go b/modules/runtime.go new file mode 100644 index 00000000..d26632a6 --- /dev/null +++ b/modules/runtime.go @@ -0,0 +1,284 @@ +// Copyright 2021 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modules + +import ( + "bytes" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "github.com/go-python/gpython/marshal" + "github.com/go-python/gpython/py" + "github.com/go-python/gpython/vm" + + _ "github.com/go-python/gpython/builtin" + _ "github.com/go-python/gpython/math" + _ "github.com/go-python/gpython/sys" + _ "github.com/go-python/gpython/time" +) + +func init() { + // Assign the base-level py.Context creation function while also preventing an import cycle. + py.NewContext = NewContext +} + +// context implements py.Context +type context struct { + store *py.ModuleStore + opts py.ContextOpts + closeOnce sync.Once + closing bool + closed bool + running sync.WaitGroup + done chan struct{} +} + +// See py.Context interface +func NewContext(opts py.ContextOpts) py.Context { + ctx := &context{ + opts: opts, + done: make(chan struct{}), + closing: false, + closed: false, + } + + ctx.store = py.NewModuleStore() + + py.Import(ctx, "builtins", "sys") + + sys_mod := ctx.Store().MustGetModule("sys") + sys_mod.Globals["argv"] = py.NewListFromStrings(opts.SysArgs) + sys_mod.Globals["path"] = py.NewListFromStrings(opts.SysPaths) + + return ctx +} + +func (ctx *context) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { + err := ctx.pushBusy() + defer ctx.popBusy() + if err != nil { + return nil, err + } + + if impl.Code == nil && len(impl.CodeSrc) > 0 { + impl.Code, err = py.Compile(string(impl.CodeSrc), impl.Info.FileDesc, py.ExecMode, 0, true) + if err != nil { + return nil, err + } + } + + if impl.Code == nil && len(impl.CodeBuf) > 0 { + codeBuf := bytes.NewBuffer(impl.CodeBuf) + obj, err := marshal.ReadObject(codeBuf) + if err != nil { + return nil, err + } + impl.Code, _ = obj.(*py.Code) + if impl.Code == nil { + return nil, py.ExceptionNewf(py.AssertionError, "Embedded code did not produce a py.Code object") + } + } + + module, err := ctx.Store().NewModule(ctx, impl) + if err != nil { + return nil, err + } + + if impl.Code != nil { + _, err = ctx.RunCode(impl.Code, module.Globals, module.Globals, nil) + if err != nil { + return nil, err + } + } + + return module, nil +} + +func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.CompileOut, error) { + err := ctx.pushBusy() + defer ctx.popBusy() + if err != nil { + return py.CompileOut{}, err + } + + tryPaths := defaultPaths + if opts.UseSysPaths { + tryPaths = ctx.Store().MustGetModule("sys").Globals["path"].(*py.List).Items + } + + out := py.CompileOut{} + + err = resolveRunPath(pathname, opts, tryPaths, func(fpath string) (bool, error) { + + stat, err := os.Stat(fpath) + if err == nil && stat.IsDir() { + // FIXME this is a massive simplification! + fpath = path.Join(fpath, "__init__.py") + _, err = os.Stat(fpath) + } + + ext := strings.ToLower(filepath.Ext(fpath)) + if ext == "" && os.IsNotExist(err) { + fpath += ".py" + ext = ".py" + _, err = os.Stat(fpath) + } + + // Keep searching while we get FNFs, stop on an error + if err != nil { + if os.IsNotExist(err) { + return true, nil + } + err = py.ExceptionNewf(py.OSError, "Error accessing %q: %v", fpath, err) + return false, err + } + + switch ext { + case ".py": + var pySrc []byte + pySrc, err = ioutil.ReadFile(fpath) + if err != nil { + return false, py.ExceptionNewf(py.OSError, "Error reading %q: %v", fpath, err) + } + + out.Code, err = py.Compile(string(pySrc), fpath, py.ExecMode, 0, true) + if err != nil { + return false, err + } + out.SrcPathname = fpath + case ".pyc": + file, err := os.Open(fpath) + if err != nil { + return false, py.ExceptionNewf(py.OSError, "Error opening %q: %v", fpath, err) + } + defer file.Close() + codeObj, err := marshal.ReadPyc(file) + if err != nil { + return false, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %v", fpath, err) + } + out.Code, _ = codeObj.(*py.Code) + out.PycPathname = fpath + } + + out.FileDesc = fpath + return false, nil + }) + + if out.Code == nil && err == nil { + err = py.ExceptionNewf(py.AssertionError, "Missing code object") + } + + if err != nil { + return py.CompileOut{}, err + } + + return out, nil +} + +func (ctx *context) pushBusy() error { + if ctx.closed { + return py.ExceptionNewf(py.RuntimeError, "Context closed") + } + ctx.running.Add(1) + return nil +} + +func (ctx *context) popBusy() { + ctx.running.Done() +} + +// Close -- see type py.Context +func (ctx *context) Close() error { + ctx.closeOnce.Do(func() { + ctx.closing = true + ctx.running.Wait() + ctx.closed = true + + // Give each module a chance to release resources + ctx.store.OnContextClosed() + close(ctx.done) + }) + return nil +} + +// Done -- see type py.Context +func (ctx *context) Done() <-chan struct{} { + return ctx.done +} + +var defaultPaths = []py.Object{ + py.String("."), +} + +func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, tryPath func(pyPath string) (bool, error)) error { + runPath = strings.TrimSuffix(runPath, "/") + + var ( + err error + cwd string + cont = true + ) + + for _, pathObj := range pathObjs { + pathStr, ok := pathObj.(py.String) + if !ok { + continue + } + + // If an absolute path, just try that. + // Otherwise, check from the passed current dir then check from the current working dir. + fpath := path.Join(string(pathStr), runPath) + if filepath.IsAbs(fpath) { + cont, err = tryPath(fpath) + } else { + if len(opts.CurDir) > 0 { + subPath := path.Join(opts.CurDir, fpath) + cont, err = tryPath(subPath) + } + if cont && err == nil { + if cwd == "" { + cwd, _ = os.Getwd() + } + subPath := path.Join(cwd, fpath) + cont, err = tryPath(subPath) + } + } + if !cont { + break + } + } + + if err != nil { + return err + } + + if cont { + return py.ExceptionNewf(py.FileNotFoundError, "Failed to resolve %q", runPath) + } + + return err +} + +func (ctx *context) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { + err := ctx.pushBusy() + defer ctx.popBusy() + if err != nil { + return nil, err + } + + return vm.EvalCode(ctx, code, globals, locals, nil, nil, nil, nil, closure) +} + +func (ctx *context) GetModule(moduleName string) (*py.Module, error) { + return ctx.store.GetModule(moduleName) +} + +func (ctx *context) Store() *py.ModuleStore { + return ctx.store +} diff --git a/parser/grammar_test.go b/parser/grammar_test.go index d3d48a50..a99beb95 100644 --- a/parser/grammar_test.go +++ b/parser/grammar_test.go @@ -21,7 +21,7 @@ var debugLevel = flag.Int("debugLevel", 0, "Debug level 0-4") func TestGrammar(t *testing.T) { SetDebug(*debugLevel) for _, test := range grammarTestData { - Ast, err := ParseString(test.in, test.mode) + Ast, err := ParseString(test.in, py.CompileMode(test.mode)) if err != nil { if test.exceptionType == nil { t.Errorf("%s: Got exception %v when not expecting one", test.in, err) diff --git a/parser/lexer.go b/parser/lexer.go index 6f9f1726..76847893 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -62,7 +62,7 @@ type yyLex struct { // can be 'exec' if source consists of a sequence of statements, // 'eval' if it consists of a single expression, or 'single' if it // consists of a single interactive statement -func NewLex(r io.Reader, filename string, mode string) (*yyLex, error) { +func NewLex(r io.Reader, filename string, mode py.CompileMode) (*yyLex, error) { x := &yyLex{ reader: bufio.NewReader(r), filename: filename, @@ -70,12 +70,12 @@ func NewLex(r io.Reader, filename string, mode string) (*yyLex, error) { state: readString, } switch mode { - case "exec": + case py.ExecMode: x.queue(FILE_INPUT) x.exec = true - case "eval": + case py.EvalMode: x.queue(EVAL_INPUT) - case "single": + case py.SingleMode: x.queue(SINGLE_INPUT) x.interactive = true default: @@ -933,7 +933,7 @@ func SetDebug(level int) { } // Parse a file -func Parse(in io.Reader, filename string, mode string) (mod ast.Mod, err error) { +func Parse(in io.Reader, filename string, mode py.CompileMode) (mod ast.Mod, err error) { lex, err := NewLex(in, filename, mode) if err != nil { return nil, err @@ -952,12 +952,12 @@ func Parse(in io.Reader, filename string, mode string) (mod ast.Mod, err error) } // Parse a string -func ParseString(in string, mode string) (ast.Ast, error) { +func ParseString(in string, mode py.CompileMode) (ast.Ast, error) { return Parse(bytes.NewBufferString(in), "", mode) } // Lex a file only, returning a sequence of tokens -func Lex(in io.Reader, filename string, mode string) (lts LexTokens, err error) { +func Lex(in io.Reader, filename string, mode py.CompileMode) (lts LexTokens, err error) { lex, err := NewLex(in, filename, mode) if err != nil { return nil, err @@ -984,6 +984,6 @@ func Lex(in io.Reader, filename string, mode string) (lts LexTokens, err error) } // Lex a string -func LexString(in string, mode string) (lts LexTokens, err error) { +func LexString(in string, mode py.CompileMode) (lts LexTokens, err error) { return Lex(bytes.NewBufferString(in), "", mode) } diff --git a/parser/lexer_test.go b/parser/lexer_test.go index 18e20e5a..ab252088 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -196,7 +196,7 @@ func TestLex(t *testing.T) { for _, test := range []struct { in string errString string - mode string + mode py.CompileMode lts LexTokens }{ {"", "", "exec", LexTokens{ diff --git a/py/args.go b/py/args.go index f38fdfde..5209a3d3 100644 --- a/py/args.go +++ b/py/args.go @@ -368,9 +368,7 @@ // $ // // PyArg_ParseTupleAndKeywords() only: Indicates that the remaining -// arguments in the Python argument list are keyword-only. Currently, -// all keyword-only arguments must also be optional arguments, so | -// must always be specified before $ in the format string. +// arguments in the Python argument list are keyword-only. // // New in version 3.3. // @@ -416,17 +414,13 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist if kwlist != nil && len(results) != len(kwlist) { return ExceptionNewf(TypeError, "Internal error: supply the same number of results and kwlist") } - min, max, name, ops := parseFormat(format) - keywordOnly := false - err := checkNumberOfArgs(name, len(args)+len(kwargs), len(results), min, max) + var opsBuf [16]formatOp + min, name, kwOnly_i, ops := parseFormat(format, opsBuf[:0]) + err := checkNumberOfArgs(name, len(args)+len(kwargs), len(results), min, len(ops)) if err != nil { return err } - if len(ops) > 0 && ops[0] == "$" { - keywordOnly = true - ops = ops[1:] - } // Check all the kwargs are in kwlist // O(N^2) Slow but kwlist is usually short for kwargName := range kwargs { @@ -439,46 +433,60 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist found: } - // Create args tuple with all the arguments we have in - args = args.Copy() - for i, kw := range kwlist { - if value, ok := kwargs[kw]; ok { - if len(args) > i { + // Walk through all the results we want + for i, op := range ops { + + var ( + arg Object + kw string + ) + if i < len(kwlist) { + kw = kwlist[i] + arg = kwargs[kw] + } + + // Consume ordered args first -- they should not require keyword only or also be specified via keyword + if i < len(args) { + if i >= kwOnly_i { + return ExceptionNewf(TypeError, "%s() specifies argument '%s' that is keyword only", name, kw) + } + if arg != nil { return ExceptionNewf(TypeError, "%s() got multiple values for argument '%s'", name, kw) } - args = append(args, value) - } else if keywordOnly { - args = append(args, nil) + arg = args[i] } - } - for i, arg := range args { - op := ops[i] + + // Unspecified args retain their default value + if arg == nil { + continue + } + result := results[i] - switch op { - case "O": + switch op.code { + case 'O': *result = arg - case "Z", "z": + case 'Z', 'z': if _, ok := arg.(NoneType); ok { *result = arg break } fallthrough - case "U", "s": + case 'U', 's': if _, ok := arg.(String); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be str, not %s", name, i+1, arg.Type().Name) } *result = arg - case "i": + case 'i': if _, ok := arg.(Int); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be int, not %s", name, i+1, arg.Type().Name) } *result = arg - case "p": + case 'p': if _, ok := arg.(Bool); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be bool, not %s", name, i+1, arg.Type().Name) } *result = arg - case "d": + case 'd': switch x := arg.(type) { case Int: *result = Float(x) @@ -500,30 +508,42 @@ func ParseTuple(args Tuple, format string, results ...*Object) error { return ParseTupleAndKeywords(args, nil, format, nil, results...) } +type formatOp struct { + code byte + modifier byte +} + // Parse the format -func parseFormat(format string) (min, max int, name string, ops []string) { +func parseFormat(format string, in []formatOp) (min int, name string, kwOnly_i int, ops []formatOp) { name = "function" min = -1 - for format != "" { - op := string(format[0]) - format = format[1:] - if len(format) > 1 && (format[1] == '*' || format[1] == '#') { - op += string(format[0]) - format = format[1:] + kwOnly_i = 0xFFFF + ops = in[:0] + + N := len(format) + for i := 0; i < N; { + op := formatOp{code: format[i]} + i++ + if i < N { + if mod := format[i]; mod == '*' || mod == '#' { + op.modifier = mod + i++ + } } - switch op { - case ":", ";": - name = format - format = "" - case "|": + switch op.code { + case ':', ';': + name = format[i:] + i = N + case '$': + kwOnly_i = len(ops) + case '|': min = len(ops) default: ops = append(ops, op) } } - max = len(ops) if min < 0 { - min = max + min = len(ops) } return } diff --git a/py/bytes.go b/py/bytes.go index b9ae3abe..2c653455 100644 --- a/py/bytes.go +++ b/py/bytes.go @@ -214,14 +214,14 @@ func (a Bytes) M__le__(other Object) (Object, error) { func (a Bytes) M__eq__(other Object) (Object, error) { if b, ok := convertToBytes(other); ok { - return NewBool(bytes.Compare(a, b) == 0), nil + return NewBool(bytes.Equal(a, b)), nil } return NotImplemented, nil } func (a Bytes) M__ne__(other Object) (Object, error) { if b, ok := convertToBytes(other); ok { - return NewBool(bytes.Compare(a, b) != 0), nil + return NewBool(!bytes.Equal(a, b)), nil } return NotImplemented, nil } diff --git a/py/code.go b/py/code.go index 355e88e0..09027497 100644 --- a/py/code.go +++ b/py/code.go @@ -97,7 +97,7 @@ const NAME_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvw // all_name_chars(s): true iff all chars in s are valid NAME_CHARS func all_name_chars(s String) bool { for _, c := range s { - if strings.IndexRune(NAME_CHARS, c) < 0 { + if !strings.ContainsRune(NAME_CHARS, c) { return false } } diff --git a/py/dict.go b/py/dict.go index d3df854c..4f277c47 100644 --- a/py/dict.go +++ b/py/dict.go @@ -217,3 +217,9 @@ func (a StringDict) M__contains__(other Object) (Object, error) { } return False, nil } + +func (d StringDict) GetDict() StringDict { + return d +} + +var _ IGetDict = (*StringDict)(nil) diff --git a/py/float.go b/py/float.go index 4f47759b..66d1f78f 100644 --- a/py/float.go +++ b/py/float.go @@ -309,6 +309,10 @@ func (a Float) M__bool__() (Object, error) { return NewBool(a != 0), nil } +func (a Float) M__index__() (Int, error) { + return Int(a), nil +} + func (a Float) M__int__() (Object, error) { if a >= IntMin && a <= IntMax { return Int(a), nil @@ -413,3 +417,4 @@ var _ floatArithmetic = Float(0) var _ conversionBetweenTypes = Float(0) var _ I__bool__ = Float(0) var _ richComparison = Float(0) +var _ I__index__ = Float(0) diff --git a/py/frame.go b/py/frame.go index f91438a0..ce595d8a 100644 --- a/py/frame.go +++ b/py/frame.go @@ -26,6 +26,7 @@ type TryBlock struct { // A python Frame object type Frame struct { // Back *Frame // previous frame, or nil + Context Context // host module (state) context Code *Code // code segment Builtins StringDict // builtin symbol table Globals StringDict // global symbol table @@ -77,7 +78,7 @@ func (o *Frame) Type() *Type { } // Make a new frame for a code object -func NewFrame(globals, locals StringDict, code *Code, closure Tuple) *Frame { +func NewFrame(ctx Context, globals, locals StringDict, code *Code, closure Tuple) *Frame { nlocals := int(code.Nlocals) ncells := len(code.Cellvars) nfrees := len(code.Freevars) @@ -90,12 +91,13 @@ func NewFrame(globals, locals StringDict, code *Code, closure Tuple) *Frame { cellAndFreeVars := allocation[nlocals:varsize] return &Frame{ + Context: ctx, Globals: globals, Locals: locals, Code: code, LocalVars: localVars, CellAndFreeVars: cellAndFreeVars, - Builtins: Builtins.Globals, + Builtins: ctx.Store().Builtins.Globals, Localsplus: allocation, Stack: make([]Object, 0, code.Stacksize), } diff --git a/py/function.go b/py/function.go index 28eace5d..2c37499d 100644 --- a/py/function.go +++ b/py/function.go @@ -18,6 +18,7 @@ package py // A python Function object type Function struct { Code *Code // A code object, the __code__ attribute + Context Context // Host VM context Globals StringDict // A dictionary (other mappings won't do) Defaults Tuple // NULL or a tuple KwDefaults StringDict // NULL or a dict @@ -26,7 +27,6 @@ type Function struct { Name string // The __name__ attribute, a string object Dict StringDict // The __dict__ attribute, a dict or NULL Weakreflist List // List of weak references - Module Object // The __module__ attribute, can be anything Annotations StringDict // Annotations, a dict or NULL Qualname string // The qualified name } @@ -56,9 +56,8 @@ func (f *Function) GetDict() StringDict { // attribute. qualname should be a unicode object or ""; if "", the // __qualname__ attribute is set to the same value as its __name__ // attribute. -func NewFunction(code *Code, globals StringDict, qualname string) *Function { +func NewFunction(ctx Context, code *Code, globals StringDict, qualname string) *Function { var doc Object - var module Object = None if len(code.Consts) >= 1 { doc = code.Consts[0] if _, ok := doc.(String); !ok { @@ -68,29 +67,24 @@ func NewFunction(code *Code, globals StringDict, qualname string) *Function { doc = None } - // __module__: If module name is in globals, use it. Otherwise, use None. - if moduleobj, ok := globals["__name__"]; ok { - module = moduleobj - } - if qualname == "" { qualname = code.Name } return &Function{ Code: code, + Context: ctx, Qualname: qualname, Globals: globals, Name: code.Name, Doc: doc, - Module: module, Dict: make(StringDict), } } // Call a function func (f *Function) M__call__(args Tuple, kwargs StringDict) (Object, error) { - result, err := VmEvalCodeEx(f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure) + result, err := VmEvalCode(f.Context, f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure) if err != nil { return nil, err } diff --git a/py/import.go b/py/import.go index 3cd1fd64..0eaae921 100644 --- a/py/import.go +++ b/py/import.go @@ -7,17 +7,19 @@ package py import ( - "io/ioutil" - "os" "path" - "path/filepath" "strings" ) -var ( - // This will become sys.path one day ;-) - modulePath = []string{"", "/usr/lib/python3.4", "/usr/local/lib/python3.4/dist-packages", "/usr/lib/python3/dist-packages"} -) +func Import(ctx Context, names ...string) error { + for _, name := range names { + _, err := ImportModuleLevelObject(ctx, name, nil, nil, nil, 0) + if err != nil { + return err + } + } + return nil +} // The workings of __import__ // @@ -78,9 +80,18 @@ var ( // // Changed in version 3.3: Negative values for level are no longer // supported (which also changes the default value to 0). -func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist Tuple, level int) (Object, error) { +func ImportModuleLevelObject(ctx Context, name string, globals, locals StringDict, fromlist Tuple, level int) (Object, error) { // Module already loaded - return that - if module, ok := modules[name]; ok { + if module, err := ctx.GetModule(name); err == nil { + return module, nil + } + + // See if the module is a registered embeddded module that has not been loaded into this ctx yet. + if impl := GetModuleImpl(name); impl != nil { + module, err := ctx.ModuleInit(impl) + if err != nil { + return nil, err + } return module, nil } @@ -88,74 +99,24 @@ func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist T return nil, ExceptionNewf(SystemError, "Relative import not supported yet") } + // Convert import's dot separators into path seps parts := strings.Split(name, ".") - pathParts := path.Join(parts...) + srcPathname := path.Join(parts...) - for _, mpath := range modulePath { - if mpath == "" { - mpathObj, ok := globals["__file__"] - if !ok { - var err error - mpath, err = os.Getwd() - if err != nil { - return nil, err - } - } else { - mpath = path.Dir(string(mpathObj.(String))) - } - } - fullPath := path.Join(mpath, pathParts) - // FIXME Read pyc/pyo too - fullPath, err := filepath.Abs(fullPath) - if err != nil { - continue - } - if fi, err := os.Stat(fullPath); err == nil && fi.IsDir() { - // FIXME this is a massive simplification! - fullPath = path.Join(fullPath, "__init__.py") - } else { - fullPath += ".py" - } - // Check if file exists - if _, err := os.Stat(fullPath); err == nil { - str, err := ioutil.ReadFile(fullPath) - if err != nil { - return nil, ExceptionNewf(OSError, "Couldn't read %q: %v", fullPath, err) - } - codeObj, err := Compile(string(str), fullPath, "exec", 0, true) - if err != nil { - return nil, err - } - code, ok := codeObj.(*Code) - if !ok { - return nil, ExceptionNewf(ImportError, "Compile didn't return code object") - } - module := NewModule(name, "", nil, nil) - _, err = VmRun(module.Globals, module.Globals, code, nil) - if err != nil { - return nil, err - } - module.Globals["__file__"] = String(fullPath) - return module, nil - } + opts := CompileOpts{ + UseSysPaths: true, } - return nil, ExceptionNewf(ImportError, "No module named '%s'", name) - - // Convert to absolute path if relative - // Use __file__ from globals to work out what we are relative to - - // '' in path seems to mean use the current __file__ - - // Find a valid path which we need to check for the correct __init__.py in subdirectories etc - - // Look for .py and .pyc files - - // Make absolute module path too if we can for sys.modules - //How do we uniquely identify modules? + if fromFile, ok := globals["__file__"]; ok { + opts.CurDir = path.Dir(string(fromFile.(String))) + } - // SystemError: Parent module '' not loaded, cannot perform relative import + module, err := RunFile(ctx, srcPathname, opts, name) + if err != nil { + return nil, err + } + return module, nil } // Straight port of the python code @@ -163,7 +124,7 @@ func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist T // This calls functins from _bootstrap.py which is a frozen module // // Too much functionality for the moment -func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Object, level int) (Object, error) { +func XImportModuleLevelObject(ctx Context, nameObj, given_globals, locals, given_fromlist Object, level int) (Object, error) { var abs_name string var builtins_import Object var final_mod Object @@ -175,6 +136,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj var ok bool var name string var err error + store := ctx.Store() // Make sure to use default values so as to not have // PyObject_CallMethodObjArgs() truncate the parameter list because of a @@ -239,7 +201,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } } - if _, ok = modules[string(Package)]; !ok { + if _, err = ctx.GetModule(string(Package)); err != nil { return nil, ExceptionNewf(SystemError, "Parent module %q not loaded, cannot perform relative import", Package) } } else { // level == 0 */ @@ -277,16 +239,16 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj // From this point forward, goto error_with_unlock! builtins_import, ok = globals["__import__"] if !ok { - builtins_import, ok = Builtins.Globals["__import__"] + builtins_import, ok = store.Builtins.Globals["__import__"] if !ok { return nil, ExceptionNewf(ImportError, "__import__ not found") } } - mod, ok = modules[abs_name] - if mod == None { + mod, err = ctx.GetModule(abs_name) + if err != nil || mod == None { return nil, ExceptionNewf(ImportError, "import of %q halted; None in sys.modules", abs_name) - } else if ok { + } else if err == nil { var value Object var err error initializing := false @@ -306,7 +268,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } if initializing { // _bootstrap._lock_unlock_module() releases the import lock */ - value, err = Importlib.Call("_lock_unlock_module", Tuple{String(abs_name)}, nil) + _, err = store.Importlib.Call("_lock_unlock_module", Tuple{String(abs_name)}, nil) if err != nil { return nil, err } @@ -318,7 +280,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } } else { // _bootstrap._find_and_load() releases the import lock - mod, err = Importlib.Call("_find_and_load", Tuple{String(abs_name), builtins_import}, nil) + mod, err = store.Importlib.Call("_find_and_load", Tuple{String(abs_name), builtins_import}, nil) if err != nil { return nil, err } @@ -345,8 +307,8 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj cut_off := len(name) - len(front) abs_name_len := len(abs_name) to_return := abs_name[:abs_name_len-cut_off] - final_mod, ok = modules[to_return] - if !ok { + final_mod, err = ctx.GetModule(to_return) + if err != nil { return nil, ExceptionNewf(KeyError, "%q not in sys.modules as expected", to_return) } } @@ -354,7 +316,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj final_mod = mod } } else { - final_mod, err = Importlib.Call("_handle_fromlist", Tuple{mod, fromlist, builtins_import}, nil) + final_mod, err = store.Importlib.Call("_handle_fromlist", Tuple{mod, fromlist, builtins_import}, nil) if err != nil { return nil, err } @@ -376,7 +338,7 @@ error: } // The actual import code -func BuiltinImport(self Object, args Tuple, kwargs StringDict, currentGlobal StringDict) (Object, error) { +func BuiltinImport(ctx Context, self Object, args Tuple, kwargs StringDict, currentGlobal StringDict) (Object, error) { kwlist := []string{"name", "globals", "locals", "fromlist", "level"} var name Object var globals Object = currentGlobal @@ -391,5 +353,5 @@ func BuiltinImport(self Object, args Tuple, kwargs StringDict, currentGlobal Str if fromlist == None { fromlist = Tuple{} } - return ImportModuleLevelObject(string(name.(String)), globals.(StringDict), locals.(StringDict), fromlist.(Tuple), int(level.(Int))) + return ImportModuleLevelObject(ctx, string(name.(String)), globals.(StringDict), locals.(StringDict), fromlist.(Tuple), int(level.(Int))) } diff --git a/py/list.go b/py/list.go index eee7bf3e..5ea84ed7 100644 --- a/py/list.go +++ b/py/list.go @@ -41,8 +41,8 @@ func init() { ListType.Dict["sort"] = MustNewMethod("sort", func(self Object, args Tuple, kwargs StringDict) (Object, error) { const funcName = "sort" - var l *List - if self == None { + l, isList := self.(*List) + if !isList { // method called using `list.sort([], **kwargs)` var o Object err := UnpackTuple(args, nil, funcName, 1, 1, &o) @@ -60,7 +60,6 @@ func init() { if err != nil { return nil, err } - l = self.(*List) } err := SortInPlace(l, kwargs, funcName) if err != nil { @@ -121,6 +120,16 @@ func NewListFromItems(items []Object) *List { return l } +// Makes an argv into a tuple +func NewListFromStrings(items []string) *List { + l := NewListSized(len(items)) + for i, v := range items { + l.Items[i] = String(v) + } + return l +} + + // Copy a list object func (l *List) Copy() *List { return NewListFromItems(l.Items) @@ -141,6 +150,13 @@ func (l *List) Extend(items []Object) { l.Items = append(l.Items, items...) } +// Extend the list with strings +func (l *List) ExtendWithStrings(items []string) { + for _, item := range items { + l.Items = append(l.Items, Object(String(item))) + } +} + // Extends the list with the sequence passed in func (l *List) ExtendSequence(seq Object) error { return Iterate(seq, func(item Object) bool { diff --git a/py/method.go b/py/method.go index 1f8ecb72..c8b0ab03 100644 --- a/py/method.go +++ b/py/method.go @@ -70,6 +70,8 @@ type Method struct { Flags int // Go function implementation method interface{} + // Parent module of this method + Module *Module } // Internal method types implemented within eval.go @@ -224,14 +226,14 @@ func newBoundMethod(name string, fn interface{}) (Object, error) { return f(a, b, c) } default: - return nil, fmt.Errorf("Unknown bound method type for %q: %T", name, fn) + return nil, fmt.Errorf("unknown bound method type for %q: %T", name, fn) } return m, nil } // Call a method func (m *Method) M__call__(args Tuple, kwargs StringDict) (Object, error) { - self := None // FIXME should be the module + self := Object(m.Module) if kwargs != nil { return m.CallWithKeywords(self, args, kwargs) } diff --git a/py/module.go b/py/module.go index ae2a2171..e37ec4d2 100644 --- a/py/module.go +++ b/py/module.go @@ -6,24 +6,92 @@ package py -import "fmt" +import ( + "fmt" + "sync" +) + +type ModuleFlags int32 + +const ( + // ShareModule signals that an embedded module is threadsafe and read-only, meaninging it could be shared across multiple py.Context instances (for efficiency). + // Otherwise, ModuleImpl will create a separate py.Module instance for each py.Context that imports it. + // This should be used with extreme caution since any module mutation (write) means possible cross-context data corruption. + ShareModule ModuleFlags = 0x01 + + MainModuleName = "__main__" +) + +// ModuleInfo contains info and about a module and can specify flags that affect how it is imported into a py.Context +type ModuleInfo struct { + Name string // __name__ (if nil, "__main__" is used) + Doc string // __doc__ + FileDesc string // __file__ + Flags ModuleFlags +} -var ( +// ModuleImpl is used for modules that are ready to be imported into a py.Context. +// If a module is threadsafe and stateless it can be shared across multiple py.Context instances (for efficiency). +// By convention, .Code is executed when a module instance is initialized. +// If .Code == nil, then .CodeBuf or .CodeSrc will be auto-compiled to set .Code. +type ModuleImpl struct { + Info ModuleInfo + Methods []*Method // Module-bound global method functions + Globals StringDict // Module-bound global variables + CodeSrc string // Module code body (source code to be compiled) + CodeBuf []byte // Module code body (serialized py.Code object) + Code *Code // Module code body + OnContextClosed func(*Module) // Callback for when a py.Context is closing to release resources +} + +// ModuleStore is a container of Module imported into an owning py.Context. +type ModuleStore struct { // Registry of installed modules - modules = make(map[string]*Module) + modules map[string]*Module // Builtin module Builtins *Module // this should be the frozen module importlib/_bootstrap.py generated // by Modules/_freeze_importlib.c into Python/importlib.h Importlib *Module -) +} + +func RegisterModule(module *ModuleImpl) { + gRuntime.RegisterModule(module) +} -// A python Module object +func GetModuleImpl(moduleName string) *ModuleImpl { + gRuntime.mu.RLock() + defer gRuntime.mu.RUnlock() + impl := gRuntime.ModuleImpls[moduleName] + return impl +} + +type Runtime struct { + mu sync.RWMutex + ModuleImpls map[string]*ModuleImpl +} + +var gRuntime = Runtime{ + ModuleImpls: make(map[string]*ModuleImpl), +} + +func (rt *Runtime) RegisterModule(impl *ModuleImpl) { + rt.mu.Lock() + defer rt.mu.Unlock() + rt.ModuleImpls[impl.Info.Name] = impl +} + +func NewModuleStore() *ModuleStore { + return &ModuleStore{ + modules: make(map[string]*Module), + } +} + +// Module is a runtime instance of a ModuleImpl bound to the py.Context that imported it. type Module struct { - Name string - Doc string - Globals StringDict - // dict Dict + ModuleImpl *ModuleImpl // Parent implementation of this Module instance + Globals StringDict // Initialized from ModuleImpl.Globals + Context Context // Parent context that "owns" this Module instance } var ModuleType = NewType("module", "module object") @@ -34,7 +102,11 @@ func (o *Module) Type() *Type { } func (m *Module) M__repr__() (Object, error) { - return String(fmt.Sprintf("", m.Name)), nil + name, ok := m.Globals["__name__"].(String) + if !ok { + name = "???" + } + return String(fmt.Sprintf("", string(name))), nil } // Get the Dict @@ -42,60 +114,82 @@ func (m *Module) GetDict() StringDict { return m.Globals } -// Define a new module -func NewModule(name, doc string, methods []*Method, globals StringDict) *Module { +// Calls a named method of a module +func (m *Module) Call(name string, args Tuple, kwargs StringDict) (Object, error) { + attr, err := GetAttrString(m, name) + if err != nil { + return nil, err + } + return Call(attr, args, kwargs) +} + +// Interfaces +var _ IGetDict = (*Module)(nil) + +// NewModule adds a new Module instance to this ModuleStore. +// Each given Method prototype is used to create a new "live" Method bound this the newly created Module. +// This func also sets appropriate module global attribs based on the given ModuleInfo (e.g. __name__). +func (store *ModuleStore) NewModule(ctx Context, impl *ModuleImpl) (*Module, error) { + name := impl.Info.Name + if name == "" { + name = MainModuleName + } m := &Module{ - Name: name, - Doc: doc, - Globals: globals.Copy(), + ModuleImpl: impl, + Globals: impl.Globals.Copy(), + Context: ctx, } // Insert the methods into the module dictionary - for _, method := range methods { - m.Globals[method.Name] = method + // Copy each method an insert each "live" with a ptr back to the module (which can also lead us to the host Context) + for _, method := range impl.Methods { + methodInst := new(Method) + *methodInst = *method + methodInst.Module = m + m.Globals[method.Name] = methodInst } // Set some module globals m.Globals["__name__"] = String(name) - m.Globals["__doc__"] = String(doc) + m.Globals["__doc__"] = String(impl.Info.Doc) m.Globals["__package__"] = None + if len(impl.Info.FileDesc) > 0 { + m.Globals["__file__"] = String(impl.Info.FileDesc) + } // Register the module - modules[name] = m + store.modules[name] = m // Make a note of some modules switch name { case "builtins": - Builtins = m + store.Builtins = m case "importlib": - Importlib = m + store.Importlib = m } - // fmt.Printf("Registering module %q\n", name) - return m + // fmt.Printf("Registered module %q\n", moduleName) + return m, nil } // Gets a module -func GetModule(name string) (*Module, error) { - m, ok := modules[name] +func (store *ModuleStore) GetModule(name string) (*Module, error) { + m, ok := store.modules[name] if !ok { - return nil, ExceptionNewf(ImportError, "Module %q not found", name) + return nil, ExceptionNewf(ImportError, "Module '%s' not found", name) } return m, nil } // Gets a module or panics -func MustGetModule(name string) *Module { - m, err := GetModule(name) +func (store *ModuleStore) MustGetModule(name string) *Module { + m, err := store.GetModule(name) if err != nil { panic(err) } return m } -// Calls a named method of a module -func (m *Module) Call(name string, args Tuple, kwargs StringDict) (Object, error) { - attr, err := GetAttrString(m, name) - if err != nil { - return nil, err +// OnContextClosed signals all module instances that the parent py.Context has closed +func (store *ModuleStore) OnContextClosed() { + for _, m := range store.modules { + if m.ModuleImpl.OnContextClosed != nil { + m.ModuleImpl.OnContextClosed(m) + } } - return Call(attr, args, kwargs) } - -// Interfaces -var _ IGetDict = (*Module)(nil) diff --git a/py/py.go b/py/py.go index 0c48ee7b..59d0737a 100644 --- a/py/py.go +++ b/py/py.go @@ -24,15 +24,10 @@ type IGoInt64 interface { GoInt64() (int64, error) } -// Some well known objects var ( // Set in vm/eval.go - to avoid circular import - VmRun func(globals, locals StringDict, code *Code, closure Tuple) (res Object, err error) - VmRunFrame func(frame *Frame) (res Object, err error) - VmEvalCodeEx func(co *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) - - // See compile/compile.go - set to avoid circular import - Compile func(str, filename, mode string, flags int, dont_inherit bool) (Object, error) + VmEvalCode func(ctx Context, code *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) + VmRunFrame func(frame *Frame) (res Object, err error) ) // Called to create a new instance of class cls. __new__() is a static method (special-cased so you need not declare it as such) that takes the class of which an instance was requested as its first argument. The remaining arguments are those passed to the object constructor expression (the call to the class). The return value of __new__() should be the new object instance (usually an instance of cls). diff --git a/py/range.go b/py/range.go index 1f0513f8..1b261ccd 100644 --- a/py/range.go +++ b/py/range.go @@ -146,7 +146,6 @@ func computeRangeLength(start, stop, step Int) Int { if step > 0 { lo = start hi = stop - step = step } else { lo = stop hi = start diff --git a/py/run.go b/py/run.go new file mode 100644 index 00000000..e99880bf --- /dev/null +++ b/py/run.go @@ -0,0 +1,179 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +type CompileMode string + +const ( + ExecMode CompileMode = "exec" // Compile a module + EvalMode CompileMode = "eval" // Compile an expression + SingleMode CompileMode = "single" // Compile a single (interactive) statement +) + +// Context is a gpython environment instance container, providing a high-level mechanism +// for multiple gpython interpreters to run concurrently without restriction. +// +// Context instances maintain completely independent environments, namely the modules that +// have been imported and their state. Modules imported into a Context are instanced +// from a parent ModuleImpl. For example, since Contexts each have their +// own sys module instance, each can set sys.path differently and independently. +// +// If you access a Context from multiple groutines, you are responsible that access is not concurrent, +// with the exception of Close() and Done(). +// +// See examples/multi-context and examples/embedding. +type Context interface { + + // Resolves then compiles (if applicable) the given file system pathname into a py.Code ready to be executed. + ResolveAndCompile(pathname string, opts CompileOpts) (CompileOut, error) + + // Creates a new py.Module instance and initializes ModuleImpl's code in the new module (if applicable). + ModuleInit(impl *ModuleImpl) (*Module, error) + + // RunCode is a lower-level invocation to execute the given py.Code. + // Blocks until execution is complete. + RunCode(code *Code, globals, locals StringDict, closure Tuple) (result Object, err error) + + // Returns the named module for this context (or an error if not found) + GetModule(moduleName string) (*Module, error) + + // Gereric access to this context's modules / state. + Store() *ModuleStore + + // Close signals this context is about to go out of scope and any internal resources should be released. + // Code execution on a py.Context that has been closed will result in an error. + Close() error + + // Done returns a signal that can be used to detect when this Context has fully closed / completed. + // If Close() is called while execution in progress, Done() will not signal until execution is complete. + Done() <-chan struct{} +} + +// CompileOpts specifies options for high-level compilation. +type CompileOpts struct { + UseSysPaths bool // If set, sys.path will be used to resolve relative pathnames + CurDir string // If non-empty, this is the path of the current working directory. If empty, os.Getwd() is used. +} + +// CompileOut the output of high-level compilation -- e.g. ResolveAndCompile() +type CompileOut struct { + SrcPathname string // Resolved pathname the .py file that was compiled (if applicable) + PycPathname string // Pathname of the .pyc file read and/or written (if applicable) + FileDesc string // Pathname to be used for a a module's "__file__" attrib + Code *Code // Read/Output code object ready for execution +} + +// DefaultCoreSysPaths specify default search paths for module sys +// This can be changed during runtime and plays nice with others using DefaultContextOpts() +var DefaultCoreSysPaths = []string{ + ".", + "lib", +} + +// DefaultAuxSysPaths are secondary default search paths for module sys. +// This can be changed during runtime and plays nice with others using DefaultContextOpts() +// They are separated from the default core paths since they the more likley thing you will want to completely replace when using gpython. +var DefaultAuxSysPaths = []string{ + "/usr/lib/python3.4", + "/usr/local/lib/python3.4/dist-packages", + "/usr/lib/python3/dist-packages", +} + +// ContextOpts specifies fundamental environment and input settings for creating a new py.Context +type ContextOpts struct { + SysArgs []string // sys.argv initializer + SysPaths []string // sys.path initializer +} + +var ( + // DefaultContextOpts should be the default opts created for py.NewContext. + // Calling this ensure that you future proof you code for suggested/default settings. + DefaultContextOpts = func() ContextOpts { + opts := ContextOpts{ + SysPaths: DefaultCoreSysPaths, + } + opts.SysPaths = append(opts.SysPaths, DefaultAuxSysPaths...) + return opts + } + + // NewContext is a high-level call to create a new gpython interpreter context. + // See type Context interface. + NewContext func(opts ContextOpts) Context + + // Compiles a python buffer into a py.Code object. + // Returns a py.Code object or otherwise an error. + Compile func(src, srcDesc string, mode CompileMode, flags int, dont_inherit bool) (*Code, error) +) + +// RunFile resolves the given pathname, compiles as needed, executes the code in the given module, and returns the Module to indicate success. +// +// See RunCode() for description of inModule. +func RunFile(ctx Context, pathname string, opts CompileOpts, inModule interface{}) (*Module, error) { + out, err := ctx.ResolveAndCompile(pathname, opts) + if err != nil { + return nil, err + } + + return RunCode(ctx, out.Code, out.FileDesc, inModule) +} + +// RunSrc compiles the given python buffer and executes it within the given module and returns the Module to indicate success. +// +// See RunCode() for description of inModule. +func RunSrc(ctx Context, pySrc string, pySrcDesc string, inModule interface{}) (*Module, error) { + if pySrcDesc == "" { + pySrcDesc = "" + } + code, err := Compile(pySrc+"\n", pySrcDesc, SingleMode, 0, true) + if err != nil { + return nil, err + } + + return RunCode(ctx, code, pySrcDesc, inModule) +} + +// RunCode executes the given code object within the given module and returns the Module to indicate success. +// If inModule is a *Module, then the code is run in that module. +// If inModule is nil, the code is run in a new __main__ module (and the new Module is returned). +// If inModule is a string, the code is run in a new module with the given name (and the new Module is returned). +func RunCode(ctx Context, code *Code, codeDesc string, inModule interface{}) (*Module, error) { + var ( + module *Module + moduleName string + err error + ) + + createNew := false + switch mod := inModule.(type) { + + case string: + moduleName = mod + createNew = true + case nil: + createNew = true + case *Module: + _, err = ctx.RunCode(code, mod.Globals, mod.Globals, nil) + module = mod + default: + err = ExceptionNewf(TypeError, "unsupported module type: %v", inModule) + } + + if err == nil && createNew { + moduleImpl := ModuleImpl{ + Info: ModuleInfo{ + Name: moduleName, + FileDesc: codeDesc, + }, + Code: code, + } + module, err = ctx.ModuleInit(&moduleImpl) + } + + if err != nil { + return nil, err + } + + return module, nil +} diff --git a/py/string.go b/py/string.go index b985621d..0f9ddc28 100644 --- a/py/string.go +++ b/py/string.go @@ -94,8 +94,7 @@ func StringEscape(a String, ascii bool) string { func fieldsN(s string, n int) []string { out := []string{} cur := []rune{} - r := []rune(s) - for _, c := range r { + for _, c := range s { //until we have covered the first N elements, multiple white-spaces are 'merged' if n < 0 || len(out) < n { if unicode.IsSpace(c) { @@ -139,7 +138,7 @@ func init() { maxSplit = int(m) } } - valArray := []string{} + var valArray []string if valStr, ok := value.(String); ok { valArray = strings.SplitN(string(selfStr), string(valStr), maxSplit+1) } else if _, ok := value.(NoneType); ok { diff --git a/py/traceback.go b/py/traceback.go index 7cad8f04..bf7ba6db 100644 --- a/py/traceback.go +++ b/py/traceback.go @@ -52,7 +52,7 @@ RuntimeError: this is the error message func (tb *Traceback) TracebackDump(w io.Writer) { for ; tb != nil; tb = tb.Next { fmt.Fprintf(w, " File %q, line %d, in %s\n", tb.Frame.Code.Filename, tb.Lineno, tb.Frame.Code.Name) - fmt.Fprintf(w, " %s\n", "FIXME line of source goes here") + //fmt.Fprintf(w, " %s\n", "FIXME line of source goes here") } } diff --git a/py/util.go b/py/util.go new file mode 100644 index 00000000..43028bce --- /dev/null +++ b/py/util.go @@ -0,0 +1,206 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +import ( + "errors" + "strconv" +) + +var ( + ErrUnsupportedObjType = errors.New("unsupported obj type") +) + +// GetLen is a high-level convenience function that returns the length of the given Object. +func GetLen(obj Object) (Int, error) { + getlen, ok := obj.(I__len__) + if !ok { + return 0, nil + } + + lenObj, err := getlen.M__len__() + if err != nil { + return 0, err + } + + return GetInt(lenObj) +} + +// GetInt is a high-level convenience function that converts the given value to an int. +func GetInt(obj Object) (Int, error) { + toIdx, ok := obj.(I__index__) + if !ok { + _, err := cantConvert(obj, "int") + return 0, err + } + + return toIdx.M__index__() +} + +// LoadTuple attempts to convert each element of the given list and store into each destination value (based on its type). +func LoadTuple(args Tuple, vars []interface{}) error { + + if len(args) > len(vars) { + return ExceptionNewf(RuntimeError, "%d args given, expected %d", len(args), len(vars)) + } + + if len(vars) > len(args) { + vars = vars[:len(args)] + } + + for i, rval := range vars { + err := loadValue(args[i], rval) + if err == ErrUnsupportedObjType { + return ExceptionNewf(TypeError, "arg %d has unsupported object type: %s", i, args[i].Type().Name) + } + } + + return nil +} + +// LoadAttr gets the named attribute and attempts to store it into the given destination value (based on its type). +func LoadAttr(obj Object, attrName string, dst interface{}) error { + attr, err := GetAttrString(obj, attrName) + if err != nil { + return err + } + err = loadValue(attr, dst) + if err == ErrUnsupportedObjType { + return ExceptionNewf(TypeError, "attribute \"%s\" has unsupported object type: %s", attrName, attr.Type().Name) + } + return nil +} + +// LoadIntsFromList extracts a list of ints contained given a py.List or py.Tuple +func LoadIntsFromList(list Object) ([]int64, error) { + N, err := GetLen(list) + if err != nil { + return nil, err + } + + getter, ok := list.(I__getitem__) + if !ok { + return nil, nil + } + + if N <= 0 { + return nil, nil + } + + intList := make([]int64, N) + for i := Int(0); i < N; i++ { + item, err := getter.M__getitem__(i) + if err != nil { + return nil, err + } + + var intVal Int + intVal, err = GetInt(item) + if err != nil { + return nil, err + } + + intList[i] = int64(intVal) + } + + return intList, nil +} + +func loadValue(src Object, data interface{}) error { + var ( + v_str string + v_float float64 + v_int int64 + ) + + haveStr := false + + switch v := src.(type) { + case Bool: + if v { + v_int = 1 + v_float = 1 + v_str = "True" + } else { + v_str = "False" + } + haveStr = true + case Int: + v_int = int64(v) + v_float = float64(v) + case Float: + v_int = int64(v) + v_float = float64(v) + case String: + v_str = string(v) + haveStr = true + case NoneType: + // No-op + default: + return ErrUnsupportedObjType + } + + switch dst := data.(type) { + case *Int: + *dst = Int(v_int) + case *bool: + *dst = v_int != 0 + case *int8: + *dst = int8(v_int) + case *uint8: + *dst = uint8(v_int) + case *int16: + *dst = int16(v_int) + case *uint16: + *dst = uint16(v_int) + case *int32: + *dst = int32(v_int) + case *uint32: + *dst = uint32(v_int) + case *int: + *dst = int(v_int) + case *uint: + *dst = uint(v_int) + case *int64: + *dst = v_int + case *uint64: + *dst = uint64(v_int) + case *float32: + if haveStr { + v_float, _ = strconv.ParseFloat(v_str, 32) + } + *dst = float32(v_float) + case *float64: + if haveStr { + v_float, _ = strconv.ParseFloat(v_str, 64) + } + *dst = v_float + case *Float: + if haveStr { + v_float, _ = strconv.ParseFloat(v_str, 64) + } + *dst = Float(v_float) + case *string: + *dst = v_str + case *String: + *dst = String(v_str) + // case []uint64: + // for i := range data { + // dst[i] = order.Uint64(bs[8*i:]) + // } + // case []float32: + // for i := range data { + // dst[i] = math.Float32frombits(order.Uint32(bs[4*i:])) + // } + // case []float64: + // for i := range data { + // dst[i] = math.Float64frombits(order.Uint64(bs[8*i:])) + // } + + default: + return ExceptionNewf(NotImplementedError, "%s", "unsupported Go data type") + } + return nil +} diff --git a/pytest/pytest.go b/pytest/pytest.go index 7dbd6f3d..c7af7737 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -11,13 +11,14 @@ import ( "strings" "testing" - _ "github.com/go-python/gpython/builtin" + _ "github.com/go-python/gpython/modules" + "github.com/go-python/gpython/compile" "github.com/go-python/gpython/py" - _ "github.com/go-python/gpython/sys" - "github.com/go-python/gpython/vm" ) +var gContext = py.NewContext(py.DefaultContextOpts()) + // Compile the program in the file prog to code in the module that is returned func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { f, err := os.Open(prog) @@ -34,27 +35,32 @@ func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { if err != nil { t.Fatalf("%s: ReadAll failed: %v", prog, err) } + return CompileSrc(t, gContext, string(str), prog) +} - obj, err := compile.Compile(string(str), prog, "exec", 0, true) +func CompileSrc(t testing.TB, ctx py.Context, pySrc string, prog string) (*py.Module, *py.Code) { + code, err := compile.Compile(string(pySrc), prog, py.ExecMode, 0, true) if err != nil { t.Fatalf("%s: Compile failed: %v", prog, err) } - code := obj.(*py.Code) - module := py.NewModule("__main__", "", nil, nil) - module.Globals["__file__"] = py.String(prog) + module, err := ctx.Store().NewModule(ctx, &py.ModuleImpl{ + Info: py.ModuleInfo{ + FileDesc: prog, + }, + }) + if err != nil { + t.Fatalf("%s: NewModule failed: %v", prog, err) + } + return module, code } // Run the code in the module func run(t testing.TB, module *py.Module, code *py.Code) { - _, err := vm.Run(module.Globals, module.Globals, code, nil) + _, err := gContext.RunCode(code, module.Globals, module.Globals, nil) if err != nil { - if wantErr, ok := module.Globals["err"]; ok { - wantErrObj, ok := wantErr.(py.Object) - if !ok { - t.Fatalf("want err is not py.Object: %#v", wantErr) - } + if wantErrObj, ok := module.Globals["err"]; ok { gotExc, ok := err.(py.ExceptionInfo) if !ok { t.Fatalf("got err is not ExceptionInfo: %#v", err) diff --git a/repl/cli/cli.go b/repl/cli/cli.go index 2e1da88a..90f1463b 100644 --- a/repl/cli/cli.go +++ b/repl/cli/cli.go @@ -119,10 +119,12 @@ func (rl *readline) Print(out string) { } // RunREPL starts the REPL loop -func RunREPL() { - repl := repl.New() - rl := newReadline(repl) - repl.SetUI(rl) +func RunREPL(replCtx *repl.REPL) { + if replCtx == nil { + replCtx = repl.New(nil) + } + rl := newReadline(replCtx) + replCtx.SetUI(rl) defer rl.Close() err := rl.ReadHistory() if err != nil { diff --git a/repl/repl.go b/repl/repl.go index d7fd9600..f6639b25 100644 --- a/repl/repl.go +++ b/repl/repl.go @@ -10,7 +10,6 @@ import ( "sort" "strings" - "github.com/go-python/gpython/compile" "github.com/go-python/gpython/py" "github.com/go-python/gpython/vm" ) @@ -23,7 +22,8 @@ const ( // Repl state type REPL struct { - module *py.Module + Context py.Context + Module *py.Module prog string continuation bool previous string @@ -39,15 +39,23 @@ type UI interface { Print(string) } -// New create a new REPL and initialises the state machine -func New() *REPL { +// New create a new REPL and initializes the state machine +func New(ctx py.Context) *REPL { + if ctx == nil { + ctx = py.NewContext(py.DefaultContextOpts()) + } + r := &REPL{ - module: py.NewModule("__main__", "", nil, nil), + Context: ctx, prog: "", continuation: false, previous: "", } - r.module.Globals["__file__"] = py.String(r.prog) + r.Module, _ = ctx.ModuleInit(&py.ModuleImpl{ + Info: py.ModuleInfo{ + FileDesc: r.prog, + }, + }) return r } @@ -76,7 +84,7 @@ func (r *REPL) Run(line string) { if toCompile == "" { return } - obj, err := compile.Compile(toCompile+"\n", r.prog, "single", 0, true) + code, err := py.Compile(toCompile+"\n", r.prog, py.SingleMode, 0, true) if err != nil { // Detect that we should start a continuation line // FIXME detect EOF properly! @@ -99,8 +107,7 @@ func (r *REPL) Run(line string) { r.term.Print(fmt.Sprintf("Compile error: %v", err)) return } - code := obj.(*py.Code) - _, err = vm.Run(r.module.Globals, r.module.Globals, code, nil) + _, err = r.Context.RunCode(code, r.Module.Globals, r.Module.Globals, nil) if err != nil { py.TracebackDump(err) } @@ -129,8 +136,8 @@ func (r *REPL) Completer(line string, pos int) (head string, completions []strin } } } - match(r.module.Globals) - match(py.Builtins.Globals) + match(r.Module.Globals) + match(r.Context.Store().Builtins.Globals) sort.Strings(completions) return head, completions, tail } diff --git a/repl/repl_test.go b/repl/repl_test.go index a98ce9a9..9c64879e 100644 --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -6,10 +6,7 @@ import ( "testing" // import required modules - _ "github.com/go-python/gpython/builtin" - _ "github.com/go-python/gpython/math" - _ "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" + _ "github.com/go-python/gpython/modules" ) type replTest struct { @@ -38,7 +35,7 @@ func (rt *replTest) assert(t *testing.T, what, wantPrompt, wantOut string) { } func TestREPL(t *testing.T) { - r := New() + r := New(nil) rt := &replTest{} r.SetUI(rt) @@ -78,7 +75,7 @@ func TestREPL(t *testing.T) { } func TestCompleter(t *testing.T) { - r := New() + r := New(nil) rt := &replTest{} r.SetUI(rt) diff --git a/repl/web/main.go b/repl/web/main.go index bad66f16..16a2eaaa 100644 --- a/repl/web/main.go +++ b/repl/web/main.go @@ -4,6 +4,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build js // +build js package main @@ -77,7 +78,7 @@ func main() { node.Get("classList").Call("add", "active") // Make a repl referring to an empty term for the moment - REPL := repl.New() + REPL := repl.New(nil) cb := js.NewCallback(func(args []js.Value) { REPL.Run(args[0].String()) }) diff --git a/symtable/symtable_data_test.go b/symtable/symtable_data_test.go index b755841d..2ac2754a 100644 --- a/symtable/symtable_data_test.go +++ b/symtable/symtable_data_test.go @@ -12,7 +12,7 @@ import ( var symtableTestData = []struct { in string - mode string // exec, eval or single + mode py.CompileMode out *SymTable exceptionType *py.Type errString string diff --git a/sys/sys.go b/sys/sys.go index fcd1f0df..b8aa3dc2 100644 --- a/sys/sys.go +++ b/sys/sys.go @@ -652,18 +652,27 @@ func init() { py.MustNewMethod("call_tracing", sys_call_tracing, 0, call_tracing_doc), py.MustNewMethod("_debugmallocstats", sys_debugmallocstats, 0, debugmallocstats_doc), } - argv := MakeArgv(os.Args[1:]) - stdin, stdout, stderr := &py.File{os.Stdin, py.FileRead}, - &py.File{os.Stdout, py.FileWrite}, - &py.File{os.Stderr, py.FileWrite} + + stdin := &py.File{File: os.Stdin, FileMode: py.FileRead} + stdout := &py.File{File: os.Stdout, FileMode: py.FileWrite} + stderr := &py.File{File: os.Stderr, FileMode: py.FileWrite} + + executable, err := os.Executable() + if err != nil { + panic(err) + } + globals := py.StringDict{ - "argv": argv, + "path": py.NewList(), + "argv": py.NewListFromStrings(os.Args[1:]), "stdin": stdin, "stdout": stdout, "stderr": stderr, "__stdin__": stdin, "__stdout__": stdout, "__stderr__": stderr, + "executable": py.String(executable), + //"version": py.Int(MARSHAL_VERSION), // /* stdin/stdout/stderr are now set by pythonrun.c */ @@ -787,14 +796,14 @@ func init() { // SET_SYS_FROM_STRING("thread_info", PyThread_GetInfo()); // #endif } - py.NewModule("sys", module_doc, methods, globals) -} -// Makes an argv into a tuple -func MakeArgv(pyargs []string) py.Object { - argv := py.NewListSized(len(pyargs)) - for i, v := range pyargs { - argv.Items[i] = py.String(v) - } - return argv + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "sys", + Doc: module_doc, + }, + Methods: methods, + Globals: globals, + }) + } diff --git a/time/time.go b/time/time.go index c028966a..d783ae8f 100644 --- a/time/time.go +++ b/time/time.go @@ -1007,10 +1007,16 @@ func init() { py.MustNewMethod("perf_counter", time_perf_counter, 0, perf_counter_doc), py.MustNewMethod("get_clock_info", time_get_clock_info, 0, get_clock_info_doc), } - globals := py.StringDict{ - //"version": py.Int(MARSHAL_VERSION), - } - py.NewModule("time", module_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "time", + Doc: module_doc, + }, + Methods: methods, + Globals: py.StringDict{ + }, + }) } diff --git a/vm/builtin.go b/vm/builtin.go index 3fcdccac..6466b31f 100644 --- a/vm/builtin.go +++ b/vm/builtin.go @@ -12,13 +12,13 @@ import ( "github.com/go-python/gpython/py" ) -func builtinEvalOrExec(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict, mode string) (py.Object, error) { +func builtinEvalOrExec(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict, mode py.CompileMode) (py.Object, error) { var ( cmd py.Object globals py.Object = py.None locals py.Object = py.None ) - err := py.UnpackTuple(args, kwargs, mode, 1, 3, &cmd, &globals, &locals) + err := py.UnpackTuple(args, kwargs, string(mode), 1, 3, &cmd, &globals, &locals) if err != nil { return nil, err } @@ -60,24 +60,23 @@ func builtinEvalOrExec(self py.Object, args py.Tuple, kwargs, currentLocals, cur } if code == nil { codeStr = strings.TrimLeft(codeStr, " \t") - obj, err := py.Compile(codeStr, "", mode, 0, true) + code, err = py.Compile(codeStr, "", mode, 0, true) if err != nil { return nil, err } - code = obj.(*py.Code) } if code.GetNumFree() > 0 { return nil, py.ExceptionNewf(py.TypeError, "code passed to %s() may not contain free variables", mode) } - return EvalCode(code, globalsDict, localsDict) + return ctx.RunCode(code, globalsDict, localsDict, nil) } -func builtinEval(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { - return builtinEvalOrExec(self, args, kwargs, currentLocals, currentGlobals, builtins, "eval") +func builtinEval(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { + return builtinEvalOrExec(ctx, args, kwargs, currentLocals, currentGlobals, builtins, py.EvalMode) } -func builtinExec(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { - _, err := builtinEvalOrExec(self, args, kwargs, currentLocals, currentGlobals, builtins, "exec") +func builtinExec(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { + _, err := builtinEvalOrExec(ctx, args, kwargs, currentLocals, currentGlobals, builtins, py.ExecMode) if err != nil { return nil, err } diff --git a/vm/eval.go b/vm/eval.go index e98392a9..94bf6d1e 100644 --- a/vm/eval.go +++ b/vm/eval.go @@ -767,7 +767,7 @@ func do_END_FINALLY(vm *Vm, arg int32) error { // Loads the __build_class__ helper function to the stack which // creates a new class object. func do_LOAD_BUILD_CLASS(vm *Vm, arg int32) error { - vm.PUSH(py.Builtins.Globals["__build_class__"]) + vm.PUSH(vm.context.Store().Builtins.Globals["__build_class__"]) return nil } @@ -1087,9 +1087,9 @@ func do_IMPORT_NAME(vm *Vm, namei int32) error { } v := vm.POP() u := vm.TOP() - var locals py.Object = vm.frame.Locals - if locals == nil { - locals = py.None + var locals py.Object = py.None + if vm.frame.Locals != nil { + locals = vm.frame.Locals } var args py.Tuple if _, ok := u.(py.Int); ok { @@ -1435,7 +1435,7 @@ func _make_function(vm *Vm, argc int32, opcode OpCode) { num_annotations := (argc >> 16) & 0x7fff qualname := vm.POP() code := vm.POP() - function := py.NewFunction(code.(*py.Code), vm.frame.Globals, string(qualname.(py.String))) + function := py.NewFunction(vm.context, code.(*py.Code), vm.frame.Globals, string(qualname.(py.String))) if opcode == MAKE_CLOSURE { function.Closure = vm.POP().(py.Tuple) @@ -1579,7 +1579,7 @@ func EvalGetFuncDesc(fn py.Object) string { } } -// As py.Call but takes an intepreter Frame object +// As py.Call but takes an interpreter Frame object // // Used to implement some interpreter magic like locals(), globals() etc func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame) (py.Object, error) { @@ -1592,13 +1592,13 @@ func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame f.FastToLocals() return f.Locals, nil case py.InternalMethodImport: - return py.BuiltinImport(nil, args, kwargs, f.Globals) + return py.BuiltinImport(f.Context, nil, args, kwargs, f.Globals) case py.InternalMethodEval: f.FastToLocals() - return builtinEval(nil, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinEval(f.Context, args, kwargs, f.Locals, f.Globals, f.Builtins) case py.InternalMethodExec: f.FastToLocals() - return builtinExec(nil, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinExec(f.Context, args, kwargs, f.Locals, f.Globals, f.Builtins) default: return nil, py.ExceptionNewf(py.SystemError, "Internal method %v not found", x) } @@ -1731,7 +1731,8 @@ func (vm *Vm) UnwindExceptHandler(frame *py.Frame, block *py.TryBlock) { // This is the equivalent of PyEval_EvalFrame func RunFrame(frame *py.Frame) (res py.Object, err error) { var vm = Vm{ - frame: frame, + frame: frame, + context: frame.Context, } // FIXME need to do this to save the old exeption when we @@ -2033,7 +2034,14 @@ func tooManyPositional(co *py.Code, given, defcount int, fastlocals []py.Object) chooseString(given == 1 && kwonly_given == 0, "was", "were")) } -func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kws py.StringDict, defs []py.Object, kwdefs py.StringDict, closure py.Tuple) (retval py.Object, err error) { +// EvalCode runs a new virtual machine on a Code object. +// +// Any parameters are expected to have been decoded into locals +// +// Returns an Object and an error. The error will be a py.ExceptionInfo +// +// This is the equivalent of PyEval_EvalCode with closure support +func EvalCode(ctx py.Context, co *py.Code, globals, locals py.StringDict, args []py.Object, kws py.StringDict, defs []py.Object, kwdefs py.StringDict, closure py.Tuple) (retval py.Object, err error) { total_args := int(co.Argcount + co.Kwonlyargcount) n := len(args) var kwdict py.StringDict @@ -2045,7 +2053,7 @@ func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kw //assert(tstate != nil) //assert(globals != nil) // f = PyFrame_New(tstate, co, globals, locals) - f := py.NewFrame(globals, locals, co, closure) // FIXME extra closure parameter? + f := py.NewFrame(ctx, globals, locals, co, closure) // FIXME extra closure parameter? fastlocals := f.Localsplus freevars := f.CellAndFreeVars @@ -2162,34 +2170,8 @@ func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kw return RunFrame(f) } -func EvalCode(co *py.Code, globals, locals py.StringDict) (py.Object, error) { - return EvalCodeEx(co, - globals, locals, - nil, - nil, - nil, - nil, nil) -} - -// Run the virtual machine on a Code object -// -// Any parameters are expected to have been decoded into locals -// -// Returns an Object and an error. The error will be a py.ExceptionInfo -// -// This is the equivalent of PyEval_EvalCode with closure support -func Run(globals, locals py.StringDict, code *py.Code, closure py.Tuple) (res py.Object, err error) { - return EvalCodeEx(code, - globals, locals, - nil, - nil, - nil, - nil, closure) -} - // Write the py global to avoid circular import func init() { - py.VmRun = Run + py.VmEvalCode = EvalCode py.VmRunFrame = RunFrame - py.VmEvalCodeEx = EvalCodeEx } diff --git a/vm/vm.go b/vm/vm.go index 4eb3cb20..9c122f32 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -41,4 +41,6 @@ type Vm struct { curexc py.ExceptionInfo // Previous exception type, value and traceback exc py.ExceptionInfo + // This vm's access to persistent state and modules + context py.Context } diff --git a/vm/vm_test.go b/vm/vm_test.go index f1686207..a47e20dc 100644 --- a/vm/vm_test.go +++ b/vm/vm_test.go @@ -5,8 +5,13 @@ package vm_test import ( + "fmt" + "strconv" + "strings" + "sync" "testing" + "github.com/go-python/gpython/py" "github.com/go-python/gpython/pytest" ) @@ -17,3 +22,72 @@ func TestVm(t *testing.T) { func BenchmarkVM(b *testing.B) { pytest.RunBenchmarks(b, "benchmarks") } + +var jobSrcTemplate = ` + +doc="multi py.Context text" +WORKER_ID = "{{WORKER_ID}}" +def fib(n): + if n == 0: + return 0 + elif n == 1: + return 1 + return fib(n - 2) + fib(n - 1) + +x = {{FIB_TO}} +fx = fib(x) +print("%s says fib(%d) is %d" % (WORKER_ID, x, fx)) +` + +type worker struct { + name string + ctx py.Context +} + +func (w *worker) run(b testing.TB, pySrc string, countUpto int) { + pySrc = strings.Replace(pySrc, "{{WORKER_ID}}", w.name, -1) + pySrc = strings.Replace(pySrc, "{{FIB_TO}}", strconv.Itoa(countUpto), -1) + + module, code := pytest.CompileSrc(b, w.ctx, pySrc, w.name) + _, err := w.ctx.RunCode(code, module.Globals, module.Globals, nil) + if err != nil { + b.Fatal(err) + } +} + +func BenchmarkContext(b *testing.B) { + numWorkers := 4 + workersRunning := sync.WaitGroup{} + + numJobs := 35 + fmt.Printf("Starting %d workers to process %d jobs...\n", numWorkers, numJobs) + + jobPipe := make(chan int) + go func() { + for i := 0; i < numJobs; i++ { + jobPipe <- i + 1 + } + close(jobPipe) + }() + + workers := make([]worker, numWorkers) + for i := 0; i < numWorkers; i++ { + + workers[i] = worker{ + name: fmt.Sprintf("Worker #%d", i+1), + ctx: py.NewContext(py.DefaultContextOpts()), + } + + workersRunning.Add(1) + w := workers[i] + go func() { + for jobID := range jobPipe { + w.run(b, jobSrcTemplate, jobID) + //fmt.Printf("### %s finished job %v ###\n", w.name, jobID) + } + workersRunning.Done() + }() + } + + workersRunning.Wait() +}