compile: symtable: test and fix flags

Author: ncw

compile/make_symtable_test.py

@@ -8,8 +8,14 @@
 import subprocess
 import dis
 from symtable import symtable
-
-# FIXME test errors too
+try:
+    # run ./build.sh in the readsymtab directory to create this module
+    from readsymtab import readsymtab
+    use_readsymtab = True
+except ImportError:
+    import ctypes
+    use_readsymtab = False
+    print("Using compiler dependent code to read bitfields - compile readsymtab module to be sure!")
 
 inp = [
     ('''1''', "eval"),
@@ -103,12 +109,36 @@ def dump_symtable(st):
     out += 'Lineno:%s,\n' % st.get_lineno() # Return the number of the first line in the block this table represents.
     out += 'Unoptimized:%s,\n' % dump_flags(st._table.optimized, OPT_FLAGS) # Return False if the locals in this table can be optimized.
     out += 'Nested:%s,\n' % dump_bool(st.is_nested()) # Return True if the block is a nested class or function.
+
+    if use_readsymtab:
+    # Use readsymtab modules to read the bitfields which aren't normally exported
+        free, child_free, generator, varargs, varkeywords, returns_value, needs_class_closure =  readsymtab(st._table)
+        out += 'Free:%s,\n' % dump_bool(free)
+        out += 'ChildFree:%s,\n' % dump_bool(child_free)
+        out += 'Generator:%s,\n' % dump_bool(generator)
+        out += 'Varargs:%s,\n' % dump_bool(varargs)
+        out += 'Varkeywords:%s,\n' % dump_bool(varkeywords)
+        out += 'ReturnsValue:%s,\n' % dump_bool(returns_value)
+        out += 'NeedsClassClosure:%s,\n' % dump_bool(needs_class_closure)
+    else:
+        # Use ctypes to read the bitfields which aren't normally exported
+        # FIXME compiler dependent!
+        base_addr = id(st._table) + ctypes.sizeof(ctypes.c_long)*8+ ctypes.sizeof(ctypes.c_int)*3
+        flags = int.from_bytes(ctypes.c_int.from_address(base_addr), sys.byteorder)
+        out += 'Free:%s,\n' % dump_bool(flags & (1 << 0))
+        out += 'ChildFree:%s,\n' % dump_bool(flags & (1 << 1))
+        out += 'Generator:%s,\n' % dump_bool(flags & (1 << 2))
+        out += 'Varargs:%s,\n' % dump_bool(flags & (1 << 3))
+        out += 'Varkeywords:%s,\n' % dump_bool(flags & (1 << 4))
+        out += 'ReturnsValue:%s,\n' % dump_bool(flags & (1 << 5))
+        out += 'NeedsClassClosure:%s,\n' % dump_bool(flags & (1 << 6))
+
     #out += 'Exec:%s,\n' % dump_bool(st.has_exec()) # Return True if the block uses exec.
     #out += 'ImportStar:%s,\n' % dump_bool(st.has_import_star()) # Return True if the block uses a starred from-import.
     out += 'Varnames:%s,\n' % dump_strings(st._table.varnames)
     out += 'Symbols: Symbols{\n'
     children = dict()
-    for name in st.get_identifiers():
+    for name in sorted(st.get_identifiers()):
         s = st.lookup(name)
         out += '"%s":%s,\n' % (name, dump_symbol(s))
         ns = s.get_namespaces()
@@ -120,7 +150,7 @@ def dump_symtable(st):
             raise AssertionError("More than one namespace")
     out += '},\n'
     out += 'Children:map[string]*SymTable{\n'
-    for name, symtable in children.items():
+    for name, symtable in sorted(children.items()):
         out += '"%s":%s,\n' % (name, dump_symtable(symtable))
     out += '},\n'
     out += "}"

compile/readsymtab_module/build.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -e
+rm -rf build
+PYTHON=/opt/python3.4/bin/python3.4
+INCLUDE=/opt/python3.4/include/python3.4m
+$PYTHON setup.py build_ext --inplace
+cp -av *.so *.dll ..

compile/readsymtab_module/readsymtab.c

@@ -0,0 +1,54 @@
+#include <Python.h>
+#include "Python-ast.h"
+#include "code.h"
+#include "symtable.h"
+#include "structmember.h"
+
+static PyObject*
+readsymtab(PyObject* self, PyObject* args)
+{
+    PyObject* obj;
+ 
+    if (!PyArg_ParseTuple(args, "O", &obj))
+        return NULL;
+
+    if (Py_TYPE(obj) != &PySTEntry_Type) {
+        fprintf(stderr, "Ooops wrong type\n");
+    }
+
+    PySTEntryObject *st = (PySTEntryObject *)obj;
+ 
+    return Py_BuildValue("(iiiiiii)",
+                         st->ste_free,
+                         st->ste_child_free,
+                         st->ste_generator,
+                         st->ste_varargs,
+                         st->ste_varkeywords,
+                         st->ste_returns_value,
+                         st->ste_needs_class_closure);
+}
+ 
+static PyMethodDef readsymtabmethods[] =
+{
+     {"readsymtab", readsymtab, METH_VARARGS, "Read the symbol table."},
+     {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef readsymtabmodule = {
+    PyModuleDef_HEAD_INIT,
+    "readsymtab",
+    "Read the symbol table.",
+    -1,
+    readsymtabmethods,
+    NULL,
+    NULL,
+    NULL,
+    NULL
+};
+
+ 
+PyMODINIT_FUNC
+PyInit_readsymtab(void)
+{
+    return PyModule_Create(&readsymtabmodule);
+}

compile/readsymtab_module/setup.py

@@ -0,0 +1,9 @@
+from distutils.core import setup, Extension
+
+readsymtab = Extension('readsymtab', sources = ['readsymtab.c'])
+
+setup (name = 'readsymtab',
+        version = '1.0',
+        description = 'Read the symbol table',
+        ext_modules = [readsymtab]
+)

compile/symtable.go

@@ -99,12 +99,12 @@ type SymTable struct {
 	Generator         bool      // true if namespace is a generator
 	Varargs           bool      // true if block has varargs
 	Varkeywords       bool      // true if block has varkeywords
-	Returns_value     bool      // true if namespace uses return with an argument
+	ReturnsValue      bool      // true if namespace uses return with an argument
 	NeedsClassClosure bool      // for class scopes, true if a closure over __class__ should be created
-	col_offset        int       // offset of first line of block
-	opt_lineno        int       // lineno of last exec or import *
-	opt_col_offset    int       // offset of last exec or import *
-	tmpname           int       // counter for listcomp temp vars
+	// col_offset        int       // offset of first line of block
+	// opt_lineno        int       // lineno of last exec or import *
+	// opt_col_offset    int       // offset of last exec or import *
+	// tmpname           int       // counter for listcomp temp vars
 
 	Symbols  Symbols
 	Global   *SymTable // symbol table entry for module
@@ -282,6 +282,11 @@ func (st *SymTable) Parse(Ast ast.Ast) {
 				}
 				st.Unoptimized |= optImportStar
 			}
+
+		case *ast.Return:
+			if node.Value != nil {
+				st.ReturnsValue = true
+			}
 		}
 		return true
 	})