Improved __future__ parser; still more to do

Makefile.pre.in: add target future.o

Include/compile.h: define PyFutureFeaters and PyNode_Future()
                   add c_future slot to struct compiling

Include/symtable.h: add st_future slot to struct symtable

Python/future.c: implementation of PyNode_Future()

Python/compile.c: use PyNode_Future() for nested_scopes support

Python/symtable.c: include compile.h to pick up PyFutureFeatures decl

Author: jeremyhylton

Include/compile.h

@@ -7,9 +7,6 @@
 extern "C" {
 #endif
 
-#define NESTED_SCOPES_DEFAULT 0
-#define FUTURE_NESTED_SCOPES "nested_scopes"
-
 /* Bytecode object */
 typedef struct {
     PyObject_HEAD
@@ -51,6 +48,19 @@ DL_IMPORT(PyCodeObject *) PyCode_New(
         /* same as struct above */
 DL_IMPORT(int) PyCode_Addr2Line(PyCodeObject *, int);
 
+/* Future feature support */
+
+typedef struct {
+    int ff_last_lineno;
+    int ff_n_simple_stmt;
+    int ff_nested_scopes;
+} PyFutureFeatures;
+
+DL_IMPORT(PyFutureFeatures *) PyNode_Future(struct _node *, char *);
+
+#define NESTED_SCOPES_DEFAULT 0
+#define FUTURE_NESTED_SCOPES "nested_scopes"
+
 /* for internal use only */
 #define _PyCode_GETCODEPTR(co, pp) \
 	((*(co)->co_code->ob_type->tp_as_buffer->bf_getreadbuffer) \

Include/symtable.h

@@ -30,6 +30,7 @@ struct symtable {
 	int st_errors;           /* number of errors */
 	char *st_private;        /* name of current class or NULL */
 	int st_tmpname;          /* temporary name counter */
+	PyFutureFeatures *st_future; /* module's future features */
 };
 
 typedef struct _symtable_entry {

Makefile.pre.in

@@ -200,6 +200,7 @@ PYTHON_OBJS=	\
 		Python/errors.o \
 		Python/frozen.o \
 		Python/frozenmain.o \
+		Python/future.o \
 		Python/getargs.o \
 		Python/getcompiler.o \
 		Python/getcopyright.o \

Python/compile.c

@@ -55,9 +55,6 @@ int Py_OptimizeFlag = 0;
 #define ILLEGAL_DYNAMIC_SCOPE \
 "%.100s: exec or 'import *' makes names ambiguous in nested scope"
 
-#define UNDEFINED_FUTURE_FEATURE \
-"future feature %.100s is not defined"
-
 #define GLOBAL_AFTER_ASSIGN \
 "name '%.400s' is assigned to before global declaration"
 
@@ -368,6 +365,7 @@ struct compiling {
 	int c_nested;		/* Is block nested funcdef or lamdef? */
 	int c_closure;		/* Is nested w/freevars? */
 	struct symtable *c_symtable; /* pointer to module symbol table */
+        PyFutureFeatures *c_future; /* pointer to module's __future__ */
 };
 
 int is_free(int v)
@@ -3864,7 +3862,8 @@ jcompile(node *n, char *filename, struct compiling *base)
 			sc.c_nested = 1;
 	} else {
 		sc.c_private = NULL;
-		if (symtable_build(&sc, n) < 0) {
+		sc.c_future = PyNode_Future(n, filename);
+		if (sc.c_future == NULL || symtable_build(&sc, n) < 0) {
 			com_free(&sc);
 			return NULL;
 		}
@@ -3996,6 +3995,8 @@ symtable_build(struct compiling *c, node *n)
 {
 	if ((c->c_symtable = symtable_init()) == NULL)
 		return -1;
+	if (c->c_future->ff_nested_scopes)
+		c->c_symtable->st_nested_scopes = 1;
 	c->c_symtable->st_filename = c->c_filename;
 	symtable_enter_scope(c->c_symtable, TOP, TYPE(n), n->n_lineno);
 	if (c->c_symtable->st_errors > 0)
@@ -4280,44 +4281,6 @@ PySymtable_Free(struct symtable *st)
 	PyMem_Free((void *)st);
 }
 
-/* XXX this code is a placeholder for correct code.
-   from __future__ import name set language options */
-
-static int
-symtable_check_future(struct symtable *st, node *n)
-{
-	int i;
-	node *name = CHILD(n, 1);
-
-	if (strcmp(STR(CHILD(name, 0)), "__future__") != 0)
-		return 0;
-	/* It is only legal to define __future__ features at the top
-	   of a module.  If the current scope is not the module level
-	   or if there are any symbols defined, it is too late. */
-	if (st->st_cur->ste_symbols != st->st_global
-	    || PyDict_Size(st->st_cur->ste_symbols) != 0) {
-		PyErr_SetString(PyExc_SyntaxError, 
-	"imports from __future__ are only legal at the beginning of a module");
-		return -1;
-	}
-	for (i = 3; i < NCH(n); ++i) {
-		char *feature = STR(CHILD(CHILD(n, i), 0));
-		/* Do a linear search through the defined features,
-		   assuming there aren't very many of them. */ 
-		if (strcmp(feature, FUTURE_NESTED_SCOPES) == 0) {
-			st->st_nested_scopes = 1;
-		} else {
-			PyErr_Format(PyExc_SyntaxError,
-				     UNDEFINED_FUTURE_FEATURE, feature);
-			set_error_location(st->st_filename,
-					   st->st_cur->ste_lineno);
-			st->st_errors++;
-			return -1;
-		}
-	}
-	return 1;
-}
-
 /* When the compiler exits a scope, it must should update the scope's
    free variable information with the list of free variables in its
    children.
@@ -4910,7 +4873,6 @@ symtable_import(struct symtable *st, node *n)
 	   import_as_name: NAME [NAME NAME]
 	*/
 	if (STR(CHILD(n, 0))[0] == 'f') {  /* from */
-		symtable_check_future(st, n);
 		if (TYPE(CHILD(n, 3)) == STAR) {
 			st->st_cur->ste_optimized |= OPT_IMPORT_STAR;
 		} else {

Python/future.c

@@ -0,0 +1,146 @@
+#include "Python.h"
+#include "node.h"
+#include "token.h"
+#include "graminit.h"
+#include "compile.h"
+#include "symtable.h"
+
+#define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined"
+
+static int
+future_check_features(PyFutureFeatures *ff, node *n)
+{
+	int i;
+	char *feature;
+
+	REQ(n, import_stmt); /* must by from __future__ import ... */
+
+	for (i = 3; i < NCH(n); ++i) {
+		feature = STR(CHILD(CHILD(n, i), 0));
+		if (strcmp(feature, FUTURE_NESTED_SCOPES) == 0) {
+			ff->ff_nested_scopes = 1;
+		} else {
+			PyErr_Format(PyExc_SyntaxError,
+				     UNDEFINED_FUTURE_FEATURE, feature);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+/* Relevant portions of the grammar:
+
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+file_input: (NEWLINE | stmt)* ENDMARKER
+stmt: simple_stmt | compound_stmt
+simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt
+import_stmt: 'import' dotted_as_name (',' dotted_as_name)* | 'from' dotted_name 'import' ('*' | import_as_name (',' import_as_name)*)
+import_as_name: NAME [NAME NAME]
+dotted_as_name: dotted_name [NAME NAME]
+dotted_name: NAME ('.' NAME)*
+*/
+
+/* future_parse() return values:
+   -1 indicates an error occurred, e.g. unknown feature name
+   0 indicates no feature was found
+   1 indicates a feature was found
+*/
+
+static int
+future_parse(PyFutureFeatures *ff, node *n)
+{
+	int i, r, found;
+ loop:
+
+/*	fprintf(stderr, "future_parse(%d, %d, %s)\n",
+		TYPE(n), NCH(n), (n == NULL) ? "NULL" : STR(n));
+*/
+	switch (TYPE(n)) {
+
+	case file_input:
+		for (i = 0; i < NCH(n); i++) {
+			node *ch = CHILD(n, i);
+			if (TYPE(ch) == stmt) {
+				n = ch;
+				goto loop;
+			}
+		}
+		return 0;
+
+	case simple_stmt:
+		if (NCH(n) == 1) {
+			REQ(CHILD(n, 0), small_stmt);
+			n = CHILD(n, 0);
+			goto loop;
+		}
+		found = 0;
+		for (i = 0; i < NCH(n); ++i)
+			if (TYPE(CHILD(n, i)) == small_stmt) {
+				r = future_parse(ff, CHILD(n, i));
+				if (r < 1) {
+					ff->ff_last_lineno = n->n_lineno;
+					ff->ff_n_simple_stmt = i;
+					return r;
+				} else
+					found++;
+			}
+		if (found)
+			return 1;
+		else
+			return 0;
+	
+	case stmt:
+		if (TYPE(CHILD(n, 0)) == simple_stmt) {
+			n = CHILD(n, 0);
+			goto loop;
+		} else {
+			REQ(CHILD(n, 0), compound_stmt);
+			ff->ff_last_lineno = n->n_lineno;
+			return 0;
+		}
+
+	case small_stmt:
+		n = CHILD(n, 0);
+		goto loop;
+
+	case import_stmt: {
+		node *name;
+
+		if (STR(CHILD(n, 0))[0] != 'f') { /* from */
+			ff->ff_last_lineno = n->n_lineno;
+			return 0;
+		}
+		name = CHILD(n, 1);
+		if (strcmp(STR(CHILD(name, 0)), "__future__") != 0)
+			return 0;
+		if (future_check_features(ff, n) < 0)
+			return -1;
+		return 1;
+	}
+
+	default:
+		ff->ff_last_lineno = n->n_lineno;
+		return 0;
+	}
+}
+
+PyFutureFeatures *
+PyNode_Future(node *n, char *filename)
+{
+	PyFutureFeatures *ff;
+
+	ff = (PyFutureFeatures *)PyMem_Malloc(sizeof(PyFutureFeatures));
+	if (ff == NULL)
+		return NULL;
+	ff->ff_last_lineno = 0;
+	ff->ff_n_simple_stmt = -1;
+	ff->ff_nested_scopes = 0;
+
+	if (future_parse(ff, n) < 0) {
+		PyMem_Free((void *)ff);
+		return NULL;
+	}
+	return ff;
+}
+

Python/symtable.c

@@ -1,4 +1,5 @@
 #include "Python.h"
+#include "compile.h"
 #include "symtable.h"
 #include "graminit.h"
 #include "structmember.h"