Presumed correct compiler pass for future statements

XXX still need to integrate into symtable API

compile.h: Remove ff_n_simple_stmt; obsolete.

           Add ff_found_docstring used internally to skip one and only
           one string at the beginning of a module.

compile.c: Add check for from __future__ imports to far into the file.

 	   In symtable_global() check for -1 returned from
	   symtable_lookup(), which signifies name not defined.

	   Add missing DECERF in symtable_add_def.

           Free c->c_future.

future.c:  Add special handling for multiple statements joined on a
	   single line using one or more semicolons; this form can
           include an illegal future statement that would otherwise be
           hard to detect.

	   Add support for detecting and skipping doc strings.

Author: jeremyhylton

Include/compile.h

@@ -51,8 +51,8 @@ DL_IMPORT(int) PyCode_Addr2Line(PyCodeObject *, int);
 /* Future feature support */
 
 typedef struct {
+    int ff_found_docstring;
     int ff_last_lineno;
-    int ff_n_simple_stmt;
     int ff_nested_scopes;
 } PyFutureFeatures;
 

Python/compile.c

@@ -64,6 +64,9 @@ int Py_OptimizeFlag = 0;
 #define LOCAL_GLOBAL \
 "name '%.400s' is a function paramter and declared global"
 
+#define LATE_FUTURE \
+"from __future__ imports must occur at the beginning of the file"
+
 #define MANGLE_LEN 256
 
 #define OFF(x) offsetof(PyCodeObject, x)
@@ -605,6 +608,8 @@ com_free(struct compiling *c)
 	Py_XDECREF(c->c_freevars);
 	Py_XDECREF(c->c_cellvars);
 	Py_XDECREF(c->c_lnotab);
+	if (c->c_future)
+		PyMem_Free((void *)c->c_future);
 }
 
 static void
@@ -1544,9 +1549,12 @@ com_argument(struct compiling *c, node *n, PyObject **pkeywords)
 		PyObject *v = PyString_InternFromString(STR(m));
 		if (v != NULL && *pkeywords == NULL)
 			*pkeywords = PyDict_New();
-		if (v == NULL || *pkeywords == NULL)
+		if (v == NULL)
 			c->c_errors++;
-		else {
+		else if (*pkeywords == NULL) {
+			c->c_errors++;
+			Py_DECREF(v);
+		} else {
 			if (PyDict_GetItem(*pkeywords, v) != NULL)
 				com_error(c, PyExc_SyntaxError,
 					  "duplicate keyword argument");
@@ -3995,6 +4003,7 @@ symtable_build(struct compiling *c, node *n)
 {
 	if ((c->c_symtable = symtable_init()) == NULL)
 		return -1;
+	c->c_symtable->st_future = c->c_future;
 	if (c->c_future->ff_nested_scopes)
 		c->c_symtable->st_nested_scopes = 1;
 	c->c_symtable->st_filename = c->c_filename;
@@ -4482,12 +4491,15 @@ symtable_add_def(struct symtable *st, char *name, int flag)
 {
 	PyObject *s;
 	char buffer[MANGLE_LEN];
+	int ret;
 
 	if (mangle(st->st_private, name, buffer, sizeof(buffer)))
 		name = buffer;
 	if ((s = PyString_InternFromString(name)) == NULL)
 		return -1;
-	return symtable_add_def_o(st, st->st_cur->ste_symbols, s, flag);
+	ret = symtable_add_def_o(st, st->st_cur->ste_symbols, s, flag);
+	Py_DECREF(s);
+	return ret;
 }
 
 /* Must only be called with mangled names */
@@ -4819,12 +4831,14 @@ symtable_global(struct symtable *st, node *n)
 		int flags;
 
 		flags = symtable_lookup(st, name);
+		if (flags < 0)
+			continue;
 		if (flags && flags != DEF_GLOBAL) {
 			char buf[500];
 			if (flags & DEF_PARAM) {
 				PyErr_Format(PyExc_SyntaxError,
 				     "name '%.400s' is local and global",
-					     PyString_AS_STRING(name));
+					     name);
 				set_error_location(st->st_filename,
 						   st->st_cur->ste_lineno);
 				st->st_errors++;
@@ -4873,6 +4887,18 @@ symtable_import(struct symtable *st, node *n)
 	   import_as_name: NAME [NAME NAME]
 	*/
 	if (STR(CHILD(n, 0))[0] == 'f') {  /* from */
+		node *dotname = CHILD(n, 1);
+		if (strcmp(STR(CHILD(dotname, 0)), "__future__") == 0) {
+			/* check for bogus imports */
+			if (n->n_lineno >= st->st_future->ff_last_lineno) {
+				PyErr_SetString(PyExc_SyntaxError,
+						LATE_FUTURE);
+ 				set_error_location(st->st_filename,
+						   n->n_lineno);
+				st->st_errors++;
+				return;
+			}
+		}
 		if (TYPE(CHILD(n, 3)) == STAR) {
 			st->st_cur->ste_optimized |= OPT_IMPORT_STAR;
 		} else {

Python/future.c

@@ -7,6 +7,8 @@
 
 #define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined"
 
+#define FUTURE_POSSIBLE(FF) ((FF)->ff_last_lineno == -1)
+
 static int
 future_check_features(PyFutureFeatures *ff, node *n)
 {
@@ -28,6 +30,15 @@ future_check_features(PyFutureFeatures *ff, node *n)
 	return 0;
 }
 
+static void
+future_error(node *n, char *filename)
+{
+	PyErr_SetString(PyExc_SyntaxError,
+			"from __future__ imports must occur at the "
+			"beginning of the file");
+	/* XXX set filename and lineno */
+}
+
 /* Relevant portions of the grammar:
 
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
@@ -48,52 +59,82 @@ dotted_name: NAME ('.' NAME)*
 */
 
 static int
-future_parse(PyFutureFeatures *ff, node *n)
+future_parse(PyFutureFeatures *ff, node *n, char *filename)
 {
-	int i, r, found;
+	int i, r;
  loop:
 
-/*	fprintf(stderr, "future_parse(%d, %d, %s)\n",
-		TYPE(n), NCH(n), (n == NULL) ? "NULL" : STR(n));
+/*	fprintf(stderr, "future_parse(%d, %d, %s, %d)\n",
+		TYPE(n), NCH(n), (n == NULL) ? "NULL" : STR(n),
+		n->n_lineno);
 */
+
 	switch (TYPE(n)) {
 
 	case file_input:
 		for (i = 0; i < NCH(n); i++) {
 			node *ch = CHILD(n, i);
 			if (TYPE(ch) == stmt) {
-				n = ch;
-				goto loop;
+				r = future_parse(ff, ch, filename);
+				if (!FUTURE_POSSIBLE(ff))
+					return r;
 			}
 		}
 		return 0;
 
 	case simple_stmt:
-		if (NCH(n) == 1) {
+		if (NCH(n) == 2) {
 			REQ(CHILD(n, 0), small_stmt);
 			n = CHILD(n, 0);
 			goto loop;
-		}
-		found = 0;
-		for (i = 0; i < NCH(n); ++i)
-			if (TYPE(CHILD(n, i)) == small_stmt) {
-				r = future_parse(ff, CHILD(n, i));
-				if (r < 1) {
-					ff->ff_last_lineno = n->n_lineno;
-					ff->ff_n_simple_stmt = i;
-					return r;
-				} else
-					found++;
+		} else {
+			/* Deal with the special case of a series of
+			   small statements on a single line.  If a
+			   future statement follows some other
+			   statement, the SyntaxError is raised here.
+			   In all other cases, the symtable pass
+			   raises the exception.
+			*/
+			int found = 0, end_of_future = 0;
+
+			for (i = 0; i < NCH(n); i += 2) {
+				if (TYPE(CHILD(n, i)) == small_stmt) {
+					r = future_parse(ff, CHILD(n, i), 
+							 filename);
+					if (r < 1)
+						end_of_future = 1;
+					else {
+						found = 1;
+						if (end_of_future) {
+							future_error(n, 
+								     filename);
+							return -1;
+						}
+					}
+				}
 			}
-		if (found)
-			return 1;
-		else
-			return 0;
+
+			/* If we found one and only one, then the
+			   current lineno is legal. 
+			*/
+			if (found)
+				ff->ff_last_lineno = n->n_lineno + 1;
+			else
+				ff->ff_last_lineno = n->n_lineno;
+
+			if (end_of_future && found)
+				return 1;
+			else 
+				return 0;
+		}
 
 	case stmt:
 		if (TYPE(CHILD(n, 0)) == simple_stmt) {
 			n = CHILD(n, 0);
 			goto loop;
+		} else if (TYPE(CHILD(n, 0)) == expr_stmt) {
+			n = CHILD(n, 0);
+			goto loop;
 		} else {
 			REQ(CHILD(n, 0), compound_stmt);
 			ff->ff_last_lineno = n->n_lineno;
@@ -119,10 +160,42 @@ future_parse(PyFutureFeatures *ff, node *n)
 		return 1;
 	}
 
+	/* The cases below -- all of them! -- are necessary to find
+	   and skip doc strings. */
+	case expr_stmt:
+	case testlist:
+	case test:
+	case and_test:
+	case not_test:
+	case comparison:
+	case expr:
+	case xor_expr:
+	case and_expr:
+	case shift_expr:
+	case arith_expr:
+	case term:
+	case factor:
+	case power:
+		if (NCH(n) == 1) {
+			n = CHILD(n, 0);
+			goto loop;
+		}
+		break;
+
+	case atom:
+		if (TYPE(CHILD(n, 0)) == STRING 
+		    && ff->ff_found_docstring == 0) {
+			ff->ff_found_docstring = 1;
+			return 0;
+		}
+		ff->ff_last_lineno = n->n_lineno;
+		return 0;
+
 	default:
 		ff->ff_last_lineno = n->n_lineno;
 		return 0;
 	}
+	return 0;
 }
 
 PyFutureFeatures *
@@ -133,11 +206,11 @@ PyNode_Future(node *n, char *filename)
 	ff = (PyFutureFeatures *)PyMem_Malloc(sizeof(PyFutureFeatures));
 	if (ff == NULL)
 		return NULL;
-	ff->ff_last_lineno = 0;
-	ff->ff_n_simple_stmt = -1;
+	ff->ff_found_docstring = 0;
+	ff->ff_last_lineno = -1;
 	ff->ff_nested_scopes = 0;
 
-	if (future_parse(ff, n) < 0) {
+	if (future_parse(ff, n, filename) < 0) {
 		PyMem_Free((void *)ff);
 		return NULL;
 	}