parser: fix "single" mode and return correct EOF error

Author: ncw

notes.txt

@@ -18,8 +18,6 @@ Things to do before release
 ===========================
 
   * Line numbers
-  * compile single
-  * interactive interpreter
   * Subclass builtins
   * pygen
 
@@ -29,36 +27,12 @@ FIXME recursive types for __repr__ in list, dict, tuple
 >>> L
 [[...]]
 
-interactive interpreter
-calls compile(..."single) until doesn't get "SyntaxError: unexpected EOF while parsing" ?
-see pythonrun.c
-PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags)
-...
-    mod = PyParser_ASTFromFileObject(fp, filename, enc,
-                                     Py_single_input, ps1, ps2,
-                                     flags, &errcode, arena);
-    Py_XDECREF(v);
-    Py_XDECREF(w);
-    Py_XDECREF(oenc);
-    if (mod == NULL) {
-        PyArena_Free(arena);
-        if (errcode == E_EOF) {
-            PyErr_Clear();
-            return E_EOF;
-        }
-        PyErr_Print();
-        return -1;
-    }
-
-
 Limitations & Missing parts
 ===========================
   * string keys only in dictionaries
     * intermediate fix - stop it panicing!
   * line numbers missing in SyntaxErrors
   * \N{...} escapes not implemented
-  * Interactive interpreter does single lines only
-  * compile(..., "single") not working
   * lots of builtins still to implement
   * FIXME eq && ne should throw an error for a type which doesn' have eq implemented
   * repr/str

parser/grammar.y

@@ -259,18 +259,20 @@ inputs:
 	}
 
 single_input:
-	NEWLINE
+/*	NEWLINE
 	{
-		// panic("FIXME no coverage")
+		// This is in the python grammar, but the interpreter
+		// just gives "unexpected EOF while parsing" when you
+		// give it a \n
 		$$ = &ast.Interactive{ModBase: ast.ModBase{Pos: $<pos>$}}
 	}
-|	simple_stmt
+|*/	simple_stmt
 	{
 		$$ = &ast.Interactive{ModBase: ast.ModBase{Pos: $<pos>$}, Body: $1}
 	}
 |	compound_stmt NEWLINE
 	{
-		// panic("FIXME no coverage")
+		//  NB: compound_stmt in single_input is followed by extra NEWLINE!
 		$$ = &ast.Interactive{ModBase: ast.ModBase{Pos: $<pos>$}, Body: []ast.Stmt{$1}}
 	}
 

parser/grammar_data_test.go

@@ -29,8 +29,9 @@ var grammarTestData = []struct {
 	{"b'abc' b'''123'''", "eval", "Expression(body=Bytes(s=b'abc123'))", nil, ""},
 	{"1234", "eval", "Expression(body=Num(n=1234))", nil, ""},
 	{"01234", "eval", "", py.SyntaxError, "illegal decimal with leading zero"},
-	{"1234d", "eval", "", py.SyntaxError, "invalid syntax"},
+	{"1234d", "eval", "", py.SyntaxError, "unexpected EOF while parsing"},
 	{"1234d", "exec", "", py.SyntaxError, "invalid syntax"},
+	{"1234d", "single", "", py.SyntaxError, "unexpected EOF while parsing"},
 	{"0x1234", "eval", "Expression(body=Num(n=4660))", nil, ""},
 	{"12.34", "eval", "Expression(body=Num(n=12.34))", nil, ""},
 	{"1,", "eval", "Expression(body=Tuple(elts=[Num(n=1)], ctx=Load()))", nil, ""},
@@ -311,5 +312,9 @@ var grammarTestData = []struct {
 	{"@dec(a,b,c=d,*args,**kwargs)\ndef fn():\n    pass\n", "exec", "Module(body=[FunctionDef(name='fn', args=arguments(args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Call(func=Name(id='dec', ctx=Load()), args=[Name(id='a', ctx=Load()), Name(id='b', ctx=Load())], keywords=[keyword(arg='c', value=Name(id='d', ctx=Load()))], starargs=Name(id='args', ctx=Load()), kwargs=Name(id='kwargs', ctx=Load()))], returns=None)])", nil, ""},
 	{"@dec1\n@dec2()\n@dec3(a)\n@dec4(a,b)\ndef fn():\n    pass\n", "exec", "Module(body=[FunctionDef(name='fn', args=arguments(args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='dec1', ctx=Load()), Call(func=Name(id='dec2', ctx=Load()), args=[], keywords=[], starargs=None, kwargs=None), Call(func=Name(id='dec3', ctx=Load()), args=[Name(id='a', ctx=Load())], keywords=[], starargs=None, kwargs=None), Call(func=Name(id='dec4', ctx=Load()), args=[Name(id='a', ctx=Load()), Name(id='b', ctx=Load())], keywords=[], starargs=None, kwargs=None)], returns=None)])", nil, ""},
 	{"@dec1\n@dec2()\n@dec3(a)\n@dec4(a,b)\nclass A(B):\n    pass\n", "exec", "Module(body=[ClassDef(name='A', bases=[Name(id='B', ctx=Load())], keywords=[], starargs=None, kwargs=None, body=[Pass()], decorator_list=[Name(id='dec1', ctx=Load()), Call(func=Name(id='dec2', ctx=Load()), args=[], keywords=[], starargs=None, kwargs=None), Call(func=Name(id='dec3', ctx=Load()), args=[Name(id='a', ctx=Load())], keywords=[], starargs=None, kwargs=None), Call(func=Name(id='dec4', ctx=Load()), args=[Name(id='a', ctx=Load()), Name(id='b', ctx=Load())], keywords=[], starargs=None, kwargs=None)])])", nil, ""},
+	{"", "single", "", py.SyntaxError, "unexpected EOF while parsing"},
+	{"\n", "single", "", py.SyntaxError, "unexpected EOF while parsing"},
 	{"pass\n", "single", "Interactive(body=[Pass()])", nil, ""},
+	{"if True:\n   pass\n\n", "single", "Interactive(body=[If(test=NameConstant(value=True), body=[Pass()], orelse=[])])", nil, ""},
+	{"while True:\n pass\nelse:\n return\n", "single", "Interactive(body=[While(test=NameConstant(value=True), body=[Pass()], orelse=[Return(value=None)])])", nil, ""},
 }

parser/lexer.go

@@ -47,7 +47,7 @@ type yyLex struct {
 	parenthesis   int     // number of open ( )
 	brace         int     // number of open { }
 	mod           ast.Mod // output
-	startToken    int     // initial token to output
+	tokens        []int   // buffered tokens to output
 }
 
 // Create a new lexer
@@ -64,19 +64,41 @@ func NewLex(r io.Reader, mode string) (*yyLex, error) {
 	}
 	switch mode {
 	case "exec":
-		x.startToken = FILE_INPUT
+		x.queue(FILE_INPUT)
 		x.exec = true
 	case "eval":
-		x.startToken = EVAL_INPUT
+		x.queue(EVAL_INPUT)
 	case "single":
-		x.startToken = SINGLE_INPUT
+		x.queue(SINGLE_INPUT)
 		x.interactive = true
 	default:
 		return nil, py.ExceptionNewf(py.ValueError, "compile mode must be 'exec', 'eval' or 'single'")
 	}
 	return x, nil
 }
 
+// queue tokens for later return
+func (x *yyLex) queue(tokens ...int) {
+	x.tokens = append(x.tokens, tokens...)
+}
+
+// Return whether the token queue is empty
+func (x *yyLex) queueEmpty() bool {
+	return len(x.tokens) == 0
+}
+
+// dequeue a token for return
+//
+// panic if no token available
+func (x *yyLex) dequeue() int {
+	if x.queueEmpty() {
+		panic("token queue empty")
+	}
+	token := x.tokens[0]
+	x.tokens = x.tokens[1:]
+	return token
+}
+
 // Refill line
 func (x *yyLex) refill() {
 	var err error
@@ -328,6 +350,14 @@ func (lts LexTokens) String() string {
 	return buf.String()
 }
 
+// Queue any remaining DEDENTS
+func (x *yyLex) queueDedents() {
+	for i := len(x.indentStack) - 1; i >= 1; i-- {
+		x.queue(DEDENT)
+	}
+	x.indentStack = x.indentStack[:1]
+}
+
 // The parser calls this method to get each new token.  This
 // implementation returns operators and NUM.
 func (x *yyLex) Lex(yylval *yySymType) (ret int) {
@@ -340,11 +370,9 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
 		}()
 	}
 
-	// Return initial token
-	if x.startToken != eof {
-		token := x.startToken
-		x.startToken = eof
-		return token
+	// Return queued tokens if there are any
+	if !x.queueEmpty() {
+		return x.dequeue()
 	}
 
 	// FIXME keep x.pos up to date
@@ -358,10 +386,13 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
 			x.state++
 			if x.line == "" && x.eof {
 				x.state = checkEof
-				// an empty line while reading interactive input should return a NEWLINE
-				// Don't output NEWLINE if brackets are open
+				// During interactive input of statements an entirely blank logical
+				// line (i.e. one containing not even whitespace or a comment)
+				// terminates a multi-line statement.
 				if x.interactive && !x.openBrackets() {
-					return NEWLINE
+					x.queueDedents()
+					x.queue(NEWLINE)
+					return x.dequeue()
 				}
 				continue
 			}
@@ -380,32 +411,33 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
 			}
 			x.state++
 		case checkIndent:
+			x.state++
 			// Don't output INDENT or DEDENT if brackets are open
 			if x.openBrackets() {
-				x.state++
 				continue
 			}
 			// See if indent has changed and issue INDENT / DEDENT
 			indent := countIndent(x.currentIndent)
-			indentStackTop := x.indentStack[len(x.indentStack)-1]
-			switch {
-			case indent > indentStackTop:
+			i := len(x.indentStack) - 1
+			indentStackTop := x.indentStack[i]
+			if indent == indentStackTop {
+				continue
+			} else if indent > indentStackTop {
 				x.indentStack = append(x.indentStack, indent)
-				x.state++
 				return INDENT
-			case indent < indentStackTop:
-				for i := len(x.indentStack) - 1; i >= 0; i-- {
+			} else {
+				for ; i >= 0; i-- {
 					if x.indentStack[i] == indent {
 						goto foundIndent
 					}
+					x.queue(DEDENT)
 				}
 				x.SyntaxError("Inconsistent indent")
 				return eof
 			foundIndent:
-				x.indentStack = x.indentStack[:len(x.indentStack)-1]
-				return DEDENT
+				x.indentStack = x.indentStack[:i+1]
+				return x.dequeue()
 			}
-			x.state++
 		case parseTokens:
 			// Skip white space
 			x.line = strings.TrimLeft(x.line, " \t")
@@ -493,18 +525,16 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
 			return eof
 		case checkEof:
 			if x.eof {
-				// Return any remaining DEDENTS
-				if len(x.indentStack) > 1 {
-					x.indentStack = x.indentStack[:len(x.indentStack)-1]
-					x.state = checkEof
-					return DEDENT
-				}
+				x.queueDedents()
 				// then return ENDMARKER
 				x.state = isEof
-				if x.interactive {
+				if !x.interactive {
+					x.queue(ENDMARKER)
+				}
+				if x.queueEmpty() {
 					continue
 				}
-				return ENDMARKER
+				return x.dequeue()
 			}
 			x.state = readString
 		case isEof:
@@ -857,7 +887,11 @@ func (x *yyLex) SyntaxErrorf(format string, a ...interface{}) {
 func (x *yyLex) ErrorReturn() error {
 	if x.error {
 		if x.errorString == "" {
-			x.errorString = "invalid syntax"
+			if x.eof && !x.exec {
+				x.errorString = "unexpected EOF while parsing"
+			} else {
+				x.errorString = "invalid syntax"
+			}
 		}
 		return py.ExceptionNewf(py.SyntaxError, "%s", x.errorString)
 	}

parser/lexer_test.go

@@ -361,6 +361,56 @@ func TestLex(t *testing.T) {
 		{"$asdasd", "invalid syntax", "eval", LexTokens{
 			{EVAL_INPUT, nil},
 		}},
+		{"if True:\n   pass\n\n", "", "single", LexTokens{
+			{SINGLE_INPUT, nil},
+			{IF, nil},
+			{TRUE, nil},
+			{':', nil},
+			{NEWLINE, nil},
+			{INDENT, nil},
+			{PASS, nil},
+			{NEWLINE, nil},
+			{DEDENT, nil},
+			{NEWLINE, nil},
+		}},
+		{"while True:\n pass\nelse:\n return\n", "", "single", LexTokens{
+			{SINGLE_INPUT, nil},
+			{WHILE, nil},
+			{TRUE, nil},
+			{':', nil},
+			{NEWLINE, nil},
+			{INDENT, nil},
+			{PASS, nil},
+			{NEWLINE, nil},
+			{DEDENT, nil},
+			{ELSE, nil},
+			{':', nil},
+			{NEWLINE, nil},
+			{INDENT, nil},
+			{RETURN, nil},
+			{NEWLINE, nil},
+			{DEDENT, nil},
+			{NEWLINE, nil},
+		}},
+		{"while True:\n pass\nelse:\n return\n", "", "exec", LexTokens{
+			{FILE_INPUT, nil},
+			{WHILE, nil},
+			{TRUE, nil},
+			{':', nil},
+			{NEWLINE, nil},
+			{INDENT, nil},
+			{PASS, nil},
+			{NEWLINE, nil},
+			{DEDENT, nil},
+			{ELSE, nil},
+			{':', nil},
+			{NEWLINE, nil},
+			{INDENT, nil},
+			{RETURN, nil},
+			{NEWLINE, nil},
+			{DEDENT, nil},
+			{ENDMARKER, nil},
+		}},
 	} {
 		lts, err := LexString(test.in, test.mode)
 		errString := ""

parser/make_grammar_test.py

@@ -25,8 +25,9 @@
     ("b'abc' b'''123'''", "eval"),
     ("1234", "eval"),
     ("01234", "eval", SyntaxError, "illegal decimal with leading zero"),
-    ("1234d", "eval", SyntaxError, "invalid syntax"),
+    ("1234d", "eval", SyntaxError),
     ("1234d", "exec", SyntaxError),
+    ("1234d", "single", SyntaxError),
     ("0x1234", "eval"),
     ("12.34", "eval"),
     ("1,", "eval"),
@@ -461,10 +462,11 @@ class A(B):
 """, "exec"),
 
     # single input
-    #("\n", "single"),
+    ("", "single", SyntaxError),
+    ("\n", "single", SyntaxError),
     ("pass\n", "single"),
-    # FIXME ("if True:\n   pass\n\n", "single"),
-
+    ("if True:\n   pass\n\n", "single"),
+    ("while True:\n pass\nelse:\n return\n", "single"),
 ]
 
 def dump(source, mode):