parser: fix reporting of errors and parsing of decimals with leading zeros

Author: ncw

notes.txt

@@ -9,8 +9,6 @@ Limitations
   * string keys only in dictionaries
   * ints only 64 bit
 
-FIXME parsing 00007 works fine whereas it should be throwing an error
-
 FIXME interesting crash with compiling `int("42"), sausage=11)`
 Compile error: SystemError: [interface conversion: *ast.Call is not ast.SetCtxer: missing method SetCtx]
 

parser/grammar.y

@@ -1335,7 +1335,7 @@ comp_op:
 |	LTGT
 	{
 		// panic("FIXME no coverage")
-		yylex.Error("Invalid syntax")
+		yylex.(*yyLex).SyntaxError("Invalid syntax")
 	}
 |	PLINGEQ
 	{
@@ -1495,14 +1495,14 @@ strings:
 			case py.String:
 				$$ = a + b
 			default:
-				yylex.Error("SyntaxError: cannot mix string and nonstring literals")
+				yylex.(*yyLex).SyntaxError("cannot mix string and nonstring literals")
 			}
 		case py.Bytes:
 			switch b := $2.(type) {
 			case py.Bytes:
 				$$ = append(a, b...)
 			default:
-				yylex.Error("SyntaxError: cannot mix bytes and nonbytes literals")
+				yylex.(*yyLex).SyntaxError("cannot mix bytes and nonbytes literals")
 			}
 		}
 	}
@@ -1823,7 +1823,7 @@ arglist:
 		call := $1
 		call.Starargs = $3
 		if len($4.Args) != 0 {
-			yylex.Error("SyntaxError: only named arguments may follow *expression")
+			yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
 		}
 		call.Keywords = append(call.Keywords, $4.Keywords...)
 		$$ = call
@@ -1834,7 +1834,7 @@ arglist:
 		call.Starargs = $3
 		call.Kwargs = $7
 		if len($4.Args) != 0 {
-			yylex.Error("SyntaxError: only named arguments may follow *expression")
+			yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
 		}
 		call.Keywords = append(call.Keywords, $4.Keywords...)
 		$$ = call
@@ -1868,7 +1868,7 @@ argument:
 		if name, ok := test.(*ast.Name); ok {
 			$$.Keywords = []*ast.Keyword{&ast.Keyword{Pos: name.Pos, Arg: name.Id, Value: $3}}
 		} else {
-			yylex.Error("SyntaxError: keyword can't be an expression")
+			yylex.(*yyLex).SyntaxError("keyword can't be an expression")
 		}
 	}
 

parser/grammar_data_test.go

@@ -7,24 +7,44 @@ import (
 	"testing"
 
 	"github.com/ncw/gpython/ast"
+	"github.com/ncw/gpython/py"
 )
 
 var debugLevel = flag.Int("debugLevel", 0, "Debug level 0-4")
 
 // FIXME test pos is correct
 
-// FIXME add tests to test the error cases
-
 func TestGrammar(t *testing.T) {
 	SetDebug(*debugLevel)
 	for _, test := range grammarTestData {
 		Ast, err := ParseString(test.in, test.mode)
 		if err != nil {
-			t.Errorf("Parse(%q) returned error: %v", test.in, err)
+			if test.exceptionType == nil {
+				t.Errorf("%s: Got exception %v when not expecting one", test.in, err)
+				return
+			} else if exc, ok := err.(*py.Exception); !ok {
+				t.Errorf("%s: Got non python exception %T %v", test.in, err, err)
+				return
+			} else if exc.Type() != test.exceptionType {
+				t.Errorf("%s: want exception type %v got %v", test.in, test.exceptionType, exc.Type())
+				return
+			} else if exc.Type() != test.exceptionType {
+				t.Errorf("%s: want exception type %v got %v", test.in, test.exceptionType, exc.Type())
+				return
+			} else {
+				msg := string(exc.Args.(py.Tuple)[0].(py.String))
+				if msg != test.errString {
+					t.Errorf("%s: want exception text %q got %q", test.in, test.errString, msg)
+				}
+			}
 		} else {
-			out := ast.Dump(Ast)
-			if out != test.out {
-				t.Errorf("Parse(%q)\nwant> %q\n got> %q\n", test.in, test.out, out)
+			if test.exceptionType != nil {
+				t.Errorf("%s: expecting exception %q", test.in, test.errString)
+			} else {
+				out := ast.Dump(Ast)
+				if out != test.out {
+					t.Errorf("Parse(%q)\nwant> %q\n got> %q\n", test.in, test.out, out)
+				}
 			}
 		}
 	}

parser/grammar_test.go

@@ -90,7 +90,7 @@ func (x *yyLex) refill() {
 		x.eof = true
 	default:
 		x.eof = true
-		x.Errorf("Error reading input: %v", err)
+		x.SyntaxErrorf("Error reading input: %v", err)
 	}
 	// If this is exec input, add a newline to the end of the
 	// string if there isn't one already.
@@ -129,7 +129,7 @@ func countIndent(s string) int {
 			//         a       b
 			indent += tabSize - (indent & (tabSize - 1))
 		default:
-			panic("bad indent")
+			panic(py.ExceptionNewf(py.IndentationError, "unexpected indent"))
 		}
 
 	}
@@ -399,7 +399,7 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
 						goto foundIndent
 					}
 				}
-				x.Error("Inconsistent indent")
+				x.SyntaxError("Inconsistent indent")
 				return eof
 			foundIndent:
 				x.indentStack = x.indentStack[:len(x.indentStack)-1]
@@ -489,7 +489,7 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
 			}
 
 			// Nothing we recognise found
-			x.Error("invalid syntax")
+			x.SyntaxError("invalid syntax")
 			return eof
 		case checkEof:
 			if x.eof {
@@ -683,8 +683,9 @@ isNumber:
 			value = py.Complex(complex(0, f))
 		} else {
 			// Discard numbers with leading 0 except all 0s
-			if illegalDecimalInteger.FindString(x.line) != "" {
-				x.Error("illegal decimal with leading zero")
+			if illegalDecimalInteger.FindString(s) != "" {
+				// FIXME where is this error going in the grammar?
+				x.SyntaxError("illegal decimal with leading zero")
 				return eofError, nil
 			}
 			value, err = py.IntFromString(s, 10)
@@ -806,12 +807,12 @@ found:
 			}
 		}
 		if !multiLineString {
-			x.Errorf("Unterminated %sx%s string", stringEnd, stringEnd)
+			x.SyntaxErrorf("Unterminated %sx%s string", stringEnd, stringEnd)
 			return eofError, nil
 		}
 	readMore:
 		if x.eof {
-			x.Errorf("Unterminated %sx%s string", stringEnd, stringEnd)
+			x.SyntaxErrorf("Unterminated %sx%s string", stringEnd, stringEnd)
 			return eofError, nil
 		}
 		x.refill()
@@ -821,7 +822,7 @@ foundEndOfString:
 		var err error
 		buf, err = DecodeEscape(buf, byteString)
 		if err != nil {
-			x.Errorf("Decode error: %v", err)
+			x.SyntaxErrorf("Decode error: %v", err)
 			return eofError, nil
 		}
 	}
@@ -834,23 +835,31 @@ foundEndOfString:
 // The parser calls this method on a parse error.
 func (x *yyLex) Error(s string) {
 	x.error = true
-	x.errorString = s
 	if yyDebug >= 1 {
 		log.Printf("Parse error: %s", s)
 		log.Printf("Parse buffer %q", x.line)
 		log.Printf("State %#v", x)
 	}
 }
 
+// The parser calls this method on a parse error.
+func (x *yyLex) SyntaxError(s string) {
+	x.errorString = s
+	x.Error(s)
+}
+
 // Call this to write formatted errors
-func (x *yyLex) Errorf(format string, a ...interface{}) {
-	x.Error(fmt.Sprintf(format, a...))
+func (x *yyLex) SyntaxErrorf(format string, a ...interface{}) {
+	x.SyntaxError(fmt.Sprintf(format, a...))
 }
 
 // Returns an python error for the current yyLex
 func (x *yyLex) ErrorReturn() error {
 	if x.error {
-		return py.ExceptionNewf(py.SyntaxError, "Syntax Error: %s", x.errorString)
+		if x.errorString == "" {
+			x.errorString = "invalid syntax"
+		}
+		return py.ExceptionNewf(py.SyntaxError, "%s", x.errorString)
 	}
 	return nil
 }

parser/lexer.go

@@ -233,6 +233,31 @@ func TestLex(t *testing.T) {
 			{DEDENT, nil},
 			{ENDMARKER, nil},
 		}},
+		{"1", "", "eval", LexTokens{
+			{EVAL_INPUT, nil},
+			{NUMBER, py.Int(1)},
+			{ENDMARKER, nil},
+		}},
+		{"01", "illegal decimal with leading zero", "eval", LexTokens{
+			{EVAL_INPUT, nil},
+		}},
+		{"1", "", "exec", LexTokens{
+			{FILE_INPUT, nil},
+			{NUMBER, py.Int(1)},
+			{NEWLINE, nil},
+			{ENDMARKER, nil},
+		}},
+		{"1 2 3", "", "exec", LexTokens{
+			{FILE_INPUT, nil},
+			{NUMBER, py.Int(1)},
+			{NUMBER, py.Int(2)},
+			{NUMBER, py.Int(3)},
+			{NEWLINE, nil},
+			{ENDMARKER, nil},
+		}},
+		{"01", "illegal decimal with leading zero", "exec", LexTokens{
+			{FILE_INPUT, nil},
+		}},
 		{"1\n 2\n  3\n4\n", "", "exec", LexTokens{
 			{FILE_INPUT, nil},
 			{NUMBER, py.Int(1)},
@@ -343,7 +368,7 @@ func TestLex(t *testing.T) {
 			errString = err.Error()
 		}
 		if test.errString != "" {
-			test.errString = "SyntaxError: [Syntax Error: " + test.errString + "]"
+			test.errString = "SyntaxError: [" + test.errString + "]"
 		}
 		if errString != test.errString || !lts.Eq(test.lts) {
 			t.Errorf("Lex(%q) expecting (%v, %q) got (%v, %q)", test.in, test.lts, test.errString, lts, errString)

parser/lexer_test.go

@@ -24,6 +24,9 @@
     ("b'abc'", "eval"),
     ("b'abc' b'''123'''", "eval"),
     ("1234", "eval"),
+    ("01234", "eval", SyntaxError, "illegal decimal with leading zero"),
+    ("1234d", "eval", SyntaxError, "invalid syntax"),
+    ("1234d", "exec", SyntaxError),
     ("0x1234", "eval"),
     ("12.34", "eval"),
     ("1,", "eval"),
@@ -359,6 +362,7 @@
 
     # Assign
     ("a = b", "exec"),
+    ("a = 007", "exec", SyntaxError, "illegal decimal with leading zero"),
     ("a = b = c", "exec"),
     ("a, b = 1, 2", "exec"),
     ("a, b = c, d = 1, 2", "exec"),
@@ -466,13 +470,37 @@ def main():
 
 package parser
 
+import (
+"github.com/ncw/gpython/py"
+)
+
 var grammarTestData = []struct {
 in   string
 mode string
 out  string
+exceptionType *py.Type
+errString string
 }{"""]
-    for source, mode in inp:
-        out.append('{"%s", "%s", "%s"},' % (escape(source), mode, escape(dump(source, mode))))
+    for x in inp:
+        source, mode = x[:2]
+        if len(x) > 2:
+            exc = x[2]
+            errString = (x[3] if len(x) > 3 else "")
+            try:
+                dump(source, mode)
+            except exc as e:
+                error = e.msg
+            else:
+                raise ValueError("Expecting exception %s" % exc)
+            if errString != "":
+                error = errString # override error string
+            dmp = ""
+            exc_name = "py.%s" % exc.__name__
+        else:
+            dmp = dump(source, mode)
+            exc_name = "nil"
+            error = ""
+        out.append('{"%s", "%s", "%s", %s, "%s"},' % (escape(source), mode, escape(dmp), exc_name, escape(error)))
     out.append("}")
     print("Writing %s" % path)
     with open(path, "w") as f:

parser/make_grammar_test.py

@@ -2063,7 +2063,7 @@ yydefault:
 		//line grammar.y:1336
 		{
 			// panic("FIXME no coverage")
-			yylex.Error("Invalid syntax")
+			yylex.(*yyLex).SyntaxError("Invalid syntax")
 		}
 	case 209:
 		//line grammar.y:1341
@@ -2234,14 +2234,14 @@ yydefault:
 				case py.String:
 					yyVAL.obj = a + b
 				default:
-					yylex.Error("SyntaxError: cannot mix string and nonstring literals")
+					yylex.(*yyLex).SyntaxError("cannot mix string and nonstring literals")
 				}
 			case py.Bytes:
 				switch b := yyS[yypt-0].obj.(type) {
 				case py.Bytes:
 					yyVAL.obj = append(a, b...)
 				default:
-					yylex.Error("SyntaxError: cannot mix bytes and nonbytes literals")
+					yylex.(*yyLex).SyntaxError("cannot mix bytes and nonbytes literals")
 				}
 			}
 		}
@@ -2582,7 +2582,7 @@ yydefault:
 			call := yyS[yypt-3].call
 			call.Starargs = yyS[yypt-1].expr
 			if len(yyS[yypt-0].call.Args) != 0 {
-				yylex.Error("SyntaxError: only named arguments may follow *expression")
+				yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
 			}
 			call.Keywords = append(call.Keywords, yyS[yypt-0].call.Keywords...)
 			yyVAL.call = call
@@ -2594,7 +2594,7 @@ yydefault:
 			call.Starargs = yyS[yypt-4].expr
 			call.Kwargs = yyS[yypt-0].expr
 			if len(yyS[yypt-3].call.Args) != 0 {
-				yylex.Error("SyntaxError: only named arguments may follow *expression")
+				yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
 			}
 			call.Keywords = append(call.Keywords, yyS[yypt-3].call.Keywords...)
 			yyVAL.call = call
@@ -2628,7 +2628,7 @@ yydefault:
 			if name, ok := test.(*ast.Name); ok {
 				yyVAL.call.Keywords = []*ast.Keyword{&ast.Keyword{Pos: name.Pos, Arg: name.Id, Value: yyS[yypt-0].expr}}
 			} else {
-				yylex.Error("SyntaxError: keyword can't be an expression")
+				yylex.(*yyLex).SyntaxError("keyword can't be an expression")
 			}
 		}
 	case 303: