create NameConstant AST class for None, True, and False literals (closes python#16619)

Author: benjaminp

Include/Python-ast.h

@@ -182,8 +182,9 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
                   SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
                   Yield_kind=12, YieldFrom_kind=13, Compare_kind=14,
                   Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18,
-                  Ellipsis_kind=19, Attribute_kind=20, Subscript_kind=21,
-                  Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25};
+                  NameConstant_kind=19, Ellipsis_kind=20, Attribute_kind=21,
+                  Subscript_kind=22, Starred_kind=23, Name_kind=24,
+                  List_kind=25, Tuple_kind=26};
 struct _expr {
         enum _expr_kind kind;
         union {
@@ -278,6 +279,10 @@ struct _expr {
                         bytes s;
                 } Bytes;
 
+                struct {
+                        singleton value;
+                } NameConstant;
+                
                 struct {
                         expr_ty value;
                         identifier attr;
@@ -509,6 +514,9 @@ expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
 expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena);
 #define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3)
 expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
+#define NameConstant(a0, a1, a2, a3) _Py_NameConstant(a0, a1, a2, a3)
+expr_ty _Py_NameConstant(singleton value, int lineno, int col_offset, PyArena
+                         *arena);
 #define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2)
 expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena);
 #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)

Include/asdl.h

@@ -5,6 +5,7 @@ typedef PyObject * identifier;
 typedef PyObject * string;
 typedef PyObject * bytes;
 typedef PyObject * object;
+typedef PyObject * singleton;
 
 /* It would be nice if the code generated by asdl_c.py was completely
    independent of Python, but it is a goal the requires too much work

Lib/ast.py

@@ -42,7 +42,6 @@ def literal_eval(node_or_string):
     Python literal structures: strings, bytes, numbers, tuples, lists, dicts,
     sets, booleans, and None.
     """
-    _safe_names = {'None': None, 'True': True, 'False': False}
     if isinstance(node_or_string, str):
         node_or_string = parse(node_or_string, mode='eval')
     if isinstance(node_or_string, Expression):
@@ -61,9 +60,8 @@ def _convert(node):
         elif isinstance(node, Dict):
             return dict((_convert(k), _convert(v)) for k, v
                         in zip(node.keys, node.values))
-        elif isinstance(node, Name):
-            if node.id in _safe_names:
-                return _safe_names[node.id]
+        elif isinstance(node, NameConstant):
+            return node.value
         elif isinstance(node, UnaryOp) and \
              isinstance(node.op, (UAdd, USub)) and \
              isinstance(node.operand, (Num, UnaryOp, BinOp)):

Lib/test/test_ast.py

@@ -928,6 +928,9 @@ def test_list(self):
     def test_tuple(self):
         self._sequence(ast.Tuple)
 
+    def test_nameconstant(self):
+        self.expr(ast.NameConstant(4), "singleton must be True, False, or None")
+
     def test_stdlib_validates(self):
         stdlib = os.path.dirname(ast.__file__)
         tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")]
@@ -959,13 +962,13 @@ def main():
 
 #### EVERYTHING BELOW IS GENERATED #####
 exec_results = [
-('Module', [('Expr', (1, 0), ('Name', (1, 0), 'None', ('Load',)))]),
+('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None)]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [], []), [('Pass', (1, 10))], [], None)]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [('Num', (1, 8), 0)], []), [('Pass', (1, 12))], [], None)]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], 'args', None, [], None, None, [], []), [('Pass', (1, 14))], [], None)]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], 'kwargs', None, [], []), [('Pass', (1, 17))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None), ('arg', 'b', None), ('arg', 'c', None), ('arg', 'd', None), ('arg', 'e', None)], 'args', None, [], 'kwargs', None, [('Num', (1, 11), 1), ('Name', (1, 16), 'None', ('Load',)), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])], []), [('Pass', (1, 52))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None), ('arg', 'b', None), ('arg', 'c', None), ('arg', 'd', None), ('arg', 'e', None)], 'args', None, [], 'kwargs', None, [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])], []), [('Pass', (1, 52))], [], None)]),
 ('Module', [('ClassDef', (1, 0), 'C', [], [], None, None, [('Pass', (1, 8))], [])]),
 ('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], None, None, [('Pass', (1, 17))], [])]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
@@ -1002,14 +1005,14 @@ def main():
 ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
 ]
 eval_results = [
-('Expression', ('Name', (1, 0), 'None', ('Load',))),
+('Expression', ('NameConstant', (1, 0), None)),
 ('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
 ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
 ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
-('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('Name', (1, 7), 'None', ('Load',)))),
+('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('NameConstant', (1, 7), None))),
 ('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
 ('Expression', ('Dict', (1, 0), [], [])),
-('Expression', ('Set', (1, 0), [('Name', (1, 1), 'None', ('Load',))])),
+('Expression', ('Set', (1, 0), [('NameConstant', (1, 1), None)])),
 ('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])),
 ('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),

Lib/test/test_syntax.py

@@ -33,7 +33,7 @@
 
 >>> None = 1
 Traceback (most recent call last):
-SyntaxError: assignment to keyword
+SyntaxError: can't assign to keyword
 
 It's a syntax error to assign to the empty tuple.  Why isn't it an
 error to assign to the empty list?  It will always raise some error at
@@ -233,7 +233,7 @@
 SyntaxError: can't assign to generator expression
 >>> None += 1
 Traceback (most recent call last):
-SyntaxError: assignment to keyword
+SyntaxError: can't assign to keyword
 >>> f() += 1
 Traceback (most recent call last):
 SyntaxError: can't assign to function call

Misc/NEWS

@@ -10,6 +10,10 @@ What's New in Python 3.4.0 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #16619: Create NameConstant AST class to represent None, True, and False
+  literals. As a result, these constants are never loaded at runtime from
+  builtins.
+
 - Issue #16455: On FreeBSD and Solaris, if the locale is C, the
   ASCII/surrogateescape codec is now used, instead of the locale encoding, to
   decode the command line arguments. This change fixes inconsistencies with

Parser/Python.asdl

@@ -1,4 +1,4 @@
--- ASDL's five builtin types are identifier, int, string, bytes, object
+-- ASDL's six builtin types are identifier, int, string, bytes, object, singleton
 
 module Python
 {
@@ -69,8 +69,8 @@ module Python
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
          | Bytes(bytes s)
+         | NameConstant(singleton value)
          | Ellipsis
-         -- other literals? bools?
 
          -- the following expression can appear in assignment context
          | Attribute(expr value, identifier attr, expr_context ctx)

Parser/asdl.py

@@ -222,7 +222,7 @@ def p_field_5(self, type_):
         " field ::= Id ? "
         return Field(type[0], opt=True)
 
-builtin_types = ("identifier", "string", "bytes", "int", "object")
+builtin_types = ("identifier", "string", "bytes", "int", "object", "singleton")
 
 # below is a collection of classes to capture the AST of an AST :-)
 # not sure if any of the methods are useful yet, but I'm adding them

Parser/asdl_c.py

@@ -820,6 +820,7 @@ def visitModule(self, mod):
     Py_INCREF((PyObject*)o);
     return (PyObject*)o;
 }
+#define ast2obj_singleton ast2obj_object
 #define ast2obj_identifier ast2obj_object
 #define ast2obj_string ast2obj_object
 #define ast2obj_bytes ast2obj_object
@@ -831,6 +832,17 @@ def visitModule(self, mod):
 
 /* Conversion Python -> AST */
 
+static int obj2ast_singleton(PyObject *obj, PyObject** out, PyArena* arena)
+{
+    if (obj != Py_None && obj != Py_True && obj != Py_False) {
+        PyErr_SetString(PyExc_ValueError,
+                        "AST singleton must be True, False, or None");
+        return 1;
+    }
+    *out = obj;
+    return 0;
+}
+
 static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
 {
     if (obj == Py_None)

Python/Python-ast.c

@@ -271,6 +271,10 @@ static PyTypeObject *Bytes_type;
 static char *Bytes_fields[]={
         "s",
 };
+static PyTypeObject *NameConstant_type;
+static char *NameConstant_fields[]={
+        "value",
+};
 static PyTypeObject *Ellipsis_type;
 static PyTypeObject *Attribute_type;
 _Py_IDENTIFIER(attr);
@@ -673,6 +677,7 @@ static PyObject* ast2obj_object(void *o)
     Py_INCREF((PyObject*)o);
     return (PyObject*)o;
 }
+#define ast2obj_singleton ast2obj_object
 #define ast2obj_identifier ast2obj_object
 #define ast2obj_string ast2obj_object
 #define ast2obj_bytes ast2obj_object
@@ -684,6 +689,17 @@ static PyObject* ast2obj_int(long b)
 
 /* Conversion Python -> AST */
 
+static int obj2ast_singleton(PyObject *obj, PyObject** out, PyArena* arena)
+{
+    if (obj != Py_None && obj != Py_True && obj != Py_False) {
+        PyErr_SetString(PyExc_ValueError,
+                        "AST singleton must be True, False, or None");
+        return 1;
+    }
+    *out = obj;
+    return 0;
+}
+
 static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
 {
     if (obj == Py_None)
@@ -860,6 +876,9 @@ static int init_types(void)
         if (!Str_type) return 0;
         Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1);
         if (!Bytes_type) return 0;
+        NameConstant_type = make_type("NameConstant", expr_type,
+                                      NameConstant_fields, 1);
+        if (!NameConstant_type) return 0;
         Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0);
         if (!Ellipsis_type) return 0;
         Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3);
@@ -1920,6 +1939,25 @@ Bytes(bytes s, int lineno, int col_offset, PyArena *arena)
         return p;
 }
 
+expr_ty
+NameConstant(singleton value, int lineno, int col_offset, PyArena *arena)
+{
+        expr_ty p;
+        if (!value) {
+                PyErr_SetString(PyExc_ValueError,
+                                "field value is required for NameConstant");
+                return NULL;
+        }
+        p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+        if (!p)
+                return NULL;
+        p->kind = NameConstant_kind;
+        p->v.NameConstant.value = value;
+        p->lineno = lineno;
+        p->col_offset = col_offset;
+        return p;
+}
+
 expr_ty
 Ellipsis(int lineno, int col_offset, PyArena *arena)
 {
@@ -2028,6 +2066,7 @@ Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena
      *arena)
 {
         expr_ty p;
+        assert(PyUnicode_CompareWithASCIIString(id, "True") && PyUnicode_CompareWithASCIIString(id, "False") && PyUnicode_CompareWithASCIIString(id, "None"));
         if (!id) {
                 PyErr_SetString(PyExc_ValueError,
                                 "field id is required for Name");
@@ -2948,6 +2987,15 @@ ast2obj_expr(void* _o)
                         goto failed;
                 Py_DECREF(value);
                 break;
+        case NameConstant_kind:
+                result = PyType_GenericNew(NameConstant_type, NULL, NULL);
+                if (!result) goto failed;
+                value = ast2obj_singleton(o->v.NameConstant.value);
+                if (!value) goto failed;
+                if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+                        goto failed;
+                Py_DECREF(value);
+                break;
         case Ellipsis_kind:
                 result = PyType_GenericNew(Ellipsis_type, NULL, NULL);
                 if (!result) goto failed;
@@ -5688,6 +5736,29 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
                 if (*out == NULL) goto failed;
                 return 0;
         }
+        isinstance = PyObject_IsInstance(obj, (PyObject*)NameConstant_type);
+        if (isinstance == -1) {
+                return 1;
+        }
+        if (isinstance) {
+                singleton value;
+
+                if (_PyObject_HasAttrId(obj, &PyId_value)) {
+                        int res;
+                        tmp = _PyObject_GetAttrId(obj, &PyId_value);
+                        if (tmp == NULL) goto failed;
+                        res = obj2ast_singleton(tmp, &value, arena);
+                        if (res != 0) goto failed;
+                        Py_XDECREF(tmp);
+                        tmp = NULL;
+                } else {
+                        PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from NameConstant");
+                        return 1;
+                }
+                *out = NameConstant(value, lineno, col_offset, arena);
+                if (*out == NULL) goto failed;
+                return 0;
+        }
         isinstance = PyObject_IsInstance(obj, (PyObject*)Ellipsis_type);
         if (isinstance == -1) {
                 return 1;
@@ -7008,6 +7079,8 @@ PyInit__ast(void)
             NULL;
         if (PyDict_SetItemString(d, "Bytes", (PyObject*)Bytes_type) < 0) return
             NULL;
+        if (PyDict_SetItemString(d, "NameConstant",
+            (PyObject*)NameConstant_type) < 0) return NULL;
         if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0)
             return NULL;
         if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) <

Python/ast.c

@@ -282,6 +282,7 @@ validate_expr(expr_ty exp, expr_context_ty ctx)
         return validate_exprs(exp->v.Tuple.elts, ctx, 0);
     /* These last cases don't have any checking. */
     case Name_kind:
+    case NameConstant_kind:
     case Ellipsis_kind:
         return 1;
     default:
@@ -903,7 +904,7 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)
             break;
         case Name_kind:
             if (ctx == Store) {
-                if (forbidden_name(c, e->v.Name.id, n, 1))
+                if (forbidden_name(c, e->v.Name.id, n, 0))
                     return 0; /* forbidden_name() calls ast_error() */
             }
             e->v.Name.ctx = ctx;
@@ -955,6 +956,9 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)
         case Bytes_kind:
             expr_name = "literal";
             break;
+        case NameConstant_kind:
+            expr_name = "keyword";
+            break;
         case Ellipsis_kind:
             expr_name = "Ellipsis";
             break;
@@ -1819,11 +1823,21 @@ ast_for_atom(struct compiling *c, const node *n)
 
     switch (TYPE(ch)) {
     case NAME: {
-        /* All names start in Load context, but may later be
-           changed. */
-        PyObject *name = NEW_IDENTIFIER(ch);
+        PyObject *name;
+        const char *s = STR(ch);
+        size_t len = strlen(s);
+        if (len >= 4 && len <= 5) {
+            if (!strcmp(s, "None"))
+                return NameConstant(Py_None, LINENO(n), n->n_col_offset, c->c_arena);
+            if (!strcmp(s, "True"))
+                return NameConstant(Py_True, LINENO(n), n->n_col_offset, c->c_arena);
+            if (!strcmp(s, "False"))
+                return NameConstant(Py_False, LINENO(n), n->n_col_offset, c->c_arena);
+        }
+        name = new_identifier(s, c);
         if (!name)
             return NULL;
+        /* All names start in Load context, but may later be changed. */
         return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
     }
     case STRING: {