Add ast.Constant

Issue #26146: Add a new kind of AST node: ast.Constant. It can be used by
external AST optimizers, but the compiler does not emit directly such node.

An optimizer can replace the following AST nodes with ast.Constant:

* ast.NameConstant: None, False, True
* ast.Num: int, float, complex
* ast.Str: str
* ast.Bytes: bytes
* ast.Tuple if items are constants too: tuple
* frozenset

Update code to accept ast.Constant instead of ast.Num and/or ast.Str:

* compiler
* docstrings
* ast.literal_eval()
* Tools/parser/unparse.py

Author: vstinner

Include/Python-ast.h

@@ -202,9 +202,9 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
                   Await_kind=12, Yield_kind=13, YieldFrom_kind=14,
                   Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18,
                   FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21,
-                  NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24,
-                  Subscript_kind=25, Starred_kind=26, Name_kind=27,
-                  List_kind=28, Tuple_kind=29};
+                  NameConstant_kind=22, Ellipsis_kind=23, Constant_kind=24,
+                  Attribute_kind=25, Subscript_kind=26, Starred_kind=27,
+                  Name_kind=28, List_kind=29, Tuple_kind=30};
 struct _expr {
     enum _expr_kind kind;
     union {
@@ -315,6 +315,10 @@ struct _expr {
             singleton value;
         } NameConstant;
 
+        struct {
+            constant value;
+        } Constant;
+        
         struct {
             expr_ty value;
             identifier attr;
@@ -567,6 +571,9 @@ expr_ty _Py_NameConstant(singleton value, int lineno, int col_offset, PyArena
                          *arena);
 #define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2)
 expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena);
+#define Constant(a0, a1, a2, a3) _Py_Constant(a0, a1, a2, a3)
+expr_ty _Py_Constant(constant value, int lineno, int col_offset, PyArena
+                     *arena);
 #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
 expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
                       lineno, int col_offset, PyArena *arena);

Include/asdl.h

@@ -6,6 +6,7 @@ typedef PyObject * string;
 typedef PyObject * bytes;
 typedef PyObject * object;
 typedef PyObject * singleton;
+typedef PyObject * constant;
 
 /* It would be nice if the code generated by asdl_c.py was completely
    independent of Python, but it is a goal the requires too much work

Lib/ast.py

@@ -35,6 +35,8 @@ def parse(source, filename='<unknown>', mode='exec'):
     return compile(source, filename, mode, PyCF_ONLY_AST)
 
 
+_NUM_TYPES = (int, float, complex)
+
 def literal_eval(node_or_string):
     """
     Safely evaluate an expression node or a string containing a Python
@@ -47,7 +49,9 @@ def literal_eval(node_or_string):
     if isinstance(node_or_string, Expression):
         node_or_string = node_or_string.body
     def _convert(node):
-        if isinstance(node, (Str, Bytes)):
+        if isinstance(node, Constant):
+            return node.value
+        elif isinstance(node, (Str, Bytes)):
             return node.s
         elif isinstance(node, Num):
             return node.n
@@ -62,24 +66,21 @@ def _convert(node):
                         in zip(node.keys, node.values))
         elif isinstance(node, NameConstant):
             return node.value
-        elif isinstance(node, UnaryOp) and \
-             isinstance(node.op, (UAdd, USub)) and \
-             isinstance(node.operand, (Num, UnaryOp, BinOp)):
+        elif isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)):
             operand = _convert(node.operand)
-            if isinstance(node.op, UAdd):
-                return + operand
-            else:
-                return - operand
-        elif isinstance(node, BinOp) and \
-             isinstance(node.op, (Add, Sub)) and \
-             isinstance(node.right, (Num, UnaryOp, BinOp)) and \
-             isinstance(node.left, (Num, UnaryOp, BinOp)):
+            if isinstance(operand, _NUM_TYPES):
+                if isinstance(node.op, UAdd):
+                    return + operand
+                else:
+                    return - operand
+        elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)):
             left = _convert(node.left)
             right = _convert(node.right)
-            if isinstance(node.op, Add):
-                return left + right
-            else:
-                return left - right
+            if isinstance(left, _NUM_TYPES) and isinstance(right, _NUM_TYPES):
+                if isinstance(node.op, Add):
+                    return left + right
+                else:
+                    return left - right
         raise ValueError('malformed node or string: ' + repr(node))
     return _convert(node_or_string)
 
@@ -196,12 +197,19 @@ def get_docstring(node, clean=True):
     """
     if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
         raise TypeError("%r can't have docstrings" % node.__class__.__name__)
-    if node.body and isinstance(node.body[0], Expr) and \
-       isinstance(node.body[0].value, Str):
-        if clean:
-            import inspect
-            return inspect.cleandoc(node.body[0].value.s)
-        return node.body[0].value.s
+    if not(node.body and isinstance(node.body[0], Expr)):
+        return
+    node = node.body[0].value
+    if isinstance(node, Str):
+        text = node.s
+    elif isinstance(node, Constant) and isinstance(node.value, str):
+        text = node.value
+    else:
+        return
+    if clean:
+        import inspect
+        text = inspect.cleandoc(text)
+    return text
 
 
 def walk(node):

Lib/test/test_ast.py

@@ -1,7 +1,8 @@
+import ast
+import dis
 import os
 import sys
 import unittest
-import ast
 import weakref
 
 from test import support
@@ -933,6 +934,123 @@ def test_stdlib_validates(self):
             compile(mod, fn, "exec")
 
 
+class ConstantTests(unittest.TestCase):
+    """Tests on the ast.Constant node type."""
+
+    def compile_constant(self, value):
+        tree = ast.parse("x = 123")
+
+        node = tree.body[0].value
+        new_node = ast.Constant(value=value)
+        ast.copy_location(new_node, node)
+        tree.body[0].value = new_node
+
+        code = compile(tree, "<string>", "exec")
+
+        ns = {}
+        exec(code, ns)
+        return ns['x']
+
+    def test_singletons(self):
+        for const in (None, False, True, Ellipsis, b'', frozenset()):
+            with self.subTest(const=const):
+                value = self.compile_constant(const)
+                self.assertIs(value, const)
+
+    def test_values(self):
+        nested_tuple = (1,)
+        nested_frozenset = frozenset({1})
+        for level in range(3):
+            nested_tuple = (nested_tuple, 2)
+            nested_frozenset = frozenset({nested_frozenset, 2})
+        values = (123, 123.0, 123j,
+                  "unicode", b'bytes',
+                  tuple("tuple"), frozenset("frozenset"),
+                  nested_tuple, nested_frozenset)
+        for value in values:
+            with self.subTest(value=value):
+                result = self.compile_constant(value)
+                self.assertEqual(result, value)
+
+    def test_assign_to_constant(self):
+        tree = ast.parse("x = 1")
+
+        target = tree.body[0].targets[0]
+        new_target = ast.Constant(value=1)
+        ast.copy_location(new_target, target)
+        tree.body[0].targets[0] = new_target
+
+        with self.assertRaises(ValueError) as cm:
+            compile(tree, "string", "exec")
+        self.assertEqual(str(cm.exception),
+                         "expression which can't be assigned "
+                         "to in Store context")
+
+    def test_get_docstring(self):
+        tree = ast.parse("'docstring'\nx = 1")
+        self.assertEqual(ast.get_docstring(tree), 'docstring')
+
+        tree.body[0].value = ast.Constant(value='constant docstring')
+        self.assertEqual(ast.get_docstring(tree), 'constant docstring')
+
+    def get_load_const(self, tree):
+        # Compile to bytecode, disassemble and get parameter of LOAD_CONST
+        # instructions
+        co = compile(tree, '<string>', 'exec')
+        consts = []
+        for instr in dis.get_instructions(co):
+            if instr.opname == 'LOAD_CONST':
+                consts.append(instr.argval)
+        return consts
+
+    @support.cpython_only
+    def test_load_const(self):
+        consts = [None,
+                  True, False,
+                  124,
+                  2.0,
+                  3j,
+                  "unicode",
+                  b'bytes',
+                  (1, 2, 3)]
+
+        code = '\n'.join(map(repr, consts))
+        code += '\n...'
+
+        code_consts = [const for const in consts
+                       if (not isinstance(const, (str, int, float, complex))
+                           or isinstance(const, bool))]
+        code_consts.append(Ellipsis)
+        # the compiler adds a final "LOAD_CONST None"
+        code_consts.append(None)
+
+        tree = ast.parse(code)
+        self.assertEqual(self.get_load_const(tree), code_consts)
+
+        # Replace expression nodes with constants
+        for expr_node, const in zip(tree.body, consts):
+            assert isinstance(expr_node, ast.Expr)
+            new_node = ast.Constant(value=const)
+            ast.copy_location(new_node, expr_node.value)
+            expr_node.value = new_node
+
+        self.assertEqual(self.get_load_const(tree), code_consts)
+
+    def test_literal_eval(self):
+        tree = ast.parse("1 + 2")
+        binop = tree.body[0].value
+
+        new_left = ast.Constant(value=10)
+        ast.copy_location(new_left, binop.left)
+        binop.left = new_left
+
+        new_right = ast.Constant(value=20)
+        ast.copy_location(new_right, binop.right)
+        binop.right = new_right
+
+        self.assertEqual(ast.literal_eval(binop), 30)
+
+
 def main():
     if __name__ != '__main__':
         return

Misc/NEWS

@@ -10,6 +10,10 @@ Release date: tba
 Core and Builtins
 -----------------
 
+- Issue #26146: Add a new kind of AST node: ``ast.Constant``. It can be used
+  by external AST optimizers, but the compiler does not emit directly such
+  node.
+
 - Issue #18018: Import raises ImportError instead of SystemError if a relative
   import is attempted without a known parent package.
 

Parser/Python.asdl

@@ -1,4 +1,8 @@
--- ASDL's six builtin types are identifier, int, string, bytes, object, singleton
+-- ASDL's 7 builtin types are:
+-- identifier, int, string, bytes, object, singleton, constant
+--
+-- singleton: None, True or False
+-- constant can be None, whereas None means "no value" for object.
 
 module Python
 {
@@ -76,6 +80,7 @@ module Python
          | Bytes(bytes s)
          | NameConstant(singleton value)
          | Ellipsis
+         | Constant(constant value)
 
          -- the following expression can appear in assignment context
          | Attribute(expr value, identifier attr, expr_context ctx)

Parser/asdl.py

@@ -33,7 +33,8 @@
 # See the EBNF at the top of the file to understand the logical connection
 # between the various node types.
 
-builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton'}
+builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton',
+                 'constant'}
 
 class AST:
     def __repr__(self):

Parser/asdl_c.py

@@ -834,6 +834,7 @@ def visitModule(self, mod):
     return (PyObject*)o;
 }
 #define ast2obj_singleton ast2obj_object
+#define ast2obj_constant ast2obj_object
 #define ast2obj_identifier ast2obj_object
 #define ast2obj_string ast2obj_object
 #define ast2obj_bytes ast2obj_object
@@ -871,6 +872,26 @@ def visitModule(self, mod):
     return 0;
 }
 
+static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena)
+{
+    if (obj == Py_None || obj == Py_True || obj == Py_False) {
+        /* don't increment the reference counter, Constant uses a borrowed
+         * reference, not a strong reference */
+        *out = obj;
+        return 0;
+    }
+
+    if (obj) {
+        if (PyArena_AddPyObject(arena, obj) < 0) {
+            *out = NULL;
+            return -1;
+        }
+        Py_INCREF(obj);
+    }
+    *out = obj;
+    return 0;
+}
+
 static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena)
 {
     if (!PyUnicode_CheckExact(obj) && obj != Py_None) {