Issue #17807: Generators can now be finalized even when they are part of a reference cycle.

Author: pitrou

Include/frameobject.h

@@ -36,6 +36,8 @@ typedef struct _frame {
            non-generator frames. See the save_exc_state and swap_exc_state
            functions in ceval.c for details of their use. */
     PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
+    /* Borrowed referenced to a generator, or NULL */
+    PyObject *f_gen;
 
     PyThreadState *f_tstate;
     int f_lasti;                /* Last instruction if called */
@@ -84,6 +86,13 @@ PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out);
 /* Return the line of code the frame is currently executing. */
 PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *);
 
+/* Generator support */
+PyAPI_FUNC(PyObject *) _PyFrame_YieldingFrom(PyFrameObject *);
+PyAPI_FUNC(PyObject *) _PyFrame_GeneratorSend(PyFrameObject *, PyObject *, int exc);
+PyAPI_FUNC(PyObject *) _PyFrame_Finalize(PyFrameObject *);
+PyAPI_FUNC(int) _PyFrame_CloseIterator(PyObject *);
+
+
 #ifdef __cplusplus
 }
 #endif

Include/genobject.h

@@ -33,6 +33,7 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
 #define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type)
 
 PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
+/* Deprecated, kept for backwards compatibility. */
 PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
 PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
 PyObject *_PyGen_Send(PyGenObject *, PyObject *);

Lib/test/test_generators.py

@@ -1,3 +1,55 @@
+import gc
+import sys
+import unittest
+import weakref
+
+from test import support
+
+
+class FinalizationTest(unittest.TestCase):
+
+    def test_frame_resurrect(self):
+        # A generator frame can be resurrected by a generator's finalization.
+        def gen():
+            nonlocal frame
+            try:
+                yield
+            finally:
+                frame = sys._getframe()
+
+        g = gen()
+        wr = weakref.ref(g)
+        next(g)
+        del g
+        support.gc_collect()
+        self.assertIs(wr(), None)
+        self.assertTrue(frame)
+        del frame
+        support.gc_collect()
+
+    def test_refcycle(self):
+        # A generator caught in a refcycle gets finalized anyway.
+        old_garbage = gc.garbage[:]
+        finalized = False
+        def gen():
+            nonlocal finalized
+            try:
+                g = yield
+                yield 1
+            finally:
+                finalized = True
+
+        g = gen()
+        next(g)
+        g.send(g)
+        self.assertGreater(sys.getrefcount(g), 2)
+        self.assertFalse(finalized)
+        del g
+        support.gc_collect()
+        self.assertTrue(finalized)
+        self.assertEqual(gc.garbage, old_garbage)
+
+
 tutorial_tests = """
 Let's try a simple generator:
 
@@ -1880,6 +1932,7 @@ def printsolution(self, x):
 # so this works as expected in both ways of running regrtest.
 def test_main(verbose=None):
     from test import support, test_generators
+    support.run_unittest(__name__)
     support.run_doctest(test_generators, verbose)
 
 # This part isn't needed for regrtest, but for running the test directly.

Lib/test/test_sys.py

@@ -764,7 +764,7 @@ class C(object): pass
         nfrees = len(x.f_code.co_freevars)
         extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
                   ncells + nfrees - 1
-        check(x, vsize('12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
+        check(x, vsize('13P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
         # function
         def func(): pass
         check(func, size('12P'))

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 3.4.0 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #17807: Generators can now be finalized even when they are part of
+  a reference cycle.
+
 - Issue #1545463: At shutdown, defer finalization of codec modules so
   that stderr remains usable.
 

Modules/gcmodule.c

@@ -524,10 +524,7 @@ untrack_dicts(PyGC_Head *head)
 static int
 has_finalizer(PyObject *op)
 {
-    if (PyGen_CheckExact(op))
-        return PyGen_NeedsFinalizing((PyGenObject *)op);
-    else
-        return op->ob_type->tp_del != NULL;
+    return op->ob_type->tp_del != NULL;
 }
 
 /* Move the objects in unreachable with __del__ methods into `finalizers`.