Issue #12573: Add resource checks for dangling Thread and Process objects.

Author: pitrou

Lib/multiprocessing/process.py

@@ -42,6 +42,7 @@
 import sys
 import signal
 import itertools
+from _weakrefset import WeakSet
 
 #
 #
@@ -109,6 +110,7 @@ def __init__(self, group=None, target=None, name=None, args=(), kwargs={},
         self._kwargs = dict(kwargs)
         self._name = name or type(self).__name__ + '-' + \
                      ':'.join(str(i) for i in self._identity)
+        _dangling.add(self)
 
     def run(self):
         '''
@@ -344,3 +346,6 @@ def __init__(self):
 for name, signum in list(signal.__dict__.items()):
     if name[:3]=='SIG' and '_' not in name:
         _exitcode_to_name[-signum] = name
+
+# For debug and leak testing
+_dangling = WeakSet()

Lib/test/regrtest.py

@@ -182,6 +182,15 @@
 import warnings
 from inspect import isabstract
 
+try:
+    import threading
+except ImportError:
+    threading = None
+try:
+    import multiprocessing.process
+except ImportError:
+    multiprocessing = None
+
 
 # Some times __path__ and __file__ are not absolute (e.g. while running from
 # Lib/) and, if we change the CWD to run the tests in a temporary dir, some
@@ -930,7 +939,8 @@ def __init__(self, testname, verbose=0, quiet=False):
                  'os.environ', 'sys.path', 'sys.path_hooks', '__import__',
                  'warnings.filters', 'asyncore.socket_map',
                  'logging._handlers', 'logging._handlerList', 'sys.gettrace',
-                 'sys.warnoptions')
+                 'sys.warnoptions', 'threading._dangling',
+                 'multiprocessing.process._dangling')
 
     def get_sys_argv(self):
         return id(sys.argv), sys.argv, sys.argv[:]
@@ -1023,6 +1033,31 @@ def restore_sys_warnoptions(self, saved_options):
         sys.warnoptions = saved_options[1]
         sys.warnoptions[:] = saved_options[2]
 
+    # Controlling dangling references to Thread objects can make it easier
+    # to track reference leaks.
+    def get_threading__dangling(self):
+        if not threading:
+            return None
+        # This copies the weakrefs without making any strong reference
+        return threading._dangling.copy()
+    def restore_threading__dangling(self, saved):
+        if not threading:
+            return
+        threading._dangling.clear()
+        threading._dangling.update(saved)
+
+    # Same for Process objects
+    def get_multiprocessing_process__dangling(self):
+        if not multiprocessing:
+            return None
+        # This copies the weakrefs without making any strong reference
+        return multiprocessing.process._dangling.copy()
+    def restore_multiprocessing_process__dangling(self, saved):
+        if not multiprocessing:
+            return
+        multiprocessing.process._dangling.clear()
+        multiprocessing.process._dangling.update(saved)
+
     def resource_info(self):
         for name in self.resources:
             method_suffix = name.replace('.', '_')

Lib/threading.py

@@ -6,6 +6,7 @@
 from time import time as _time, sleep as _sleep
 from traceback import format_exc as _format_exc
 from collections import deque
+from _weakrefset import WeakSet
 
 # Note regarding PEP 8 compliant names
 #  This threading model was originally inspired by Java, and inherited
@@ -608,6 +609,8 @@ def _newname(template="Thread-%d"):
 _active = {}    # maps thread id to Thread object
 _limbo = {}
 
+# For debug and leak testing
+_dangling = WeakSet()
 
 # Main class for threads
 
@@ -645,6 +648,7 @@ def __init__(self, group=None, target=None, name=None,
         # sys.stderr is not stored in the class like
         # sys.exc_info since it can be changed between instances
         self._stderr = _sys.stderr
+        _dangling.add(self)
 
     def _reset_internal_locks(self):
         # private!  Called by _after_fork() to reset our internal locks as

Misc/NEWS

@@ -1036,6 +1036,8 @@ Extension Modules
 Tests
 -----
 
+- Issue #12573: Add resource checks for dangling Thread and Process objects.
+
 - Issue #12549: Correct test_platform to not fail when OS X returns 'x86_64'
   as the processor type on some Mac systems.