gh-116322: Add Py_mod_gil module slot (#116882)

This PR adds the ability to enable the GIL if it was disabled at
interpreter startup, and modifies the multi-phase module initialization
path to enable the GIL when loading a module, unless that module's spec
includes a slot indicating it can run safely without the GIL.

PEP 703 called the constant for the slot `Py_mod_gil_not_used`; I went
with `Py_MOD_GIL_NOT_USED` for consistency with gh-104148.

A warning will be issued up to once per interpreter for the first
GIL-using module that is loaded. If `-v` is given, a shorter message
will be printed to stderr every time a GIL-using module is loaded
(including the first one that issues a warning).

Author: swtaarrs

Doc/c-api/module.rst

@@ -411,6 +411,31 @@ The available slot types are:
 
    .. versionadded:: 3.12
 
+.. c:macro:: Py_mod_gil
+
+   Specifies one of the following values:
+
+   .. c:macro:: Py_MOD_GIL_USED
+
+      The module depends on the presence of the global interpreter lock (GIL),
+      and may access global state without synchronization.
+
+   .. c:macro:: Py_MOD_GIL_NOT_USED
+
+      The module is safe to run without an active GIL.
+
+   This slot is ignored by Python builds not configured with
+   :option:`--disable-gil`.  Otherwise, it determines whether or not importing
+   this module will cause the GIL to be automatically enabled. See
+   :envvar:`PYTHON_GIL` and :option:`-X gil <-X>` for more detail.
+
+   Multiple ``Py_mod_gil`` slots may not be specified in one module definition.
+
+   If ``Py_mod_gil`` is not specified, the import machinery defaults to
+   ``Py_MOD_GIL_USED``.
+
+   .. versionadded: 3.13
+
 See :PEP:`489` for more details on multi-phase initialization.
 
 Low-level module creation functions
@@ -609,6 +634,19 @@ state:
 
    .. versionadded:: 3.9
 
+.. c:function:: int PyModule_ExperimentalSetGIL(PyObject *module, void *gil)
+
+   Indicate that *module* does or does not support running without the global
+   interpreter lock (GIL), using one of the values from
+   :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization
+   function. If this function is not called during module initialization, the
+   import machinery assumes the module does not support running without the
+   GIL. This function is only available in Python builds configured with
+   :option:`--disable-gil`.
+   Return ``-1`` on error, ``0`` on success.
+
+   .. versionadded:: 3.13
+
 
 Module lookup
 ^^^^^^^^^^^^^

Include/internal/pycore_moduleobject.h

@@ -22,6 +22,9 @@ typedef struct {
     PyObject *md_weaklist;
     // for logging purposes after md_dict is cleared
     PyObject *md_name;
+#ifdef Py_GIL_DISABLED
+    void *md_gil;
+#endif
 } PyModuleObject;
 
 static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) {

Include/moduleobject.h

@@ -76,9 +76,13 @@ struct PyModuleDef_Slot {
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000
 #  define Py_mod_multiple_interpreters 3
 #endif
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000
+#  define Py_mod_gil 4
+#endif
+
 
 #ifndef Py_LIMITED_API
-#define _Py_mod_LAST_SLOT 3
+#define _Py_mod_LAST_SLOT 4
 #endif
 
 #endif /* New in 3.5 */
@@ -90,6 +94,16 @@ struct PyModuleDef_Slot {
 #  define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2)
 #endif
 
+/* for Py_mod_gil: */
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000
+#  define Py_MOD_GIL_USED ((void *)0)
+#  define Py_MOD_GIL_NOT_USED ((void *)1)
+#endif
+
+#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED)
+PyAPI_FUNC(int) PyModule_ExperimentalSetGIL(PyObject *module, void *gil);
+#endif
+
 struct PyModuleDef {
   PyModuleDef_Base m_base;
   const char* m_name;

Lib/test/test_importlib/extension/_test_nonmodule_cases.py

@@ -0,0 +1,44 @@
+import types
+import unittest
+from test.test_importlib import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+from test.test_importlib.extension.test_loader import MultiPhaseExtensionModuleTests
+
+
+class NonModuleExtensionTests:
+    setUp = MultiPhaseExtensionModuleTests.setUp
+    load_module_by_name = MultiPhaseExtensionModuleTests.load_module_by_name
+
+    def _test_nonmodule(self):
+        # Test returning a non-module object from create works.
+        name = self.name + '_nonmodule'
+        mod = self.load_module_by_name(name)
+        self.assertNotEqual(type(mod), type(unittest))
+        self.assertEqual(mod.three, 3)
+
+    # issue 27782
+    def test_nonmodule_with_methods(self):
+        # Test creating a non-module object with methods defined.
+        name = self.name + '_nonmodule_with_methods'
+        mod = self.load_module_by_name(name)
+        self.assertNotEqual(type(mod), type(unittest))
+        self.assertEqual(mod.three, 3)
+        self.assertEqual(mod.bar(10, 1), 9)
+
+    def test_null_slots(self):
+        # Test that NULL slots aren't a problem.
+        name = self.name + '_null_slots'
+        module = self.load_module_by_name(name)
+        self.assertIsInstance(module, types.ModuleType)
+        self.assertEqual(module.__name__, name)
+
+
+(Frozen_NonModuleExtensionTests,
+ Source_NonModuleExtensionTests
+ ) = util.test_both(NonModuleExtensionTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/extension/test_loader.py

@@ -10,7 +10,8 @@
 import warnings
 import importlib.util
 import importlib
-from test.support import MISSING_C_DOCSTRINGS
+from test import support
+from test.support import MISSING_C_DOCSTRINGS, script_helper
 
 
 class LoaderTests:
@@ -325,29 +326,6 @@ def test_unloadable_nonascii(self):
             self.load_module_by_name(name)
         self.assertEqual(cm.exception.name, name)
 
-    def test_nonmodule(self):
-        # Test returning a non-module object from create works.
-        name = self.name + '_nonmodule'
-        mod = self.load_module_by_name(name)
-        self.assertNotEqual(type(mod), type(unittest))
-        self.assertEqual(mod.three, 3)
-
-    # issue 27782
-    def test_nonmodule_with_methods(self):
-        # Test creating a non-module object with methods defined.
-        name = self.name + '_nonmodule_with_methods'
-        mod = self.load_module_by_name(name)
-        self.assertNotEqual(type(mod), type(unittest))
-        self.assertEqual(mod.three, 3)
-        self.assertEqual(mod.bar(10, 1), 9)
-
-    def test_null_slots(self):
-        # Test that NULL slots aren't a problem.
-        name = self.name + '_null_slots'
-        module = self.load_module_by_name(name)
-        self.assertIsInstance(module, types.ModuleType)
-        self.assertEqual(module.__name__, name)
-
     def test_bad_modules(self):
         # Test SystemError is raised for misbehaving extensions.
         for name_base in [
@@ -401,5 +379,14 @@ def test_nonascii(self):
  ) = util.test_both(MultiPhaseExtensionModuleTests, machinery=machinery)
 
 
+class NonModuleExtensionTests(unittest.TestCase):
+    def test_nonmodule_cases(self):
+        # The test cases in this file cause the GIL to be enabled permanently
+        # in free-threaded builds, so they are run in a subprocess to isolate
+        # this effect.
+        script = support.findfile("test_importlib/extension/_test_nonmodule_cases.py")
+        script_helper.run_test_script(script)
+
+
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_sys.py

@@ -1606,7 +1606,10 @@ def get_gen(): yield 1
         check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit)
         check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit)
         # module
-        check(unittest, size('PnPPP'))
+        if support.Py_GIL_DISABLED:
+            check(unittest, size('PPPPPP'))
+        else:
+            check(unittest, size('PPPPP'))
         # None
         check(None, size(''))
         # NotImplementedType

Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst

@@ -0,0 +1,5 @@
+Extension modules may indicate to the runtime that they can run without the
+GIL. Multi-phase init modules do so by calling providing
+``Py_MOD_GIL_NOT_USED`` for the ``Py_mod_gil`` slot, while single-phase init
+modules call ``PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED)`` from
+their init function.

Modules/_abc.c

@@ -970,6 +970,7 @@ _abcmodule_free(void *module)
 static PyModuleDef_Slot _abcmodule_slots[] = {
     {Py_mod_exec, _abcmodule_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_asynciomodule.c

@@ -3795,6 +3795,7 @@ module_exec(PyObject *mod)
 static struct PyModuleDef_Slot module_slots[] = {
     {Py_mod_exec, module_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL},
 };
 

Modules/_bisectmodule.c

@@ -462,6 +462,7 @@ bisect_modexec(PyObject *m)
 static PyModuleDef_Slot bisect_slots[] = {
     {Py_mod_exec, bisect_modexec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_blake2/blake2module.c

@@ -137,6 +137,7 @@ blake2_exec(PyObject *m)
 static PyModuleDef_Slot _blake2_slots[] = {
     {Py_mod_exec, blake2_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_bz2module.c

@@ -802,6 +802,7 @@ _bz2_free(void *module)
 static struct PyModuleDef_Slot _bz2_slots[] = {
     {Py_mod_exec, _bz2_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_codecsmodule.c

@@ -1050,6 +1050,7 @@ static PyMethodDef _codecs_functions[] = {
 
 static PyModuleDef_Slot _codecs_slots[] = {
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_collectionsmodule.c

@@ -2817,6 +2817,7 @@ collections_exec(PyObject *module) {
 static struct PyModuleDef_Slot collections_slots[] = {
     {Py_mod_exec, collections_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_contextvarsmodule.c

@@ -45,6 +45,7 @@ _contextvars_exec(PyObject *m)
 static struct PyModuleDef_Slot _contextvars_slots[] = {
     {Py_mod_exec, _contextvars_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_csv.c

@@ -1796,6 +1796,7 @@ csv_exec(PyObject *module) {
 static PyModuleDef_Slot csv_slots[] = {
     {Py_mod_exec, csv_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_ctypes/_ctypes.c

@@ -5948,6 +5948,7 @@ module_free(void *module)
 static PyModuleDef_Slot module_slots[] = {
     {Py_mod_exec, _ctypes_mod_exec},
     {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_ctypes/_ctypes_test.c

@@ -1,7 +1,7 @@
-// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED
+// Need limited C API version 3.13 for Py_mod_gil
 #include "pyconfig.h"   // Py_GIL_DISABLED
 #ifndef Py_GIL_DISABLED
-#  define Py_LIMITED_API 0x030c0000
+#  define Py_LIMITED_API 0x030d0000
 #endif
 
 // gh-85283: On Windows, Py_LIMITED_API requires Py_BUILD_CORE to not attempt
@@ -1167,6 +1167,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item)
 
 static struct PyModuleDef_Slot _ctypes_test_slots[] = {
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_curses_panel.c

@@ -697,6 +697,7 @@ static PyModuleDef_Slot _curses_slots[] = {
     // XXX gh-103092: fix isolation.
     {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},
     //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_cursesmodule.c

@@ -4743,6 +4743,9 @@ PyInit__curses(void)
     m = PyModule_Create(&_cursesmodule);
     if (m == NULL)
         return NULL;
+#ifdef Py_GIL_DISABLED
+    PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED);
+#endif
 
     /* Add some symbolic constants to the module */
     d = PyModule_GetDict(m);

Modules/_datetimemodule.c

@@ -6984,6 +6984,9 @@ PyInit__datetime(void)
     PyObject *mod = PyModule_Create(&datetimemodule);
     if (mod == NULL)
         return NULL;
+#ifdef Py_GIL_DISABLED
+    PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED);
+#endif
 
     if (_datetime_exec(mod) < 0) {
         Py_DECREF(mod);

Modules/_dbmmodule.c

@@ -616,6 +616,7 @@ _dbm_module_free(void *module)
 static PyModuleDef_Slot _dbmmodule_slots[] = {
     {Py_mod_exec, _dbm_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_decimal/_decimal.c

@@ -6157,6 +6157,7 @@ decimal_free(void *module)
 static struct PyModuleDef_Slot _decimal_slots[] = {
     {Py_mod_exec, _decimal_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL},
 };
 

Modules/_elementtree.c

@@ -4463,6 +4463,7 @@ module_exec(PyObject *m)
 static struct PyModuleDef_Slot elementtree_slots[] = {
     {Py_mod_exec, module_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL},
 };
 

Modules/_functoolsmodule.c

@@ -1559,6 +1559,7 @@ _functools_free(void *module)
 static struct PyModuleDef_Slot _functools_slots[] = {
     {Py_mod_exec, _functools_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_gdbmmodule.c

@@ -825,6 +825,7 @@ _gdbm_module_free(void *module)
 static PyModuleDef_Slot _gdbm_module_slots[] = {
     {Py_mod_exec, _gdbm_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_hashopenssl.c

@@ -2289,6 +2289,7 @@ static PyModuleDef_Slot hashlib_slots[] = {
     {Py_mod_exec, hashlib_init_constructors},
     {Py_mod_exec, hashlib_exception},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_heapqmodule.c

@@ -681,6 +681,7 @@ heapq_exec(PyObject *m)
 static struct PyModuleDef_Slot heapq_slots[] = {
     {Py_mod_exec, heapq_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };