Skip to content

Commit dd36b71

Browse files
gh-81057: Move the Extension Modules Cache to _PyRuntimeState (gh-99355)
We also move the closely related max_module_number and add comments documenting the group of struct members. #81057
1 parent fe55ff3 commit dd36b71

File tree

7 files changed

+104
-33
lines changed

7 files changed

+104
-33
lines changed

Include/internal/pycore_import.h

+17
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,23 @@
55
extern "C" {
66
#endif
77

8+
9+
struct _import_runtime_state {
10+
/* The most recent value assigned to a PyModuleDef.m_base.m_index.
11+
This is incremented each time PyModuleDef_Init() is called,
12+
which is just about every time an extension module is imported.
13+
See PyInterpreterState.modules_by_index for more info. */
14+
Py_ssize_t last_module_index;
15+
/* A dict mapping (filename, name) to PyModuleDef for modules.
16+
Only legacy (single-phase init) extension modules are added
17+
and only if they support multiple initialization (m_size >- 0)
18+
or are imported in the main interpreter.
19+
This is initialized lazily in _PyImport_FixupExtensionObject().
20+
Modules are added there and looked up in _imp.find_extension(). */
21+
PyObject *extensions;
22+
};
23+
24+
825
#ifdef HAVE_FORK
926
extern PyStatus _PyImport_ReInitLock(void);
1027
#endif

Include/internal/pycore_interp.h

+19
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,25 @@ struct _is {
123123

124124
// sys.modules dictionary
125125
PyObject *modules;
126+
/* This is the list of module objects for all legacy (single-phase init)
127+
extension modules ever loaded in this process (i.e. imported
128+
in this interpreter or in any other). Py_None stands in for
129+
modules that haven't actually been imported in this interpreter.
130+
131+
A module's index (PyModuleDef.m_base.m_index) is used to look up
132+
the corresponding module object for this interpreter, if any.
133+
(See PyState_FindModule().) When any extension module
134+
is initialized during import, its moduledef gets initialized by
135+
PyModuleDef_Init(), and the first time that happens for each
136+
PyModuleDef, its index gets set to the current value of
137+
a global counter (see _PyRuntimeState.imports.last_module_index).
138+
The entry for that index in this interpreter remains unset until
139+
the module is actually imported here. (Py_None is used as
140+
a placeholder.) Note that multi-phase init modules always get
141+
an index for which there will never be a module set.
142+
143+
This is initialized lazily in _PyState_AddModule(), which is also
144+
where modules get added. */
126145
PyObject *modules_by_index;
127146
// Dictionary of the sys module
128147
PyObject *sysdict;

Include/internal/pycore_runtime.h

+2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ extern "C" {
1111
#include "pycore_atomic.h" /* _Py_atomic_address */
1212
#include "pycore_gil.h" // struct _gil_runtime_state
1313
#include "pycore_global_objects.h" // struct _Py_global_objects
14+
#include "pycore_import.h" // struct _import_runtime_state
1415
#include "pycore_interp.h" // PyInterpreterState
1516
#include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids
1617

@@ -115,6 +116,7 @@ typedef struct pyruntimestate {
115116
void (*exitfuncs[NEXITFUNCS])(void);
116117
int nexitfuncs;
117118

119+
struct _import_runtime_state imports;
118120
struct _ceval_runtime_state ceval;
119121
struct _gilstate_runtime_state gilstate;
120122
struct _getargs_runtime_state getargs;

Include/moduleobject.h

+14
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,22 @@ PyAPI_DATA(PyTypeObject) PyModuleDef_Type;
4343

4444
typedef struct PyModuleDef_Base {
4545
PyObject_HEAD
46+
/* The function used to re-initialize the module.
47+
This is only set for legacy (single-phase init) extension modules
48+
and only used for those that support multiple initializations
49+
(m_size >= 0).
50+
It is set by _PyImport_LoadDynamicModuleWithSpec()
51+
and _imp.create_builtin(). */
4652
PyObject* (*m_init)(void);
53+
/* The module's index into its interpreter's modules_by_index cache.
54+
This is set for all extension modules but only used for legacy ones.
55+
(See PyInterpreterState.modules_by_index for more info.)
56+
It is set by PyModuleDef_Init(). */
4757
Py_ssize_t m_index;
58+
/* A copy of the module's __dict__ after the first time it was loaded.
59+
This is only set/used for legacy modules that do not support
60+
multiple initializations.
61+
It is set by _PyImport_FixupExtensionObject(). */
4862
PyObject* m_copy;
4963
} PyModuleDef_Base;
5064

Objects/moduleobject.c

+2-3
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
#include "pycore_moduleobject.h" // _PyModule_GetDef()
1010
#include "structmember.h" // PyMemberDef
1111

12-
static Py_ssize_t max_module_number;
1312

1413
static PyMemberDef module_members[] = {
1514
{"__dict__", T_OBJECT, offsetof(PyModuleObject, md_dict), READONLY},
@@ -43,10 +42,10 @@ PyModuleDef_Init(PyModuleDef* def)
4342
{
4443
assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY);
4544
if (def->m_base.m_index == 0) {
46-
max_module_number++;
45+
_PyRuntime.imports.last_module_index++;
4746
Py_SET_REFCNT(def, 1);
4847
Py_SET_TYPE(def, &PyModuleDef_Type);
49-
def->m_base.m_index = max_module_number;
48+
def->m_base.m_index = _PyRuntime.imports.last_module_index;
5049
}
5150
return (PyObject*)def;
5251
}

Python/import.c

+50-28
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@ extern "C" {
2727
/* Forward references */
2828
static PyObject *import_add_module(PyThreadState *tstate, PyObject *name);
2929

30-
/* See _PyImport_FixupExtensionObject() below */
31-
static PyObject *extensions = NULL;
32-
3330
/* This table is defined in config.c: */
3431
extern struct _inittab _PyImport_Inittab[];
3532

@@ -221,10 +218,12 @@ _imp_release_lock_impl(PyObject *module)
221218
Py_RETURN_NONE;
222219
}
223220

221+
static inline void _extensions_cache_clear(void);
222+
224223
void
225224
_PyImport_Fini(void)
226225
{
227-
Py_CLEAR(extensions);
226+
_extensions_cache_clear();
228227
if (import_lock != NULL) {
229228
PyThread_free_lock(import_lock);
230229
import_lock = NULL;
@@ -398,6 +397,51 @@ PyImport_GetMagicTag(void)
398397
dictionary, to avoid loading shared libraries twice.
399398
*/
400399

400+
static PyModuleDef *
401+
_extensions_cache_get(PyObject *filename, PyObject *name)
402+
{
403+
PyObject *extensions = _PyRuntime.imports.extensions;
404+
if (extensions == NULL) {
405+
return NULL;
406+
}
407+
PyObject *key = PyTuple_Pack(2, filename, name);
408+
if (key == NULL) {
409+
return NULL;
410+
}
411+
PyModuleDef *def = (PyModuleDef *)PyDict_GetItemWithError(extensions, key);
412+
Py_DECREF(key);
413+
return def;
414+
}
415+
416+
static int
417+
_extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def)
418+
{
419+
PyObject *extensions = _PyRuntime.imports.extensions;
420+
if (extensions == NULL) {
421+
extensions = PyDict_New();
422+
if (extensions == NULL) {
423+
return -1;
424+
}
425+
_PyRuntime.imports.extensions = extensions;
426+
}
427+
PyObject *key = PyTuple_Pack(2, filename, name);
428+
if (key == NULL) {
429+
return -1;
430+
}
431+
int res = PyDict_SetItem(extensions, key, (PyObject *)def);
432+
Py_DECREF(key);
433+
if (res < 0) {
434+
return -1;
435+
}
436+
return 0;
437+
}
438+
439+
static void
440+
_extensions_cache_clear(void)
441+
{
442+
Py_CLEAR(_PyRuntime.imports.extensions);
443+
}
444+
401445
int
402446
_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name,
403447
PyObject *filename, PyObject *modules)
@@ -442,20 +486,7 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name,
442486
}
443487
}
444488

445-
if (extensions == NULL) {
446-
extensions = PyDict_New();
447-
if (extensions == NULL) {
448-
return -1;
449-
}
450-
}
451-
452-
PyObject *key = PyTuple_Pack(2, filename, name);
453-
if (key == NULL) {
454-
return -1;
455-
}
456-
int res = PyDict_SetItem(extensions, key, (PyObject *)def);
457-
Py_DECREF(key);
458-
if (res < 0) {
489+
if (_extensions_cache_set(filename, name, def) < 0) {
459490
return -1;
460491
}
461492
}
@@ -480,16 +511,7 @@ static PyObject *
480511
import_find_extension(PyThreadState *tstate, PyObject *name,
481512
PyObject *filename)
482513
{
483-
if (extensions == NULL) {
484-
return NULL;
485-
}
486-
487-
PyObject *key = PyTuple_Pack(2, filename, name);
488-
if (key == NULL) {
489-
return NULL;
490-
}
491-
PyModuleDef* def = (PyModuleDef *)PyDict_GetItemWithError(extensions, key);
492-
Py_DECREF(key);
514+
PyModuleDef *def = _extensions_cache_get(filename, name);
493515
if (def == NULL) {
494516
return NULL;
495517
}

Tools/c-analyzer/cpython/globals-to-fix.tsv

-2
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,6 @@ Python/hamt.c - _empty_hamt -
317317

318318
# state
319319
Objects/typeobject.c resolve_slotdups pname -
320-
Python/import.c - extensions -
321320

322321

323322
##################################
@@ -449,7 +448,6 @@ Python/getargs.c - static_arg_parsers -
449448
Objects/dictobject.c - _pydict_global_version -
450449
Objects/dictobject.c - next_dict_keys_version -
451450
Objects/funcobject.c - next_func_version -
452-
Objects/moduleobject.c - max_module_number -
453451
Objects/object.c - _Py_RefTotal -
454452
Python/perf_trampoline.c - perf_status -
455453
Python/perf_trampoline.c - extra_code_index -

0 commit comments

Comments
 (0)