gh-101758: Clean Up Uses of Import State (gh-101919)

This change is almost entirely moving code around and hiding import state behind internal API.  We introduce no changes to behavior, nor to non-internal API.  (Since there was already going to be a lot of churn, I took this as an opportunity to re-organize import.c into topically-grouped sections of code.)  The motivation is to simplify a number of upcoming changes.

Specific changes:

* move existing import-related code to import.c, wherever possible
* add internal API for interacting with import state (both global and per-interpreter)
* use only API outside of import.c (to limit churn there when changing the location, etc.)
* consolidate the import-related state of PyInterpreterState into a single struct field (this changes layout slightly)
* add macros for import state in import.c (to simplify changing the location)
* group code in import.c into sections
*remove _PyState_AddModule()

#101758

Author: ericsnowcurrently

Include/internal/pycore_import.h

@@ -36,11 +36,112 @@ struct _import_runtime_state {
     const char * pkgcontext;
 };
 
+struct _import_state {
+    /* cached sys.modules dictionary */
+    PyObject *modules;
+    /* This is the list of module objects for all legacy (single-phase init)
+       extension modules ever loaded in this process (i.e. imported
+       in this interpreter or in any other).  Py_None stands in for
+       modules that haven't actually been imported in this interpreter.
+
+       A module's index (PyModuleDef.m_base.m_index) is used to look up
+       the corresponding module object for this interpreter, if any.
+       (See PyState_FindModule().)  When any extension module
+       is initialized during import, its moduledef gets initialized by
+       PyModuleDef_Init(), and the first time that happens for each
+       PyModuleDef, its index gets set to the current value of
+       a global counter (see _PyRuntimeState.imports.last_module_index).
+       The entry for that index in this interpreter remains unset until
+       the module is actually imported here.  (Py_None is used as
+       a placeholder.)  Note that multi-phase init modules always get
+       an index for which there will never be a module set.
+
+       This is initialized lazily in PyState_AddModule(), which is also
+       where modules get added. */
+    PyObject *modules_by_index;
+    /* importlib module._bootstrap */
+    PyObject *importlib;
+    /* override for config->use_frozen_modules (for tests)
+       (-1: "off", 1: "on", 0: no override) */
+    int override_frozen_modules;
+#ifdef HAVE_DLOPEN
+    int dlopenflags;
+#endif
+    PyObject *import_func;
+};
+
+#ifdef HAVE_DLOPEN
+#  include <dlfcn.h>
+#  if HAVE_DECL_RTLD_NOW
+#    define _Py_DLOPEN_FLAGS RTLD_NOW
+#  else
+#    define _Py_DLOPEN_FLAGS RTLD_LAZY
+#  endif
+#  define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS,
+#else
+#  define _Py_DLOPEN_FLAGS 0
+#  define DLOPENFLAGS_INIT
+#endif
+
+#define IMPORTS_INIT \
+    { \
+        .override_frozen_modules = 0, \
+        DLOPENFLAGS_INIT \
+    }
+
+extern void _PyImport_ClearCore(PyInterpreterState *interp);
+
+extern Py_ssize_t _PyImport_GetNextModuleIndex(void);
+extern const char * _PyImport_ResolveNameWithPackageContext(const char *name);
+extern const char * _PyImport_SwapPackageContext(const char *newcontext);
+
+extern int _PyImport_GetDLOpenFlags(PyInterpreterState *interp);
+extern void _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val);
+
+extern PyObject * _PyImport_InitModules(PyInterpreterState *interp);
+extern PyObject * _PyImport_GetModules(PyInterpreterState *interp);
+extern void _PyImport_ClearModules(PyInterpreterState *interp);
+
+extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp);
+
+extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp);
+extern int _PyImport_IsDefaultImportFunc(
+        PyInterpreterState *interp,
+        PyObject *func);
+
+extern PyObject * _PyImport_GetImportlibLoader(
+        PyInterpreterState *interp,
+        const char *loader_name);
+extern PyObject * _PyImport_GetImportlibExternalLoader(
+        PyInterpreterState *interp,
+        const char *loader_name);
+extern PyObject * _PyImport_BlessMyLoader(
+        PyInterpreterState *interp,
+        PyObject *module_globals);
+extern PyObject * _PyImport_ImportlibModuleRepr(
+        PyInterpreterState *interp,
+        PyObject *module);
+
+
+extern PyStatus _PyImport_Init(void);
+extern void _PyImport_Fini(void);
+extern void _PyImport_Fini2(void);
+
+extern PyStatus _PyImport_InitCore(
+        PyThreadState *tstate,
+        PyObject *sysmod,
+        int importlib);
+extern PyStatus _PyImport_InitExternal(PyThreadState *tstate);
+extern void _PyImport_FiniCore(PyInterpreterState *interp);
+extern void _PyImport_FiniExternal(PyInterpreterState *interp);
+
 
 #ifdef HAVE_FORK
 extern PyStatus _PyImport_ReInitLock(void);
 #endif
-extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate);
+
+
+extern PyObject* _PyImport_GetBuiltinModuleNames(void);
 
 struct _module_alias {
     const char *name;                 /* ASCII encoded string */

Include/internal/pycore_interp.h

@@ -21,6 +21,7 @@ extern "C" {
 #include "pycore_function.h"      // FUNC_MAX_WATCHERS
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_gc.h"            // struct _gc_runtime_state
+#include "pycore_import.h"        // struct _import_state
 #include "pycore_list.h"          // struct _Py_list_state
 #include "pycore_global_objects.h"  // struct _Py_interp_static_objects
 #include "pycore_tuple.h"         // struct _Py_tuple_state
@@ -92,53 +93,24 @@ struct _is {
     struct _ceval_state ceval;
     struct _gc_runtime_state gc;
 
-    // sys.modules dictionary
-    PyObject *modules;
-    /* This is the list of module objects for all legacy (single-phase init)
-       extension modules ever loaded in this process (i.e. imported
-       in this interpreter or in any other).  Py_None stands in for
-       modules that haven't actually been imported in this interpreter.
-
-       A module's index (PyModuleDef.m_base.m_index) is used to look up
-       the corresponding module object for this interpreter, if any.
-       (See PyState_FindModule().)  When any extension module
-       is initialized during import, its moduledef gets initialized by
-       PyModuleDef_Init(), and the first time that happens for each
-       PyModuleDef, its index gets set to the current value of
-       a global counter (see _PyRuntimeState.imports.last_module_index).
-       The entry for that index in this interpreter remains unset until
-       the module is actually imported here.  (Py_None is used as
-       a placeholder.)  Note that multi-phase init modules always get
-       an index for which there will never be a module set.
-
-       This is initialized lazily in _PyState_AddModule(), which is also
-       where modules get added. */
-    PyObject *modules_by_index;
+    struct _import_state imports;
+
     // Dictionary of the sys module
     PyObject *sysdict;
     // Dictionary of the builtins module
     PyObject *builtins;
-    // importlib module
-    PyObject *importlib;
-    // override for config->use_frozen_modules (for tests)
-    // (-1: "off", 1: "on", 0: no override)
-    int override_frozen_modules;
 
     PyObject *codec_search_path;
     PyObject *codec_search_cache;
     PyObject *codec_error_registry;
     int codecs_initialized;
 
     PyConfig config;
-#ifdef HAVE_DLOPEN
-    int dlopenflags;
-#endif
     unsigned long feature_flags;
 
     PyObject *dict;  /* Stores per-interpreter state */
 
     PyObject *builtins_copy;
-    PyObject *import_func;
     // Initialized to _PyEval_EvalFrameDefault().
     _PyFrameEvalFunction eval_frame;
 
@@ -205,7 +177,6 @@ struct _is {
 
 /* other API */
 
-extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp);
 extern void _PyInterpreterState_Clear(PyThreadState *tstate);
 
 

Include/internal/pycore_pylifecycle.h

@@ -30,7 +30,6 @@ PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc);
 /* Various one-time initializers */
 
 extern void _Py_InitVersion(void);
-extern PyStatus _PyImport_Init(void);
 extern PyStatus _PyFaulthandler_Init(int enable);
 extern int _PyTraceMalloc_Init(int enable);
 extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp);
@@ -45,7 +44,6 @@ extern int _PyBuiltins_AddExceptions(PyObject * bltinmod);
 extern PyStatus _Py_HashRandomization_Init(const PyConfig *);
 
 extern PyStatus _PyTime_Init(void);
-extern PyStatus _PyImportZip_Init(PyThreadState *tstate);
 extern PyStatus _PyGC_Init(PyInterpreterState *interp);
 extern PyStatus _PyAtExit_Init(PyInterpreterState *interp);
 extern int _Py_Deepfreeze_Init(void);
@@ -55,8 +53,6 @@ extern int _Py_Deepfreeze_Init(void);
 extern int _PySignal_Init(int install_signal_handlers);
 extern void _PySignal_Fini(void);
 
-extern void _PyImport_Fini(void);
-extern void _PyImport_Fini2(void);
 extern void _PyGC_Fini(PyInterpreterState *interp);
 extern void _Py_HashRandomization_Fini(void);
 extern void _PyFaulthandler_Fini(void);

Include/internal/pycore_pystate.h

@@ -152,12 +152,6 @@ extern void _PySignal_AfterFork(void);
 #endif
 
 
-PyAPI_FUNC(int) _PyState_AddModule(
-    PyThreadState *tstate,
-    PyObject* module,
-    PyModuleDef* def);
-
-
 PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate);
 
 #define HEAD_LOCK(runtime) \

Include/internal/pycore_runtime_init.h

@@ -97,23 +97,10 @@ extern "C" {
         ._main_interpreter = _PyInterpreterState_INIT, \
     }
 
-#ifdef HAVE_DLOPEN
-#  include <dlfcn.h>
-#  if HAVE_DECL_RTLD_NOW
-#    define _Py_DLOPEN_FLAGS RTLD_NOW
-#  else
-#    define _Py_DLOPEN_FLAGS RTLD_LAZY
-#  endif
-#  define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS,
-#else
-#  define _Py_DLOPEN_FLAGS 0
-#  define DLOPENFLAGS_INIT
-#endif
-
 #define _PyInterpreterState_INIT \
     { \
         .id_refcount = -1, \
-        DLOPENFLAGS_INIT \
+        .imports = IMPORTS_INIT, \
         .ceval = { \
             .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
         }, \

Include/internal/pycore_sysmodule.h

@@ -20,6 +20,9 @@ extern void _PySys_ClearAuditHooks(PyThreadState *tstate);
 
 PyAPI_FUNC(int) _PySys_SetAttr(PyObject *, PyObject *);
 
+extern int _PySys_ClearAttrString(PyInterpreterState *interp,
+                                  const char *name, int verbose);
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_imp.py

@@ -387,7 +387,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized,
             check_basic_reloaded(mod, lookedup, initialized, init_count,
                                  before, reloaded)
 
-            # Currently _PyState_AddModule() always replaces the cached module.
+            # Currently PyState_AddModule() always replaces the cached module.
             self.assertIs(basic.look_up_self(), mod)
             self.assertEqual(basic.initialized_count(), expected_init_count)
 

Lib/test/test_stable_abi_ctypes.py

@@ -1684,9 +1684,6 @@
 [function._PyObject_NewVar]
     added = '3.2'
     abi_only = true
-[function._PyState_AddModule]
-    added = '3.2'
-    abi_only = true
 [function._PyThreadState_Init]
     added = '3.2'
     abi_only = true

Misc/stable_abi.toml

@@ -42,10 +42,9 @@ PyModuleDef_Init(PyModuleDef* def)
 {
     assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY);
     if (def->m_base.m_index == 0) {
-        _PyRuntime.imports.last_module_index++;
         Py_SET_REFCNT(def, 1);
         Py_SET_TYPE(def, &PyModuleDef_Type);
-        def->m_base.m_index = _PyRuntime.imports.last_module_index;
+        def->m_base.m_index = _PyImport_GetNextModuleIndex();
     }
     return (PyObject*)def;
 }
@@ -209,24 +208,7 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version)
             "module %s: PyModule_Create is incompatible with m_slots", name);
         return NULL;
     }
-    /* Make sure name is fully qualified.
-
-       This is a bit of a hack: when the shared library is loaded,
-       the module name is "package.module", but the module calls
-       PyModule_Create*() with just "module" for the name.  The shared
-       library loader squirrels away the true name of the module in
-       _Py_PackageContext, and PyModule_Create*() will substitute this
-       (if the name actually matches).
-    */
-#define _Py_PackageContext (_PyRuntime.imports.pkgcontext)
-    if (_Py_PackageContext != NULL) {
-        const char *p = strrchr(_Py_PackageContext, '.');
-        if (p != NULL && strcmp(module->m_name, p+1) == 0) {
-            name = _Py_PackageContext;
-            _Py_PackageContext = NULL;
-        }
-    }
-#undef _Py_PackageContext
+    name = _PyImport_ResolveNameWithPackageContext(name);
     if ((m = (PyModuleObject*)PyModule_New(name)) == NULL)
         return NULL;
 
@@ -710,8 +692,7 @@ static PyObject *
 module_repr(PyModuleObject *m)
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-
-    return PyObject_CallMethod(interp->importlib, "_module_repr", "O", m);
+    return _PyImport_ImportlibModuleRepr(interp, (PyObject *)m);
 }
 
 /* Check if the "_initializing" attribute of the module spec is set to true.

Objects/moduleobject.c

@@ -214,7 +214,7 @@ get_warnings_attr(PyInterpreterState *interp, PyObject *attr, int try_import)
            gone, then we can't even use PyImport_GetModule without triggering
            an interpreter abort.
         */
-        if (!interp->modules) {
+        if (!_PyImport_GetModules(interp)) {
             return NULL;
         }
         warnings_module = PyImport_GetModule(&_Py_ID(warnings));
@@ -1050,7 +1050,6 @@ warnings_warn_impl(PyObject *module, PyObject *message, PyObject *category,
 static PyObject *
 get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno)
 {
-    PyObject *external;
     PyObject *loader;
     PyObject *module_name;
     PyObject *get_source;
@@ -1059,13 +1058,7 @@ get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno
     PyObject *source_line;
 
     /* stolen from import.c */
-    external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external");
-    if (external == NULL) {
-        return NULL;
-    }
-
-    loader = PyObject_CallMethod(external, "_bless_my_loader", "O", module_globals, NULL);
-    Py_DECREF(external);
+    loader = _PyImport_BlessMyLoader(interp, module_globals);
     if (loader == NULL) {
         return NULL;
     }

PC/python3dll.c

@@ -2688,7 +2688,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame,
     }
     PyObject *locals = frame->f_locals;
     /* Fast path for not overloaded __import__. */
-    if (import_func == tstate->interp->import_func) {
+    if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) {
         int ilevel = _PyLong_AsInt(level);
         if (ilevel == -1 && _PyErr_Occurred(tstate)) {
             return NULL;

Python/_warnings.c

@@ -75,7 +75,7 @@ _PyImport_FindSharedFuncptr(const char *prefix,
             return NULL;
     }
 
-    dlopenflags = _PyInterpreterState_GET()->dlopenflags;
+    dlopenflags = _PyImport_GetDLOpenFlags(_PyInterpreterState_GET());
 
     handle = dlopen(pathname, dlopenflags);