Skip to content

Commit e377416

Browse files
bpo-29102: Add a unique ID to PyInterpreterState. (#1639)
1 parent 93fc20b commit e377416

File tree

8 files changed

+152
-7
lines changed

8 files changed

+152
-7
lines changed

Doc/c-api/init.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -821,6 +821,14 @@ been created.
821821
:c:func:`PyThreadState_Clear`.
822822
823823
824+
.. c:function:: PY_INT64_T PyInterpreterState_GetID(PyInterpreterState *interp)
825+
826+
Return the interpreter's unique ID. If there was any error in doing
827+
so then -1 is returned and an error is set.
828+
829+
.. versionadded:: 3.7
830+
831+
824832
.. c:function:: PyObject* PyThreadState_GetDict()
825833
826834
Return a dictionary in which extensions can store thread-specific state

Include/pystate.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ typedef struct _is {
2828
struct _is *next;
2929
struct _ts *tstate_head;
3030

31+
int64_t id;
32+
3133
PyObject *modules;
3234
PyObject *modules_by_index;
3335
PyObject *sysdict;
@@ -154,9 +156,16 @@ typedef struct _ts {
154156
#endif
155157

156158

159+
#ifndef Py_LIMITED_API
160+
PyAPI_FUNC(void) _PyInterpreterState_Init(void);
161+
#endif /* !Py_LIMITED_API */
157162
PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void);
158163
PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *);
159164
PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *);
165+
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000
166+
/* New in 3.7 */
167+
PyAPI_FUNC(int64_t) PyInterpreterState_GetID(PyInterpreterState *);
168+
#endif
160169
#ifndef Py_LIMITED_API
161170
PyAPI_FUNC(int) _PyState_AddModule(PyObject*, struct PyModuleDef*);
162171
#endif /* !Py_LIMITED_API */

Lib/test/test_capi.py

Lines changed: 86 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
# Run the _testcapi module tests (tests for the Python/C API): by defn,
22
# these are all functions _testcapi exports whose name begins with 'test_'.
33

4+
from collections import namedtuple
45
import os
56
import pickle
7+
import platform
68
import random
79
import re
810
import subprocess
@@ -384,12 +386,91 @@ def run_embedded_interpreter(self, *args):
384386
return out, err
385387

386388
def test_subinterps(self):
387-
# This is just a "don't crash" test
388389
out, err = self.run_embedded_interpreter("repeated_init_and_subinterpreters")
389-
if support.verbose:
390-
print()
391-
print(out)
392-
print(err)
390+
self.assertEqual(err, "")
391+
392+
# The output from _testembed looks like this:
393+
# --- Pass 0 ---
394+
# interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728
395+
# interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784
396+
# interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368
397+
# interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200
398+
# interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728
399+
# --- Pass 1 ---
400+
# ...
401+
402+
interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, "
403+
r"thread state <(0x[\dA-F]+)>: "
404+
r"id\(modules\) = ([\d]+)$")
405+
Interp = namedtuple("Interp", "id interp tstate modules")
406+
407+
main = None
408+
lastmain = None
409+
numinner = None
410+
numloops = 0
411+
for line in out.splitlines():
412+
if line == "--- Pass {} ---".format(numloops):
413+
if numinner is not None:
414+
self.assertEqual(numinner, 5)
415+
if support.verbose:
416+
print(line)
417+
lastmain = main
418+
main = None
419+
mainid = 0
420+
numloops += 1
421+
numinner = 0
422+
continue
423+
numinner += 1
424+
425+
self.assertLessEqual(numinner, 5)
426+
match = re.match(interp_pat, line)
427+
if match is None:
428+
self.assertRegex(line, interp_pat)
429+
430+
# The last line in the loop should be the same as the first.
431+
if numinner == 5:
432+
self.assertEqual(match.groups(), main)
433+
continue
434+
435+
# Parse the line from the loop. The first line is the main
436+
# interpreter and the 3 afterward are subinterpreters.
437+
interp = Interp(*match.groups())
438+
if support.verbose:
439+
print(interp)
440+
if numinner == 1:
441+
main = interp
442+
id = str(mainid)
443+
else:
444+
subid = mainid + numinner - 1
445+
id = str(subid)
446+
447+
# Validate the loop line for each interpreter.
448+
self.assertEqual(interp.id, id)
449+
self.assertTrue(interp.interp)
450+
self.assertTrue(interp.tstate)
451+
self.assertTrue(interp.modules)
452+
if platform.system() == 'Windows':
453+
# XXX Fix on Windows: something is going on with the
454+
# pointers in Programs/_testembed.c. interp.interp
455+
# is 0x0 and # interp.modules is the same between
456+
# interpreters.
457+
continue
458+
if interp is main:
459+
if lastmain is not None:
460+
# A new main interpreter may have the same interp
461+
# and/or tstate pointer as an earlier finalized/
462+
# destroyed one. So we do not check interp or
463+
# tstate here.
464+
self.assertNotEqual(interp.modules, lastmain.modules)
465+
else:
466+
# A new subinterpreter may have the same
467+
# PyInterpreterState pointer as a previous one if
468+
# the earlier one has already been destroyed. So
469+
# we compare with the main interpreter. The same
470+
# applies to tstate.
471+
self.assertNotEqual(interp.interp, main.interp)
472+
self.assertNotEqual(interp.tstate, main.tstate)
473+
self.assertNotEqual(interp.modules, main.modules)
393474

394475
@staticmethod
395476
def _get_default_pipe_encoding():

Misc/NEWS

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ Core and Builtins
5353
- bpo-24821: Fixed the slowing down to 25 times in the searching of some
5454
unlucky Unicode characters.
5555

56+
- bpo-29102: Add a unique ID to PyInterpreterState. This makes it easier
57+
to identify each subinterpreter.
58+
5659
- bpo-29894: The deprecation warning is emitted if __complex__ returns an
5760
instance of a strict subclass of complex. In a future versions of Python
5861
this can be an error.

PC/python3.def

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -538,6 +538,7 @@ EXPORTS
538538
PySlice_Type=python37.PySlice_Type DATA
539539
PySlice_Unpack=python37.PySlice_Unpack
540540
PySortWrapper_Type=python37.PySortWrapper_Type DATA
541+
PyInterpreterState_GetID=python37.PyInterpreterState_GetID
541542
PyState_AddModule=python37.PyState_AddModule
542543
PyState_FindModule=python37.PyState_FindModule
543544
PyState_RemoveModule=python37.PyState_RemoveModule

Programs/_testembed.c

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include <Python.h>
2+
#include <inttypes.h>
23
#include <stdio.h>
34

45
/*********************************************************
@@ -22,9 +23,13 @@ static void _testembed_Py_Initialize(void)
2223

2324
static void print_subinterp(void)
2425
{
25-
/* Just output some debug stuff */
26+
/* Output information about the interpreter in the format
27+
expected in Lib/test/test_capi.py (test_subinterps). */
2628
PyThreadState *ts = PyThreadState_Get();
27-
printf("interp %p, thread state %p: ", ts->interp, ts);
29+
PyInterpreterState *interp = ts->interp;
30+
int64_t id = PyInterpreterState_GetID(interp);
31+
printf("interp %lu <0x%" PRIXPTR ">, thread state <0x%" PRIXPTR ">: ",
32+
id, (uintptr_t)interp, (uintptr_t)ts);
2833
fflush(stdout);
2934
PyRun_SimpleString(
3035
"import sys;"

Python/pylifecycle.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
344344

345345
_PyRandom_Init();
346346

347+
_PyInterpreterState_Init();
347348
interp = PyInterpreterState_New();
348349
if (interp == NULL)
349350
Py_FatalError("Py_Initialize: can't make first interpreter");

Python/pystate.c

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,23 @@ PyThreadFrameGetter _PyThreadState_GetFrame = NULL;
6565
static void _PyGILState_NoteThreadState(PyThreadState* tstate);
6666
#endif
6767

68+
/* _next_interp_id is an auto-numbered sequence of small integers.
69+
It gets initialized in _PyInterpreterState_Init(), which is called
70+
in Py_Initialize(), and used in PyInterpreterState_New(). A negative
71+
interpreter ID indicates an error occurred. The main interpreter
72+
will always have an ID of 0. Overflow results in a RuntimeError.
73+
If that becomes a problem later then we can adjust, e.g. by using
74+
a Python int.
75+
76+
We initialize this to -1 so that the pre-Py_Initialize() value
77+
results in an error. */
78+
static int64_t _next_interp_id = -1;
79+
80+
void
81+
_PyInterpreterState_Init(void)
82+
{
83+
_next_interp_id = 0;
84+
}
6885

6986
PyInterpreterState *
7087
PyInterpreterState_New(void)
@@ -103,6 +120,15 @@ PyInterpreterState_New(void)
103120
HEAD_LOCK();
104121
interp->next = interp_head;
105122
interp_head = interp;
123+
if (_next_interp_id < 0) {
124+
/* overflow or Py_Initialize() not called! */
125+
PyErr_SetString(PyExc_RuntimeError,
126+
"failed to get an interpreter ID");
127+
interp = NULL;
128+
} else {
129+
interp->id = _next_interp_id;
130+
_next_interp_id += 1;
131+
}
106132
HEAD_UNLOCK();
107133
}
108134

@@ -170,6 +196,17 @@ PyInterpreterState_Delete(PyInterpreterState *interp)
170196
}
171197

172198

199+
int64_t
200+
PyInterpreterState_GetID(PyInterpreterState *interp)
201+
{
202+
if (interp == NULL) {
203+
PyErr_SetString(PyExc_RuntimeError, "no interpreter provided");
204+
return -1;
205+
}
206+
return interp->id;
207+
}
208+
209+
173210
/* Default implementation for _PyThreadState_GetFrame */
174211
static struct _frame *
175212
threadstate_getframe(PyThreadState *self)

0 commit comments

Comments
 (0)