From 1b01a914f7c8fef34921cc2e4b4cdc8dc90625dc Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Tue, 24 Sep 2024 15:13:32 -0700 Subject: [PATCH 01/84] Add `f_generator` property to Python frame objects `f_generator` returns the generator / coroutine / async generator object that owns the frame. For all other kinds of frames it will return `None`. This is useful to reconstruct call stack for async/await code. --- Doc/library/inspect.rst | 10 +++++++++ Lib/test/test_frame.py | 50 +++++++++++++++++++++++++++++++++++++++++ Objects/frameobject.c | 11 +++++++++ 3 files changed, 71 insertions(+) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 853671856b2a14..6ed76faa6ebcdd 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -162,6 +162,12 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | per-opcode events are | | | | requested | +-----------------+-------------------+---------------------------+ +| | f_generator | returns the generator or | +| | | coroutine object that | +| | | owns this frame, or | +| | | ``None`` if the frame is | +| | | of a regular function | ++-----------------+-------------------+---------------------------+ | | clear() | used to clear all | | | | references to local | | | | variables | @@ -310,6 +316,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Add ``__builtins__`` attribute to functions. +.. versionchanged:: 3.14 + + Add ``f_generator`` attribute to frames. + .. function:: getmembers(object[, predicate]) Return all the members of an object in a list of ``(name, value)`` diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 32de8ed9a13f80..bec5655180855d 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -222,6 +222,56 @@ def test_f_lineno_del_segfault(self): with self.assertRaises(AttributeError): del f.f_lineno + def test_f_generator(self): + # Test f_generator in different contexts. + + def t0(): + def nested(): + frame = sys._getframe() + return frame.f_generator + + def gen(): + yield nested() + + g = gen() + try: + return next(g) + finally: + g.close() + + def t1(): + frame = sys._getframe() + return frame.f_generator + + def t2(): + frame = sys._getframe() + yield frame.f_generator + + async def t3(): + frame = sys._getframe() + return frame.f_generator + + # For regular functions f_generator is None + self.assertIsNone(t0()) + self.assertIsNone(t1()) + + # For generators f_generator is equal to self + g = t2() + try: + frame_g = next(g) + self.assertIs(g, frame_g) + finally: + g.close() + + # Ditto for coroutines + c = t3() + try: + c.send(None) + except StopIteration as ex: + self.assertIs(ex.value, c) + else: + raise AssertionError('coroutine did not exit') + class ReprTest(unittest.TestCase): """ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f3a66ffc9aac8f..7119405d4971f7 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1616,6 +1616,16 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) return 0; } +static PyObject * +frame_getgenerator(PyFrameObject *f, void *arg) { + if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { + PyObject *gen = (PyObject *)_PyGen_GetGeneratorFromFrame(f->f_frame); + Py_INCREF(gen); + return gen; + } + Py_RETURN_NONE; +} + static PyGetSetDef frame_getsetlist[] = { {"f_back", (getter)frame_getback, NULL, NULL}, @@ -1628,6 +1638,7 @@ static PyGetSetDef frame_getsetlist[] = { {"f_builtins", (getter)frame_getbuiltins, NULL, NULL}, {"f_code", (getter)frame_getcode, NULL, NULL}, {"f_trace_opcodes", (getter)frame_gettrace_opcodes, (setter)frame_settrace_opcodes, NULL}, + {"f_generator", (getter)frame_getgenerator, NULL, NULL}, {0} }; From 0fc5511578b2c145c6d0fd23615d2004b8ba524e Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 25 Sep 2024 14:00:55 -0700 Subject: [PATCH 02/84] Working implementation of `asyncio.capture_call_stack()` --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 4 + Lib/asyncio/__init__.py | 2 + Lib/asyncio/futures.py | 2 + Lib/asyncio/stack.py | 120 ++++++++ Lib/asyncio/taskgroups.py | 6 + Lib/asyncio/tasks.py | 37 ++- Lib/test/test_asyncio/test_stack.py | 262 ++++++++++++++++++ Modules/_asynciomodule.c | 232 +++++++++++++++- 11 files changed, 663 insertions(+), 5 deletions(-) create mode 100644 Lib/asyncio/stack.py create mode 100644 Lib/test/test_asyncio/test_stack.py diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 28a76c36801b4b..98a48bce511be4 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -741,6 +741,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_asyncio_future_blocking)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_awaited_by)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_blksize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_bootstrap)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_check_retval_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index ac789b06fb8a61..eddea908ed9709 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -230,6 +230,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) STRUCT_FOR_ID(_asyncio_future_blocking) + STRUCT_FOR_ID(_awaited_by) STRUCT_FOR_ID(_blksize) STRUCT_FOR_ID(_bootstrap) STRUCT_FOR_ID(_check_retval_) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 7847a5c63ebf3f..3f23898566c6d5 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -739,6 +739,7 @@ extern "C" { INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ INIT_ID(_asyncio_future_blocking), \ + INIT_ID(_awaited_by), \ INIT_ID(_blksize), \ INIT_ID(_bootstrap), \ INIT_ID(_check_retval_), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index a688f70a2ba36f..87f7c090f57e03 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -720,6 +720,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_awaited_by); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_blksize); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 03165a425eb7d2..b05c3fdbdf9641 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -14,6 +14,7 @@ from .protocols import * from .runners import * from .queues import * +from .stack import * from .streams import * from .subprocess import * from .tasks import * @@ -31,6 +32,7 @@ protocols.__all__ + runners.__all__ + queues.__all__ + + stack.__all__ + streams.__all__ + subprocess.__all__ + tasks.__all__ + diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 5f6fa2348726cf..bc3e65b35c337a 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -45,6 +45,8 @@ class Future: """ + _awaited_by = None + # Class variables serving as defaults for instance variables. _state = _PENDING _result = None diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py new file mode 100644 index 00000000000000..e25d9e0a35957a --- /dev/null +++ b/Lib/asyncio/stack.py @@ -0,0 +1,120 @@ +"""Introspection utils for tasks call stacks.""" + +import dataclasses +import sys +import types + +from . import base_futures +from . import futures +from . import tasks + +__all__ = ( + 'capture_call_stack', + 'FrameCallStackEntry', + 'CoroutineCallStackEntry', + 'FutureCallStack', +) + +# Sadly, we can't re-use the traceback's module datastructures as those +# are tailored for error reporting, whereas we need to represent an +# async call stack. +# +# Going with pretty verbose names as we'd like to export them to the +# top level asyncio namespace, and want to avoid future name clashes. + +@dataclasses.dataclass(slots=True) +class FrameCallStackEntry: + frame: types.FrameType + + +@dataclasses.dataclass(slots=True) +class CoroutineCallStackEntry: + coroutine: types.CoroutineType + + +@dataclasses.dataclass(slots=True) +class FutureCallStack: + future: futures.Future + call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] + awaited_by: list[FutureCallStack] + + +def _build_stack_for_future(future: any) -> FutureCallStack: + if not base_futures.isfuture(future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + try: + get_coro = future.get_coro + except AttributeError: + coro = None + else: + coro = get_coro() + + st: list[CoroutineCallStackEntry] = [] + awaited_by: list[FutureCallStack] = [] + + while coro is not None: + if hasattr(coro, 'cr_await'): + # A native coroutine or duck-type compatible iterator + st.append(CoroutineCallStackEntry(coro)) + coro = coro.cr_await + elif hasattr(coro, 'ag_await'): + # A native async generator or duck-type compatible iterator + st.append(CoroutineCallStackEntry(coro)) + coro = coro.ag_await + else: + break + + if fut_waiters := getattr(future, '_awaited_by', None): + for parent in fut_waiters: + awaited_by.append(_build_stack_for_future(parent)) + + st.reverse() + return FutureCallStack(future, st, awaited_by) + + +def capture_call_stack(*, future: any = None) -> FutureCallStack | None: + """Capture async call stack for the current task or the provided Future.""" + + if future is not None: + if future is not tasks.current_task(): + return _build_stack_for_future(future) + # else: future is the current task, move on. + else: + future = tasks.current_task() + + if future is None: + # This isn't a generic call stack introspection utility. If we + # can't determine the current task and none was provided, we + # just return. + return None + + call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] = [] + + f = sys._getframe(1) + try: + while f is not None: + is_async = f.f_generator is not None + + if is_async: + call_stack.append(CoroutineCallStackEntry(f.f_generator)) + if f.f_back is not None and f.f_back.f_generator is None: + # We've reached the bottom of the coroutine stack, which + # must be the Task that runs it. + break + else: + call_stack.append(FrameCallStackEntry(f)) + + f = f.f_back + finally: + del f + + awaited_by = [] + if getattr(future, '_awaited_by', None): + for parent in future._awaited_by: + awaited_by.append(_build_stack_for_future(parent)) + + return FutureCallStack(future, call_stack, awaited_by) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index f2ee9648c43876..a5c936ef85f63c 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -174,6 +174,9 @@ def create_task(self, coro, *, name=None, context=None): else: task = self._loop.create_task(coro, name=name, context=context) + if hasattr(task, '_awaited_by'): + task._awaited_by = {self._parent_task} + # optimization: Immediately call the done callback if the task is # already done (e.g. if the coro was able to complete eagerly), # and skip scheduling a done callback @@ -202,6 +205,9 @@ def _abort(self): def _on_task_done(self, task): self._tasks.discard(task) + if hasattr(task, '_awaited_by'): + task._awaited_by = None + if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): self._on_completed_fut.set_result(True) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 2112dd4b99d17f..ac2e18adfdabb0 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -322,6 +322,7 @@ def __step_run_and_handle_result(self, exc): self._loop.call_soon( self.__step, new_exc, context=self._context) else: + _add_to_awaited_by(result, self) result._asyncio_future_blocking = False result.add_done_callback( self.__wakeup, context=self._context) @@ -356,6 +357,7 @@ def __step_run_and_handle_result(self, exc): self = None # Needed to break cycles when an exception occurs. def __wakeup(self, future): + _discard_from_awaited_by(future, self) try: future.result() except BaseException as exc: @@ -502,6 +504,7 @@ async def _wait(fs, timeout, return_when, loop): if timeout is not None: timeout_handle = loop.call_later(timeout, _release_waiter, waiter) counter = len(fs) + cur_task = current_task() def _on_completion(f): nonlocal counter @@ -514,9 +517,11 @@ def _on_completion(f): timeout_handle.cancel() if not waiter.done(): waiter.set_result(None) + _discard_from_awaited_by(f, cur_task) for f in fs: f.add_done_callback(_on_completion) + _add_to_awaited_by(f, cur_task) try: await waiter @@ -802,10 +807,13 @@ def gather(*coros_or_futures, return_exceptions=False): outer.set_result([]) return outer - def _done_callback(fut): + def _done_callback(fut, cur_task): nonlocal nfinished nfinished += 1 + if cur_task is not None: + _discard_from_awaited_by(fut, cur_task) + if outer is None or outer.done(): if not fut.cancelled(): # Mark exception retrieved. @@ -864,6 +872,7 @@ def _done_callback(fut): done_futs = [] loop = None outer = None # bpo-46672 + cur_task = current_task() for arg in coros_or_futures: if arg not in arg_to_fut: fut = ensure_future(arg, loop=loop) @@ -875,13 +884,14 @@ def _done_callback(fut): # can't control it, disable the "destroy pending task" # warning. fut._log_destroy_pending = False - + if cur_task is not None: + _add_to_awaited_by(fut, cur_task) nfuts += 1 arg_to_fut[arg] = fut if fut.done(): done_futs.append(fut) else: - fut.add_done_callback(_done_callback) + fut.add_done_callback(lambda fut: _done_callback(fut, cur_task)) else: # There's a duplicate Future object in coros_or_futures. @@ -940,7 +950,14 @@ def shield(arg): loop = futures._get_loop(inner) outer = loop.create_future() - def _inner_done_callback(inner): + cur_task = current_task() + if cur_task is not None: + _add_to_awaited_by(inner, cur_task) + + def _inner_done_callback(inner, cur_task=cur_task): + if cur_task is not None: + _discard_from_awaited_by(inner, cur_task) + if outer.cancelled(): if not inner.cancelled(): # Mark inner's result as retrieved. @@ -1074,6 +1091,18 @@ def _unregister_eager_task(task): _eager_tasks.discard(task) +def _add_to_awaited_by(fut, waiter): + if hasattr(fut, '_awaited_by'): + if fut._awaited_by is None: + fut._awaited_by = set() + fut._awaited_by.add(waiter) + + +def _discard_from_awaited_by(fut, waiter): + if awaited_by := getattr(fut, '_awaited_by', None): + awaited_by.discard(waiter) + + _py_current_task = current_task _py_register_task = _register_task _py_register_eager_task = _register_eager_task diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py new file mode 100644 index 00000000000000..f669772932f84d --- /dev/null +++ b/Lib/test/test_asyncio/test_stack.py @@ -0,0 +1,262 @@ +import asyncio +import unittest + +import pprint + + +# To prevent a warning "test altered the execution environment" +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + +def capture_test_stack(*, fut=None): + + def walk(s): + ret = [ + f"T<{n}>" if '-' not in (n := s.future.get_name()) else 'T' + if isinstance(s.future, asyncio.Task) else 'F' + ] + + ret.append( + [ + ( + f"s {entry.frame.f_code.co_name}" + if isinstance(entry, asyncio.FrameCallStackEntry) else + ( + f"a {entry.coroutine.cr_code.co_name}" + if hasattr(entry.coroutine, 'cr_code') else + f"ag {entry.coroutine.ag_code.co_name}" + ) + ) for entry in s.call_stack + ] + ) + + ret.append( + sorted([ + walk(ab) for ab in s.awaited_by + ], key=lambda entry: entry[0]) + ) + + return ret + + stack = asyncio.capture_call_stack(future=fut) + return walk(stack) + + +class TestCallStack(unittest.IsolatedAsyncioTestCase): + + + async def test_stack_tgroup(self): + + stack_for_c5 = None + + def c5(): + nonlocal stack_for_c5 + stack_for_c5 = capture_test_stack() + + async def c4(): + await asyncio.sleep(0) + c5() + + async def c3(): + await c4() + + async def c2(): + await c3() + + async def c1(task): + await task + + async def main(): + async with asyncio.TaskGroup() as tg: + task = tg.create_task(c2(), name="c2_root") + tg.create_task(c1(task), name="sub_main_1") + tg.create_task(c1(task), name="sub_main_2") + + await main() + + self.assertEqual(stack_for_c5, [ + # task name + 'T', + # call stack + ['s capture_test_stack', 's c5', 'a c4', 'a c3', 'a c2'], + # awaited by + [ + ['T', + ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ], + ['T', + ['a c1'], + [ + ['T', + ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ], + ['T', + ['a c1'], + [ + ['T', + ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ] + ] + ]) + + async def test_stack_async_gen(self): + + stack_for_gen_nested_call = None + + async def gen_nested_call(): + nonlocal stack_for_gen_nested_call + stack_for_gen_nested_call = capture_test_stack() + + async def gen(): + for num in range(2): + yield num + if num == 1: + await gen_nested_call() + + async def main(): + async for el in gen(): + pass + + await main() + + self.assertEqual(stack_for_gen_nested_call, [ + 'T', + [ + 's capture_test_stack', + 'a gen_nested_call', + 'ag gen', + 'a main', + 'a test_stack_async_gen' + ], + [] + ]) + + async def test_stack_gather(self): + + stack_for_deep = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_deep + stack_for_deep = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(0) + + async def main(): + await asyncio.gather(c1(), c2()) + + await main() + + self.assertEqual(stack_for_deep, [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_gather'], []] + ] + ]) + + async def test_stack_shield(self): + + stack_for_shield = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_shield + stack_for_shield = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_shield, [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_shield'], []] + ] + ]) + + async def test_stack_timeout(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_inner, [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', ['a main', 'a test_stack_timeout'], []] + ] + ]) + + async def test_stack_wait(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def c2(): + for i in range(3): + await asyncio.sleep(0) + + async def main(t1, t2): + while True: + _, pending = await asyncio.wait([t1, t2]) + if not pending: + break + + t1 = asyncio.create_task(c1()) + t2 = asyncio.create_task(c2()) + try: + await main(t1, t2) + finally: + await t1 + await t2 + + self.assertEqual(stack_for_inner, [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', + ['a _wait', 'a wait', 'a main', 'a test_stack_wait'], + [] + ] + ] + ]) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 870084100a1b85..d8b822ef5b53a9 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -40,6 +40,7 @@ typedef enum { PyObject *prefix##_source_tb; \ PyObject *prefix##_cancel_msg; \ PyObject *prefix##_cancelled_exc; \ + PyObject *prefix##_awaited_by; \ fut_state prefix##_state; \ /* These bitfields need to be at the end of the struct so that these and bitfields from TaskObj are contiguous. @@ -475,6 +476,7 @@ future_init(FutureObj *fut, PyObject *loop) Py_CLEAR(fut->fut_source_tb); Py_CLEAR(fut->fut_cancel_msg); Py_CLEAR(fut->fut_cancelled_exc); + Py_CLEAR(fut->fut_awaited_by); fut->fut_state = STATE_PENDING; fut->fut_log_tb = 0; @@ -518,6 +520,216 @@ future_init(FutureObj *fut, PyObject *loop) return 0; } +static int +future_awaited_by_add(FutureObj *fut, PyObject *thing) +{ + /* Most futures/task are only awaited by one entity, so we want + to avoid always creating a set for `fut_awaited_by`. + */ + if (fut->fut_awaited_by == NULL) { + Py_INCREF(thing); + fut->fut_awaited_by = thing; + return 0; + } + + if (PySet_Check(fut->fut_awaited_by)) { + return PySet_Add(fut->fut_awaited_by, thing); + } + + PyObject *set = PySet_New(NULL); + if (set == NULL) { + return -1; + } + if (PySet_Add(set, thing)) { + Py_DECREF(set); + return -1; + } + if (PySet_Add(set, fut->fut_awaited_by)) { + Py_DECREF(set); + return -1; + } + Py_SETREF(fut->fut_awaited_by, set); + return 0; +} + +static int +future_awaited_by_discard(FutureObj *fut, PyObject *thing) +{ + /* Following the semantics of 'set.discard()' here in not + raising an error if `thing` isn't in the `awaited_by` "set". + */ + if (fut->fut_awaited_by == NULL) { + return 0; + } + if (fut->fut_awaited_by == thing) { + Py_CLEAR(fut->fut_awaited_by); + return 0; + } + if (PySet_Check(fut->fut_awaited_by)) { + int err = PySet_Discard(fut->fut_awaited_by, thing); + if (err < 0 && PyErr_Occurred()) { + return -1; + } else { + return 0; + } + } + return 0; +} + +static int +awaited_by_add(asyncio_state *state, PyObject *maybe_fut, PyObject *thing) +{ + if (Future_CheckExact(state, maybe_fut) + || Task_CheckExact(state, maybe_fut) + ) { + return future_awaited_by_add((FutureObj *)maybe_fut, thing); + } + + PyObject *awaited_by; + int err = PyObject_GetOptionalAttr( + maybe_fut, &_Py_ID(_awaited_by), &awaited_by); + if (err < 0) { + return err; + } + + if (err == 1) { + if (PySet_Check(awaited_by)) { + if (PySet_Add(awaited_by, thing)) { + Py_DECREF(awaited_by); + return -1; + } else { + Py_DECREF(awaited_by); + return 0; + } + } else if (awaited_by == Py_None) { + Py_DECREF(awaited_by); + goto new_set; + } else { + Py_DECREF(awaited_by); + PyErr_SetString(PyExc_RuntimeError, + "_awaited_by is not a set or None"); + return -1; + } + } + + assert(err == 0); + assert(awaited_by == NULL); + +new_set: + awaited_by = PySet_New(NULL); + if (awaited_by == NULL) { + return -1; + } + if (PySet_Add(awaited_by, thing)) { + Py_DECREF(awaited_by); + return -1; + } + + err = PyObject_SetAttr(maybe_fut, &_Py_ID(_awaited_by), awaited_by); + Py_DECREF(awaited_by); + return err; +} + +static int +awaited_by_discard(asyncio_state *state, PyObject *maybe_fut, PyObject *thing) +{ + if (Future_CheckExact(state, maybe_fut) + || Task_CheckExact(state, maybe_fut) + ) { + return future_awaited_by_discard((FutureObj *)maybe_fut, thing); + } + + PyObject *awaited_by; + int err = PyObject_GetOptionalAttr( + maybe_fut, &_Py_ID(_awaited_by), &awaited_by); + if (err < 0) { + return err; + } + + if (err == 0) { + return 0; + } + + assert(err == 1); + + if (PySet_Check(awaited_by)) { + err = PySet_Discard(awaited_by, thing); + Py_DECREF(awaited_by); + if (err < 0 && PyErr_Occurred()) { + return -1; + } else { + return 0; + } + } else if (awaited_by == Py_None) { + Py_DECREF(awaited_by); + return 0; + } else { + Py_DECREF(awaited_by); + PyErr_SetString(PyExc_RuntimeError, + "_awaited_by is not a set or None"); + return -1; + } +} + +static PyObject * +future_get_awaited_by(FutureObj *fut) +{ + /* Implementation of a Python getter. */ + if (fut->fut_awaited_by == NULL) { + Py_RETURN_NONE; + } + if (PySet_Check(fut->fut_awaited_by)) { + Py_INCREF(fut->fut_awaited_by); + return fut->fut_awaited_by; + } + + /* We don't want to "leak" our optimization that we don't always create + a set to the pure-Python land. Accessing `_awaited_by` from Python + can mean two things: + + (a) asyncio TaskGroup or gather or a similar primitive uses it + to ensure correct call stack. In this case, the TaskGroup + will attempt to mutate the set. + + (b) an async call stack is being rendered and needs to infer + what tasks are awaiting on this task or future. In this case + we don't want to micro-optimize things. + + The bottom line: it's easier to make a set, use it and return it. + */ + + PyObject *set = PySet_New(NULL); + if (set == NULL) { + return NULL; + } + if (PySet_Add(set, fut->fut_awaited_by)) { + Py_DECREF(set); + return NULL; + } + + Py_SETREF(fut->fut_awaited_by, set); + + Py_INCREF(set); + return set; +} + +static int +future_set_awaited_by(FutureObj *fut, PyObject *set) +{ + /* Implementation of a Python setter. */ + if (set == Py_None) { + Py_CLEAR(fut->fut_awaited_by); + return 0; + } + if (!PySet_Check(set)) { + PyErr_SetString(PyExc_ValueError, "_awaited_by expects a set"); + return -1; + } + Py_XSETREF(fut->fut_awaited_by, set); + Py_INCREF(set); + return 0; +} + static PyObject * future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) { @@ -804,6 +1016,7 @@ FutureObj_clear(FutureObj *fut) Py_CLEAR(fut->fut_source_tb); Py_CLEAR(fut->fut_cancel_msg); Py_CLEAR(fut->fut_cancelled_exc); + Py_CLEAR(fut->fut_awaited_by); PyObject_ClearManagedDict((PyObject *)fut); return 0; } @@ -822,6 +1035,7 @@ FutureObj_traverse(FutureObj *fut, visitproc visit, void *arg) Py_VISIT(fut->fut_source_tb); Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->fut_cancelled_exc); + Py_VISIT(fut->fut_awaited_by); PyObject_VisitManagedDict((PyObject *)fut, visit, arg); return 0; } @@ -1504,7 +1718,9 @@ static PyMethodDef FutureType_methods[] = { {"_source_traceback", (getter)FutureObj_get_source_traceback, \ NULL, NULL}, \ {"_cancel_message", (getter)FutureObj_get_cancel_message, \ - (setter)FutureObj_set_cancel_message, NULL}, + (setter)FutureObj_set_cancel_message, NULL}, \ + {"_awaited_by", (getter)future_get_awaited_by, \ + (setter)future_set_awaited_by, NULL}, static PyGetSetDef FutureType_getsetlist[] = { FUTURE_COMMON_GETSETLIST @@ -2198,6 +2414,7 @@ TaskObj_traverse(TaskObj *task, visitproc visit, void *arg) Py_VISIT(fut->fut_source_tb); Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->fut_cancelled_exc); + Py_VISIT(fut->fut_awaited_by); PyObject_VisitManagedDict((PyObject *)fut, visit, arg); return 0; } @@ -2938,6 +3155,10 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu goto yield_insteadof_yf; } + if (awaited_by_add(state, result, (PyObject *)task)) { + goto fail; + } + fut->fut_blocking = 0; /* result.add_done_callback(task._wakeup) */ @@ -3016,6 +3237,10 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu goto yield_insteadof_yf; } + if (awaited_by_add(state, result, (PyObject *)task)) { + goto fail; + } + /* result._asyncio_future_blocking = False */ if (PyObject_SetAttr( result, &_Py_ID(_asyncio_future_blocking), Py_False) == -1) { @@ -3213,6 +3438,11 @@ task_wakeup(TaskObj *task, PyObject *o) assert(o); asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); + + if (awaited_by_discard(state, o, (PyObject *)task)) { + return NULL; + } + if (Future_CheckExact(state, o) || Task_CheckExact(state, o)) { PyObject *fut_result = NULL; int res = future_get_result(state, (FutureObj*)o, &fut_result); From 1d20a51dc903459a6b7c02be7959d78c07404ef5 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 12:12:12 -0700 Subject: [PATCH 03/84] Address Guido's comments --- Lib/asyncio/futures.py | 50 ++++++ Lib/asyncio/stack.py | 27 ++-- Lib/asyncio/taskgroups.py | 7 +- Lib/asyncio/tasks.py | 31 ++-- Lib/test/test_asyncio/test_stack.py | 2 - Modules/_asynciomodule.c | 234 ++++++++++++---------------- Modules/clinic/_asynciomodule.c.h | 118 +++++++++++++- 7 files changed, 291 insertions(+), 178 deletions(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index bc3e65b35c337a..f2150c89d60160 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -2,6 +2,7 @@ __all__ = ( 'Future', 'wrap_future', 'isfuture', + 'future_add_to_awaited_by', 'future_discard_from_awaited_by', ) import concurrent.futures @@ -68,6 +69,9 @@ class Future: # `yield Future()` (incorrect). _asyncio_future_blocking = False + # Used by the capture_call_stack() API. + _asyncio_awaited_by = None + __log_traceback = False def __init__(self, *, loop=None): @@ -419,6 +423,46 @@ def wrap_future(future, *, loop=None): return new_future +def future_add_to_awaited_by(fut, waiter): + """Record that `fut` is awaited on by `waiter`.""" + # For the sake of keeping the implementation minimal and assuming + # that 99.9% of asyncio users use the built-in Futures and Tasks + # (or their subclasses), we only support native Future objects + # and their subclasses. + # + # Longer version: tracking requires storing the caller-callee + # dependency somewhere. One obvious choice is to store that + # information right in the future itself in a dedicated attribute. + # This means that we'd have to require all duck-type compatible + # futures to implement a specific attribute used by asyncio for + # the book keeping. Another solution would be to store that in + # a global dictionary. The downside here is that that would create + # strong references and any scenario where the "add" call isn't + # followed by a "discard" call would lead to a memory leak. + # Using WeakDict would resolve that issue, but would complicate + # the C code (_asynciomodule.c). The bottom line here is that + # it's not clear that all this work would be worth the effort. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._asyncio_awaited_by is None: + fut._asyncio_awaited_by = set() + fut._asyncio_awaited_by.add(waiter) + + +def future_discard_from_awaited_by(fut, waiter): + """Record that `fut` is no longer awaited on by `waiter`.""" + # See the comment in "future_add_to_awaited_by()" body for + # details on implemntation. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._asyncio_awaited_by is not None: + fut._asyncio_awaited_by.discard(waiter) + + try: import _asyncio except ImportError: @@ -426,3 +470,9 @@ def wrap_future(future, *, loop=None): else: # _CFuture is needed for tests. Future = _CFuture = _asyncio.Future + +try: + from _asyncio import future_add_to_awaited_by, \ + future_discard_from_awaited_by +except ImportError: + pass diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index e25d9e0a35957a..fe650d6d0a3cd5 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -1,10 +1,9 @@ """Introspection utils for tasks call stacks.""" -import dataclasses import sys import types +import typing -from . import base_futures from . import futures from . import tasks @@ -22,25 +21,23 @@ # Going with pretty verbose names as we'd like to export them to the # top level asyncio namespace, and want to avoid future name clashes. -@dataclasses.dataclass(slots=True) -class FrameCallStackEntry: + +class FrameCallStackEntry(typing.NamedTuple): frame: types.FrameType -@dataclasses.dataclass(slots=True) -class CoroutineCallStackEntry: +class CoroutineCallStackEntry(typing.NamedTuple): coroutine: types.CoroutineType -@dataclasses.dataclass(slots=True) -class FutureCallStack: +class FutureCallStack(typing.NamedTuple): future: futures.Future call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] awaited_by: list[FutureCallStack] def _build_stack_for_future(future: any) -> FutureCallStack: - if not base_futures.isfuture(future): + if not isinstance(future, futures.Future): raise TypeError( f"{future!r} object does not appear to be compatible " f"with asyncio.Future" @@ -68,7 +65,7 @@ def _build_stack_for_future(future: any) -> FutureCallStack: else: break - if fut_waiters := getattr(future, '_awaited_by', None): + if fut_waiters := getattr(future, '_asyncio_awaited_by', None): for parent in fut_waiters: awaited_by.append(_build_stack_for_future(parent)) @@ -92,6 +89,12 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: # just return. return None + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] = [] f = sys._getframe(1) @@ -113,8 +116,8 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: del f awaited_by = [] - if getattr(future, '_awaited_by', None): - for parent in future._awaited_by: + if fut_waiters := getattr(future, '_asyncio_awaited_by', None): + for parent in fut_waiters: awaited_by.append(_build_stack_for_future(parent)) return FutureCallStack(future, call_stack, awaited_by) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index a5c936ef85f63c..dad5d15256a4ef 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -6,6 +6,7 @@ from . import events from . import exceptions +from . import futures from . import tasks @@ -174,8 +175,7 @@ def create_task(self, coro, *, name=None, context=None): else: task = self._loop.create_task(coro, name=name, context=context) - if hasattr(task, '_awaited_by'): - task._awaited_by = {self._parent_task} + futures.future_add_to_awaited_by(task, self._parent_task) # optimization: Immediately call the done callback if the task is # already done (e.g. if the coro was able to complete eagerly), @@ -205,8 +205,7 @@ def _abort(self): def _on_task_done(self, task): self._tasks.discard(task) - if hasattr(task, '_awaited_by'): - task._awaited_by = None + futures.future_discard_from_awaited_by(task, self._parent_task) if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index ac2e18adfdabb0..a90fc23ff0f551 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -322,7 +322,7 @@ def __step_run_and_handle_result(self, exc): self._loop.call_soon( self.__step, new_exc, context=self._context) else: - _add_to_awaited_by(result, self) + futures.future_add_to_awaited_by(result, self) result._asyncio_future_blocking = False result.add_done_callback( self.__wakeup, context=self._context) @@ -357,7 +357,7 @@ def __step_run_and_handle_result(self, exc): self = None # Needed to break cycles when an exception occurs. def __wakeup(self, future): - _discard_from_awaited_by(future, self) + futures.future_discard_from_awaited_by(future, self) try: future.result() except BaseException as exc: @@ -517,11 +517,11 @@ def _on_completion(f): timeout_handle.cancel() if not waiter.done(): waiter.set_result(None) - _discard_from_awaited_by(f, cur_task) + futures.future_discard_from_awaited_by(f, cur_task) for f in fs: f.add_done_callback(_on_completion) - _add_to_awaited_by(f, cur_task) + futures.future_add_to_awaited_by(f, cur_task) try: await waiter @@ -812,7 +812,7 @@ def _done_callback(fut, cur_task): nfinished += 1 if cur_task is not None: - _discard_from_awaited_by(fut, cur_task) + futures.future_discard_from_awaited_by(fut, cur_task) if outer is None or outer.done(): if not fut.cancelled(): @@ -885,7 +885,7 @@ def _done_callback(fut, cur_task): # warning. fut._log_destroy_pending = False if cur_task is not None: - _add_to_awaited_by(fut, cur_task) + futures.future_add_to_awaited_by(fut, cur_task) nfuts += 1 arg_to_fut[arg] = fut if fut.done(): @@ -950,13 +950,12 @@ def shield(arg): loop = futures._get_loop(inner) outer = loop.create_future() - cur_task = current_task() - if cur_task is not None: - _add_to_awaited_by(inner, cur_task) + if (cur_task := current_task()) is not None: + futures.future_add_to_awaited_by(inner, cur_task) def _inner_done_callback(inner, cur_task=cur_task): if cur_task is not None: - _discard_from_awaited_by(inner, cur_task) + futures.future_discard_from_awaited_by(inner, cur_task) if outer.cancelled(): if not inner.cancelled(): @@ -1091,18 +1090,6 @@ def _unregister_eager_task(task): _eager_tasks.discard(task) -def _add_to_awaited_by(fut, waiter): - if hasattr(fut, '_awaited_by'): - if fut._awaited_by is None: - fut._awaited_by = set() - fut._awaited_by.add(waiter) - - -def _discard_from_awaited_by(fut, waiter): - if awaited_by := getattr(fut, '_awaited_by', None): - awaited_by.discard(waiter) - - _py_current_task = current_task _py_register_task = _register_task _py_register_eager_task = _register_eager_task diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index f669772932f84d..d7ff9e21b9f256 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -1,8 +1,6 @@ import asyncio import unittest -import pprint - # To prevent a warning "test altered the execution environment" def tearDownModule(): diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index d8b822ef5b53a9..7726f572608476 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -75,8 +75,19 @@ typedef struct { #define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) #define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) -#define Future_Check(state, obj) PyObject_TypeCheck(obj, state->FutureType) -#define Task_Check(state, obj) PyObject_TypeCheck(obj, state->TaskType) +#define Future_Check(state, obj) \ + (Future_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->FutureType)) + +#define Task_Check(state, obj) \ + (Task_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->TaskType)) + +#define TaskOrFuture_Check(state, obj) \ + (Task_CheckExact(state, obj) \ + || Future_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->FutureType) \ + || PyObject_TypeCheck(obj, state->TaskType)) #ifdef Py_GIL_DISABLED # define ASYNCIO_STATE_LOCK(state) Py_BEGIN_CRITICAL_SECTION_MUT(&state->mutex) @@ -521,19 +532,28 @@ future_init(FutureObj *fut, PyObject *loop) } static int -future_awaited_by_add(FutureObj *fut, PyObject *thing) +future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) { + if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { + // We only want to support native asyncio Futures. + // For further insight see the comment in the Python + // implementation of "future_add_to_awaited_by()". + return 0; + } + + FutureObj *_fut = (FutureObj *)fut; + /* Most futures/task are only awaited by one entity, so we want to avoid always creating a set for `fut_awaited_by`. */ - if (fut->fut_awaited_by == NULL) { + if (_fut->fut_awaited_by == NULL) { Py_INCREF(thing); - fut->fut_awaited_by = thing; + _fut->fut_awaited_by = thing; return 0; } - if (PySet_Check(fut->fut_awaited_by)) { - return PySet_Add(fut->fut_awaited_by, thing); + if (PySet_Check(_fut->fut_awaited_by)) { + return PySet_Add(_fut->fut_awaited_by, thing); } PyObject *set = PySet_New(NULL); @@ -544,29 +564,38 @@ future_awaited_by_add(FutureObj *fut, PyObject *thing) Py_DECREF(set); return -1; } - if (PySet_Add(set, fut->fut_awaited_by)) { + if (PySet_Add(set, _fut->fut_awaited_by)) { Py_DECREF(set); return -1; } - Py_SETREF(fut->fut_awaited_by, set); + Py_SETREF(_fut->fut_awaited_by, set); return 0; } static int -future_awaited_by_discard(FutureObj *fut, PyObject *thing) +future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) { + if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { + // We only want to support native asyncio Futures. + // For further insight see the comment in the Python + // implementation of "future_add_to_awaited_by()". + return 0; + } + + FutureObj *_fut = (FutureObj *)fut; + /* Following the semantics of 'set.discard()' here in not raising an error if `thing` isn't in the `awaited_by` "set". */ - if (fut->fut_awaited_by == NULL) { + if (_fut->fut_awaited_by == NULL) { return 0; } - if (fut->fut_awaited_by == thing) { - Py_CLEAR(fut->fut_awaited_by); + if (_fut->fut_awaited_by == thing) { + Py_CLEAR(_fut->fut_awaited_by); return 0; } - if (PySet_Check(fut->fut_awaited_by)) { - int err = PySet_Discard(fut->fut_awaited_by, thing); + if (PySet_Check(_fut->fut_awaited_by)) { + int err = PySet_Discard(_fut->fut_awaited_by, thing); if (err < 0 && PyErr_Occurred()) { return -1; } else { @@ -576,101 +605,6 @@ future_awaited_by_discard(FutureObj *fut, PyObject *thing) return 0; } -static int -awaited_by_add(asyncio_state *state, PyObject *maybe_fut, PyObject *thing) -{ - if (Future_CheckExact(state, maybe_fut) - || Task_CheckExact(state, maybe_fut) - ) { - return future_awaited_by_add((FutureObj *)maybe_fut, thing); - } - - PyObject *awaited_by; - int err = PyObject_GetOptionalAttr( - maybe_fut, &_Py_ID(_awaited_by), &awaited_by); - if (err < 0) { - return err; - } - - if (err == 1) { - if (PySet_Check(awaited_by)) { - if (PySet_Add(awaited_by, thing)) { - Py_DECREF(awaited_by); - return -1; - } else { - Py_DECREF(awaited_by); - return 0; - } - } else if (awaited_by == Py_None) { - Py_DECREF(awaited_by); - goto new_set; - } else { - Py_DECREF(awaited_by); - PyErr_SetString(PyExc_RuntimeError, - "_awaited_by is not a set or None"); - return -1; - } - } - - assert(err == 0); - assert(awaited_by == NULL); - -new_set: - awaited_by = PySet_New(NULL); - if (awaited_by == NULL) { - return -1; - } - if (PySet_Add(awaited_by, thing)) { - Py_DECREF(awaited_by); - return -1; - } - - err = PyObject_SetAttr(maybe_fut, &_Py_ID(_awaited_by), awaited_by); - Py_DECREF(awaited_by); - return err; -} - -static int -awaited_by_discard(asyncio_state *state, PyObject *maybe_fut, PyObject *thing) -{ - if (Future_CheckExact(state, maybe_fut) - || Task_CheckExact(state, maybe_fut) - ) { - return future_awaited_by_discard((FutureObj *)maybe_fut, thing); - } - - PyObject *awaited_by; - int err = PyObject_GetOptionalAttr( - maybe_fut, &_Py_ID(_awaited_by), &awaited_by); - if (err < 0) { - return err; - } - - if (err == 0) { - return 0; - } - - assert(err == 1); - - if (PySet_Check(awaited_by)) { - err = PySet_Discard(awaited_by, thing); - Py_DECREF(awaited_by); - if (err < 0 && PyErr_Occurred()) { - return -1; - } else { - return 0; - } - } else if (awaited_by == Py_None) { - Py_DECREF(awaited_by); - return 0; - } else { - Py_DECREF(awaited_by); - PyErr_SetString(PyExc_RuntimeError, - "_awaited_by is not a set or None"); - return -1; - } -} - static PyObject * future_get_awaited_by(FutureObj *fut) { @@ -679,26 +613,10 @@ future_get_awaited_by(FutureObj *fut) Py_RETURN_NONE; } if (PySet_Check(fut->fut_awaited_by)) { - Py_INCREF(fut->fut_awaited_by); - return fut->fut_awaited_by; + return PyFrozenSet_New(fut->fut_awaited_by); } - /* We don't want to "leak" our optimization that we don't always create - a set to the pure-Python land. Accessing `_awaited_by` from Python - can mean two things: - - (a) asyncio TaskGroup or gather or a similar primitive uses it - to ensure correct call stack. In this case, the TaskGroup - will attempt to mutate the set. - - (b) an async call stack is being rendered and needs to infer - what tasks are awaiting on this task or future. In this case - we don't want to micro-optimize things. - - The bottom line: it's easier to make a set, use it and return it. - */ - - PyObject *set = PySet_New(NULL); + PyObject *set = PyFrozenSet_New(NULL); if (set == NULL) { return NULL; } @@ -706,10 +624,6 @@ future_get_awaited_by(FutureObj *fut) Py_DECREF(set); return NULL; } - - Py_SETREF(fut->fut_awaited_by, set); - - Py_INCREF(set); return set; } @@ -1719,8 +1633,8 @@ static PyMethodDef FutureType_methods[] = { NULL, NULL}, \ {"_cancel_message", (getter)FutureObj_get_cancel_message, \ (setter)FutureObj_set_cancel_message, NULL}, \ - {"_awaited_by", (getter)future_get_awaited_by, \ - (setter)future_set_awaited_by, NULL}, + {"_asyncio_awaited_by", (getter)future_get_awaited_by, \ + (setter)future_set_awaited_by, NULL}, static PyGetSetDef FutureType_getsetlist[] = { FUTURE_COMMON_GETSETLIST @@ -3155,7 +3069,7 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu goto yield_insteadof_yf; } - if (awaited_by_add(state, result, (PyObject *)task)) { + if (future_awaited_by_add(state, result, (PyObject *)task)) { goto fail; } @@ -3237,7 +3151,7 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu goto yield_insteadof_yf; } - if (awaited_by_add(state, result, (PyObject *)task)) { + if (future_awaited_by_add(state, result, (PyObject *)task)) { goto fail; } @@ -3439,7 +3353,7 @@ task_wakeup(TaskObj *task, PyObject *o) asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); - if (awaited_by_discard(state, o, (PyObject *)task)) { + if (future_awaited_by_discard(state, o, (PyObject *)task)) { return NULL; } @@ -3903,6 +3817,50 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) return tasks; } +/*[clinic input] +_asyncio.future_add_to_awaited_by + + fut: object + waiter: object + +Record that `fut` is awaited on by `waiter`. + +[clinic start generated code]*/ + +static PyObject * +_asyncio_future_add_to_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter) +/*[clinic end generated code: output=0ab9a1a63389e4df input=29259cdbafe9e7bf]*/ +{ + asyncio_state *state = get_asyncio_state(module); + if (future_awaited_by_add(state, fut, waiter)) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +_asyncio.future_discard_from_awaited_by + + fut: object + waiter: object + +Record that `fut` is no longer awaited on by `waiter`. + +[clinic start generated code]*/ + +static PyObject * +_asyncio_future_discard_from_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter) +/*[clinic end generated code: output=a03b0b4323b779de input=5d67a3edc79b6094]*/ +{ + asyncio_state *state = get_asyncio_state(module); + if (future_awaited_by_discard(state, fut, waiter)) { + return NULL; + } + Py_RETURN_NONE; +} + static int module_traverse(PyObject *mod, visitproc visit, void *arg) { @@ -4072,6 +4030,8 @@ static PyMethodDef asyncio_methods[] = { _ASYNCIO__LEAVE_TASK_METHODDEF _ASYNCIO__SWAP_CURRENT_TASK_METHODDEF _ASYNCIO_ALL_TASKS_METHODDEF + _ASYNCIO_FUTURE_ADD_TO_AWAITED_BY_METHODDEF + _ASYNCIO_FUTURE_DISCARD_FROM_AWAITED_BY_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index d619a124ccead5..8a645d636e8e96 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -1547,4 +1547,120 @@ _asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=ffe9b71bc65888b3 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_asyncio_future_add_to_awaited_by__doc__, +"future_add_to_awaited_by($module, /, fut, waiter)\n" +"--\n" +"\n" +"Record that `fut` is awaited on by `waiter`."); + +#define _ASYNCIO_FUTURE_ADD_TO_AWAITED_BY_METHODDEF \ + {"future_add_to_awaited_by", _PyCFunction_CAST(_asyncio_future_add_to_awaited_by), METH_FASTCALL|METH_KEYWORDS, _asyncio_future_add_to_awaited_by__doc__}, + +static PyObject * +_asyncio_future_add_to_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter); + +static PyObject * +_asyncio_future_add_to_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(fut), &_Py_ID(waiter), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"fut", "waiter", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "future_add_to_awaited_by", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *fut; + PyObject *waiter; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + fut = args[0]; + waiter = args[1]; + return_value = _asyncio_future_add_to_awaited_by_impl(module, fut, waiter); + +exit: + return return_value; +} + +PyDoc_STRVAR(_asyncio_future_discard_from_awaited_by__doc__, +"future_discard_from_awaited_by($module, /, fut, waiter)\n" +"--\n" +"\n" +"Record that `fut` is no longer awaited on by `waiter`."); + +#define _ASYNCIO_FUTURE_DISCARD_FROM_AWAITED_BY_METHODDEF \ + {"future_discard_from_awaited_by", _PyCFunction_CAST(_asyncio_future_discard_from_awaited_by), METH_FASTCALL|METH_KEYWORDS, _asyncio_future_discard_from_awaited_by__doc__}, + +static PyObject * +_asyncio_future_discard_from_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter); + +static PyObject * +_asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(fut), &_Py_ID(waiter), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"fut", "waiter", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "future_discard_from_awaited_by", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *fut; + PyObject *waiter; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + fut = args[0]; + waiter = args[1]; + return_value = _asyncio_future_discard_from_awaited_by_impl(module, fut, waiter); + +exit: + return return_value; +} +/*[clinic end generated code: output=a05f7308434b488c input=a9049054013a1b77]*/ From c8be18e6387f52a757843ef959ca873d106d13bb Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 12:17:41 -0700 Subject: [PATCH 04/84] Add a comment for capture_call_stack() --- Lib/asyncio/stack.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index fe650d6d0a3cd5..09dbe1d38de4c8 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -74,7 +74,34 @@ def _build_stack_for_future(future: any) -> FutureCallStack: def capture_call_stack(*, future: any = None) -> FutureCallStack | None: - """Capture async call stack for the current task or the provided Future.""" + """Capture async call stack for the current task or the provided Future. + + The stack is represented with three data structures: + + * FutureCallStack(future, call_stack, awaited_by) + + Where 'future' is a reference to an asyncio.Future or asyncio.Task + (or their subclasses.) + + 'call_stack' is a list of FrameCallStackEntry and CoroutineCallStackEntry + objects (more on them below.) + + 'awaited_by' is a list of FutureCallStack objects. + + * FrameCallStackEntry(frame) + + Where 'frame' is a frame object of a regular Python function + in the call stack. + + * CoroutineCallStackEntry(coroutine) + + Where 'coroutine' is a coroutine object of an awaiting coroutine + or asyncronous generator. + + Receives an optional keyword-only "future" argument. If not passed, + the current task will be used. If there's no current task, the function + returns None. + """ if future is not None: if future is not tasks.current_task(): From abf2cb9b42c5618f7a6b2208bf9c89548b9a95cc Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 12:30:41 -0700 Subject: [PATCH 05/84] Add a couple more tests --- Lib/test/test_asyncio/test_stack.py | 74 ++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index d7ff9e21b9f256..f64a444f0a7697 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -11,7 +11,7 @@ def capture_test_stack(*, fut=None): def walk(s): ret = [ - f"T<{n}>" if '-' not in (n := s.future.get_name()) else 'T' + (f"T<{n}>" if '-' not in (n := s.future.get_name()) else 'T') if isinstance(s.future, asyncio.Task) else 'F' ] @@ -258,3 +258,75 @@ async def main(t1, t2): ] ] ]) + + async def test_stack_task(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + await inner() + + async def c2(): + await asyncio.create_task(c1(), name='there there') + + async def main(): + await c2() + + await main() + + self.assertEqual(stack_for_inner, [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [['T', ['a c2', 'a main', 'a test_stack_task'], []]] + ]) + + async def test_stack_future(self): + + stack_for_fut = None + + async def a2(fut): + await fut + + async def a1(fut): + await a2(fut) + + async def b1(fut): + await fut + + async def main(): + nonlocal stack_for_fut + + fut = asyncio.Future() + async with asyncio.TaskGroup() as g: + g.create_task(a1(fut), name="task A") + g.create_task(b1(fut), name='task B') + + for _ in range(5): + # Do a few iterations to ensure that both a1 and b1 + # await on the future + await asyncio.sleep(0) + + stack_for_fut = capture_test_stack(fut=fut) + fut.set_result(None) + + await main() + + self.assertEqual(stack_for_fut, + ['F', + [], + [ + ['T', + ['a a2', 'a a1'], + [['T', ['a test_stack_future'], []]] + ], + ['T', + ['a b1'], + [['T', ['a test_stack_future'], []]] + ], + ]] + ) From 20ceab7ba5a7391c3ef07754d71a84141d7f8d9e Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 12:50:54 -0700 Subject: [PATCH 06/84] Remove setter for C impl of Task._awaited_by --- Modules/_asynciomodule.c | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 7726f572608476..105c8cb64e6326 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -627,23 +627,6 @@ future_get_awaited_by(FutureObj *fut) return set; } -static int -future_set_awaited_by(FutureObj *fut, PyObject *set) -{ - /* Implementation of a Python setter. */ - if (set == Py_None) { - Py_CLEAR(fut->fut_awaited_by); - return 0; - } - if (!PySet_Check(set)) { - PyErr_SetString(PyExc_ValueError, "_awaited_by expects a set"); - return -1; - } - Py_XSETREF(fut->fut_awaited_by, set); - Py_INCREF(set); - return 0; -} - static PyObject * future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) { @@ -1633,8 +1616,7 @@ static PyMethodDef FutureType_methods[] = { NULL, NULL}, \ {"_cancel_message", (getter)FutureObj_get_cancel_message, \ (setter)FutureObj_set_cancel_message, NULL}, \ - {"_asyncio_awaited_by", (getter)future_get_awaited_by, \ - (setter)future_set_awaited_by, NULL}, + {"_asyncio_awaited_by", (getter)future_get_awaited_by, NULL, NULL}, static PyGetSetDef FutureType_getsetlist[] = { FUTURE_COMMON_GETSETLIST From 72d9321d8ad08571fd73ea672f68498af014bb8d Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 15:06:30 -0700 Subject: [PATCH 07/84] Intoduce cr_task --- Include/cpython/genobject.h | 2 ++ Include/internal/pycore_genobject.h | 7 +++++++ Lib/test/test_sys.py | 2 +- Modules/_asynciomodule.c | 28 ++++++++++++++++++++++++---- Objects/genobject.c | 15 ++++++++++++++- 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index f75884e597e2c2..f0c36081d120cc 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -32,6 +32,8 @@ PyAPI_DATA(PyTypeObject) PyCoro_Type; PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, PyObject *name, PyObject *qualname); +PyAPI_FUNC(void) _PyCoro_SetTask(PyObject *coro, PyObject *task); + /* --- Asynchronous Generators -------------------------------------------- */ diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index f6d7e6d367177b..40ef9098124753 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -22,6 +22,13 @@ extern "C" { PyObject *prefix##_qualname; \ _PyErr_StackItem prefix##_exc_state; \ PyObject *prefix##_origin_or_finalizer; \ + /* A *borrowed* reference to a task that drives the coroutine. \ + The field is meant to be used by profilers and debuggers only. \ + The main invariant is that a task can't get GC'ed while \ + the coroutine it drives is alive and vice versa. \ + Profilers can use this field to reconstruct the full async \ + call stack of program. */ \ + PyObject *prefix##_task; \ char prefix##_hooks_inited; \ char prefix##_closed; \ char prefix##_running_async; \ diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 9689ef8e96e072..49767964a01977 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1617,7 +1617,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('6P4c' + INTERPRETER_FRAME + 'P')) + check(get_gen(), size('7P4c' + INTERPRETER_FRAME + 'P')) # iterator check(iter('abc'), size('lP')) # callable-iterator diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 105c8cb64e6326..553620b1b66cbd 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -239,6 +239,27 @@ static PyObject * task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *result); +static void +clear_task_coro(TaskObj *task) +{ + if (task->task_coro != NULL &&PyCoro_CheckExact(task->task_coro)) { + _PyCoro_SetTask(task->task_coro, (PyObject *)task); + } + Py_CLEAR(task->task_coro); +} + + +static void +set_task_coro(TaskObj *task, PyObject *coro) +{ + if (PyCoro_CheckExact(coro)) { + _PyCoro_SetTask(coro, (PyObject *)task); + } + Py_INCREF(coro); + Py_XSETREF(task->task_coro, coro); +} + + static int _is_coroutine(asyncio_state *state, PyObject *coro) { @@ -2235,8 +2256,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, self->task_must_cancel = 0; self->task_log_destroy_pending = 1; self->task_num_cancels_requested = 0; - Py_INCREF(coro); - Py_XSETREF(self->task_coro, coro); + set_task_coro(self, coro); if (name == Py_None) { // optimization: defer task name formatting @@ -2284,8 +2304,8 @@ static int TaskObj_clear(TaskObj *task) { (void)FutureObj_clear((FutureObj*) task); + clear_task_coro(task); Py_CLEAR(task->task_context); - Py_CLEAR(task->task_coro); Py_CLEAR(task->task_name); Py_CLEAR(task->task_fut_waiter); return 0; @@ -3321,7 +3341,7 @@ task_eager_start(asyncio_state *state, TaskObj *task) register_task(state, task); } else { // This seems to really help performance on pyperformance benchmarks - Py_CLEAR(task->task_coro); + clear_task_coro(task); } return retval; diff --git a/Objects/genobject.c b/Objects/genobject.c index 41cf8fdcc9dee8..bd7090226ec5fa 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -127,6 +127,10 @@ gen_dealloc(PyGenObject *gen) { PyObject *self = (PyObject *) gen; + /* A borrowed reference used only by coroutines and async + frameworks. Just set it to NULL. */ + gen->gi_task = NULL; + _PyObject_GC_UNTRACK(gen); if (gen->gi_weakreflist != NULL) @@ -961,6 +965,7 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, frame->owner = FRAME_OWNED_BY_GENERATOR; assert(PyObject_GC_IsTracked((PyObject *)f)); Py_DECREF(f); + gen->gi_task = NULL; gen->gi_weakreflist = NULL; gen->gi_exc_state.exc_value = NULL; gen->gi_exc_state.previous_item = NULL; @@ -976,6 +981,13 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, return (PyObject *)gen; } +void +_PyCoro_SetTask(PyObject *coro, PyObject *task) +{ + assert(PyCoro_CheckExact(coro)); + ((PyCoroObject *)coro)->cr_task = task; +} + PyObject * PyGen_NewWithQualName(PyFrameObject *f, PyObject *name, PyObject *qualname) { @@ -1114,7 +1126,6 @@ cr_getcode(PyCoroObject *coro, void *Py_UNUSED(ignored)) return _gen_getcode((PyGenObject *)coro, "cr_code"); } - static PyGetSetDef coro_getsetlist[] = { {"__name__", (getter)gen_get_name, (setter)gen_set_name, PyDoc_STR("name of the coroutine")}, @@ -1348,6 +1359,8 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname) return NULL; } + ((PyCoroObject *)coro)->cr_task = NULL; + PyThreadState *tstate = _PyThreadState_GET(); int origin_depth = tstate->coroutine_origin_tracking_depth; From c9475f6fe8ae047b1e9bd88a7173d00e95718cb9 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 15:27:59 -0700 Subject: [PATCH 08/84] Unbreak shield() and gather() --- Lib/asyncio/tasks.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index a90fc23ff0f551..4025163416cde3 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -870,9 +870,12 @@ def _done_callback(fut, cur_task): nfuts = 0 nfinished = 0 done_futs = [] - loop = None outer = None # bpo-46672 - cur_task = current_task() + loop = events._get_running_loop() + if loop is not None: + cur_task = current_task(loop) + else: + cur_task = None for arg in coros_or_futures: if arg not in arg_to_fut: fut = ensure_future(arg, loop=loop) @@ -906,7 +909,7 @@ def _done_callback(fut, cur_task): # this will effectively complete the gather eagerly, with the last # callback setting the result (or exception) on outer before returning it for fut in done_futs: - _done_callback(fut) + _done_callback(fut, cur_task) return outer @@ -950,8 +953,10 @@ def shield(arg): loop = futures._get_loop(inner) outer = loop.create_future() - if (cur_task := current_task()) is not None: + if loop is not None and (cur_task := current_task(loop)) is not None: futures.future_add_to_awaited_by(inner, cur_task) + else: + cur_task = None def _inner_done_callback(inner, cur_task=cur_task): if cur_task is not None: From e1099e93538708c505cdc545aba16f7a93d02182 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 15:28:11 -0700 Subject: [PATCH 09/84] Add convinience fields to C Task/Future for profilers --- Modules/_asynciomodule.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 553620b1b66cbd..657ff0f0c2cfe7 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -46,7 +46,11 @@ typedef enum { so that these and bitfields from TaskObj are contiguous. */ \ unsigned prefix##_log_tb: 1; \ - unsigned prefix##_blocking: 1; + unsigned prefix##_blocking: 1; \ + /* Used by profilers to make traversing the stack from an external \ + process faster. */ \ + unsigned prefix##_is_task: 1; \ + unsigned prefix##_awaited_by_is_set: 1; typedef struct { FutureObj_HEAD(fut) @@ -513,6 +517,8 @@ future_init(FutureObj *fut, PyObject *loop) fut->fut_state = STATE_PENDING; fut->fut_log_tb = 0; fut->fut_blocking = 0; + fut->fut_awaited_by_is_set = 0; + fut->fut_is_task = 0; if (loop == Py_None) { asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); @@ -568,12 +574,14 @@ future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) to avoid always creating a set for `fut_awaited_by`. */ if (_fut->fut_awaited_by == NULL) { + assert(!_fut->fut_awaited_by_is_set); Py_INCREF(thing); _fut->fut_awaited_by = thing; return 0; } - if (PySet_Check(_fut->fut_awaited_by)) { + if (_fut->fut_awaited_by_is_set) { + assert(PySet_Check(_fut->fut_awaited_by)); return PySet_Add(_fut->fut_awaited_by, thing); } @@ -590,6 +598,7 @@ future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) return -1; } Py_SETREF(_fut->fut_awaited_by, set); + _fut->fut_awaited_by_is_set = 1; return 0; } @@ -615,7 +624,8 @@ future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) Py_CLEAR(_fut->fut_awaited_by); return 0; } - if (PySet_Check(_fut->fut_awaited_by)) { + if (_fut->fut_awaited_by_is_set) { + assert(PySet_Check(_fut->fut_awaited_by)); int err = PySet_Discard(_fut->fut_awaited_by, thing); if (err < 0 && PyErr_Occurred()) { return -1; @@ -633,7 +643,9 @@ future_get_awaited_by(FutureObj *fut) if (fut->fut_awaited_by == NULL) { Py_RETURN_NONE; } - if (PySet_Check(fut->fut_awaited_by)) { + if (fut->fut_awaited_by_is_set) { + /* Already a set, just wrap it into a frozen set and return. */ + assert(PySet_Check(fut->fut_awaited_by)); return PyFrozenSet_New(fut->fut_awaited_by); } @@ -935,6 +947,7 @@ FutureObj_clear(FutureObj *fut) Py_CLEAR(fut->fut_cancel_msg); Py_CLEAR(fut->fut_cancelled_exc); Py_CLEAR(fut->fut_awaited_by); + fut->fut_awaited_by_is_set = 0; PyObject_ClearManagedDict((PyObject *)fut); return 0; } @@ -2229,6 +2242,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, if (future_init((FutureObj*)self, loop)) { return -1; } + self->task_is_task = 1; asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); int is_coro = is_coroutine(state, coro); From 817f88bba1fdafb79b59adf279ced6cce99c0e26 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 27 Sep 2024 10:06:13 -0700 Subject: [PATCH 10/84] Fix ups --- Modules/_asynciomodule.c | 5 +++-- Objects/genobject.c | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 657ff0f0c2cfe7..7e39841d72d89d 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -246,8 +246,8 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu static void clear_task_coro(TaskObj *task) { - if (task->task_coro != NULL &&PyCoro_CheckExact(task->task_coro)) { - _PyCoro_SetTask(task->task_coro, (PyObject *)task); + if (task->task_coro != NULL && PyCoro_CheckExact(task->task_coro)) { + _PyCoro_SetTask(task->task_coro, NULL); } Py_CLEAR(task->task_coro); } @@ -256,6 +256,7 @@ clear_task_coro(TaskObj *task) static void set_task_coro(TaskObj *task, PyObject *coro) { + assert(coro != NULL); if (PyCoro_CheckExact(coro)) { _PyCoro_SetTask(coro, (PyObject *)task); } diff --git a/Objects/genobject.c b/Objects/genobject.c index bd7090226ec5fa..1f2c02884e28ae 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -890,6 +890,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) gen->gi_name = Py_NewRef(func->func_name); assert(func->func_qualname != NULL); gen->gi_qualname = Py_NewRef(func->func_qualname); + gen->gi_task = NULL; _PyObject_GC_TRACK(gen); return (PyObject *)gen; } From 54386ac6a46794d07afaf0da27c62a9e34eac744 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 27 Sep 2024 15:41:11 -0700 Subject: [PATCH 11/84] Add basic docs --- Doc/library/asyncio-future.rst | 2 +- Doc/library/asyncio-stack.rst | 80 +++++++++++++++++++++++++++++++ Doc/library/asyncio.rst | 1 + Lib/asyncio/futures.py | 4 +- Lib/asyncio/stack.py | 12 ++++- Modules/_asynciomodule.c | 6 ++- Modules/clinic/_asynciomodule.c.h | 72 ++++------------------------ 7 files changed, 108 insertions(+), 69 deletions(-) create mode 100644 Doc/library/asyncio-stack.rst diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 9dce0731411940..f1f23b8021b29f 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -65,7 +65,7 @@ Future Functions and *loop* is not specified and there is no running event loop. -.. function:: wrap_future(future, *, loop=None) +.. function:: wrap_future(future, /, *, loop=None) Wrap a :class:`concurrent.futures.Future` object in a :class:`asyncio.Future` object. diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst new file mode 100644 index 00000000000000..274f64fa2ecfcf --- /dev/null +++ b/Doc/library/asyncio-stack.rst @@ -0,0 +1,80 @@ +.. currentmodule:: asyncio + + +.. _asyncio-stack: + +=================== +Stack Introspection +=================== + +**Source code:** :source:`Lib/asyncio/stack.py` + +------------------------------------- + +asyncio has powerful runtime call stack introspection utilities +to trace the entire call graph of a running coroutine or task, or +a suspended *future*. + +.. versionadded:: 3.14 + + +.. function:: capture_call_stack(*, future=None) + + Capture the async call stack for the current task or the provided + :class:`Task` or :class:`Future`. + + The function receives an optional keyword-only *future* argument. + If not passed, the current task will be used. If there's no current task, + the function returns ``None``. + + Returns a ``FutureCallStack`` named tuple: + + * ``FutureCallStack(future, call_stack, awaited_by)`` + + Where 'future' is a reference to a *Future* or a *Task* + (or their subclasses.) + + ``call_stack`` is a list of ``FrameCallStackEntry`` and + ``CoroutineCallStackEntry`` objects (more on them below.) + + ``awaited_by`` is a list of ``FutureCallStack`` tuples. + + * ``FrameCallStackEntry(frame)`` + + Where ``frame`` is a frame object of a regular Python function + in the call stack. + + * ``CoroutineCallStackEntry(coroutine)`` + + Where ``coroutine`` is a coroutine object of an awaiting coroutine + or asyncronous generator. + + +Low level utility functions +=========================== + +To introspect an async call stack asyncio requires cooperation from +control flow structures, such as :func:`shield` or :class:`TaskGroup`. +Any time an intermediate ``Future`` object with low-level APIs like +:meth:`Future.add_done_callback() ` is +involved, the following two functions should be used to inform *asyncio* +about how exactly such intermediate future objects are connected with +the tasks they wrap or control. + + +.. function:: future_add_to_awaited_by(future, waiter, /) + + Record that *future* is awaited on by *waiter*. + + Both *future* and *waiter* must be instances of + :class:`asyncio.Future ` or :class:`asyncio.Task ` or + their subclasses, otherwise the call would have no effect. + + +.. function:: future_discard_from_awaited_by(future, waiter, /) + + Record that *future* is no longer awaited on by *waiter*. + + Both *future* and *waiter* must be instances of + :class:`asyncio.Future ` or :class:`asyncio.Task ` or + their subclasses, otherwise the call would have no effect. diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 5f83b3a2658da4..5098805f26cbd5 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -99,6 +99,7 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst + asyncio-stack.rst .. toctree:: :caption: Low-level APIs diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index f2150c89d60160..5785477248a8d9 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -423,7 +423,7 @@ def wrap_future(future, *, loop=None): return new_future -def future_add_to_awaited_by(fut, waiter): +def future_add_to_awaited_by(fut, waiter, /): """Record that `fut` is awaited on by `waiter`.""" # For the sake of keeping the implementation minimal and assuming # that 99.9% of asyncio users use the built-in Futures and Tasks @@ -451,7 +451,7 @@ def future_add_to_awaited_by(fut, waiter): fut._asyncio_awaited_by.add(waiter) -def future_discard_from_awaited_by(fut, waiter): +def future_discard_from_awaited_by(fut, waiter, /): """Record that `fut` is no longer awaited on by `waiter`.""" # See the comment in "future_add_to_awaited_by()" body for # details on implemntation. diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 09dbe1d38de4c8..15086df8f63a3a 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -4,6 +4,7 @@ import types import typing +from . import events from . import futures from . import tasks @@ -103,11 +104,20 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: returns None. """ + loop = events._get_running_loop() + if future is not None: - if future is not tasks.current_task(): + # Check if we're in a context of a running event loop; + # if yes - check if the passed future is the currently + # running task or not. + if loop is None or future is not tasks.current_task(): return _build_stack_for_future(future) # else: future is the current task, move on. else: + if loop is None: + raise RuntimeError( + 'capture_call_stack() is called outside of a running ' + 'event loop and no *future* to introspect was provided') future = tasks.current_task() if future is None: diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 7e39841d72d89d..0b7d98b0d586b1 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3839,6 +3839,7 @@ _asyncio.future_add_to_awaited_by fut: object waiter: object + / Record that `fut` is awaited on by `waiter`. @@ -3847,7 +3848,7 @@ Record that `fut` is awaited on by `waiter`. static PyObject * _asyncio_future_add_to_awaited_by_impl(PyObject *module, PyObject *fut, PyObject *waiter) -/*[clinic end generated code: output=0ab9a1a63389e4df input=29259cdbafe9e7bf]*/ +/*[clinic end generated code: output=0ab9a1a63389e4df input=06e6eaac51f532b9]*/ { asyncio_state *state = get_asyncio_state(module); if (future_awaited_by_add(state, fut, waiter)) { @@ -3861,6 +3862,7 @@ _asyncio.future_discard_from_awaited_by fut: object waiter: object + / Record that `fut` is no longer awaited on by `waiter`. @@ -3869,7 +3871,7 @@ Record that `fut` is no longer awaited on by `waiter`. static PyObject * _asyncio_future_discard_from_awaited_by_impl(PyObject *module, PyObject *fut, PyObject *waiter) -/*[clinic end generated code: output=a03b0b4323b779de input=5d67a3edc79b6094]*/ +/*[clinic end generated code: output=a03b0b4323b779de input=b5f7a39ccd36b5db]*/ { asyncio_state *state = get_asyncio_state(module); if (future_awaited_by_discard(state, fut, waiter)) { diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 8a645d636e8e96..f7643f8676cc69 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -1549,53 +1549,26 @@ _asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } PyDoc_STRVAR(_asyncio_future_add_to_awaited_by__doc__, -"future_add_to_awaited_by($module, /, fut, waiter)\n" +"future_add_to_awaited_by($module, fut, waiter, /)\n" "--\n" "\n" "Record that `fut` is awaited on by `waiter`."); #define _ASYNCIO_FUTURE_ADD_TO_AWAITED_BY_METHODDEF \ - {"future_add_to_awaited_by", _PyCFunction_CAST(_asyncio_future_add_to_awaited_by), METH_FASTCALL|METH_KEYWORDS, _asyncio_future_add_to_awaited_by__doc__}, + {"future_add_to_awaited_by", _PyCFunction_CAST(_asyncio_future_add_to_awaited_by), METH_FASTCALL, _asyncio_future_add_to_awaited_by__doc__}, static PyObject * _asyncio_future_add_to_awaited_by_impl(PyObject *module, PyObject *fut, PyObject *waiter); static PyObject * -_asyncio_future_add_to_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_future_add_to_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(fut), &_Py_ID(waiter), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"fut", "waiter", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "future_add_to_awaited_by", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; PyObject *fut; PyObject *waiter; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("future_add_to_awaited_by", nargs, 2, 2)) { goto exit; } fut = args[0]; @@ -1607,53 +1580,26 @@ _asyncio_future_add_to_awaited_by(PyObject *module, PyObject *const *args, Py_ss } PyDoc_STRVAR(_asyncio_future_discard_from_awaited_by__doc__, -"future_discard_from_awaited_by($module, /, fut, waiter)\n" +"future_discard_from_awaited_by($module, fut, waiter, /)\n" "--\n" "\n" "Record that `fut` is no longer awaited on by `waiter`."); #define _ASYNCIO_FUTURE_DISCARD_FROM_AWAITED_BY_METHODDEF \ - {"future_discard_from_awaited_by", _PyCFunction_CAST(_asyncio_future_discard_from_awaited_by), METH_FASTCALL|METH_KEYWORDS, _asyncio_future_discard_from_awaited_by__doc__}, + {"future_discard_from_awaited_by", _PyCFunction_CAST(_asyncio_future_discard_from_awaited_by), METH_FASTCALL, _asyncio_future_discard_from_awaited_by__doc__}, static PyObject * _asyncio_future_discard_from_awaited_by_impl(PyObject *module, PyObject *fut, PyObject *waiter); static PyObject * -_asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(fut), &_Py_ID(waiter), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"fut", "waiter", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "future_discard_from_awaited_by", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; PyObject *fut; PyObject *waiter; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("future_discard_from_awaited_by", nargs, 2, 2)) { goto exit; } fut = args[0]; @@ -1663,4 +1609,4 @@ _asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=a05f7308434b488c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e164592826f13567 input=a9049054013a1b77]*/ From 98434f0107d4a81544b70fcb38065f4da2b61cd8 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 27 Sep 2024 17:02:44 -0700 Subject: [PATCH 12/84] Implement, test, and document asyncio.print_call_stack() --- Doc/library/asyncio-stack.rst | 21 ++++++-- Lib/asyncio/stack.py | 78 ++++++++++++++++++++++++++++- Lib/test/test_asyncio/test_stack.py | 33 ++++++++---- 3 files changed, 118 insertions(+), 14 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 274f64fa2ecfcf..89a3d2b8b21894 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -23,9 +23,9 @@ a suspended *future*. Capture the async call stack for the current task or the provided :class:`Task` or :class:`Future`. - The function receives an optional keyword-only *future* argument. - If not passed, the current task will be used. If there's no current task, - the function returns ``None``. + The function recieves an optional keyword-only *future* argument. + If not passed, the current running task will be used. If there's no + current task, the function returns ``None``. Returns a ``FutureCallStack`` named tuple: @@ -49,6 +49,17 @@ a suspended *future*. Where ``coroutine`` is a coroutine object of an awaiting coroutine or asyncronous generator. +.. function:: print_call_stack(*, future=None, file=None) + + Print the async call stack for the current task or the provided + :class:`Task` or :class:`Future`. + + The function recieves an optional keyword-only *future* argument. + If not passed, the current running task will be used. If there's no + current task, the function returns ``None``. + + If *file* is not specified the function will print to :data:`sys.stdout`. + Low level utility functions =========================== @@ -70,6 +81,10 @@ the tasks they wrap or control. :class:`asyncio.Future ` or :class:`asyncio.Task ` or their subclasses, otherwise the call would have no effect. + A call to ``future_add_to_awaited_by()`` must be followed by an + eventual call to the ``future_discard_from_awaited_by()`` function + with the same arguments. + .. function:: future_discard_from_awaited_by(future, waiter, /) diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 15086df8f63a3a..16d718e2e9839b 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -10,6 +10,7 @@ __all__ = ( 'capture_call_stack', + 'print_call_stack', 'FrameCallStackEntry', 'CoroutineCallStackEntry', 'FutureCallStack', @@ -110,7 +111,7 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: # Check if we're in a context of a running event loop; # if yes - check if the passed future is the currently # running task or not. - if loop is None or future is not tasks.current_task(): + if loop is None or future is not tasks.current_task(loop=loop): return _build_stack_for_future(future) # else: future is the current task, move on. else: @@ -118,7 +119,7 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: raise RuntimeError( 'capture_call_stack() is called outside of a running ' 'event loop and no *future* to introspect was provided') - future = tasks.current_task() + future = tasks.current_task(loop=loop) if future is None: # This isn't a generic call stack introspection utility. If we @@ -158,3 +159,76 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: awaited_by.append(_build_stack_for_future(parent)) return FutureCallStack(future, call_stack, awaited_by) + + +def print_call_stack(*, future: any = None, file=None) -> None: + """Print async call stack for the current task or the provided Future.""" + + stack = capture_call_stack(future=future) + if stack is None: + return + + buf = [] + + def render_level(st: FutureCallStack, level: int = 0): + def add_line(line: str): + buf.append(level * ' ' + line) + + if isinstance(st.future, tasks.Task): + add_line( + f'* Task(name={st.future.get_name()!r}, id=0x{id(st.future):x})' + ) + else: + add_line( + f'* Future(id=0x{id(st.future):x})' + ) + + if st.call_stack: + add_line( + f' + Call stack:' + ) + for ste in st.call_stack: + if isinstance(ste, FrameCallStackEntry): + f = ste.frame + add_line( + f' | * {f.f_code.co_qualname}()' + ) + add_line( + f' | {f.f_code.co_filename}:{f.f_lineno}' + ) + else: + assert isinstance(ste, CoroutineCallStackEntry) + c = ste.coroutine + + try: + f = c.cr_frame + code = c.cr_code + tag = 'async' + except AttributeError: + try: + f = c.ag_frame + code = c.ag_code + tag = 'async generator' + except AttributeError: + f = c.gi_frame + code = c.gi_code + tag = 'generator' + + add_line( + f' | * {tag} {code.co_qualname}()' + ) + add_line( + f' | {f.f_code.co_filename}:{f.f_lineno}' + ) + + if st.awaited_by: + add_line( + f' + Awaited by:' + ) + for fut in st.awaited_by: + render_level(fut, level + 1) + + render_level(stack) + rendered = '\n'.join(buf) + + print(rendered, file=file) diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index f64a444f0a7697..7f434b242f5e44 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -1,4 +1,5 @@ import asyncio +import io import unittest @@ -37,8 +38,11 @@ def walk(s): return ret + buf = io.StringIO() + asyncio.print_call_stack(future=fut, file=buf) + stack = asyncio.capture_call_stack(future=fut) - return walk(stack) + return walk(stack), buf.getvalue() class TestCallStack(unittest.IsolatedAsyncioTestCase): @@ -73,7 +77,7 @@ async def main(): await main() - self.assertEqual(stack_for_c5, [ + self.assertEqual(stack_for_c5[0], [ # task name 'T', # call stack @@ -102,6 +106,11 @@ async def main(): ] ]) + self.assertIn( + '* async TestCallStack.test_stack_tgroup()', + stack_for_c5[1]) + + async def test_stack_async_gen(self): stack_for_gen_nested_call = None @@ -122,7 +131,7 @@ async def main(): await main() - self.assertEqual(stack_for_gen_nested_call, [ + self.assertEqual(stack_for_gen_nested_call[0], [ 'T', [ 's capture_test_stack', @@ -134,6 +143,10 @@ async def main(): [] ]) + self.assertIn( + 'async generator TestCallStack.test_stack_async_gen..gen()', + stack_for_gen_nested_call[1]) + async def test_stack_gather(self): stack_for_deep = None @@ -155,7 +168,7 @@ async def main(): await main() - self.assertEqual(stack_for_deep, [ + self.assertEqual(stack_for_deep[0], [ 'T', ['s capture_test_stack', 'a deep', 'a c1'], [ @@ -181,7 +194,7 @@ async def main(): await main() - self.assertEqual(stack_for_shield, [ + self.assertEqual(stack_for_shield[0], [ 'T', ['s capture_test_stack', 'a deep', 'a c1'], [ @@ -208,7 +221,7 @@ async def main(): await main() - self.assertEqual(stack_for_inner, [ + self.assertEqual(stack_for_inner[0], [ 'T', ['s capture_test_stack', 'a inner', 'a c1'], [ @@ -248,7 +261,7 @@ async def main(t1, t2): await t1 await t2 - self.assertEqual(stack_for_inner, [ + self.assertEqual(stack_for_inner[0], [ 'T', ['s capture_test_stack', 'a inner', 'a c1'], [ @@ -279,7 +292,7 @@ async def main(): await main() - self.assertEqual(stack_for_inner, [ + self.assertEqual(stack_for_inner[0], [ 'T', ['s capture_test_stack', 'a inner', 'a c1'], [['T', ['a c2', 'a main', 'a test_stack_task'], []]] @@ -316,7 +329,7 @@ async def main(): await main() - self.assertEqual(stack_for_fut, + self.assertEqual(stack_for_fut[0], ['F', [], [ @@ -330,3 +343,5 @@ async def main(): ], ]] ) + + self.assertTrue(stack_for_fut[1].startswith('* Future(id=')) From 485c1663dc451c12a2c5e6b301eb86e62c410a31 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sat, 28 Sep 2024 15:20:43 -0700 Subject: [PATCH 13/84] Reorder a few things --- Doc/library/asyncio-stack.rst | 69 +++++++++++++++++++++++++++++------ Lib/asyncio/stack.py | 25 +++++++------ 2 files changed, 72 insertions(+), 22 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 89a3d2b8b21894..c04d80488d3547 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -18,6 +18,64 @@ a suspended *future*. .. versionadded:: 3.14 +.. function:: print_call_stack(*, future=None, file=None) + + Print the async call stack for the current task or the provided + :class:`Task` or :class:`Future`. + + The function recieves an optional keyword-only *future* argument. + If not passed, the current running task will be used. If there's no + current task, the function returns ``None``. + + If *file* is not specified the function will print to :data:`sys.stdout`. + + **Example:** + + The following Python code: + + .. code-block:: python + + import asyncio + + async def test(): + asyncio.print_call_stack() + + async def main(): + async with asyncio.TaskGroup() as g: + g.create_task(test()) + + asyncio.run(main()) + + will print:: + + * Task(name='Task-2', id=0x105038fe0) + + Call stack: + | * print_call_stack() + | asyncio/stack.py:231 + | * async test() + | test.py:4 + + Awaited by: + * Task(name='Task-1', id=0x1050a6060) + + Call stack: + | * async TaskGroup.__aexit__() + | asyncio/taskgroups.py:107 + | * async main() + | test.py:7 + + For rendering the call stack to a string the following pattern + should be used: + + .. code-block:: python + + import io + + ... + + buf = io.StringIO() + asyncio.print_call_stack(file=buf) + output = buf.getvalue() + + .. function:: capture_call_stack(*, future=None) Capture the async call stack for the current task or the provided @@ -49,17 +107,6 @@ a suspended *future*. Where ``coroutine`` is a coroutine object of an awaiting coroutine or asyncronous generator. -.. function:: print_call_stack(*, future=None, file=None) - - Print the async call stack for the current task or the provided - :class:`Task` or :class:`Future`. - - The function recieves an optional keyword-only *future* argument. - If not passed, the current running task will be used. If there's no - current task, the function returns ``None``. - - If *file* is not specified the function will print to :data:`sys.stdout`. - Low level utility functions =========================== diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 16d718e2e9839b..652e2eb399463c 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -164,13 +164,7 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: def print_call_stack(*, future: any = None, file=None) -> None: """Print async call stack for the current task or the provided Future.""" - stack = capture_call_stack(future=future) - if stack is None: - return - - buf = [] - - def render_level(st: FutureCallStack, level: int = 0): + def render_level(st: FutureCallStack, buf: list[str], level: int): def add_line(line: str): buf.append(level * ' ' + line) @@ -226,9 +220,18 @@ def add_line(line: str): f' + Awaited by:' ) for fut in st.awaited_by: - render_level(fut, level + 1) + render_level(fut, buf, level + 1) - render_level(stack) - rendered = '\n'.join(buf) + stack = capture_call_stack(future=future) + if stack is None: + return - print(rendered, file=file) + try: + buf = [] + render_level(stack, buf, 0) + rendered = '\n'.join(buf) + print(rendered, file=file) + finally: + # 'stack' has references to frames so we should + # make sure it's GC'ed as soon as we don't need it. + del stack From 8802be7a6db4326568f41adbe1c0db7ff2e0cbac Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 29 Sep 2024 12:23:45 -0700 Subject: [PATCH 14/84] Add a test to exercise asyncio stack traces in out-of-process profilers Signed-off-by: Pablo Galindo --- Include/internal/pycore_runtime.h | 16 + Include/internal/pycore_runtime_init.h | 14 + Lib/test/test_external_inspection.py | 74 ++ Modules/_asynciomodule.c | 49 +- Modules/_testexternalinspection.c | 1117 ++++++++++++++++++++---- 5 files changed, 1112 insertions(+), 158 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d4291b87261ae0..c7437a2ad1515c 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -108,6 +108,7 @@ typedef struct _Py_DebugOffsets { uint64_t instr_ptr; uint64_t localsplus; uint64_t owner; + uint64_t stackpointer; } interpreter_frame; // Code object offset; @@ -152,6 +153,13 @@ typedef struct _Py_DebugOffsets { uint64_t ob_size; } list_object; + // PySet object offset; + struct _set_object { + uint64_t size; + uint64_t used; + uint64_t table; + } set_object; + // PyDict object offset; struct _dict_object { uint64_t size; @@ -192,6 +200,14 @@ typedef struct _Py_DebugOffsets { uint64_t size; uint64_t collecting; } gc; + + struct _gen_object { + uint64_t size; + uint64_t gi_name; + uint64_t gi_iframe; + uint64_t gi_task; + uint64_t gi_frame_state; + } gen_object; } _Py_DebugOffsets; /* Reference tracer state */ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index e6adb98eb19130..1072196abe4fc9 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -21,6 +21,7 @@ extern "C" { #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT +#include "pycore_genobject.h" extern PyTypeObject _PyExc_MemoryError; @@ -73,6 +74,7 @@ extern PyTypeObject _PyExc_MemoryError; .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ + .stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \ }, \ .code_object = { \ .size = sizeof(PyCodeObject), \ @@ -106,6 +108,11 @@ extern PyTypeObject _PyExc_MemoryError; .ob_item = offsetof(PyListObject, ob_item), \ .ob_size = offsetof(PyListObject, ob_base.ob_size), \ }, \ + .set_object = { \ + .size = sizeof(PySetObject), \ + .used = offsetof(PySetObject, used), \ + .table = offsetof(PySetObject, table), \ + }, \ .dict_object = { \ .size = sizeof(PyDictObject), \ .ma_keys = offsetof(PyDictObject, ma_keys), \ @@ -135,6 +142,13 @@ extern PyTypeObject _PyExc_MemoryError; .size = sizeof(struct _gc_runtime_state), \ .collecting = offsetof(struct _gc_runtime_state, collecting), \ }, \ + .gen_object = { \ + .size = sizeof(PyGenObject), \ + .gi_name = offsetof(PyGenObject, gi_name), \ + .gi_iframe = offsetof(PyGenObject, gi_iframe), \ + .gi_task = offsetof(PyGenObject, gi_task), \ + .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \ + }, \ }, \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index d896fec73d1971..d7f55fc6b06841 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -13,6 +13,7 @@ try: from _testexternalinspection import PROCESS_VM_READV_SUPPORTED from _testexternalinspection import get_stack_trace + from _testexternalinspection import get_async_stack_trace except ImportError: raise unittest.SkipTest("Test only runs when _testexternalinspection is available") @@ -74,6 +75,79 @@ def foo(): ] self.assertEqual(stack_trace, expected_stack_trace) + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_async_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import asyncio + import time + import os + import sys + import test.test_asyncio.test_stack as ts + + def c5(): + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(10000) + + async def c4(): + await asyncio.sleep(0) + c5() + + async def c3(): + await c4() + + async def c2(): + await c3() + + async def c1(task): + await task + + async def main(): + async with asyncio.TaskGroup() as tg: + task = tg.create_task(c2(), name="c2_root") + tg.create_task(c1(task), name="sub_main_1") + tg.create_task(c1(task), name="sub_main_2") + + asyncio.run(main()) + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + + expected_stack_trace = [ + ["c5", "c4", "c3", "c2"], + "c2_root", + [ + [["main"], "Task-1", []], + [["c1"], "sub_main_2", [[["main"], "Task-1", []]]], + [["c1"], "sub_main_1", [[["main"], "Task-1", []]]], + ], + ] + self.assertEqual(stack_trace, expected_stack_trace) + + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") def test_self_trace(self): diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 0b7d98b0d586b1..376664e922fe38 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -16,6 +16,9 @@ #include // offsetof() +#if defined(__APPLE__) +# include +#endif /*[clinic input] module _asyncio @@ -42,15 +45,15 @@ typedef enum { PyObject *prefix##_cancelled_exc; \ PyObject *prefix##_awaited_by; \ fut_state prefix##_state; \ - /* These bitfields need to be at the end of the struct - so that these and bitfields from TaskObj are contiguous. + /* Used by profilers to make traversing the stack from an external \ + process faster. */ \ + char prefix##_is_task; \ + char prefix##_awaited_by_is_set; \ + /* These bitfields need to be at the end of the struct \ + so that these and bitfields from TaskObj are contiguous. \ */ \ unsigned prefix##_log_tb: 1; \ unsigned prefix##_blocking: 1; \ - /* Used by profilers to make traversing the stack from an external \ - process faster. */ \ - unsigned prefix##_is_task: 1; \ - unsigned prefix##_awaited_by_is_set: 1; typedef struct { FutureObj_HEAD(fut) @@ -102,6 +105,40 @@ typedef struct { #endif typedef struct futureiterobject futureiterobject; +typedef struct _Py_AsyncioModuleDebugOffsets { + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; +} Py_AsyncioModuleDebugOffsets; + +#if defined(MS_WINDOWS) + +#pragma section("AsyncioDebug", read, write) +__declspec(allocate("AsyncioDebug")) + +#elif defined(__APPLE__) + +__attribute__((section(SEG_DATA ",AsyncioDebug"))) + +#endif + +Py_AsyncioModuleDebugOffsets AsyncioDebug +#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) + __attribute__((section(".AsyncioDebug"))) +#endif + = {.asyncio_task_object = { + .size = sizeof(TaskObj), + .task_name = offsetof(TaskObj, task_name), + .task_awaited_by = offsetof(TaskObj, task_awaited_by), + .task_is_task = offsetof(TaskObj, task_is_task), + .task_awaited_by_is_set = offsetof(TaskObj, task_awaited_by_is_set), + .task_coro = offsetof(TaskObj, task_coro), + }}; /* State of the _asyncio module */ typedef struct { diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 2476346777c319..6a14c399933b05 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -57,9 +57,20 @@ # define HAVE_PROCESS_VM_READV 0 #endif +struct _Py_AsyncioModuleDebugOffsets { + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; +}; + #if defined(__APPLE__) && TARGET_OS_OSX -static void* -analyze_macho64(mach_port_t proc_ref, void* base, void* map) +static uintptr_t +return_section_address(const char* section, mach_port_t proc_ref, uintptr_t base, void* map) { struct mach_header_64* hdr = (struct mach_header_64*)map; int ncmds = hdr->ncmds; @@ -72,8 +83,12 @@ analyze_macho64(mach_port_t proc_ref, void* base, void* map) mach_vm_address_t address = (mach_vm_address_t)base; vm_region_basic_info_data_64_t region_info; mach_port_t object_name; + uintptr_t vmaddr = 0; for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__TEXT") == 0) { + vmaddr = cmd->vmaddr; + } if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { while (cmd->filesize != size) { address += size; @@ -88,17 +103,16 @@ analyze_macho64(mach_port_t proc_ref, void* base, void* map) != KERN_SUCCESS) { PyErr_SetString(PyExc_RuntimeError, "Cannot get any more VM maps.\n"); - return NULL; + return 0; } } - base = (void*)address - cmd->vmaddr; int nsects = cmd->nsects; struct section_64* sec = (struct section_64*)((void*)cmd + sizeof(struct segment_command_64)); for (int j = 0; j < nsects; j++) { - if (strcmp(sec[j].sectname, "PyRuntime") == 0) { - return base + sec[j].addr; + if (strcmp(sec[j].sectname, section) == 0) { + return base + sec[j].addr - vmaddr; } } cmd_cnt++; @@ -106,33 +120,33 @@ analyze_macho64(mach_port_t proc_ref, void* base, void* map) cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); } - return NULL; + return 0; } -static void* -analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) +static uintptr_t +search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_size_t size, mach_port_t proc_ref) { int fd = open(path, O_RDONLY); if (fd == -1) { PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); - return NULL; + return 0; } struct stat fs; if (fstat(fd, &fs) == -1) { PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); close(fd); - return NULL; + return 0; } void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path); close(fd); - return NULL; + return 0; } - void* result = NULL; + uintptr_t result = 0; struct mach_header_64* hdr = (struct mach_header_64*)map; switch (hdr->magic) { @@ -144,7 +158,7 @@ analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) break; case MH_MAGIC_64: case MH_CIGAM_64: - result = analyze_macho64(proc_ref, base, map); + result = return_section_address(secname, proc_ref, base, map); break; default: PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic"); @@ -172,9 +186,8 @@ pid_to_task(pid_t pid) return task; } -static void* -get_py_runtime_macos(pid_t pid) -{ +static uintptr_t +search_map_for_section(pid_t pid, const char* secname, const char* substr) { mach_vm_address_t address = 0; mach_vm_size_t size = 0; mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); @@ -184,12 +197,11 @@ get_py_runtime_macos(pid_t pid) mach_port_t proc_ref = pid_to_task(pid); if (proc_ref == 0) { PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID"); - return NULL; + return 0; } int match_found = 0; char map_filename[MAXPATHLEN + 1]; - void* result_address = NULL; while (mach_vm_region( proc_ref, &address, @@ -213,26 +225,21 @@ get_py_runtime_macos(pid_t pid) filename = map_filename; // No path, use the whole string } - // Check if the filename starts with "python" or "libpython" - if (!match_found && strncmp(filename, "python", 6) == 0) { + if (!match_found && strncmp(filename, substr, strlen(substr)) == 0) { match_found = 1; - result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); - } - if (strncmp(filename, "libpython", 9) == 0) { - match_found = 1; - result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); - break; + return search_section_in_file(secname, map_filename, address, size, proc_ref); } address += size; } - return result_address; + return 0; } + #endif #ifdef __linux__ -void* -find_python_map_start_address(pid_t pid, char* result_filename) +static uintptr_t +find_map_start_address(pid_t pid, char* result_filename, const char* map) { char maps_file_path[64]; sprintf(maps_file_path, "/proc/%d/maps", pid); @@ -240,14 +247,14 @@ find_python_map_start_address(pid_t pid, char* result_filename) FILE* maps_file = fopen(maps_file_path, "r"); if (maps_file == NULL) { PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return 0; } int match_found = 0; char line[256]; char map_filename[PATH_MAX]; - void* result_address = 0; + uintptr_t result_address = 0; while (fgets(line, sizeof(line), maps_file) != NULL) { unsigned long start_address = 0; sscanf(line, "%lx-%*x %*s %*s %*s %*s %s", &start_address, map_filename); @@ -258,15 +265,9 @@ find_python_map_start_address(pid_t pid, char* result_filename) filename = map_filename; // No path, use the whole string } - // Check if the filename starts with "python" or "libpython" - if (!match_found && strncmp(filename, "python", 6) == 0) { - match_found = 1; - result_address = (void*)start_address; - strcpy(result_filename, map_filename); - } - if (strncmp(filename, "libpython", 9) == 0) { + if (!match_found && strncmp(filename, map, strlen(map)) == 0) { match_found = 1; - result_address = (void*)start_address; + result_address = start_address; strcpy(result_filename, map_filename); break; } @@ -281,18 +282,17 @@ find_python_map_start_address(pid_t pid, char* result_filename) return result_address; } -void* -get_py_runtime_linux(pid_t pid) +static uintptr_t +search_map_for_section(pid_t pid, const char* secname, const char* map) { char elf_file[256]; - void* start_address = (void*)find_python_map_start_address(pid, elf_file); + uintptr_t start_address = find_map_start_address(pid, elf_file, map); if (start_address == 0) { - PyErr_SetString(PyExc_RuntimeError, "No memory map associated with python or libpython found"); - return NULL; + return 0; } - void* result = NULL; + uintptr_t result = 0; void* file_memory = NULL; int fd = open(elf_file, O_RDONLY); @@ -320,10 +320,13 @@ get_py_runtime_linux(pid_t pid) Elf_Shdr* shstrtab_section = §ion_header_table[elf_header->e_shstrndx]; char* shstrtab = (char*)(file_memory + shstrtab_section->sh_offset); - Elf_Shdr* py_runtime_section = NULL; + Elf_Shdr* section = NULL; for (int i = 0; i < elf_header->e_shnum; i++) { - if (strcmp(".PyRuntime", shstrtab + section_header_table[i].sh_name) == 0) { - py_runtime_section = §ion_header_table[i]; + char* this_sec_name = shstrtab + section_header_table[i].sh_name; + // Move 1 character to account for the leading "." + this_sec_name += 1; + if (strcmp(secname, this_sec_name) == 0) { + section = §ion_header_table[i]; break; } } @@ -338,10 +341,10 @@ get_py_runtime_linux(pid_t pid) } } - if (py_runtime_section != NULL && first_load_segment != NULL) { + if (section != NULL && first_load_segment != NULL) { uintptr_t elf_load_addr = first_load_segment->p_vaddr - (first_load_segment->p_vaddr % first_load_segment->p_align); - result = start_address + py_runtime_section->sh_addr - elf_load_addr; + result = start_address + (uintptr_t)section->sh_addr - elf_load_addr; } exit: @@ -353,10 +356,28 @@ get_py_runtime_linux(pid_t pid) } return result; } + #endif -ssize_t -read_memory(pid_t pid, void* remote_address, size_t len, void* dst) +static uintptr_t +get_py_runtime(pid_t pid) +{ + uintptr_t address = search_map_for_section(pid, "PyRuntime", "libpython"); + if (address == 0) { + address = search_map_for_section(pid, "PyRuntime", "python"); + } + return address; +} + +static uintptr_t +get_async_debug(pid_t pid) +{ + return search_map_for_section(pid, "AsyncioDebug", "_asyncio.cpython"); +} + + +static ssize_t +read_memory(pid_t pid, uintptr_t remote_address, size_t len, void* dst) { ssize_t total_bytes_read = 0; #if defined(__linux__) && HAVE_PROCESS_VM_READV @@ -409,12 +430,16 @@ read_memory(pid_t pid, void* remote_address, size_t len, void* dst) return total_bytes_read; } -int -read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buffer, Py_ssize_t size) +static int +read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, uintptr_t address, char* buffer, Py_ssize_t size) { Py_ssize_t len; - ssize_t bytes_read = - read_memory(pid, address + debug_offsets->unicode_object.length, sizeof(Py_ssize_t), &len); + ssize_t bytes_read = read_memory( + pid, + address + debug_offsets->unicode_object.length, + sizeof(Py_ssize_t), + &len + ); if (bytes_read == -1) { return -1; } @@ -431,44 +456,604 @@ read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buf return 0; } -void* -get_py_runtime(pid_t pid) + +static inline int +read_ptr(pid_t pid, uintptr_t address, uintptr_t *ptr_addr) { -#if defined(__linux__) - return get_py_runtime_linux(pid); -#elif defined(__APPLE__) && TARGET_OS_OSX - return get_py_runtime_macos(pid); -#else - return NULL; -#endif + int bytes_read = read_memory(pid, address, sizeof(void*), ptr_addr); + if (bytes_read == -1) { + return -1; + } + return 0; +} + +static inline int +read_ssize_t(pid_t pid, uintptr_t address, Py_ssize_t *size) +{ + int bytes_read = read_memory(pid, address, sizeof(Py_ssize_t), size); + if (bytes_read == -1) { + return -1; + } + return 0; } static int -parse_code_object( - int pid, - PyObject* result, - struct _Py_DebugOffsets* offsets, - void* address, - void** previous_frame) +read_py_ptr(pid_t pid, uintptr_t address, uintptr_t *ptr_addr) +{ + if (read_ptr(pid, address, ptr_addr)) { + return -1; + } + *ptr_addr &= ~Py_TAG_BITS; + return 0; +} + +static int +read_char(pid_t pid, uintptr_t address, char *result) { - void* address_of_function_name; - read_memory( + int bytes_read = read_memory(pid, address, sizeof(char), result); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static int +read_int(pid_t pid, uintptr_t address, int *result) +{ + int bytes_read = read_memory(pid, address, sizeof(int), result); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static int +read_pyobj(pid_t pid, uintptr_t address, PyObject *ptr_addr) +{ + int bytes_read = read_memory(pid, address, sizeof(PyObject), ptr_addr); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static PyObject * +read_py_str( + pid_t pid, + _Py_DebugOffsets* debug_offsets, + uintptr_t address, + ssize_t max_len +) { + assert(max_len > 0); + + PyObject *result = NULL; + + char *buf = (char *)PyMem_RawMalloc(max_len); + if (buf == NULL) { + PyErr_NoMemory(); + return NULL; + } + if (read_string(pid, debug_offsets, address, buf, max_len)) { + goto err; + } + + result = PyUnicode_FromString(buf); + if (result == NULL) { + goto err; + } + + PyMem_RawFree(buf); + assert(result != NULL); + return result; + +err: + PyMem_RawFree(buf); + return NULL; +} + +static long +read_py_long( + pid_t pid, + _Py_DebugOffsets* offsets, + uintptr_t address) { + unsigned int shift = PYLONG_BITS_IN_DIGIT; + + ssize_t size; + uintptr_t lv_tag; + int bytes_read = read_memory(pid, address + offsets->long_object.lv_tag, sizeof(uintptr_t), &lv_tag); + if (bytes_read == -1) { + return -1; + } + int negative = (lv_tag & 3) == 2; + size = lv_tag >> 3; + + if (size == 0) { + return 0; + } + + char *digits = (char *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + return -1; + } + bytes_read = read_memory(pid, address + offsets->long_object.ob_digit, sizeof(digit) * size, digits); + if (bytes_read < 0) { + return -1; + } + + long value = 0; + + for (ssize_t i = 0; i < size; ++i) { + long long factor; + if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), &factor)) { + return -1; + } + if (__builtin_add_overflow(value, factor, &value)) { + return -1; + } + } + if (negative) { + value = -1 * value; + } + + return value; +} + +static PyObject * +parse_task_name( + int pid, + _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address +) { + uintptr_t task_name_addr; + int err = read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_name, + &task_name_addr); + if (err) { + return NULL; + } + + // The task name can be a long or a string so we need to check the type + + PyObject task_name_obj; + err = read_pyobj( + pid, + task_name_addr, + &task_name_obj); + if (err) { + return NULL; + } + + int flags; + err = read_int( + pid, + (uintptr_t)task_name_obj.ob_type + offsets->type_object.tp_flags, + &flags); + if (err) { + return NULL; + } + + if ((flags & Py_TPFLAGS_LONG_SUBCLASS)) { + long res = read_py_long(pid, offsets, task_name_addr); + if (res == -1) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get task name"); + return NULL; + } + return PyUnicode_FromFormat("Task-%d", res); + } + + if(!(flags & Py_TPFLAGS_UNICODE_SUBCLASS)) { + PyErr_SetString(PyExc_RuntimeError, "Invalid task name object"); + return NULL; + } + + return read_py_str( + pid, + offsets, + task_name_addr, + 255 + ); +} + +static int +parse_coro_chain( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t coro_address, + PyObject *render_to +) { + assert((void*)coro_address != NULL); + + uintptr_t gen_type_addr; + int err = read_ptr( + pid, + coro_address + sizeof(void*), + &gen_type_addr); + if (err) { + return -1; + } + + uintptr_t gen_name_addr; + err = read_py_ptr( + pid, + coro_address + offsets->gen_object.gi_name, + &gen_name_addr); + if (err) { + return -1; + } + + PyObject *name = read_py_str( + pid, + offsets, + gen_name_addr, + 255 + ); + if (name == NULL) { + return -1; + } + + if (PyList_Append(render_to, name)) { + return -1; + } + + int gi_frame_state; + err = read_int( + pid, + coro_address + offsets->gen_object.gi_frame_state, + &gi_frame_state); + + if (gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) { + char owner; + err = read_char( pid, - (void*)(address + offsets->code_object.name), - sizeof(void*), - &address_of_function_name); + coro_address + offsets->gen_object.gi_iframe + + offsets->interpreter_frame.owner, + &owner + ); + if (err) { + return -1; + } + if (owner != FRAME_OWNED_BY_GENERATOR) { + PyErr_SetString( + PyExc_RuntimeError, + "generator doesn't own its frame \\_o_/"); + return -1; + } - if (address_of_function_name == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No function name found"); + uintptr_t stackpointer_addr; + err = read_py_ptr( + pid, + coro_address + offsets->gen_object.gi_iframe + + offsets->interpreter_frame.stackpointer, + &stackpointer_addr); + if (err) { + return -1; + } + + if ((void*)stackpointer_addr != NULL) { + uintptr_t gi_await_addr; + err = read_py_ptr( + pid, + stackpointer_addr - sizeof(void*), + &gi_await_addr); + if (err) { + return -1; + } + + if ((void*)gi_await_addr != NULL) { + uintptr_t gi_await_addr_type_addr; + int err = read_ptr( + pid, + gi_await_addr + sizeof(void*), + &gi_await_addr_type_addr); + if (err) { + return -1; + } + + if (gen_type_addr == gi_await_addr_type_addr) { + /* This needs an explanation. We always start with parsing + native coroutine / generator frames. Ultimately they + are awaiting on something. That something can be + a native coroutine frame or... an iterator. + If it's the latter -- we can't continue building + our chain. So the condition to bail out of this is + to do that when the type of the current coroutine + doesn't match the type of whatever it points to + in its cr_await. + */ + err = parse_coro_chain( + pid, + offsets, + async_offsets, + gi_await_addr, + render_to + ); + if (err) { + return -1; + } + } + } + } + + } + + return 0; +} + + +static int +parse_task_awaited_by( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address, + PyObject *awaited_by +); + + +static int +parse_task( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address, + PyObject *render_to +) { + char is_task; + int err = read_char( + pid, + task_address + async_offsets->asyncio_task_object.task_is_task, + &is_task); + if (err) { + return -1; + } + + uintptr_t refcnt; + read_ptr(pid, task_address + sizeof(Py_ssize_t), &refcnt); + + PyObject* result = PyList_New(0); + if (result == NULL) { return -1; } - char function_name[256]; - if (read_string(pid, offsets, address_of_function_name, function_name, sizeof(function_name)) != 0) { + PyObject *call_stack = PyList_New(0); + if (call_stack == NULL) { return -1; } + if (PyList_Append(result, call_stack)) { + Py_DECREF(call_stack); + goto err; + } + /* we can operate on a borrowed one to simplify cleanup */ + Py_DECREF(call_stack); + + if (is_task) { + PyObject *tn = parse_task_name(pid, offsets, async_offsets, task_address); + if (tn == NULL) { + goto err; + } + if (PyList_Append(result, tn)) { + Py_DECREF(tn); + goto err; + } + Py_DECREF(tn); + + uintptr_t coro_addr; + err = read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_coro, + &coro_addr); + if (err) { + goto err; + } + + if ((void*)coro_addr != NULL) { + err = parse_coro_chain( + pid, + offsets, + async_offsets, + coro_addr, + call_stack + ); + if (err) { + goto err; + } + + if (PyList_Reverse(call_stack)) { + goto err; + } + } + } - PyObject* py_function_name = PyUnicode_FromString(function_name); + if (PyList_Append(render_to, result)) { + goto err; + } + + PyObject *awaited_by = PyList_New(0); + if (awaited_by == NULL) { + goto err; + } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + goto err; + } + /* we can operate on a borrowed one to simplify cleanup */ + Py_DECREF(awaited_by); + + if (parse_task_awaited_by(pid, offsets, async_offsets, task_address, awaited_by)) { + goto err; + } + + return 0; + +err: + Py_DECREF(result); + return -1; +} + +static int +parse_tasks_in_set( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t set_addr, + PyObject *awaited_by +) { + uintptr_t set_obj; + if (read_py_ptr( + pid, + set_addr, + &set_obj) + ) { + return -1; + } + + Py_ssize_t set_len; + if (read_ssize_t( + pid, + set_obj + offsets->set_object.used, + &set_len) + ) { + return -1; + } + + Py_ssize_t cnt = 0; + uintptr_t table_ptr; + if (read_ptr( + pid, + set_obj + offsets->set_object.table, + &table_ptr) + ) { + return -1; + } + + while (cnt < set_len) { + uintptr_t key_addr; + if (read_py_ptr(pid, table_ptr, &key_addr)) { + return -1; + } + + if ((void*)key_addr != NULL) { + Py_ssize_t ref_cnt; + if (read_ssize_t(pid, table_ptr, &ref_cnt)) { + return -1; + } + + if (ref_cnt) { + // if 'ref_cnt=0' it's a set dummy marker + + if (parse_task( + pid, + offsets, + async_offsets, + key_addr, + awaited_by) + ) { + return -1; + } + + cnt++; + } + } + + table_ptr += sizeof(void*) * 2; + } + return 0; +} + + +static int +parse_task_awaited_by( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address, + PyObject *awaited_by +) { + uintptr_t task_ab_addr; + int err = read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_awaited_by, + &task_ab_addr); + if (err) { + return -1; + } + + if ((void*)task_ab_addr == NULL) { + return 0; + } + + char awaited_by_is_a_set; + err = read_char( + pid, + task_address + async_offsets->asyncio_task_object.task_awaited_by_is_set, + &awaited_by_is_a_set); + if (err) { + return -1; + } + + if (awaited_by_is_a_set) { + if (parse_tasks_in_set( + pid, + offsets, + async_offsets, + task_address + async_offsets->asyncio_task_object.task_awaited_by, + awaited_by) + ) { + return -1; + } + } else { + uintptr_t sub_task; + if (read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_awaited_by, + &sub_task) + ) { + return -1; + } + + if (parse_task( + pid, + offsets, + async_offsets, + sub_task, + awaited_by) + ) { + return -1; + } + } + + return 0; +} + +static int +parse_code_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + uintptr_t address, + uintptr_t* previous_frame +) { + uintptr_t address_of_function_name; + int bytes_read = read_memory( + pid, + address + offsets->code_object.name, + sizeof(void*), + &address_of_function_name + ); + if (bytes_read == -1) { + return -1; + } + + if ((void*)address_of_function_name == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No function name found"); + return -1; + } + + PyObject* py_function_name = read_py_str( + pid, offsets, address_of_function_name, 256); if (py_function_name == NULL) { return -1; } @@ -484,24 +1069,77 @@ parse_code_object( static int parse_frame_object( - int pid, - PyObject* result, - struct _Py_DebugOffsets* offsets, - void* address, - void** previous_frame) -{ + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + uintptr_t address, + uintptr_t* previous_frame +) { + int err; + ssize_t bytes_read = read_memory( - pid, - (void*)(address + offsets->interpreter_frame.previous), - sizeof(void*), - previous_frame); + pid, + address + offsets->interpreter_frame.previous, + sizeof(void*), + previous_frame + ); + if (bytes_read == -1) { + return -1; + } + + char owner; + if (read_char(pid, address + offsets->interpreter_frame.owner, &owner)) { + return -1; + } + + if (owner == FRAME_OWNED_BY_CSTACK) { + return 0; + } + + uintptr_t address_of_code_object; + err = read_py_ptr( + pid, + address + offsets->interpreter_frame.executable, + &address_of_code_object + ); + if (err) { + return -1; + } + + if ((void*)address_of_code_object == NULL) { + return 0; + } + + return parse_code_object( + pid, result, offsets, address_of_code_object, previous_frame); +} + +static int +parse_async_frame_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + uintptr_t address, + uintptr_t* task, + uintptr_t* previous_frame +) { + int err; + + *task = (uintptr_t)NULL; + + ssize_t bytes_read = read_memory( + pid, + address + offsets->interpreter_frame.previous, + sizeof(void*), + previous_frame + ); if (bytes_read == -1) { return -1; } char owner; - bytes_read = - read_memory(pid, (void*)(address + offsets->interpreter_frame.owner), sizeof(char), &owner); + bytes_read = read_memory( + pid, address + offsets->interpreter_frame.owner, sizeof(char), &owner); if (bytes_read < 0) { return -1; } @@ -510,21 +1148,125 @@ parse_frame_object( return 0; } + if (owner == FRAME_OWNED_BY_GENERATOR) { + err = read_py_ptr( + pid, + address - offsets->gen_object.gi_iframe + offsets->gen_object.gi_task, + task); + if (err) { + return -1; + } + } + uintptr_t address_of_code_object; + err = read_py_ptr( + pid, + address + offsets->interpreter_frame.executable, + &address_of_code_object + ); + if (err) { + return -1; + } + + if ((void*)address_of_code_object == NULL) { + return 0; + } + + return parse_code_object( + pid, result, offsets, address_of_code_object, previous_frame); +} + +static int +read_offsets( + int pid, + uintptr_t *runtime_start_address, + _Py_DebugOffsets* debug_offsets +) { + *runtime_start_address = get_py_runtime(pid); + if (!*runtime_start_address) { + if (!PyErr_Occurred()) { + PyErr_SetString( + PyExc_RuntimeError, "Failed to get .PyRuntime address"); + } + return -1; + } + size_t size = sizeof(struct _Py_DebugOffsets); + ssize_t bytes_read = read_memory( + pid, *runtime_start_address, size, debug_offsets); + if (bytes_read == -1) { + return -1; + } + return 0; +} + +static int +read_async_debug( + int pid, + struct _Py_AsyncioModuleDebugOffsets* async_debug +) { + uintptr_t async_debug_addr = get_async_debug(pid); + if (!async_debug) { + return -1; + } + size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); + ssize_t bytes_read = read_memory( + pid, async_debug_addr, size, async_debug); + if (bytes_read == -1) { + return -1; + } + return 0; +} + +static int +find_running_frame( + int pid, + uintptr_t runtime_start_address, + _Py_DebugOffsets* local_debug_offsets, + uintptr_t *frame +) { + off_t interpreter_state_list_head = + local_debug_offsets->runtime_state.interpreters_head; + + uintptr_t address_of_interpreter_state; + int bytes_read = read_memory( + pid, + runtime_start_address + interpreter_state_list_head, + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read == -1) { + return -1; + } + + if (address_of_interpreter_state == 0) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return -1; + } + + uintptr_t address_of_thread; bytes_read = read_memory( pid, - (void*)(address + offsets->interpreter_frame.executable), + address_of_interpreter_state + + local_debug_offsets->interpreter_state.threads_head, sizeof(void*), - &address_of_code_object); + &address_of_thread); if (bytes_read == -1) { return -1; } - if (address_of_code_object == 0) { + // No Python frames are available for us (can happen at tear-down). + if ((void*)address_of_thread != NULL) { + int err = read_ptr( + pid, + address_of_thread + local_debug_offsets->thread_state.current_frame, + frame); + if (err) { + return -1; + } return 0; } - address_of_code_object &= ~Py_TAG_BITS; - return parse_code_object(pid, result, offsets, (void *)address_of_code_object, previous_frame); + + *frame = (uintptr_t)NULL; + return 0; } static PyObject* @@ -540,88 +1282,159 @@ get_stack_trace(PyObject* self, PyObject* args) return NULL; } - void* runtime_start_address = get_py_runtime(pid); - if (runtime_start_address == NULL) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } + uintptr_t runtime_start_address = get_py_runtime(pid); + struct _Py_DebugOffsets local_debug_offsets; + + if (read_offsets(pid, &runtime_start_address, &local_debug_offsets)) { return NULL; } - size_t size = sizeof(struct _Py_DebugOffsets); - struct _Py_DebugOffsets local_debug_offsets; - ssize_t bytes_read = read_memory(pid, runtime_start_address, size, &local_debug_offsets); - if (bytes_read == -1) { + uintptr_t address_of_current_frame; + if (find_running_frame( + pid, runtime_start_address, &local_debug_offsets, + &address_of_current_frame) + ) { return NULL; } - off_t interpreter_state_list_head = local_debug_offsets.runtime_state.interpreters_head; - void* address_of_interpreter_state; - bytes_read = read_memory( - pid, - (void*)(runtime_start_address + interpreter_state_list_head), - sizeof(void*), - &address_of_interpreter_state); - if (bytes_read == -1) { + PyObject* result = PyList_New(0); + if (result == NULL) { return NULL; } - if (address_of_interpreter_state == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + while ((void*)address_of_current_frame != NULL) { + if (parse_frame_object( + pid, + result, + &local_debug_offsets, + address_of_current_frame, + &address_of_current_frame) + < 0) + { + Py_DECREF(result); + return NULL; + } + } + + return result; +} + +static PyObject* +get_async_stack_trace(PyObject* self, PyObject* args) +{ +#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); + return NULL; +#endif + int pid; + + if (!PyArg_ParseTuple(args, "i", &pid)) { return NULL; } - void* address_of_thread; - bytes_read = read_memory( - pid, - (void*)(address_of_interpreter_state + local_debug_offsets.interpreter_state.threads_head), - sizeof(void*), - &address_of_thread); - if (bytes_read == -1) { + uintptr_t runtime_start_address = get_py_runtime(pid); + struct _Py_DebugOffsets local_debug_offsets; + + if (read_offsets(pid, &runtime_start_address, &local_debug_offsets)) { return NULL; } - PyObject* result = PyList_New(0); + struct _Py_AsyncioModuleDebugOffsets local_async_debug; + if (read_async_debug(pid, &local_async_debug)) { + return NULL; + } + + uintptr_t address_of_current_frame; + if (find_running_frame( + pid, runtime_start_address, &local_debug_offsets, + &address_of_current_frame) + ) { + return NULL; + } + + PyObject* result = PyList_New(1); if (result == NULL) { return NULL; } + PyObject* calls = PyList_New(0); + if (calls == NULL) { + return NULL; + } + if (PyList_SetItem(result, 0, calls)) { /* steals ref to 'calls' */ + Py_DECREF(result); + Py_DECREF(calls); + return NULL; + } - // No Python frames are available for us (can happen at tear-down). - if (address_of_thread != NULL) { - void* address_of_current_frame; - (void)read_memory( - pid, - (void*)(address_of_thread + local_debug_offsets.thread_state.current_frame), - sizeof(void*), - &address_of_current_frame); - while (address_of_current_frame != NULL) { - if (parse_frame_object( - pid, - result, - &local_debug_offsets, - address_of_current_frame, - &address_of_current_frame) - < 0) - { - Py_DECREF(result); - return NULL; - } + uintptr_t root_task_addr = (uintptr_t)NULL; + while ((void*)address_of_current_frame != NULL) { + int err = parse_async_frame_object( + pid, + calls, + &local_debug_offsets, + address_of_current_frame, + &root_task_addr, + &address_of_current_frame + ); + if (err) { + goto result_err; + } + + if ((void*)root_task_addr != NULL) { + break; } } + if ((void*)root_task_addr != NULL) { + PyObject *tn = parse_task_name( + pid, &local_debug_offsets, &local_async_debug, root_task_addr); + if (tn == NULL) { + goto result_err; + } + if (PyList_Append(result, tn)) { + Py_DECREF(tn); + goto result_err; + } + Py_DECREF(tn); + + PyObject* awaited_by = PyList_New(0); + if (awaited_by == NULL) { + goto result_err; + } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + goto result_err; + } + + if (parse_task_awaited_by( + pid, &local_debug_offsets, &local_async_debug, root_task_addr, awaited_by) + ) { + goto result_err; + } + } + + return result; + +result_err: + Py_DECREF(result); + return NULL; } + static PyMethodDef methods[] = { - {"get_stack_trace", get_stack_trace, METH_VARARGS, "Get the Python stack from a given PID"}, - {NULL, NULL, 0, NULL}, + {"get_stack_trace", get_stack_trace, METH_VARARGS, + "Get the Python stack from a given PID"}, + {"get_async_stack_trace", get_async_stack_trace, METH_VARARGS, + "Get the asyncio stack from a given PID"}, + {NULL, NULL, 0, NULL}, }; static struct PyModuleDef module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "_testexternalinspection", - .m_size = -1, - .m_methods = methods, + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_testexternalinspection", + .m_size = -1, + .m_methods = methods, }; PyMODINIT_FUNC From 1ddc9cfad5006f1396d387ecfa7f40b79a8b3fe4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 30 Sep 2024 12:26:38 +0100 Subject: [PATCH 15/84] Refactor the section-generated code into an evil macro Signed-off-by: Pablo Galindo --- Include/internal/pycore_runtime.h | 35 +++++++++++++++++++++++++++++++ Modules/_asynciomodule.c | 20 +----------------- Python/pylifecycle.c | 22 +------------------ 3 files changed, 37 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index c7437a2ad1515c..b51f427dfa1579 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -25,6 +25,10 @@ extern "C" { #include "pycore_typeobject.h" // struct _types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state +#if defined(__APPLE__) +# include +#endif + struct _getargs_runtime_state { struct _PyArg_Parser *static_parsers; }; @@ -59,6 +63,37 @@ typedef struct _Py_AuditHookEntry { void *userData; } _Py_AuditHookEntry; +// Macros to burn global values in custom sections so out-of-process +// profilers can locate them easily + +#define GENERATE_DEBUG_SECTION(name, declaration) \ + _GENERATE_DEBUG_SECTION_WINDOWS(name) \ + _GENERATE_DEBUG_SECTION_APPLE(name) \ + declaration \ + _GENERATE_DEBUG_SECTION_LINUX(name) + +#if defined(MS_WINDOWS) +#define _GENERATE_DEBUG_SECTION_WINDOWS(name) \ + _Pragma(Py_STRINGIFY(section(Py_STRINGIFY(name), read, write))) \ + __declspec(allocate(Py_STRINGIFY(name))) +#else +#define _GENERATE_DEBUG_SECTION_WINDOWS(name) +#endif + +#if defined(__APPLE__) +#define _GENERATE_DEBUG_SECTION_APPLE(name) \ + __attribute__((section(SEG_DATA "," Py_STRINGIFY(name)))) +#else +#define _GENERATE_DEBUG_SECTION_APPLE(name) +#endif + +#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) +#define _GENERATE_DEBUG_SECTION_LINUX(name) \ + __attribute__((section("." Py_STRINGIFY(name)))) +#else +#define _GENERATE_DEBUG_SECTION_LINUX(name) +#endif + typedef struct _Py_DebugOffsets { char cookie[8]; uint64_t version; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 376664e922fe38..032a7a50cf9c79 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -16,10 +16,6 @@ #include // offsetof() -#if defined(__APPLE__) -# include -#endif - /*[clinic input] module _asyncio [clinic start generated code]*/ @@ -116,21 +112,7 @@ typedef struct _Py_AsyncioModuleDebugOffsets { } asyncio_task_object; } Py_AsyncioModuleDebugOffsets; -#if defined(MS_WINDOWS) - -#pragma section("AsyncioDebug", read, write) -__declspec(allocate("AsyncioDebug")) - -#elif defined(__APPLE__) - -__attribute__((section(SEG_DATA ",AsyncioDebug"))) - -#endif - -Py_AsyncioModuleDebugOffsets AsyncioDebug -#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) - __attribute__((section(".AsyncioDebug"))) -#endif +GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets AsyncioDebug) = {.asyncio_task_object = { .size = sizeof(TaskObj), .task_name = offsetof(TaskObj, task_name), diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ebeee4f41d795d..1c40c9fb2767c6 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -43,10 +43,6 @@ # include // isatty() #endif -#if defined(__APPLE__) -# include -#endif - #ifdef HAVE_SIGNAL_H # include // SIG_IGN #endif @@ -87,23 +83,7 @@ static void call_ll_exitfuncs(_PyRuntimeState *runtime); _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS -#if defined(MS_WINDOWS) - -#pragma section("PyRuntime", read, write) -__declspec(allocate("PyRuntime")) - -#elif defined(__APPLE__) - -__attribute__(( - section(SEG_DATA ",PyRuntime") -)) - -#endif - -_PyRuntimeState _PyRuntime -#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) -__attribute__ ((section (".PyRuntime"))) -#endif +GENERATE_DEBUG_SECTION(PyRuntime, _PyRuntimeState _PyRuntime) = _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); _Py_COMP_DIAG_POP From bc9beb85aaa7c0631227a5dede062f6f5fdbfe12 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 11:15:55 -0700 Subject: [PATCH 16/84] Rename "capture_call_stack()" et al to "capture_call_graph()" --- Doc/library/asyncio-stack.rst | 30 ++++++------- Lib/asyncio/stack.py | 68 ++++++++++++++--------------- Lib/test/test_asyncio/test_stack.py | 8 ++-- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index c04d80488d3547..675422354e5e92 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -18,9 +18,9 @@ a suspended *future*. .. versionadded:: 3.14 -.. function:: print_call_stack(*, future=None, file=None) +.. function:: print_call_graph(*, future=None, file=None) - Print the async call stack for the current task or the provided + Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. The function recieves an optional keyword-only *future* argument. @@ -38,7 +38,7 @@ a suspended *future*. import asyncio async def test(): - asyncio.print_call_stack() + asyncio.print_call_graph() async def main(): async with asyncio.TaskGroup() as g: @@ -50,7 +50,7 @@ a suspended *future*. * Task(name='Task-2', id=0x105038fe0) + Call stack: - | * print_call_stack() + | * print_call_graph() | asyncio/stack.py:231 | * async test() | test.py:4 @@ -72,37 +72,37 @@ a suspended *future*. ... buf = io.StringIO() - asyncio.print_call_stack(file=buf) + asyncio.print_call_graph(file=buf) output = buf.getvalue() -.. function:: capture_call_stack(*, future=None) +.. function:: capture_call_graph(*, future=None) - Capture the async call stack for the current task or the provided + Capture the async call graph for the current task or the provided :class:`Task` or :class:`Future`. The function recieves an optional keyword-only *future* argument. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. - Returns a ``FutureCallStack`` named tuple: + Returns a ``FutureCallGraph`` named tuple: - * ``FutureCallStack(future, call_stack, awaited_by)`` + * ``FutureCallGraph(future, call_graph, awaited_by)`` Where 'future' is a reference to a *Future* or a *Task* (or their subclasses.) - ``call_stack`` is a list of ``FrameCallStackEntry`` and - ``CoroutineCallStackEntry`` objects (more on them below.) + ``call_graph`` is a list of ``FrameCallGraphEntry`` and + ``CoroutineCallGraphEntry`` objects (more on them below.) - ``awaited_by`` is a list of ``FutureCallStack`` tuples. + ``awaited_by`` is a list of ``FutureCallGraph`` tuples. - * ``FrameCallStackEntry(frame)`` + * ``FrameCallGraphEntry(frame)`` Where ``frame`` is a frame object of a regular Python function in the call stack. - * ``CoroutineCallStackEntry(coroutine)`` + * ``CoroutineCallGraphEntry(coroutine)`` Where ``coroutine`` is a coroutine object of an awaiting coroutine or asyncronous generator. @@ -111,7 +111,7 @@ a suspended *future*. Low level utility functions =========================== -To introspect an async call stack asyncio requires cooperation from +To introspect an async call graph asyncio requires cooperation from control flow structures, such as :func:`shield` or :class:`TaskGroup`. Any time an intermediate ``Future`` object with low-level APIs like :meth:`Future.add_done_callback() ` is diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 652e2eb399463c..150638365360ff 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -9,11 +9,11 @@ from . import tasks __all__ = ( - 'capture_call_stack', - 'print_call_stack', - 'FrameCallStackEntry', - 'CoroutineCallStackEntry', - 'FutureCallStack', + 'capture_call_graph', + 'print_call_graph', + 'FrameCallGraphEntry', + 'CoroutineCallGraphEntry', + 'FutureCallGraph', ) # Sadly, we can't re-use the traceback's module datastructures as those @@ -24,21 +24,21 @@ # top level asyncio namespace, and want to avoid future name clashes. -class FrameCallStackEntry(typing.NamedTuple): +class FrameCallGraphEntry(typing.NamedTuple): frame: types.FrameType -class CoroutineCallStackEntry(typing.NamedTuple): +class CoroutineCallGraphEntry(typing.NamedTuple): coroutine: types.CoroutineType -class FutureCallStack(typing.NamedTuple): +class FutureCallGraph(typing.NamedTuple): future: futures.Future - call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] - awaited_by: list[FutureCallStack] + call_graph: list[FrameCallGraphEntry | CoroutineCallGraphEntry] + awaited_by: list[FutureCallGraph] -def _build_stack_for_future(future: any) -> FutureCallStack: +def _build_stack_for_future(future: any) -> FutureCallGraph: if not isinstance(future, futures.Future): raise TypeError( f"{future!r} object does not appear to be compatible " @@ -52,17 +52,17 @@ def _build_stack_for_future(future: any) -> FutureCallStack: else: coro = get_coro() - st: list[CoroutineCallStackEntry] = [] - awaited_by: list[FutureCallStack] = [] + st: list[CoroutineCallGraphEntry] = [] + awaited_by: list[FutureCallGraph] = [] while coro is not None: if hasattr(coro, 'cr_await'): # A native coroutine or duck-type compatible iterator - st.append(CoroutineCallStackEntry(coro)) + st.append(CoroutineCallGraphEntry(coro)) coro = coro.cr_await elif hasattr(coro, 'ag_await'): # A native async generator or duck-type compatible iterator - st.append(CoroutineCallStackEntry(coro)) + st.append(CoroutineCallGraphEntry(coro)) coro = coro.ag_await else: break @@ -72,30 +72,30 @@ def _build_stack_for_future(future: any) -> FutureCallStack: awaited_by.append(_build_stack_for_future(parent)) st.reverse() - return FutureCallStack(future, st, awaited_by) + return FutureCallGraph(future, st, awaited_by) -def capture_call_stack(*, future: any = None) -> FutureCallStack | None: +def capture_call_graph(*, future: any = None) -> FutureCallGraph | None: """Capture async call stack for the current task or the provided Future. The stack is represented with three data structures: - * FutureCallStack(future, call_stack, awaited_by) + * FutureCallGraph(future, call_graph, awaited_by) Where 'future' is a reference to an asyncio.Future or asyncio.Task (or their subclasses.) - 'call_stack' is a list of FrameCallStackEntry and CoroutineCallStackEntry + 'call_graph' is a list of FrameCallGraphEntry and CoroutineCallGraphEntry objects (more on them below.) - 'awaited_by' is a list of FutureCallStack objects. + 'awaited_by' is a list of FutureCallGraph objects. - * FrameCallStackEntry(frame) + * FrameCallGraphEntry(frame) Where 'frame' is a frame object of a regular Python function in the call stack. - * CoroutineCallStackEntry(coroutine) + * CoroutineCallGraphEntry(coroutine) Where 'coroutine' is a coroutine object of an awaiting coroutine or asyncronous generator. @@ -117,7 +117,7 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: else: if loop is None: raise RuntimeError( - 'capture_call_stack() is called outside of a running ' + 'capture_call_graph() is called outside of a running ' 'event loop and no *future* to introspect was provided') future = tasks.current_task(loop=loop) @@ -133,7 +133,7 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: f"with asyncio.Future" ) - call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] = [] + call_graph: list[FrameCallGraphEntry | CoroutineCallGraphEntry] = [] f = sys._getframe(1) try: @@ -141,13 +141,13 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: is_async = f.f_generator is not None if is_async: - call_stack.append(CoroutineCallStackEntry(f.f_generator)) + call_graph.append(CoroutineCallGraphEntry(f.f_generator)) if f.f_back is not None and f.f_back.f_generator is None: # We've reached the bottom of the coroutine stack, which # must be the Task that runs it. break else: - call_stack.append(FrameCallStackEntry(f)) + call_graph.append(FrameCallGraphEntry(f)) f = f.f_back finally: @@ -158,13 +158,13 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: for parent in fut_waiters: awaited_by.append(_build_stack_for_future(parent)) - return FutureCallStack(future, call_stack, awaited_by) + return FutureCallGraph(future, call_graph, awaited_by) -def print_call_stack(*, future: any = None, file=None) -> None: +def print_call_graph(*, future: any = None, file=None) -> None: """Print async call stack for the current task or the provided Future.""" - def render_level(st: FutureCallStack, buf: list[str], level: int): + def render_level(st: FutureCallGraph, buf: list[str], level: int): def add_line(line: str): buf.append(level * ' ' + line) @@ -177,12 +177,12 @@ def add_line(line: str): f'* Future(id=0x{id(st.future):x})' ) - if st.call_stack: + if st.call_graph: add_line( f' + Call stack:' ) - for ste in st.call_stack: - if isinstance(ste, FrameCallStackEntry): + for ste in st.call_graph: + if isinstance(ste, FrameCallGraphEntry): f = ste.frame add_line( f' | * {f.f_code.co_qualname}()' @@ -191,7 +191,7 @@ def add_line(line: str): f' | {f.f_code.co_filename}:{f.f_lineno}' ) else: - assert isinstance(ste, CoroutineCallStackEntry) + assert isinstance(ste, CoroutineCallGraphEntry) c = ste.coroutine try: @@ -222,7 +222,7 @@ def add_line(line: str): for fut in st.awaited_by: render_level(fut, buf, level + 1) - stack = capture_call_stack(future=future) + stack = capture_call_graph(future=future) if stack is None: return diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index 7f434b242f5e44..6d551b32525a5e 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -20,13 +20,13 @@ def walk(s): [ ( f"s {entry.frame.f_code.co_name}" - if isinstance(entry, asyncio.FrameCallStackEntry) else + if isinstance(entry, asyncio.FrameCallGraphEntry) else ( f"a {entry.coroutine.cr_code.co_name}" if hasattr(entry.coroutine, 'cr_code') else f"ag {entry.coroutine.ag_code.co_name}" ) - ) for entry in s.call_stack + ) for entry in s.call_graph ] ) @@ -39,9 +39,9 @@ def walk(s): return ret buf = io.StringIO() - asyncio.print_call_stack(future=fut, file=buf) + asyncio.print_call_graph(future=fut, file=buf) - stack = asyncio.capture_call_stack(future=fut) + stack = asyncio.capture_call_graph(future=fut) return walk(stack), buf.getvalue() From fd141d41eeab7648c45a189aca7c9016ab3dfe88 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 11:17:28 -0700 Subject: [PATCH 17/84] Add NEWS --- .../next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst diff --git a/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst b/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst new file mode 100644 index 00000000000000..071dd9695d4566 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst @@ -0,0 +1 @@ +Add asyncio.capture_call_graph() and asyncio.print_call_graph() functions. From 2d72f2402b83485a535e136365a7429ad22e78b7 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 11:31:56 -0700 Subject: [PATCH 18/84] Per mpage's suggestion: use traceback style formatring for frames --- Doc/library/asyncio-stack.rst | 31 ++++++++++++++++------------- Lib/asyncio/stack.py | 30 +++++++++++++++++----------- Lib/test/test_asyncio/test_stack.py | 2 +- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 675422354e5e92..fc523d0a7f10a0 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -18,7 +18,7 @@ a suspended *future*. .. versionadded:: 3.14 -.. function:: print_call_graph(*, future=None, file=None) +.. function:: print_call_graph(*, future=None, file=None, depth=1) Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. @@ -27,6 +27,10 @@ a suspended *future*. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. + If the function is called on *the current task*, the optional + keyword-only ``depth`` argument can be used to skip the specified + number of frames from top of the stack. + If *file* is not specified the function will print to :data:`sys.stdout`. **Example:** @@ -48,19 +52,14 @@ a suspended *future*. will print:: - * Task(name='Task-2', id=0x105038fe0) - + Call stack: - | * print_call_graph() - | asyncio/stack.py:231 - | * async test() - | test.py:4 - + Awaited by: - * Task(name='Task-1', id=0x1050a6060) - + Call stack: - | * async TaskGroup.__aexit__() - | asyncio/taskgroups.py:107 - | * async main() - | test.py:7 + * Task(name='Task-2', id=0x1039f0fe0) + + Call stack: + | File 't2.py', line 4, in async test() + + Awaited by: + * Task(name='Task-1', id=0x103a5e060) + + Call stack: + | File 'taskgroups.py', line 107, in async TaskGroup.__aexit__() + | File 't2.py', line 7, in async main() For rendering the call stack to a string the following pattern should be used: @@ -85,6 +84,10 @@ a suspended *future*. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. + If the function is called on *the current task*, the optional + keyword-only ``depth`` argument can be used to skip the specified + number of frames from top of the stack. + Returns a ``FutureCallGraph`` named tuple: * ``FutureCallGraph(future, call_graph, awaited_by)`` diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 150638365360ff..c8c7e39fc5aa0d 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -75,7 +75,11 @@ def _build_stack_for_future(future: any) -> FutureCallGraph: return FutureCallGraph(future, st, awaited_by) -def capture_call_graph(*, future: any = None) -> FutureCallGraph | None: +def capture_call_graph( + *, + future: any = None, + depth: int = 1, +) -> FutureCallGraph | None: """Capture async call stack for the current task or the provided Future. The stack is represented with three data structures: @@ -103,6 +107,10 @@ def capture_call_graph(*, future: any = None) -> FutureCallGraph | None: Receives an optional keyword-only "future" argument. If not passed, the current task will be used. If there's no current task, the function returns None. + + If "capture_call_graph()" is introspecting *the current task*, the + optional keyword-only "depth" argument can be used to skip the specified + number of frames from top of the stack. """ loop = events._get_running_loop() @@ -135,7 +143,7 @@ def capture_call_graph(*, future: any = None) -> FutureCallGraph | None: call_graph: list[FrameCallGraphEntry | CoroutineCallGraphEntry] = [] - f = sys._getframe(1) + f = sys._getframe(depth) try: while f is not None: is_async = f.f_generator is not None @@ -161,7 +169,7 @@ def capture_call_graph(*, future: any = None) -> FutureCallGraph | None: return FutureCallGraph(future, call_graph, awaited_by) -def print_call_graph(*, future: any = None, file=None) -> None: +def print_call_graph(*, future: any = None, file=None, depth: int = 1) -> None: """Print async call stack for the current task or the provided Future.""" def render_level(st: FutureCallGraph, buf: list[str], level: int): @@ -185,10 +193,9 @@ def add_line(line: str): if isinstance(ste, FrameCallGraphEntry): f = ste.frame add_line( - f' | * {f.f_code.co_qualname}()' - ) - add_line( - f' | {f.f_code.co_filename}:{f.f_lineno}' + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {f.f_code.co_qualname}()' ) else: assert isinstance(ste, CoroutineCallGraphEntry) @@ -209,10 +216,9 @@ def add_line(line: str): tag = 'generator' add_line( - f' | * {tag} {code.co_qualname}()' - ) - add_line( - f' | {f.f_code.co_filename}:{f.f_lineno}' + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag} {code.co_qualname}()' ) if st.awaited_by: @@ -222,7 +228,7 @@ def add_line(line: str): for fut in st.awaited_by: render_level(fut, buf, level + 1) - stack = capture_call_graph(future=future) + stack = capture_call_graph(future=future, depth=depth + 1) if stack is None: return diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index 6d551b32525a5e..f9244aab9b4052 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -107,7 +107,7 @@ async def main(): ]) self.assertIn( - '* async TestCallStack.test_stack_tgroup()', + ' async TestCallStack.test_stack_tgroup()', stack_for_c5[1]) From 391defa235f3d29cf4fef76e295ef8dd0fef32a1 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 11:35:50 -0700 Subject: [PATCH 19/84] mpage feedback: fix typo --- Doc/library/asyncio-stack.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index fc523d0a7f10a0..02eea8ffc64b69 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -23,7 +23,7 @@ a suspended *future*. Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. - The function recieves an optional keyword-only *future* argument. + The function receives an optional keyword-only *future* argument. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. @@ -80,7 +80,7 @@ a suspended *future*. Capture the async call graph for the current task or the provided :class:`Task` or :class:`Future`. - The function recieves an optional keyword-only *future* argument. + The function receives an optional keyword-only *future* argument. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. From d6357fdad1553273c35c46f6a5bc2f70fbcc2a77 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 11:47:06 -0700 Subject: [PATCH 20/84] Per mpage suggestion: get rid of CoroutineCallGraphEntry --- Doc/library/asyncio-stack.rst | 10 ++----- Lib/asyncio/stack.py | 42 +++++++++++------------------ Lib/test/test_asyncio/test_stack.py | 10 +++---- 3 files changed, 22 insertions(+), 40 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 02eea8ffc64b69..285a86fb1fbeb5 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -90,13 +90,12 @@ a suspended *future*. Returns a ``FutureCallGraph`` named tuple: - * ``FutureCallGraph(future, call_graph, awaited_by)`` + * ``FutureCallGraph(future, call_stack, awaited_by)`` Where 'future' is a reference to a *Future* or a *Task* (or their subclasses.) - ``call_graph`` is a list of ``FrameCallGraphEntry`` and - ``CoroutineCallGraphEntry`` objects (more on them below.) + ``call_stack`` is a list of ``FrameCallGraphEntry`` objects. ``awaited_by`` is a list of ``FutureCallGraph`` tuples. @@ -105,11 +104,6 @@ a suspended *future*. Where ``frame`` is a frame object of a regular Python function in the call stack. - * ``CoroutineCallGraphEntry(coroutine)`` - - Where ``coroutine`` is a coroutine object of an awaiting coroutine - or asyncronous generator. - Low level utility functions =========================== diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index c8c7e39fc5aa0d..97a1b209aa6e00 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -12,7 +12,6 @@ 'capture_call_graph', 'print_call_graph', 'FrameCallGraphEntry', - 'CoroutineCallGraphEntry', 'FutureCallGraph', ) @@ -28,13 +27,9 @@ class FrameCallGraphEntry(typing.NamedTuple): frame: types.FrameType -class CoroutineCallGraphEntry(typing.NamedTuple): - coroutine: types.CoroutineType - - class FutureCallGraph(typing.NamedTuple): future: futures.Future - call_graph: list[FrameCallGraphEntry | CoroutineCallGraphEntry] + call_stack: list[FrameCallGraphEntry] awaited_by: list[FutureCallGraph] @@ -52,17 +47,17 @@ def _build_stack_for_future(future: any) -> FutureCallGraph: else: coro = get_coro() - st: list[CoroutineCallGraphEntry] = [] + st: list[FrameCallGraphEntry] = [] awaited_by: list[FutureCallGraph] = [] while coro is not None: if hasattr(coro, 'cr_await'): # A native coroutine or duck-type compatible iterator - st.append(CoroutineCallGraphEntry(coro)) + st.append(FrameCallGraphEntry(coro.cr_frame)) coro = coro.cr_await elif hasattr(coro, 'ag_await'): # A native async generator or duck-type compatible iterator - st.append(CoroutineCallGraphEntry(coro)) + st.append(FrameCallGraphEntry(coro.cr_frame)) coro = coro.ag_await else: break @@ -84,13 +79,12 @@ def capture_call_graph( The stack is represented with three data structures: - * FutureCallGraph(future, call_graph, awaited_by) + * FutureCallGraph(future, call_stack, awaited_by) Where 'future' is a reference to an asyncio.Future or asyncio.Task (or their subclasses.) - 'call_graph' is a list of FrameCallGraphEntry and CoroutineCallGraphEntry - objects (more on them below.) + 'call_stack' is a list of FrameGraphEntry objects. 'awaited_by' is a list of FutureCallGraph objects. @@ -99,11 +93,6 @@ def capture_call_graph( Where 'frame' is a frame object of a regular Python function in the call stack. - * CoroutineCallGraphEntry(coroutine) - - Where 'coroutine' is a coroutine object of an awaiting coroutine - or asyncronous generator. - Receives an optional keyword-only "future" argument. If not passed, the current task will be used. If there's no current task, the function returns None. @@ -141,21 +130,19 @@ def capture_call_graph( f"with asyncio.Future" ) - call_graph: list[FrameCallGraphEntry | CoroutineCallGraphEntry] = [] + call_stack: list[FrameCallGraphEntry] = [] f = sys._getframe(depth) try: while f is not None: is_async = f.f_generator is not None + call_stack.append(FrameCallGraphEntry(f)) if is_async: - call_graph.append(CoroutineCallGraphEntry(f.f_generator)) if f.f_back is not None and f.f_back.f_generator is None: # We've reached the bottom of the coroutine stack, which # must be the Task that runs it. break - else: - call_graph.append(FrameCallGraphEntry(f)) f = f.f_back finally: @@ -166,7 +153,7 @@ def capture_call_graph( for parent in fut_waiters: awaited_by.append(_build_stack_for_future(parent)) - return FutureCallGraph(future, call_graph, awaited_by) + return FutureCallGraph(future, call_stack, awaited_by) def print_call_graph(*, future: any = None, file=None, depth: int = 1) -> None: @@ -185,12 +172,14 @@ def add_line(line: str): f'* Future(id=0x{id(st.future):x})' ) - if st.call_graph: + if st.call_stack: add_line( f' + Call stack:' ) - for ste in st.call_graph: - if isinstance(ste, FrameCallGraphEntry): + for ste in st.call_stack: + f = ste.frame + + if f.f_generator is None: f = ste.frame add_line( f' | File {f.f_code.co_filename!r},' @@ -198,8 +187,7 @@ def add_line(line: str): f' {f.f_code.co_qualname}()' ) else: - assert isinstance(ste, CoroutineCallGraphEntry) - c = ste.coroutine + c = f.f_generator try: f = c.cr_frame diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index f9244aab9b4052..9bb675270e4be6 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -20,13 +20,13 @@ def walk(s): [ ( f"s {entry.frame.f_code.co_name}" - if isinstance(entry, asyncio.FrameCallGraphEntry) else + if entry.frame.f_generator is None else ( - f"a {entry.coroutine.cr_code.co_name}" - if hasattr(entry.coroutine, 'cr_code') else - f"ag {entry.coroutine.ag_code.co_name}" + f"a {entry.frame.f_generator.cr_code.co_name}" + if hasattr(entry.frame.f_generator, 'cr_code') else + f"ag {entry.frame.f_generator.ag_code.co_name}" ) - ) for entry in s.call_graph + ) for entry in s.call_stack ] ) From bb3b6df709f146d88eff702ed27e79f858dda463 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 12:06:30 -0700 Subject: [PATCH 21/84] Strip sloppy white space! --- Include/internal/pycore_runtime.h | 2 +- Modules/_testexternalinspection.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index b51f427dfa1579..f9bbabfce2f232 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -70,7 +70,7 @@ typedef struct _Py_AuditHookEntry { _GENERATE_DEBUG_SECTION_WINDOWS(name) \ _GENERATE_DEBUG_SECTION_APPLE(name) \ declaration \ - _GENERATE_DEBUG_SECTION_LINUX(name) + _GENERATE_DEBUG_SECTION_LINUX(name) #if defined(MS_WINDOWS) #define _GENERATE_DEBUG_SECTION_WINDOWS(name) \ diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 6a14c399933b05..2cd59135406475 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -557,7 +557,7 @@ read_py_long( _Py_DebugOffsets* offsets, uintptr_t address) { unsigned int shift = PYLONG_BITS_IN_DIGIT; - + ssize_t size; uintptr_t lv_tag; int bytes_read = read_memory(pid, address + offsets->long_object.lv_tag, sizeof(uintptr_t), &lv_tag); @@ -581,7 +581,7 @@ read_py_long( } long value = 0; - + for (ssize_t i = 0; i < size; ++i) { long long factor; if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), &factor)) { @@ -642,7 +642,7 @@ parse_task_name( } return PyUnicode_FromFormat("Task-%d", res); } - + if(!(flags & Py_TPFLAGS_UNICODE_SUBCLASS)) { PyErr_SetString(PyExc_RuntimeError, "Invalid task name object"); return NULL; From 54c99ec94246d24f54385ab37d029fde86273666 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 12:48:35 -0700 Subject: [PATCH 22/84] Fix sloppy set iteration! --- Include/internal/pycore_runtime.h | 1 + Include/internal/pycore_runtime_init.h | 1 + Modules/_testexternalinspection.c | 22 ++++++++++++++++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index f9bbabfce2f232..0b12132a3edf10 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -193,6 +193,7 @@ typedef struct _Py_DebugOffsets { uint64_t size; uint64_t used; uint64_t table; + uint64_t mask; } set_object; // PyDict object offset; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 1072196abe4fc9..04c09f5e96067d 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -112,6 +112,7 @@ extern PyTypeObject _PyExc_MemoryError; .size = sizeof(PySetObject), \ .used = offsetof(PySetObject, used), \ .table = offsetof(PySetObject, table), \ + .mask = offsetof(PySetObject, mask), \ }, \ .dict_object = { \ .size = sizeof(PyDictObject), \ diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 2cd59135406475..daf7aad6804452 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -910,16 +910,25 @@ parse_tasks_in_set( return -1; } - Py_ssize_t set_len; + Py_ssize_t num_els; if (read_ssize_t( pid, set_obj + offsets->set_object.used, + &num_els) + ) { + return -1; + } + + Py_ssize_t set_len; + if (read_ssize_t( + pid, + set_obj + offsets->set_object.mask, &set_len) ) { return -1; } + set_len++; // The set contains the `mask+1` element slots. - Py_ssize_t cnt = 0; uintptr_t table_ptr; if (read_ptr( pid, @@ -929,7 +938,9 @@ parse_tasks_in_set( return -1; } - while (cnt < set_len) { + Py_ssize_t i = 0; + Py_ssize_t els = 0; + while (i < set_len) { uintptr_t key_addr; if (read_py_ptr(pid, table_ptr, &key_addr)) { return -1; @@ -954,11 +965,14 @@ parse_tasks_in_set( return -1; } - cnt++; + if (++els == num_els) { + break; + } } } table_ptr += sizeof(void*) * 2; + i++; } return 0; } From c1a4f09b45c27b526e4a21fad35c9403029e69ca Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 12:56:26 -0700 Subject: [PATCH 23/84] Fix a sloppy test! --- Lib/test/test_external_inspection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index d7f55fc6b06841..227ac574e9631d 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -134,14 +134,16 @@ async def main(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) expected_stack_trace = [ ["c5", "c4", "c3", "c2"], "c2_root", [ [["main"], "Task-1", []], - [["c1"], "sub_main_2", [[["main"], "Task-1", []]]], [["c1"], "sub_main_1", [[["main"], "Task-1", []]]], + [["c1"], "sub_main_2", [[["main"], "Task-1", []]]], ], ] self.assertEqual(stack_trace, expected_stack_trace) From 027d5229a91ef4b0f1ea092f13543acf2be6bd0f Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 13:02:29 -0700 Subject: [PATCH 24/84] Run 'make regen-all' --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 4 ---- 4 files changed, 7 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 98a48bce511be4..28a76c36801b4b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -741,7 +741,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_asyncio_future_blocking)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_awaited_by)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_blksize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_bootstrap)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_check_retval_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index eddea908ed9709..ac789b06fb8a61 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -230,7 +230,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) STRUCT_FOR_ID(_asyncio_future_blocking) - STRUCT_FOR_ID(_awaited_by) STRUCT_FOR_ID(_blksize) STRUCT_FOR_ID(_bootstrap) STRUCT_FOR_ID(_check_retval_) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 3f23898566c6d5..7847a5c63ebf3f 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -739,7 +739,6 @@ extern "C" { INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ INIT_ID(_asyncio_future_blocking), \ - INIT_ID(_awaited_by), \ INIT_ID(_blksize), \ INIT_ID(_bootstrap), \ INIT_ID(_check_retval_), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 87f7c090f57e03..a688f70a2ba36f 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -720,10 +720,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_awaited_by); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_blksize); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); From c2d5ec6b20be4b44baf5abcedb93fb2030126aab Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 13:05:03 -0700 Subject: [PATCH 25/84] Add a what's new entry --- Doc/whatsnew/3.14.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 67d8d389b58082..ea6557f4d8fa27 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -385,6 +385,11 @@ asyncio reduces memory usage. (Contributed by Kumar Aditya in :gh:`107803`.) +* :mod:`asyncio` has new utility functions for introspecting and printing + the program's call graph. + (Contributed by Yury Selivanov and Pablo Galindo Salgado in :gh:`91048`.) + + Deprecated ========== From 08d09eb87151d9bd32f28423eabd439f16ce8e37 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 13:11:56 -0700 Subject: [PATCH 26/84] Fix (hopefully) a compiler warning --- Modules/_asynciomodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 032a7a50cf9c79..20fbfff46a9de3 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -657,7 +657,7 @@ future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) } static PyObject * -future_get_awaited_by(FutureObj *fut) +FutureObj_get_awaited_by(FutureObj *fut, void *Py_UNUSED(ignored)) { /* Implementation of a Python getter. */ if (fut->fut_awaited_by == NULL) { @@ -1670,7 +1670,7 @@ static PyMethodDef FutureType_methods[] = { NULL, NULL}, \ {"_cancel_message", (getter)FutureObj_get_cancel_message, \ (setter)FutureObj_set_cancel_message, NULL}, \ - {"_asyncio_awaited_by", (getter)future_get_awaited_by, NULL, NULL}, + {"_asyncio_awaited_by", (getter)FutureObj_get_awaited_by, NULL, NULL}, static PyGetSetDef FutureType_getsetlist[] = { FUTURE_COMMON_GETSETLIST From fe3113bce50f8da8209be3e6cdd5c31bd19fd77b Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 13:36:33 -0700 Subject: [PATCH 27/84] Fix sloppy what's new! --- Doc/whatsnew/3.14.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index ea6557f4d8fa27..cf7bba1bf42d2e 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -386,7 +386,8 @@ asyncio (Contributed by Kumar Aditya in :gh:`107803`.) * :mod:`asyncio` has new utility functions for introspecting and printing - the program's call graph. + the program's call graph: :func:`asyncio.capture_call_graph` and + :func:`asyncio.print_call_graph`. (Contributed by Yury Selivanov and Pablo Galindo Salgado in :gh:`91048`.) From 18ec26da2a316d02347085d7b69414525c713836 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 14:16:47 -0700 Subject: [PATCH 28/84] Fix CI complaining about suspicious global in C --- Tools/c-analyzer/cpython/ignored.tsv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index e6c599a2ac4a46..c36c06e9d9fcf4 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -53,6 +53,9 @@ Python/pyhash.c - _Py_HashSecret - ## thread-safe hashtable (internal locks) Python/parking_lot.c - buckets - +## data needed for introspecting asyncio state from debuggers and profilers +Modules/_asynciomodule.c - AsyncioDebug - + ################################## ## state tied to Py_Main() From e4cc46245cb705f20729168983cfa5d14b22d194 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 14:43:10 -0700 Subject: [PATCH 29/84] Add a test for depth=2 --- Lib/test/test_asyncio/test_stack.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index 9bb675270e4be6..62ec000b1c62f4 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -8,7 +8,7 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) -def capture_test_stack(*, fut=None): +def capture_test_stack(*, fut=None, depth=1): def walk(s): ret = [ @@ -39,9 +39,9 @@ def walk(s): return ret buf = io.StringIO() - asyncio.print_call_graph(future=fut, file=buf) + asyncio.print_call_graph(future=fut, file=buf, depth=depth+1) - stack = asyncio.capture_call_graph(future=fut) + stack = asyncio.capture_call_graph(future=fut, depth=depth) return walk(stack), buf.getvalue() @@ -54,7 +54,7 @@ async def test_stack_tgroup(self): def c5(): nonlocal stack_for_c5 - stack_for_c5 = capture_test_stack() + stack_for_c5 = capture_test_stack(depth=2) async def c4(): await asyncio.sleep(0) @@ -81,7 +81,7 @@ async def main(): # task name 'T', # call stack - ['s capture_test_stack', 's c5', 'a c4', 'a c3', 'a c2'], + ['s c5', 'a c4', 'a c3', 'a c2'], # awaited by [ ['T', From d5cdc36181809235cc6b2b38f86946075eddb6d9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 2 Oct 2024 22:43:54 +0100 Subject: [PATCH 30/84] Add critical sections for free threaded builds --- Modules/_asynciomodule.c | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 20fbfff46a9de3..4acbb6c30644ff 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -579,15 +579,8 @@ future_init(FutureObj *fut, PyObject *loop) } static int -future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) +future_awaited_by_add_lock_held(asyncio_state *state, PyObject *fut, PyObject *thing) { - if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { - // We only want to support native asyncio Futures. - // For further insight see the comment in the Python - // implementation of "future_add_to_awaited_by()". - return 0; - } - FutureObj *_fut = (FutureObj *)fut; /* Most futures/task are only awaited by one entity, so we want @@ -623,7 +616,7 @@ future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) } static int -future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) +future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) { if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { // We only want to support native asyncio Futures. @@ -632,6 +625,17 @@ future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) return 0; } + int result; + Py_BEGIN_CRITICAL_SECTION(fut); + result = future_awaited_by_add_lock_held(state, fut, thing); + Py_END_CRITICAL_SECTION(); + return result; +} + +static int +future_awaited_by_discard_lock_held(asyncio_state *state, PyObject *fut, PyObject *thing) +{ + FutureObj *_fut = (FutureObj *)fut; /* Following the semantics of 'set.discard()' here in not @@ -656,6 +660,23 @@ future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) return 0; } +static int +future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) +{ + if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { + // We only want to support native asyncio Futures. + // For further insight see the comment in the Python + // implementation of "future_add_to_awaited_by()". + return 0; + } + + int result; + Py_BEGIN_CRITICAL_SECTION(fut); + result = future_awaited_by_discard_lock_held(state, fut, thing); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * FutureObj_get_awaited_by(FutureObj *fut, void *Py_UNUSED(ignored)) { From 83606f2b9686e879058934c427cb57de834ab28f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 2 Oct 2024 22:54:06 +0100 Subject: [PATCH 31/84] Add more slopy tests --- Lib/test/test_external_inspection.py | 111 +++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 227ac574e9631d..c4c3bbe54ac19f 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -148,7 +148,118 @@ async def main(): ] self.assertEqual(stack_trace, expected_stack_trace) + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_asyncgen_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import asyncio + import time + import os + import sys + import test.test_asyncio.test_stack as ts + + async def gen_nested_call(): + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(10000) + + async def gen(): + for num in range(2): + yield num + if num == 1: + await gen_nested_call() + + async def main(): + async for el in gen(): + pass + + asyncio.run(main()) + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + expected_stack_trace = [['gen_nested_call', 'gen', 'main'], 'Task-1', []] + self.assertEqual(stack_trace, expected_stack_trace) + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_async_gather_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import asyncio + import time + import os + import sys + import test.test_asyncio.test_stack as ts + + async def deep(): + await asyncio.sleep(0) + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(10000) + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(0) + + async def main(): + await asyncio.gather(c1(), c2()) + + asyncio.run(main()) + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + expected_stack_trace = [['deep', 'c1'], 'Task-2', [[['main'], 'Task-1', []]]] + self.assertEqual(stack_trace, expected_stack_trace) @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") From 5edac41fc485eac7fa1f3127f9b999b95c8c2675 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 10:37:08 -0700 Subject: [PATCH 32/84] Apply suggestions from code review Co-authored-by: Kumar Aditya --- Doc/library/asyncio-future.rst | 2 +- Doc/library/asyncio-stack.rst | 2 +- Lib/asyncio/futures.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index f1f23b8021b29f..9dce0731411940 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -65,7 +65,7 @@ Future Functions and *loop* is not specified and there is no running event loop. -.. function:: wrap_future(future, /, *, loop=None) +.. function:: wrap_future(future, *, loop=None) Wrap a :class:`concurrent.futures.Future` object in a :class:`asyncio.Future` object. diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 285a86fb1fbeb5..1553e3fd0a891e 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -28,7 +28,7 @@ a suspended *future*. current task, the function returns ``None``. If the function is called on *the current task*, the optional - keyword-only ``depth`` argument can be used to skip the specified + keyword-only *depth* argument can be used to skip the specified number of frames from top of the stack. If *file* is not specified the function will print to :data:`sys.stdout`. diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 5785477248a8d9..ba16ae8e154b5a 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -426,7 +426,7 @@ def wrap_future(future, *, loop=None): def future_add_to_awaited_by(fut, waiter, /): """Record that `fut` is awaited on by `waiter`.""" # For the sake of keeping the implementation minimal and assuming - # that 99.9% of asyncio users use the built-in Futures and Tasks + # that most of asyncio users use the built-in Futures and Tasks # (or their subclasses), we only support native Future objects # and their subclasses. # From 8dc6d340e906b6e6fc896816a253229b1ddfc2e7 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 10:44:40 -0700 Subject: [PATCH 33/84] Apply suggestions from code review --- Modules/_asynciomodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 4acbb6c30644ff..e51a9bd1d6a38c 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -649,7 +649,7 @@ future_awaited_by_discard_lock_held(asyncio_state *state, PyObject *fut, PyObjec return 0; } if (_fut->fut_awaited_by_is_set) { - assert(PySet_Check(_fut->fut_awaited_by)); + assert(PySet_CheckExact(_fut->fut_awaited_by)); int err = PySet_Discard(_fut->fut_awaited_by, thing); if (err < 0 && PyErr_Occurred()) { return -1; @@ -686,7 +686,7 @@ FutureObj_get_awaited_by(FutureObj *fut, void *Py_UNUSED(ignored)) } if (fut->fut_awaited_by_is_set) { /* Already a set, just wrap it into a frozen set and return. */ - assert(PySet_Check(fut->fut_awaited_by)); + assert(PySet_CheckExact(fut->fut_awaited_by)); return PyFrozenSet_New(fut->fut_awaited_by); } From 30884ea949c4f74833de441478d2ccdc98c01d33 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 11:09:25 -0700 Subject: [PATCH 34/84] Update Modules/_asynciomodule.c --- Modules/_asynciomodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index e51a9bd1d6a38c..ded7990d7bf98a 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -651,7 +651,7 @@ future_awaited_by_discard_lock_held(asyncio_state *state, PyObject *fut, PyObjec if (_fut->fut_awaited_by_is_set) { assert(PySet_CheckExact(_fut->fut_awaited_by)); int err = PySet_Discard(_fut->fut_awaited_by, thing); - if (err < 0 && PyErr_Occurred()) { + if (err < 0) { return -1; } else { return 0; From 131765849eabb1722b0edf4aacf6a1d8c032a6f4 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 11:10:19 -0700 Subject: [PATCH 35/84] Update Modules/_asynciomodule.c Co-authored-by: Kumar Aditya --- Modules/_asynciomodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index ded7990d7bf98a..6f51109e972db3 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -594,7 +594,7 @@ future_awaited_by_add_lock_held(asyncio_state *state, PyObject *fut, PyObject *t } if (_fut->fut_awaited_by_is_set) { - assert(PySet_Check(_fut->fut_awaited_by)); + assert(PySet_CheckExact(_fut->fut_awaited_by)); return PySet_Add(_fut->fut_awaited_by, thing); } From 81b0a310cbdcf60fc32dfa779e534bc34715e8fa Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 11:20:26 -0700 Subject: [PATCH 36/84] Use dataclasses instead of tuples for asyncio.stack --- Doc/library/asyncio-stack.rst | 4 ++-- Lib/asyncio/stack.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 1553e3fd0a891e..5263522cca0b71 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -88,7 +88,7 @@ a suspended *future*. keyword-only ``depth`` argument can be used to skip the specified number of frames from top of the stack. - Returns a ``FutureCallGraph`` named tuple: + Returns a ``FutureCallGraph`` data class object: * ``FutureCallGraph(future, call_stack, awaited_by)`` @@ -97,7 +97,7 @@ a suspended *future*. ``call_stack`` is a list of ``FrameCallGraphEntry`` objects. - ``awaited_by`` is a list of ``FutureCallGraph`` tuples. + ``awaited_by`` is a list of ``FutureCallGraph`` objects. * ``FrameCallGraphEntry(frame)`` diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 97a1b209aa6e00..dbb92fd5d5ff23 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -1,8 +1,8 @@ """Introspection utils for tasks call stacks.""" +import dataclasses import sys import types -import typing from . import events from . import futures @@ -23,11 +23,13 @@ # top level asyncio namespace, and want to avoid future name clashes. -class FrameCallGraphEntry(typing.NamedTuple): +@dataclasses.dataclass(frozen=True) +class FrameCallGraphEntry: frame: types.FrameType -class FutureCallGraph(typing.NamedTuple): +@dataclasses.dataclass(frozen=True) +class FutureCallGraph: future: futures.Future call_stack: list[FrameCallGraphEntry] awaited_by: list[FutureCallGraph] From 258ce3dcc36645fb88693ab99a035aac4b3ae970 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 11:21:01 -0700 Subject: [PATCH 37/84] Fix sloppy whitespace! --- Modules/_asynciomodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 6f51109e972db3..7700f1b7a0f56f 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -16,6 +16,7 @@ #include // offsetof() + /*[clinic input] module _asyncio [clinic start generated code]*/ From b9ecefb0477e12c70ead97bc628513cf12f07f43 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 16 Oct 2024 19:28:29 +0100 Subject: [PATCH 38/84] Remove critical sections for now --- Modules/_asynciomodule.c | 39 +++++++++------------------------------ 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 7700f1b7a0f56f..f684f5cf4a1ca7 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -580,8 +580,15 @@ future_init(FutureObj *fut, PyObject *loop) } static int -future_awaited_by_add_lock_held(asyncio_state *state, PyObject *fut, PyObject *thing) +future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) { + if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { + // We only want to support native asyncio Futures. + // For further insight see the comment in the Python + // implementation of "future_add_to_awaited_by()". + return 0; + } + FutureObj *_fut = (FutureObj *)fut; /* Most futures/task are only awaited by one entity, so we want @@ -617,7 +624,7 @@ future_awaited_by_add_lock_held(asyncio_state *state, PyObject *fut, PyObject *t } static int -future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) +future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) { if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { // We only want to support native asyncio Futures. @@ -626,17 +633,6 @@ future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) return 0; } - int result; - Py_BEGIN_CRITICAL_SECTION(fut); - result = future_awaited_by_add_lock_held(state, fut, thing); - Py_END_CRITICAL_SECTION(); - return result; -} - -static int -future_awaited_by_discard_lock_held(asyncio_state *state, PyObject *fut, PyObject *thing) -{ - FutureObj *_fut = (FutureObj *)fut; /* Following the semantics of 'set.discard()' here in not @@ -661,23 +657,6 @@ future_awaited_by_discard_lock_held(asyncio_state *state, PyObject *fut, PyObjec return 0; } -static int -future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) -{ - if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { - // We only want to support native asyncio Futures. - // For further insight see the comment in the Python - // implementation of "future_add_to_awaited_by()". - return 0; - } - - int result; - Py_BEGIN_CRITICAL_SECTION(fut); - result = future_awaited_by_discard_lock_held(state, fut, thing); - Py_END_CRITICAL_SECTION(); - return result; -} - static PyObject * FutureObj_get_awaited_by(FutureObj *fut, void *Py_UNUSED(ignored)) { From b77dcb0fbcdfd94976ac11c76a4063194e86217c Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 11:36:26 -0700 Subject: [PATCH 39/84] Eplain the weird macro --- Modules/_asynciomodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index f684f5cf4a1ca7..07f5783cbe77d2 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -87,6 +87,8 @@ typedef struct { (Task_CheckExact(state, obj) \ || PyObject_TypeCheck(obj, state->TaskType)) +// This macro is optimized to quickly return for native Future *or* Task +// objects by inlining fast "exact" checks to be called first. #define TaskOrFuture_Check(state, obj) \ (Task_CheckExact(state, obj) \ || Future_CheckExact(state, obj) \ From 8867946e7e72d9c4c2cc5a9e71289986c0544cf9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 16 Oct 2024 19:55:30 +0100 Subject: [PATCH 40/84] Fix test --- Lib/test/test_asyncio/test_stack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index 62ec000b1c62f4..9536451e0efa85 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -85,13 +85,13 @@ async def main(): # awaited by [ ['T', - ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] ], ['T', ['a c1'], [ ['T', - ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] ] ] ], @@ -99,7 +99,7 @@ async def main(): ['a c1'], [ ['T', - ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] ] ] ] From b47bef111963903c57e62ad1179293fc5fe772a5 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 12:26:28 -0700 Subject: [PATCH 41/84] Add a docs clarification --- Doc/library/asyncio-stack.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 5263522cca0b71..fab72521bb9df8 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -13,7 +13,9 @@ Stack Introspection asyncio has powerful runtime call stack introspection utilities to trace the entire call graph of a running coroutine or task, or -a suspended *future*. +a suspended *future*. These utilities and the underlying machinery +can be used by users in their Python code or by external profilers +and debuggers. .. versionadded:: 3.14 From 230b7ecd69c9a319604055d734aafdc82ae244a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 16 Oct 2024 23:53:25 +0200 Subject: [PATCH 42/84] Fix typing in asyncio.stack --- Lib/asyncio/stack.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index dbb92fd5d5ff23..451f0639e9f88b 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -3,6 +3,7 @@ import dataclasses import sys import types +import typing from . import events from . import futures @@ -35,18 +36,15 @@ class FutureCallGraph: awaited_by: list[FutureCallGraph] -def _build_stack_for_future(future: any) -> FutureCallGraph: +def _build_stack_for_future(future: futures.Future) -> FutureCallGraph: if not isinstance(future, futures.Future): raise TypeError( f"{future!r} object does not appear to be compatible " f"with asyncio.Future" ) - try: - get_coro = future.get_coro - except AttributeError: - coro = None - else: + coro = None + if get_coro := getattr(future, 'get_coro', None): coro = get_coro() st: list[FrameCallGraphEntry] = [] @@ -74,7 +72,7 @@ def _build_stack_for_future(future: any) -> FutureCallGraph: def capture_call_graph( *, - future: any = None, + future: futures.Future | None = None, depth: int = 1, ) -> FutureCallGraph | None: """Capture async call stack for the current task or the provided Future. @@ -158,11 +156,16 @@ def capture_call_graph( return FutureCallGraph(future, call_stack, awaited_by) -def print_call_graph(*, future: any = None, file=None, depth: int = 1) -> None: +def print_call_graph( + *, + future: futures.Future | None = None, + file: typing.TextIO | None = None, + depth: int = 1, +) -> None: """Print async call stack for the current task or the provided Future.""" - def render_level(st: FutureCallGraph, buf: list[str], level: int): - def add_line(line: str): + def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: + def add_line(line: str) -> None: buf.append(level * ' ' + line) if isinstance(st.future, tasks.Task): @@ -223,7 +226,7 @@ def add_line(line: str): return try: - buf = [] + buf: list[str] = [] render_level(stack, buf, 0) rendered = '\n'.join(buf) print(rendered, file=file) From b1d61582407563a3e3bc46954cd42720801a93ad Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 17 Oct 2024 01:03:39 +0100 Subject: [PATCH 43/84] Fix memory leak --- Modules/_testexternalinspection.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index daf7aad6804452..390c5caf5b575b 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -577,7 +577,7 @@ read_py_long( } bytes_read = read_memory(pid, address + offsets->long_object.ob_digit, sizeof(digit) * size, digits); if (bytes_read < 0) { - return -1; + goto error; } long value = 0; @@ -585,17 +585,20 @@ read_py_long( for (ssize_t i = 0; i < size; ++i) { long long factor; if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), &factor)) { - return -1; + goto error; } if (__builtin_add_overflow(value, factor, &value)) { - return -1; + goto error; } } + PyMem_RawFree(digits); if (negative) { value = -1 * value; } - return value; +error: + PyMem_RawFree(digits); + return -1; } static PyObject * From ac5136433de3ba6f58a0447886aef50710d0ab21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 17 Oct 2024 16:37:55 +0200 Subject: [PATCH 44/84] Address comments from Kumar's review --- Lib/asyncio/futures.py | 2 -- Lib/asyncio/stack.py | 16 ++++++++-------- ...2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst | 3 ++- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index c9cbcc6f6d8173..4e6d0cf39cd3c5 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -46,8 +46,6 @@ class Future: """ - _awaited_by = None - # Class variables serving as defaults for instance variables. _state = _PENDING _result = None diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 451f0639e9f88b..f68e80f55063af 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -24,12 +24,12 @@ # top level asyncio namespace, and want to avoid future name clashes. -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass(frozen=True, slots=True) class FrameCallGraphEntry: frame: types.FrameType -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass(frozen=True, slots=True) class FutureCallGraph: future: futures.Future call_stack: list[FrameCallGraphEntry] @@ -62,8 +62,8 @@ def _build_stack_for_future(future: futures.Future) -> FutureCallGraph: else: break - if fut_waiters := getattr(future, '_asyncio_awaited_by', None): - for parent in fut_waiters: + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: awaited_by.append(_build_stack_for_future(parent)) st.reverse() @@ -149,8 +149,8 @@ def capture_call_graph( del f awaited_by = [] - if fut_waiters := getattr(future, '_asyncio_awaited_by', None): - for parent in fut_waiters: + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: awaited_by.append(_build_stack_for_future(parent)) return FutureCallGraph(future, call_stack, awaited_by) @@ -170,11 +170,11 @@ def add_line(line: str) -> None: if isinstance(st.future, tasks.Task): add_line( - f'* Task(name={st.future.get_name()!r}, id=0x{id(st.future):x})' + f'* Task(name={st.future.get_name()!r}, id={id(st.future):#x})' ) else: add_line( - f'* Future(id=0x{id(st.future):x})' + f'* Future(id={id(st.future):#x})' ) if st.call_stack: diff --git a/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst b/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst index 071dd9695d4566..c2faf470ffc9cf 100644 --- a/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst +++ b/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst @@ -1 +1,2 @@ -Add asyncio.capture_call_graph() and asyncio.print_call_graph() functions. +Add :func:`asyncio.capture_call_graph` and +:func:`asyncio.print_call_graph` functions. From c7e59eb5c599cd8bcebc443227d536ad5720b9c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 22 Oct 2024 21:23:07 +0200 Subject: [PATCH 45/84] Fix rare Mach-O linker bug when .bss is uninitialized data --- Lib/test/test_external_inspection.py | 18 ++++++++++-------- Modules/_testexternalinspection.c | 8 +++++++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index c4c3bbe54ac19f..3391b48554fc1d 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -38,8 +38,8 @@ def baz(): foo() def foo(): - fifo = sys.argv[1] - with open(sys.argv[1], "w") as fifo: + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: fifo.write("ready") time.sleep(1000) @@ -87,8 +87,8 @@ def test_async_remote_stack_trace(self): import test.test_asyncio.test_stack as ts def c5(): - fifo = sys.argv[1] - with open(sys.argv[1], "w") as fifo: + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: fifo.write("ready") time.sleep(10000) @@ -128,6 +128,8 @@ async def main(): stack_trace = get_async_stack_trace(p.pid) except PermissionError: self.skipTest("Insufficient permissions to read the stack trace") + except RuntimeError: + breakpoint() finally: os.remove(fifo) p.kill() @@ -160,8 +162,8 @@ def test_asyncgen_remote_stack_trace(self): import test.test_asyncio.test_stack as ts async def gen_nested_call(): - fifo = sys.argv[1] - with open(sys.argv[1], "w") as fifo: + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: fifo.write("ready") time.sleep(10000) @@ -217,8 +219,8 @@ def test_async_gather_remote_stack_trace(self): async def deep(): await asyncio.sleep(0) - fifo = sys.argv[1] - with open(sys.argv[1], "w") as fifo: + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: fifo.write("ready") time.sleep(10000) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 390c5caf5b575b..ddbc9dd53a6930 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -218,6 +218,12 @@ search_map_for_section(pid_t pid, const char* secname, const char* substr) { continue; } + if ((region_info.protection & VM_PROT_READ) == 0 + || (region_info.protection & VM_PROT_EXECUTE) == 0) { + address += size; + continue; + } + char* filename = strrchr(map_filename, '/'); if (filename != NULL) { filename++; // Move past the '/' @@ -1222,7 +1228,7 @@ read_async_debug( struct _Py_AsyncioModuleDebugOffsets* async_debug ) { uintptr_t async_debug_addr = get_async_debug(pid); - if (!async_debug) { + if (!async_debug_addr) { return -1; } size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); From 59121f64fee619eb06282af8473122bab0fabb2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 22 Oct 2024 22:32:20 +0200 Subject: [PATCH 46/84] you saw nothing, okay --- Lib/test/test_external_inspection.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 3391b48554fc1d..6be41042e517d5 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -128,8 +128,6 @@ async def main(): stack_trace = get_async_stack_trace(p.pid) except PermissionError: self.skipTest("Insufficient permissions to read the stack trace") - except RuntimeError: - breakpoint() finally: os.remove(fifo) p.kill() From f8f48f0db7188e57a7ad9e6430bd9d4b23b1db1f Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Tue, 22 Oct 2024 22:18:21 -0700 Subject: [PATCH 47/84] Update Lib/asyncio/futures.py Co-authored-by: Savannah Ostrowski --- Lib/asyncio/futures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 4e6d0cf39cd3c5..8bd47a90c7a83a 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -450,7 +450,7 @@ def future_add_to_awaited_by(fut, waiter, /): def future_discard_from_awaited_by(fut, waiter, /): """Record that `fut` is no longer awaited on by `waiter`.""" # See the comment in "future_add_to_awaited_by()" body for - # details on implemntation. + # details on implementation. # # Note that there's an accelerated version of this function # shadowing this implementation later in this file. From 74c5ad141ba6a365d99228374880e6b0ca63d863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 30 Oct 2024 17:53:45 +0100 Subject: [PATCH 48/84] Remove unnecessary imports from tests --- Lib/test/test_external_inspection.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 6be41042e517d5..9ef3ea86dd002f 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -29,7 +29,7 @@ class TestGetStackTrace(unittest.TestCase): def test_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ - import time, sys, os + import time, sys def bar(): for x in range(100): if x == 50: @@ -82,9 +82,7 @@ def test_async_remote_stack_trace(self): script = textwrap.dedent("""\ import asyncio import time - import os import sys - import test.test_asyncio.test_stack as ts def c5(): fifo_path = sys.argv[1] @@ -155,9 +153,7 @@ def test_asyncgen_remote_stack_trace(self): script = textwrap.dedent("""\ import asyncio import time - import os import sys - import test.test_asyncio.test_stack as ts async def gen_nested_call(): fifo_path = sys.argv[1] @@ -211,9 +207,7 @@ def test_async_gather_remote_stack_trace(self): script = textwrap.dedent("""\ import asyncio import time - import os import sys - import test.test_asyncio.test_stack as ts async def deep(): await asyncio.sleep(0) From 067c043cc4792fa19a04a905b16a2da3731a7ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 31 Oct 2024 23:36:09 +0100 Subject: [PATCH 49/84] Remove cr_task/gi_task --- Include/cpython/genobject.h | 2 - Include/internal/pycore_debug_offsets.h | 2 - Include/internal/pycore_genobject.h | 7 - Include/internal/pycore_tstate.h | 1 + Lib/test/test_sys.py | 2 +- Modules/_asynciomodule.c | 32 ++-- Modules/_testexternalinspection.c | 244 +++++++++++++++++------- Objects/genobject.c | 15 -- Python/pystate.c | 2 + 9 files changed, 204 insertions(+), 103 deletions(-) diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index f0c36081d120cc..f75884e597e2c2 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -32,8 +32,6 @@ PyAPI_DATA(PyTypeObject) PyCoro_Type; PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, PyObject *name, PyObject *qualname); -PyAPI_FUNC(void) _PyCoro_SetTask(PyObject *coro, PyObject *task); - /* --- Asynchronous Generators -------------------------------------------- */ diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 0afa2ce6349a14..38d0cd57ab2c9b 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -203,7 +203,6 @@ typedef struct _Py_DebugOffsets { uint64_t size; uint64_t gi_name; uint64_t gi_iframe; - uint64_t gi_task; uint64_t gi_frame_state; } gen_object; } _Py_DebugOffsets; @@ -324,7 +323,6 @@ typedef struct _Py_DebugOffsets { .size = sizeof(PyGenObject), \ .gi_name = offsetof(PyGenObject, gi_name), \ .gi_iframe = offsetof(PyGenObject, gi_iframe), \ - .gi_task = offsetof(PyGenObject, gi_task), \ .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \ }, \ } diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index 40ef9098124753..f6d7e6d367177b 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -22,13 +22,6 @@ extern "C" { PyObject *prefix##_qualname; \ _PyErr_StackItem prefix##_exc_state; \ PyObject *prefix##_origin_or_finalizer; \ - /* A *borrowed* reference to a task that drives the coroutine. \ - The field is meant to be used by profilers and debuggers only. \ - The main invariant is that a task can't get GC'ed while \ - the coroutine it drives is alive and vice versa. \ - Profilers can use this field to reconstruct the full async \ - call stack of program. */ \ - PyObject *prefix##_task; \ char prefix##_hooks_inited; \ char prefix##_closed; \ char prefix##_running_async; \ diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index a72ef4493b77ca..d2a171c07c2111 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -22,6 +22,7 @@ typedef struct _PyThreadStateImpl { PyThreadState base; PyObject *asyncio_running_loop; // Strong reference + PyObject *asyncio_running_task; // Strong reference struct _qsbr_thread_state *qsbr; // only used by free-threaded build struct llist_node mem_free_queue; // delayed free queue diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 49767964a01977..9689ef8e96e072 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1617,7 +1617,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('7P4c' + INTERPRETER_FRAME + 'P')) + check(get_gen(), size('6P4c' + INTERPRETER_FRAME + 'P')) # iterator check(iter('abc'), size('lP')) # callable-iterator diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a0eca5c181cf90..dbf20c60c47006 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -75,7 +75,6 @@ typedef struct { PyObject *sw_arg; } TaskStepMethWrapper; - #define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) #define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) @@ -113,6 +112,11 @@ typedef struct _Py_AsyncioModuleDebugOffsets { uint64_t task_awaited_by_is_set; uint64_t task_coro; } asyncio_task_object; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + } asyncio_thread_state; } Py_AsyncioModuleDebugOffsets; GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets AsyncioDebug) @@ -123,6 +127,11 @@ GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets AsyncioDebug) .task_is_task = offsetof(TaskObj, task_is_task), .task_awaited_by_is_set = offsetof(TaskObj, task_awaited_by_is_set), .task_coro = offsetof(TaskObj, task_coro), + }, + .asyncio_thread_state = { + .size = sizeof(_PyThreadStateImpl), + .asyncio_running_loop = offsetof(_PyThreadStateImpl, asyncio_running_loop), + .asyncio_running_task = offsetof(_PyThreadStateImpl, asyncio_running_task), }}; /* State of the _asyncio module */ @@ -219,7 +228,6 @@ typedef struct { TaskObj tail; TaskObj *head; } asyncio_tasks; - } asyncio_state; static inline asyncio_state * @@ -268,9 +276,6 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu static void clear_task_coro(TaskObj *task) { - if (task->task_coro != NULL && PyCoro_CheckExact(task->task_coro)) { - _PyCoro_SetTask(task->task_coro, NULL); - } Py_CLEAR(task->task_coro); } @@ -279,9 +284,6 @@ static void set_task_coro(TaskObj *task, PyObject *coro) { assert(coro != NULL); - if (PyCoro_CheckExact(coro)) { - _PyCoro_SetTask(coro, (PyObject *)task); - } Py_INCREF(coro); Py_XSETREF(task->task_coro, coro); } @@ -2160,7 +2162,10 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) Py_DECREF(item); return -1; } - Py_DECREF(item); + + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + assert(ts->asyncio_running_task == NULL); + ts->asyncio_running_task = item; // strong ref return 0; } @@ -2185,7 +2190,6 @@ leave_task_predicate(PyObject *item, void *task) static int leave_task(asyncio_state *state, PyObject *loop, PyObject *task) -/*[clinic end generated code: output=0ebf6db4b858fb41 input=51296a46313d1ad8]*/ { int res = _PyDict_DelItemIf(state->current_tasks, loop, leave_task_predicate, task); @@ -2193,6 +2197,9 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) // task was not found return err_leave_task(Py_None, task); } + + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + Py_CLEAR(ts->asyncio_running_task); return res; } @@ -3960,7 +3967,9 @@ module_clear(PyObject *mod) Py_CLEAR(state->iscoroutine_typecache); Py_CLEAR(state->context_kwname); - + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + Py_CLEAR(ts->asyncio_running_loop); + Py_CLEAR(ts->asyncio_running_task); return 0; } @@ -3990,7 +3999,6 @@ module_init(asyncio_state *state) goto fail; } - state->context_kwname = Py_BuildValue("(s)", "context"); if (state->context_kwname == NULL) { goto fail; diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 8f4bb2f59b2f72..ebe92e64d9a22a 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -60,14 +60,19 @@ #endif struct _Py_AsyncioModuleDebugOffsets { - struct _asyncio_task_object { - uint64_t size; - uint64_t task_name; - uint64_t task_awaited_by; - uint64_t task_is_task; - uint64_t task_awaited_by_is_set; - uint64_t task_coro; - } asyncio_task_object; + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + } asyncio_thread_state; }; #if defined(__APPLE__) && TARGET_OS_OSX @@ -1145,13 +1150,11 @@ parse_async_frame_object( PyObject* result, struct _Py_DebugOffsets* offsets, uintptr_t address, - uintptr_t* task, - uintptr_t* previous_frame + uintptr_t* previous_frame, + uintptr_t* code_object ) { int err; - *task = (uintptr_t)NULL; - ssize_t bytes_read = read_memory( pid, address + offsets->interpreter_frame.previous, @@ -1170,35 +1173,34 @@ parse_async_frame_object( } if (owner == FRAME_OWNED_BY_CSTACK) { - return 0; + return 0; // C frame } - if (owner == FRAME_OWNED_BY_GENERATOR) { - err = read_py_ptr( - pid, - address - offsets->gen_object.gi_iframe + offsets->gen_object.gi_task, - task); - if (err) { - return -1; - } + if (owner != FRAME_OWNED_BY_GENERATOR + && owner != FRAME_OWNED_BY_THREAD) { + PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", owner); + return -1; } - uintptr_t address_of_code_object; err = read_py_ptr( pid, address + offsets->interpreter_frame.executable, - &address_of_code_object + code_object ); if (err) { return -1; } - if ((void*)address_of_code_object == NULL) { + if ((void*)*code_object == NULL) { return 0; } - return parse_code_object( - pid, result, offsets, address_of_code_object, previous_frame); + if (parse_code_object( + pid, result, offsets, *code_object, previous_frame)) { + return -1; + } + + return 1; } static int @@ -1294,6 +1296,77 @@ find_running_frame( return 0; } +static int +find_running_task( + int pid, + uintptr_t runtime_start_address, + _Py_DebugOffsets *local_debug_offsets, + struct _Py_AsyncioModuleDebugOffsets *async_offsets, + uintptr_t *running_task_addr +) { + *running_task_addr = (uintptr_t)NULL; + + off_t interpreter_state_list_head = + local_debug_offsets->runtime_state.interpreters_head; + + uintptr_t address_of_interpreter_state; + int bytes_read = read_memory( + pid, + runtime_start_address + interpreter_state_list_head, + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read == -1) { + return -1; + } + + if (address_of_interpreter_state == 0) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return -1; + } + + uintptr_t address_of_thread; + bytes_read = read_memory( + pid, + address_of_interpreter_state + + local_debug_offsets->interpreter_state.threads_head, + sizeof(void*), + &address_of_thread); + if (bytes_read == -1) { + return -1; + } + + uintptr_t address_of_running_loop; + // No Python frames are available for us (can happen at tear-down). + if ((void*)address_of_thread == NULL) { + return 0; + } + + bytes_read = read_py_ptr( + pid, + address_of_thread + + async_offsets->asyncio_thread_state.asyncio_running_loop, + &address_of_running_loop); + if (bytes_read == -1) { + return -1; + } + + // no asyncio loop is now running + if ((void*)address_of_running_loop == NULL) { + return 0; + } + + int err = read_ptr( + pid, + address_of_thread + + async_offsets->asyncio_thread_state.asyncio_running_task, + running_task_addr); + if (err) { + return -1; + } + + return 0; +} + static PyObject* get_stack_trace(PyObject* self, PyObject* args) { @@ -1369,14 +1442,6 @@ get_async_stack_trace(PyObject* self, PyObject* args) return NULL; } - uintptr_t address_of_current_frame; - if (find_running_frame( - pid, runtime_start_address, &local_debug_offsets, - &address_of_current_frame) - ) { - return NULL; - } - PyObject* result = PyList_New(1); if (result == NULL) { return NULL; @@ -1391,53 +1456,104 @@ get_async_stack_trace(PyObject* self, PyObject* args) return NULL; } - uintptr_t root_task_addr = (uintptr_t)NULL; + uintptr_t running_task_addr = (uintptr_t)NULL; + if (find_running_task( + pid, runtime_start_address, &local_debug_offsets, &local_async_debug, + &running_task_addr) + ) { + goto result_err; + } + + if ((void*)running_task_addr == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No running task found"); + goto result_err; + } + + uintptr_t running_coro_addr; + if (read_py_ptr( + pid, + running_task_addr + local_async_debug.asyncio_task_object.task_coro, + &running_coro_addr + )) { + goto result_err; + } + + if ((void*)running_coro_addr == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); + goto result_err; + } + + // note: genobject's gi_iframe is an embedded struct so the address to the offset + // leads directly to its first field: f_executable + uintptr_t address_of_running_task_code_obj; + if (read_py_ptr( + pid, + running_coro_addr + local_debug_offsets.gen_object.gi_iframe, + &address_of_running_task_code_obj + )) { + goto result_err; + } + + if ((void*)address_of_running_task_code_obj == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); + goto result_err; + } + + uintptr_t address_of_current_frame; + if (find_running_frame( + pid, runtime_start_address, &local_debug_offsets, + &address_of_current_frame) + ) { + goto result_err; + } + + uintptr_t address_of_code_object; while ((void*)address_of_current_frame != NULL) { - int err = parse_async_frame_object( + int res = parse_async_frame_object( pid, calls, &local_debug_offsets, address_of_current_frame, - &root_task_addr, - &address_of_current_frame + &address_of_current_frame, + &address_of_code_object ); - if (err) { + + if (res < 0) { goto result_err; } - if ((void*)root_task_addr != NULL) { + if (address_of_code_object == address_of_running_task_code_obj) { break; } } - if ((void*)root_task_addr != NULL) { - PyObject *tn = parse_task_name( - pid, &local_debug_offsets, &local_async_debug, root_task_addr); - if (tn == NULL) { - goto result_err; - } - if (PyList_Append(result, tn)) { - Py_DECREF(tn); - goto result_err; - } + PyObject *tn = parse_task_name( + pid, &local_debug_offsets, &local_async_debug, running_task_addr); + if (tn == NULL) { + goto result_err; + } + if (PyList_Append(result, tn)) { Py_DECREF(tn); + goto result_err; + } + Py_DECREF(tn); - PyObject* awaited_by = PyList_New(0); - if (awaited_by == NULL) { - goto result_err; - } - if (PyList_Append(result, awaited_by)) { - Py_DECREF(awaited_by); - goto result_err; - } - - if (parse_task_awaited_by( - pid, &local_debug_offsets, &local_async_debug, root_task_addr, awaited_by) - ) { - goto result_err; - } + PyObject* awaited_by = PyList_New(0); + if (awaited_by == NULL) { + goto result_err; + } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + goto result_err; } + Py_DECREF(awaited_by); + + if (parse_task_awaited_by( + pid, &local_debug_offsets, &local_async_debug, running_task_addr, awaited_by) + ) { + goto result_err; + } return result; diff --git a/Objects/genobject.c b/Objects/genobject.c index 51786cb47fd88d..49d902ea954d79 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -139,10 +139,6 @@ gen_dealloc(PyObject *self) { PyGenObject *gen = _PyGen_CAST(self); - /* A borrowed reference used only by coroutines and async - frameworks. Just set it to NULL. */ - gen->gi_task = NULL; - _PyObject_GC_UNTRACK(gen); if (gen->gi_weakreflist != NULL) @@ -918,7 +914,6 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) gen->gi_name = Py_NewRef(func->func_name); assert(func->func_qualname != NULL); gen->gi_qualname = Py_NewRef(func->func_qualname); - gen->gi_task = NULL; _PyObject_GC_TRACK(gen); return (PyObject *)gen; } @@ -995,7 +990,6 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, frame->owner = FRAME_OWNED_BY_GENERATOR; assert(PyObject_GC_IsTracked((PyObject *)f)); Py_DECREF(f); - gen->gi_task = NULL; gen->gi_weakreflist = NULL; gen->gi_exc_state.exc_value = NULL; gen->gi_exc_state.previous_item = NULL; @@ -1011,13 +1005,6 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, return (PyObject *)gen; } -void -_PyCoro_SetTask(PyObject *coro, PyObject *task) -{ - assert(PyCoro_CheckExact(coro)); - ((PyCoroObject *)coro)->cr_task = task; -} - PyObject * PyGen_NewWithQualName(PyFrameObject *f, PyObject *name, PyObject *qualname) { @@ -1403,8 +1390,6 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname) return NULL; } - ((PyCoroObject *)coro)->cr_task = NULL; - PyThreadState *tstate = _PyThreadState_GET(); int origin_depth = tstate->coroutine_origin_tracking_depth; diff --git a/Python/pystate.c b/Python/pystate.c index 7df872cd6d7d8a..59b684a361fb20 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1468,6 +1468,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->dict_global_version = 0; _tstate->asyncio_running_loop = NULL; + _tstate->asyncio_running_task = NULL; tstate->delete_later = NULL; @@ -1667,6 +1668,7 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(tstate->threading_local_sentinel); Py_CLEAR(((_PyThreadStateImpl *)tstate)->asyncio_running_loop); + Py_CLEAR(((_PyThreadStateImpl *)tstate)->asyncio_running_task); Py_CLEAR(tstate->dict); Py_CLEAR(tstate->async_exc); From 9f04911d416276b9cc4e6d4f9f4eb2b220a5c28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 13 Nov 2024 16:56:50 +0100 Subject: [PATCH 50/84] Fix refleaks --- Modules/_testexternalinspection.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index ebe92e64d9a22a..7481dac58df5d6 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -713,6 +713,7 @@ parse_coro_chain( if (PyList_Append(render_to, name)) { return -1; } + Py_DECREF(name); int gi_frame_state; err = read_int( @@ -886,6 +887,7 @@ parse_task( if (PyList_Append(render_to, result)) { goto err; } + Py_DECREF(result); PyObject *awaited_by = PyList_New(0); if (awaited_by == NULL) { @@ -1546,7 +1548,6 @@ get_async_stack_trace(PyObject* self, PyObject* args) Py_DECREF(awaited_by); goto result_err; } - Py_DECREF(awaited_by); if (parse_task_awaited_by( From 07748059f4ba983b3f65f2e4855c2461813f5566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 13 Nov 2024 17:17:31 +0100 Subject: [PATCH 51/84] Rename "asyncio.stack" to "asyncio.graph" --- .../{asyncio-stack.rst => asyncio-graph.rst} | 6 ++-- Doc/library/asyncio.rst | 2 +- Lib/asyncio/__init__.py | 4 +-- Lib/asyncio/{stack.py => graph.py} | 28 +++++++++---------- .../{test_stack.py => test_graph.py} | 0 5 files changed, 20 insertions(+), 20 deletions(-) rename Doc/library/{asyncio-stack.rst => asyncio-graph.rst} (96%) rename Lib/asyncio/{stack.py => graph.py} (90%) rename Lib/test/test_asyncio/{test_stack.py => test_graph.py} (100%) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-graph.rst similarity index 96% rename from Doc/library/asyncio-stack.rst rename to Doc/library/asyncio-graph.rst index fab72521bb9df8..55f431cb200f85 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-graph.rst @@ -1,17 +1,17 @@ .. currentmodule:: asyncio -.. _asyncio-stack: +.. _asyncio-graph: =================== Stack Introspection =================== -**Source code:** :source:`Lib/asyncio/stack.py` +**Source code:** :source:`Lib/asyncio/graph.py` ------------------------------------- -asyncio has powerful runtime call stack introspection utilities +asyncio has powerful runtime call graph introspection utilities to trace the entire call graph of a running coroutine or task, or a suspended *future*. These utilities and the underlying machinery can be used by users in their Python code or by external profilers diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 5098805f26cbd5..7d368dae49dc1d 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -99,7 +99,7 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst - asyncio-stack.rst + asyncio-graph.rst .. toctree:: :caption: Low-level APIs diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index b05c3fdbdf9641..6576070ed77a2b 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -14,7 +14,7 @@ from .protocols import * from .runners import * from .queues import * -from .stack import * +from .graph import * from .streams import * from .subprocess import * from .tasks import * @@ -32,7 +32,7 @@ protocols.__all__ + runners.__all__ + queues.__all__ + - stack.__all__ + + graph.__all__ + streams.__all__ + subprocess.__all__ + tasks.__all__ + diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/graph.py similarity index 90% rename from Lib/asyncio/stack.py rename to Lib/asyncio/graph.py index f68e80f55063af..557e4348634897 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/graph.py @@ -1,4 +1,4 @@ -"""Introspection utils for tasks call stacks.""" +"""Introspection utils for tasks call graphs.""" import dataclasses import sys @@ -18,7 +18,7 @@ # Sadly, we can't re-use the traceback's module datastructures as those # are tailored for error reporting, whereas we need to represent an -# async call stack. +# async call graph. # # Going with pretty verbose names as we'd like to export them to the # top level asyncio namespace, and want to avoid future name clashes. @@ -36,7 +36,7 @@ class FutureCallGraph: awaited_by: list[FutureCallGraph] -def _build_stack_for_future(future: futures.Future) -> FutureCallGraph: +def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: if not isinstance(future, futures.Future): raise TypeError( f"{future!r} object does not appear to be compatible " @@ -64,7 +64,7 @@ def _build_stack_for_future(future: futures.Future) -> FutureCallGraph: if future._asyncio_awaited_by: for parent in future._asyncio_awaited_by: - awaited_by.append(_build_stack_for_future(parent)) + awaited_by.append(_build_graph_for_future(parent)) st.reverse() return FutureCallGraph(future, st, awaited_by) @@ -75,9 +75,9 @@ def capture_call_graph( future: futures.Future | None = None, depth: int = 1, ) -> FutureCallGraph | None: - """Capture async call stack for the current task or the provided Future. + """Capture async call graph for the current task or the provided Future. - The stack is represented with three data structures: + The graph is represented with three data structures: * FutureCallGraph(future, call_stack, awaited_by) @@ -109,7 +109,7 @@ def capture_call_graph( # if yes - check if the passed future is the currently # running task or not. if loop is None or future is not tasks.current_task(loop=loop): - return _build_stack_for_future(future) + return _build_graph_for_future(future) # else: future is the current task, move on. else: if loop is None: @@ -151,7 +151,7 @@ def capture_call_graph( awaited_by = [] if future._asyncio_awaited_by: for parent in future._asyncio_awaited_by: - awaited_by.append(_build_stack_for_future(parent)) + awaited_by.append(_build_graph_for_future(parent)) return FutureCallGraph(future, call_stack, awaited_by) @@ -162,7 +162,7 @@ def print_call_graph( file: typing.TextIO | None = None, depth: int = 1, ) -> None: - """Print async call stack for the current task or the provided Future.""" + """Print async call graph for the current task or the provided Future.""" def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: def add_line(line: str) -> None: @@ -221,16 +221,16 @@ def add_line(line: str) -> None: for fut in st.awaited_by: render_level(fut, buf, level + 1) - stack = capture_call_graph(future=future, depth=depth + 1) - if stack is None: + graph = capture_call_graph(future=future, depth=depth + 1) + if graph is None: return try: buf: list[str] = [] - render_level(stack, buf, 0) + render_level(graph, buf, 0) rendered = '\n'.join(buf) print(rendered, file=file) finally: - # 'stack' has references to frames so we should + # 'graph' has references to frames so we should # make sure it's GC'ed as soon as we don't need it. - del stack + del graph diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_graph.py similarity index 100% rename from Lib/test/test_asyncio/test_stack.py rename to Lib/test/test_asyncio/test_graph.py From 3048493dfeaebd03281b0121c0573a3191e47986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 13 Nov 2024 17:24:23 +0100 Subject: [PATCH 52/84] Allow returning a string with `format_call_graph` --- Doc/library/asyncio-graph.rst | 14 ++------------ Lib/asyncio/graph.py | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 55f431cb200f85..f080ab0e3198aa 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -63,19 +63,9 @@ and debuggers. | File 'taskgroups.py', line 107, in async TaskGroup.__aexit__() | File 't2.py', line 7, in async main() - For rendering the call stack to a string the following pattern - should be used: - - .. code-block:: python - - import io - - ... - - buf = io.StringIO() - asyncio.print_call_graph(file=buf) - output = buf.getvalue() +.. function:: format_call_graph(*, future=None, depth=1) + Like :func:`print_call_graph`, but returns a string. .. function:: capture_call_graph(*, future=None) diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index 557e4348634897..fb613f034ba2d3 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -11,6 +11,7 @@ __all__ = ( 'capture_call_graph', + 'format_call_graph', 'print_call_graph', 'FrameCallGraphEntry', 'FutureCallGraph', @@ -156,13 +157,15 @@ def capture_call_graph( return FutureCallGraph(future, call_stack, awaited_by) -def print_call_graph( +def format_call_graph( *, future: futures.Future | None = None, - file: typing.TextIO | None = None, depth: int = 1, -) -> None: - """Print async call graph for the current task or the provided Future.""" +) -> str: + """Return async call graph as a string for `future`. + + If `future` is not provided, format the call graph for the current task. + """ def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: def add_line(line: str) -> None: @@ -228,9 +231,17 @@ def add_line(line: str) -> None: try: buf: list[str] = [] render_level(graph, buf, 0) - rendered = '\n'.join(buf) - print(rendered, file=file) + return '\n'.join(buf) finally: # 'graph' has references to frames so we should # make sure it's GC'ed as soon as we don't need it. del graph + +def print_call_graph( + *, + future: futures.Future | None = None, + file: typing.TextIO | None = None, + depth: int = 1, +) -> None: + """Print async call graph for the current task or the provided Future.""" + print(format_call_graph(future=future, depth=depth), file=file) From 1f42873bcb656f05816d155671e9c7dd84dd3968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 13 Nov 2024 17:34:08 +0100 Subject: [PATCH 53/84] Add test for eager task factory support --- Lib/test/test_external_inspection.py | 81 ++++++++++++++++------------ 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 9ef3ea86dd002f..720a6f5170c5ed 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -109,42 +109,55 @@ async def main(): tg.create_task(c1(task), name="sub_main_1") tg.create_task(c1(task), name="sub_main_2") - asyncio.run(main()) + def new_eager_loop(): + loop = asyncio.new_event_loop() + eager_task_factory = asyncio.create_eager_task_factory(asyncio.Task) + loop.set_task_factory(eager_task_factory) + return loop + + asyncio.run(main(), loop_factory={TASK_FACTORY}) """) stack_trace = None - with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - fifo = f"{work_dir}/the_fifo" - os.mkfifo(fifo) - script_name = _make_test_script(script_dir, 'script', script) - try: - p = subprocess.Popen([sys.executable, script_name, str(fifo)]) - with open(fifo, "r") as fifo_file: - response = fifo_file.read() - self.assertEqual(response, "ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") - finally: - os.remove(fifo) - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - - expected_stack_trace = [ - ["c5", "c4", "c3", "c2"], - "c2_root", - [ - [["main"], "Task-1", []], - [["c1"], "sub_main_1", [[["main"], "Task-1", []]]], - [["c1"], "sub_main_2", [[["main"], "Task-1", []]]], - ], - ] - self.assertEqual(stack_trace, expected_stack_trace) + for task_factory_variant in "asyncio.new_event_loop", "new_eager_loop": + with ( + self.subTest(task_factory_variant=task_factory_variant), + os_helper.temp_dir() as work_dir, + ): + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script.format(TASK_FACTORY=task_factory_variant)) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + root_task = "Task-1" + if task_factory_variant == "new_eager_loop": + root_task = "None" + expected_stack_trace = [ + ["c5", "c4", "c3", "c2"], + "c2_root", + [ + [["main"], root_task, []], + [["c1"], "sub_main_1", [[["main"], root_task, []]]], + [["c1"], "sub_main_2", [[["main"], root_task, []]]], + ], + ] + self.assertEqual(stack_trace, expected_stack_trace) @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") From 03ed5c188e9a15a709622909b2c2d75edc586543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 13 Nov 2024 17:59:51 +0100 Subject: [PATCH 54/84] Make _asyncio_awaited_by a frozenset in the Python version as well per Kumar's wishes --- Lib/asyncio/futures.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 8bd47a90c7a83a..71fd283acfa563 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -68,7 +68,7 @@ class Future: _asyncio_future_blocking = False # Used by the capture_call_stack() API. - _asyncio_awaited_by = None + __asyncio_awaited_by = None __log_traceback = False @@ -119,6 +119,12 @@ def _log_traceback(self, val): raise ValueError('_log_traceback can only be set to False') self.__log_traceback = False + @property + def _asyncio_awaited_by(self): + if self.__asyncio_awaited_by is None: + return None + return frozenset(self.__asyncio_awaited_by) + def get_loop(self): """Return the event loop the Future is bound to.""" loop = self._loop @@ -442,9 +448,9 @@ def future_add_to_awaited_by(fut, waiter, /): # Note that there's an accelerated version of this function # shadowing this implementation later in this file. if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): - if fut._asyncio_awaited_by is None: - fut._asyncio_awaited_by = set() - fut._asyncio_awaited_by.add(waiter) + if fut._Future__asyncio_awaited_by is None: + fut._Future__asyncio_awaited_by = set() + fut._Future__asyncio_awaited_by.add(waiter) def future_discard_from_awaited_by(fut, waiter, /): @@ -455,8 +461,8 @@ def future_discard_from_awaited_by(fut, waiter, /): # Note that there's an accelerated version of this function # shadowing this implementation later in this file. if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): - if fut._asyncio_awaited_by is not None: - fut._asyncio_awaited_by.discard(waiter) + if fut._Future__asyncio_awaited_by is not None: + fut._Future__asyncio_awaited_by.discard(waiter) try: From 21f9ea91eb95b856ac05bb3e6189b0d53368e458 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 22 Nov 2024 17:01:59 -0800 Subject: [PATCH 55/84] Address picnixz' feedback --- Doc/library/asyncio-graph.rst | 28 ++++++++++++------------- Doc/library/inspect.rst | 2 +- Include/internal/pycore_debug_offsets.h | 2 +- Lib/asyncio/__init__.py | 4 ++-- Objects/frameobject.c | 3 +-- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index f080ab0e3198aa..4ed5b2609b8b4d 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -17,7 +17,7 @@ a suspended *future*. These utilities and the underlying machinery can be used by users in their Python code or by external profilers and debuggers. -.. versionadded:: 3.14 +.. versionadded:: next .. function:: print_call_graph(*, future=None, file=None, depth=1) @@ -44,11 +44,11 @@ and debuggers. import asyncio async def test(): - asyncio.print_call_graph() + asyncio.print_call_graph() async def main(): - async with asyncio.TaskGroup() as g: - g.create_task(test()) + async with asyncio.TaskGroup() as g: + g.create_task(test()) asyncio.run(main()) @@ -77,15 +77,15 @@ and debuggers. current task, the function returns ``None``. If the function is called on *the current task*, the optional - keyword-only ``depth`` argument can be used to skip the specified + keyword-only *depth* argument can be used to skip the specified number of frames from top of the stack. Returns a ``FutureCallGraph`` data class object: * ``FutureCallGraph(future, call_stack, awaited_by)`` - Where 'future' is a reference to a *Future* or a *Task* - (or their subclasses.) + Where *future* is a reference to a :class:`Future` or + a :class:`Task` (or their subclasses.) ``call_stack`` is a list of ``FrameCallGraphEntry`` objects. @@ -93,7 +93,7 @@ and debuggers. * ``FrameCallGraphEntry(frame)`` - Where ``frame`` is a frame object of a regular Python function + Where *frame* is a frame object of a regular Python function in the call stack. @@ -102,7 +102,7 @@ Low level utility functions To introspect an async call graph asyncio requires cooperation from control flow structures, such as :func:`shield` or :class:`TaskGroup`. -Any time an intermediate ``Future`` object with low-level APIs like +Any time an intermediate :class:`Future` object with low-level APIs like :meth:`Future.add_done_callback() ` is involved, the following two functions should be used to inform *asyncio* about how exactly such intermediate future objects are connected with @@ -114,11 +114,11 @@ the tasks they wrap or control. Record that *future* is awaited on by *waiter*. Both *future* and *waiter* must be instances of - :class:`asyncio.Future ` or :class:`asyncio.Task ` or - their subclasses, otherwise the call would have no effect. + :class:`Future` or :class:`Task` or their subclasses, + otherwise the call would have no effect. A call to ``future_add_to_awaited_by()`` must be followed by an - eventual call to the ``future_discard_from_awaited_by()`` function + eventual call to the :func:`future_discard_from_awaited_by` function with the same arguments. @@ -127,5 +127,5 @@ the tasks they wrap or control. Record that *future* is no longer awaited on by *waiter*. Both *future* and *waiter* must be instances of - :class:`asyncio.Future ` or :class:`asyncio.Task ` or - their subclasses, otherwise the call would have no effect. + :class:`Future` or :class:`Task` or their subclasses, otherwise + the call would have no effect. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 91c740f433decc..0902d64f9bd22a 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -316,7 +316,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Add ``__builtins__`` attribute to functions. -.. versionchanged:: 3.14 +.. versionchanged:: next Add ``f_generator`` attribute to frames. diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 38d0cd57ab2c9b..34debf35d14df4 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -16,7 +16,7 @@ extern "C" { #endif // Macros to burn global values in custom sections so out-of-process -// profilers can locate them easily +// profilers can locate them easily. #define GENERATE_DEBUG_SECTION(name, declaration) \ _GENERATE_DEBUG_SECTION_WINDOWS(name) \ diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 6576070ed77a2b..2432e2dad74c23 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -10,11 +10,11 @@ from .events import * from .exceptions import * from .futures import * +from .graph import * from .locks import * from .protocols import * from .runners import * from .queues import * -from .graph import * from .streams import * from .subprocess import * from .tasks import * @@ -28,11 +28,11 @@ events.__all__ + exceptions.__all__ + futures.__all__ + + graph.__all__ + locks.__all__ + protocols.__all__ + runners.__all__ + queues.__all__ + - graph.__all__ + streams.__all__ + subprocess.__all__ + tasks.__all__ + diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 92f3aea5d45472..da6a9b453c7028 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1683,8 +1683,7 @@ static PyObject * frame_getgenerator(PyFrameObject *f, void *arg) { if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { PyObject *gen = (PyObject *)_PyGen_GetGeneratorFromFrame(f->f_frame); - Py_INCREF(gen); - return gen; + return Py_NewRef(gen); } Py_RETURN_NONE; } From 8a43dfa0b6e84649fd4a9ca1d9d3b895a4ac4ce2 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 22 Nov 2024 20:22:03 -0800 Subject: [PATCH 56/84] Blacken the C/Py test code by hand, by request from picnixz --- Lib/test/test_external_inspection.py | 58 +++++++--- Modules/_testexternalinspection.c | 159 ++++++++++++++++++--------- 2 files changed, 148 insertions(+), 69 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 720a6f5170c5ed..1c54599de6ef27 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -15,7 +15,8 @@ from _testexternalinspection import get_stack_trace from _testexternalinspection import get_async_stack_trace except ImportError: - raise unittest.SkipTest("Test only runs when _testexternalinspection is available") + raise unittest.SkipTest( + "Test only runs when _testexternalinspection is available") def _make_test_script(script_dir, script_basename, source): to_return = make_script(script_dir, script_basename, source) @@ -24,8 +25,10 @@ def _make_test_script(script_dir, script_basename, source): class TestGetStackTrace(unittest.TestCase): - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ @@ -75,8 +78,10 @@ def foo(): ] self.assertEqual(stack_trace, expected_stack_trace) - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_async_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ @@ -111,7 +116,8 @@ async def main(): def new_eager_loop(): loop = asyncio.new_event_loop() - eager_task_factory = asyncio.create_eager_task_factory(asyncio.Task) + eager_task_factory = asyncio.create_eager_task_factory( + asyncio.Task) loop.set_task_factory(eager_task_factory) return loop @@ -127,15 +133,20 @@ def new_eager_loop(): os.mkdir(script_dir) fifo = f"{work_dir}/the_fifo" os.mkfifo(fifo) - script_name = _make_test_script(script_dir, 'script', script.format(TASK_FACTORY=task_factory_variant)) + script_name = _make_test_script( + script_dir, 'script', + script.format(TASK_FACTORY=task_factory_variant)) try: - p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + p = subprocess.Popen( + [sys.executable, script_name, str(fifo)] + ) with open(fifo, "r") as fifo_file: response = fifo_file.read() self.assertEqual(response, "ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace") finally: os.remove(fifo) p.kill() @@ -159,8 +170,10 @@ def new_eager_loop(): ] self.assertEqual(stack_trace, expected_stack_trace) - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_asyncgen_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ @@ -210,11 +223,15 @@ async def main(): # sets are unordered, so we want to sort "awaited_by"s stack_trace[2].sort(key=lambda x: x[1]) - expected_stack_trace = [['gen_nested_call', 'gen', 'main'], 'Task-1', []] + expected_stack_trace = [ + ['gen_nested_call', 'gen', 'main'], 'Task-1', [] + ] self.assertEqual(stack_trace, expected_stack_trace) - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_async_gather_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ @@ -255,7 +272,8 @@ async def main(): self.assertEqual(response, "ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace") finally: os.remove(fifo) p.kill() @@ -265,11 +283,15 @@ async def main(): # sets are unordered, so we want to sort "awaited_by"s stack_trace[2].sort(key=lambda x: x[1]) - expected_stack_trace = [['deep', 'c1'], 'Task-2', [[['main'], 'Task-1', []]]] + expected_stack_trace = [ + ['deep', 'c1'], 'Task-2', [[['main'], 'Task-1', []]] + ] self.assertEqual(stack_trace, expected_stack_trace) - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_self_trace(self): stack_trace = get_stack_trace(os.getpid()) self.assertEqual(stack_trace[0], "test_self_trace") diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 7481dac58df5d6..82b0f95a416b3c 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -77,8 +77,12 @@ struct _Py_AsyncioModuleDebugOffsets { #if defined(__APPLE__) && TARGET_OS_OSX static uintptr_t -return_section_address(const char* section, mach_port_t proc_ref, uintptr_t base, void* map) -{ +return_section_address( + const char* section, + mach_port_t proc_ref, + uintptr_t base, + void* map +) { struct mach_header_64* hdr = (struct mach_header_64*)map; int ncmds = hdr->ncmds; @@ -88,7 +92,7 @@ return_section_address(const char* section, mach_port_t proc_ref, uintptr_t base mach_vm_size_t size = 0; mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); mach_vm_address_t address = (mach_vm_address_t)base; - vm_region_basic_info_data_64_t region_info; + vm_region_basic_info_data_64_t r_info; mach_port_t object_name; uintptr_t vmaddr = 0; @@ -99,24 +103,26 @@ return_section_address(const char* section, mach_port_t proc_ref, uintptr_t base if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { while (cmd->filesize != size) { address += size; - if (mach_vm_region( - proc_ref, - &address, - &size, - VM_REGION_BASIC_INFO_64, - (vm_region_info_t)®ion_info, // cppcheck-suppress [uninitvar] - &count, - &object_name) - != KERN_SUCCESS) - { - PyErr_SetString(PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + kern_return_t ret = mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)&r_info, // cppcheck-suppress [uninitvar] + &count, + &object_name + ); + if (ret != KERN_SUCCESS) { + PyErr_SetString( + PyExc_RuntimeError, "Cannot get any more VM maps.\n"); return 0; } } int nsects = cmd->nsects; - struct section_64* sec = - (struct section_64*)((void*)cmd + sizeof(struct segment_command_64)); + struct section_64* sec = (struct section_64*)( + (void*)cmd + sizeof(struct segment_command_64) + ); for (int j = 0; j < nsects; j++) { if (strcmp(sec[j].sectname, section) == 0) { return base + sec[j].addr - vmaddr; @@ -131,8 +137,13 @@ return_section_address(const char* section, mach_port_t proc_ref, uintptr_t base } static uintptr_t -search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_size_t size, mach_port_t proc_ref) -{ +search_section_in_file( + const char* secname, + char* path, + uintptr_t base, + mach_vm_size_t size, + mach_port_t proc_ref +) { int fd = open(path, O_RDONLY); if (fd == -1) { PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); @@ -141,7 +152,8 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ struct stat fs; if (fstat(fd, &fs) == -1) { - PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); + PyErr_Format( + PyExc_RuntimeError, "Cannot get size of binary %s\n", path); close(fd); return 0; } @@ -161,7 +173,9 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ case MH_CIGAM: case FAT_MAGIC: case FAT_CIGAM: - PyErr_SetString(PyExc_RuntimeError, "32-bit Mach-O binaries are not supported"); + PyErr_SetString( + PyExc_RuntimeError, + "32-bit Mach-O binaries are not supported"); break; case MH_MAGIC_64: case MH_CIGAM_64: @@ -216,10 +230,10 @@ search_map_for_section(pid_t pid, const char* secname, const char* substr) { VM_REGION_BASIC_INFO_64, (vm_region_info_t)®ion_info, &count, - &object_name) - == KERN_SUCCESS) + &object_name) == KERN_SUCCESS) { - int path_len = proc_regionfilename(pid, address, map_filename, MAXPATHLEN); + int path_len = proc_regionfilename( + pid, address, map_filename, MAXPATHLEN); if (path_len == 0) { address += size; continue; @@ -240,7 +254,8 @@ search_map_for_section(pid_t pid, const char* secname, const char* substr) { if (!match_found && strncmp(filename, substr, strlen(substr)) == 0) { match_found = 1; - return search_section_in_file(secname, map_filename, address, size, proc_ref); + return search_section_in_file( + secname, map_filename, address, size, proc_ref); } address += size; @@ -270,7 +285,10 @@ find_map_start_address(pid_t pid, char* result_filename, const char* map) uintptr_t result_address = 0; while (fgets(line, sizeof(line), maps_file) != NULL) { unsigned long start_address = 0; - sscanf(line, "%lx-%*x %*s %*s %*s %*s %s", &start_address, map_filename); + sscanf( + line, "%lx-%*x %*s %*s %*s %*s %s", + &start_address, map_filename + ); char* filename = strrchr(map_filename, '/'); if (filename != NULL) { filename++; // Move past the '/' @@ -328,7 +346,8 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory; - Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff); + Elf_Shdr* section_header_table = + (Elf_Shdr*)(file_memory + elf_header->e_shoff); Elf_Shdr* shstrtab_section = §ion_header_table[elf_header->e_shstrndx]; char* shstrtab = (char*)(file_memory + shstrtab_section->sh_offset); @@ -344,7 +363,9 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) } } - Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff); + Elf_Phdr* program_header_table = + (Elf_Phdr*)(file_memory + elf_header->e_phoff); + // Find the first PT_LOAD segment Elf_Phdr* first_load_segment = NULL; for (int i = 0; i < elf_header->e_phnum; i++) { @@ -355,8 +376,10 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) } if (section != NULL && first_load_segment != NULL) { - uintptr_t elf_load_addr = first_load_segment->p_vaddr - - (first_load_segment->p_vaddr % first_load_segment->p_align); + uintptr_t elf_load_addr = + first_load_segment->p_vaddr - ( + first_load_segment->p_vaddr % first_load_segment->p_align + ); result = start_address + (uintptr_t)section->sh_addr - elf_load_addr; } @@ -426,13 +449,19 @@ read_memory(pid_t pid, uintptr_t remote_address, size_t len, void* dst) if (kr != KERN_SUCCESS) { switch (kr) { case KERN_PROTECTION_FAILURE: - PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory"); + PyErr_SetString( + PyExc_PermissionError, + "Not enough permissions to read memory"); break; case KERN_INVALID_ARGUMENT: - PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite"); + PyErr_SetString( + PyExc_PermissionError, + "Invalid argument to mach_vm_read_overwrite"); break; default: - PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory"); + PyErr_SetString( + PyExc_RuntimeError, + "Unknown error reading memory"); } return -1; } @@ -444,8 +473,13 @@ read_memory(pid_t pid, uintptr_t remote_address, size_t len, void* dst) } static int -read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, uintptr_t address, char* buffer, Py_ssize_t size) -{ +read_string( + pid_t pid, + _Py_DebugOffsets* debug_offsets, + uintptr_t address, + char* buffer, + Py_ssize_t size +) { Py_ssize_t len; ssize_t bytes_read = read_memory( pid, @@ -565,18 +599,21 @@ read_py_str( } static long -read_py_long( - pid_t pid, - _Py_DebugOffsets* offsets, - uintptr_t address) { +read_py_long(pid_t pid, _Py_DebugOffsets* offsets, uintptr_t address) +{ unsigned int shift = PYLONG_BITS_IN_DIGIT; ssize_t size; uintptr_t lv_tag; - int bytes_read = read_memory(pid, address + offsets->long_object.lv_tag, sizeof(uintptr_t), &lv_tag); + + int bytes_read = read_memory( + pid, address + offsets->long_object.lv_tag, + sizeof(uintptr_t), + &lv_tag); if (bytes_read == -1) { return -1; } + int negative = (lv_tag & 3) == 2; size = lv_tag >> 3; @@ -586,9 +623,16 @@ read_py_long( char *digits = (char *)PyMem_RawMalloc(size * sizeof(digit)); if (!digits) { + PyErr_NoMemory(); return -1; } - bytes_read = read_memory(pid, address + offsets->long_object.ob_digit, sizeof(digit) * size, digits); + + bytes_read = read_memory( + pid, + address + offsets->long_object.ob_digit, + sizeof(digit) * size, + digits + ); if (bytes_read < 0) { goto error; } @@ -597,7 +641,9 @@ read_py_long( for (ssize_t i = 0; i < size; ++i) { long long factor; - if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), &factor)) { + if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), + &factor) + ) { goto error; } if (__builtin_add_overflow(value, factor, &value)) { @@ -847,7 +893,8 @@ parse_task( Py_DECREF(call_stack); if (is_task) { - PyObject *tn = parse_task_name(pid, offsets, async_offsets, task_address); + PyObject *tn = parse_task_name( + pid, offsets, async_offsets, task_address); if (tn == NULL) { goto err; } @@ -900,7 +947,9 @@ parse_task( /* we can operate on a borrowed one to simplify cleanup */ Py_DECREF(awaited_by); - if (parse_task_awaited_by(pid, offsets, async_offsets, task_address, awaited_by)) { + if (parse_task_awaited_by(pid, offsets, async_offsets, + task_address, awaited_by) + ) { goto err; } @@ -1372,8 +1421,11 @@ find_running_task( static PyObject* get_stack_trace(PyObject* self, PyObject* args) { -#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); +#if (!defined(__linux__) && !defined(__APPLE__)) || \ + (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString( + PyExc_RuntimeError, + "get_stack_trace is not supported on this platform"); return NULL; #endif int pid; @@ -1422,8 +1474,11 @@ get_stack_trace(PyObject* self, PyObject* args) static PyObject* get_async_stack_trace(PyObject* self, PyObject* args) { -#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); +#if (!defined(__linux__) && !defined(__APPLE__)) || \ + (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString( + PyExc_RuntimeError, + "get_stack_trace is not supported on this platform"); return NULL; #endif int pid; @@ -1485,8 +1540,8 @@ get_async_stack_trace(PyObject* self, PyObject* args) goto result_err; } - // note: genobject's gi_iframe is an embedded struct so the address to the offset - // leads directly to its first field: f_executable + // note: genobject's gi_iframe is an embedded struct so the address to + // the offset leads directly to its first field: f_executable uintptr_t address_of_running_task_code_obj; if (read_py_ptr( pid, @@ -1551,7 +1606,8 @@ get_async_stack_trace(PyObject* self, PyObject* args) Py_DECREF(awaited_by); if (parse_task_awaited_by( - pid, &local_debug_offsets, &local_async_debug, running_task_addr, awaited_by) + pid, &local_debug_offsets, &local_async_debug, + running_task_addr, awaited_by) ) { goto result_err; } @@ -1589,7 +1645,8 @@ PyInit__testexternalinspection(void) #ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); #endif - int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + int rc = PyModule_AddIntConstant( + mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); if (rc < 0) { Py_DECREF(mod); return NULL; From b3fae687af7a594f1166ea5892d5ec4ab495f7dd Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 22 Nov 2024 20:33:21 -0800 Subject: [PATCH 57/84] More style fixes per picnixz' suggestions --- Modules/_testexternalinspection.c | 47 +++++++++++++++++-------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 82b0f95a416b3c..d0fb3ca2736c6d 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -98,7 +98,7 @@ return_section_address( for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__TEXT") == 0) { - vmaddr = cmd->vmaddr; + vmaddr = cmd->vmaddr; } if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { while (cmd->filesize != size) { @@ -354,8 +354,11 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) Elf_Shdr* section = NULL; for (int i = 0; i < elf_header->e_shnum; i++) { - char* this_sec_name = shstrtab + section_header_table[i].sh_name; - // Move 1 character to account for the leading "." + const char* this_sec_name = ( + shstrtab + + section_header_table[i].sh_name + + 1 // "+1" accounts for the leading "." + ); this_sec_name += 1; if (strcmp(secname, this_sec_name) == 0) { section = §ion_header_table[i]; @@ -487,7 +490,7 @@ read_string( sizeof(Py_ssize_t), &len ); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } if (len >= size) { @@ -496,7 +499,7 @@ read_string( } size_t offset = debug_offsets->unicode_object.asciiobject_size; bytes_read = read_memory(pid, address + offset, len, buffer); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } buffer[len] = '\0'; @@ -508,7 +511,7 @@ static inline int read_ptr(pid_t pid, uintptr_t address, uintptr_t *ptr_addr) { int bytes_read = read_memory(pid, address, sizeof(void*), ptr_addr); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } return 0; @@ -518,7 +521,7 @@ static inline int read_ssize_t(pid_t pid, uintptr_t address, Py_ssize_t *size) { int bytes_read = read_memory(pid, address, sizeof(Py_ssize_t), size); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } return 0; @@ -610,7 +613,7 @@ read_py_long(pid_t pid, _Py_DebugOffsets* offsets, uintptr_t address) pid, address + offsets->long_object.lv_tag, sizeof(uintptr_t), &lv_tag); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -641,7 +644,7 @@ read_py_long(pid_t pid, _Py_DebugOffsets* offsets, uintptr_t address) for (ssize_t i = 0; i < size; ++i) { long long factor; - if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), + if (__builtin_mul_overflow(digits[i], (1UL << (ssize_t)(shift * i)), &factor) ) { goto error; @@ -652,7 +655,7 @@ read_py_long(pid_t pid, _Py_DebugOffsets* offsets, uintptr_t address) } PyMem_RawFree(digits); if (negative) { - value = -1 * value; + value *= -1; } return value; error: @@ -883,7 +886,7 @@ parse_task( PyObject *call_stack = PyList_New(0); if (call_stack == NULL) { - return -1; + goto err; } if (PyList_Append(result, call_stack)) { Py_DECREF(call_stack); @@ -1124,7 +1127,7 @@ parse_code_object( sizeof(void*), &address_of_function_name ); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1164,7 +1167,7 @@ parse_frame_object( sizeof(void*), previous_frame ); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1212,7 +1215,7 @@ parse_async_frame_object( sizeof(void*), previous_frame ); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1242,6 +1245,7 @@ parse_async_frame_object( return -1; } + assert(code_object != NULL); if ((void*)*code_object == NULL) { return 0; } @@ -1261,7 +1265,8 @@ read_offsets( _Py_DebugOffsets* debug_offsets ) { *runtime_start_address = get_py_runtime(pid); - if (!*runtime_start_address) { + assert(runtime_start_address != NULL); + if ((void*)*runtime_start_address == NULL) { if (!PyErr_Occurred()) { PyErr_SetString( PyExc_RuntimeError, "Failed to get .PyRuntime address"); @@ -1271,7 +1276,7 @@ read_offsets( size_t size = sizeof(struct _Py_DebugOffsets); ssize_t bytes_read = read_memory( pid, *runtime_start_address, size, debug_offsets); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } return 0; @@ -1289,7 +1294,7 @@ read_async_debug( size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); ssize_t bytes_read = read_memory( pid, async_debug_addr, size, async_debug); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } return 0; @@ -1311,7 +1316,7 @@ find_running_frame( runtime_start_address + interpreter_state_list_head, sizeof(void*), &address_of_interpreter_state); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1327,7 +1332,7 @@ find_running_frame( local_debug_offsets->interpreter_state.threads_head, sizeof(void*), &address_of_thread); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1366,7 +1371,7 @@ find_running_task( runtime_start_address + interpreter_state_list_head, sizeof(void*), &address_of_interpreter_state); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1382,7 +1387,7 @@ find_running_task( local_debug_offsets->interpreter_state.threads_head, sizeof(void*), &address_of_thread); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } From d0aedf084d578df55050553752a868099539e585 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 22 Nov 2024 20:53:57 -0800 Subject: [PATCH 58/84] Address Kumar's latest comment --- Modules/_asynciomodule.c | 53 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 985b10c3e0eb8b..aab57603a6a541 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2136,9 +2136,49 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) return -1; } + assert(task == item); + Py_CLEAR(item); + + // This block is needed to enable `asyncio.capture_call_graph()` API. + // We want to be enable debuggers and profilers to be able to quickly + // introspect the asyncio running state from another process. + // When we do that, we need to essentially traverse the address space + // of a Python process and understand what every Python thread in it is + // currently doing, mainly: + // + // * current frame + // * current asyncio task + // + // A naive solution would be to require profilers and debuggers to + // find the current task in the "_asynciomodule" module state, but + // unfortunately that would require a lot of complicated remote + // memory reads and logic, as Python's dict is a notoriously complex + // and ever-changing data structure. + // + // So the actual solution is to put a reference to the currently + // running asyncio Task to the interpreter thread state (we already + // have some asyncio state there.) _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); - assert(ts->asyncio_running_task == NULL); - ts->asyncio_running_task = item; // strong ref + if (ts->asyncio_running_loop == loop) { + // Protect from a situation when someone calls this method + // from another thread. This shouldn't ever happen though, + // as `enter_task` and `leave_task` can either be called by: + // + // - `asyncio.Task` itself, in `Task.__step()`. That method + // can only be called by the event loop itself. + // + // - third-party Task "from scratch" implementations, that + // our `capture_call_graph` API doesn't support anyway. + // + // That said, we still want to make sure we don't end up in + // a broken state, so we check that we're in the correct thread + // by comparing the *loop* argument to the event loop set + // in the current thread. If they match we know we're in the + // right thread, as asyncio event loops don't change threads. + assert(ts->asyncio_running_task == NULL); + ts->asyncio_running_task = Py_NewRef(task); + } + return 0; } @@ -2171,9 +2211,14 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) return err_leave_task(Py_None, task); } + // See the comment in `enter_task` for the explanation of why + // the following is needed. _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); - Py_CLEAR(ts->asyncio_running_task); - return res; + if (ts->asyncio_running_loop == loop) { + Py_CLEAR(ts->asyncio_running_task); + } + + return 0; } static PyObject * From df0032a09e6a8e31dea9871901757b904a3188e7 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 12:45:21 +0530 Subject: [PATCH 59/84] diff cleanup --- Include/internal/pycore_runtime.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 7a4f94152c8f22..2f2cec22cf1589 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -27,6 +27,7 @@ extern "C" { #include "pycore_typeobject.h" // struct _types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state + /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. From 0ce241bc29927529dc0d66a6458aa43ccabd55dc Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 12:59:52 +0530 Subject: [PATCH 60/84] change dataclasses to use tuples --- Lib/asyncio/graph.py | 27 +++++++++++++++------------ Lib/test/test_asyncio/test_graph.py | 4 ++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index fb613f034ba2d3..fc84531818ddfa 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -33,8 +33,8 @@ class FrameCallGraphEntry: @dataclasses.dataclass(frozen=True, slots=True) class FutureCallGraph: future: futures.Future - call_stack: list[FrameCallGraphEntry] - awaited_by: list[FutureCallGraph] + call_stack: tuple["FrameCallGraphEntry", ...] + awaited_by: tuple["FutureCallGraph", ...] def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: @@ -68,12 +68,13 @@ def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: awaited_by.append(_build_graph_for_future(parent)) st.reverse() - return FutureCallGraph(future, st, awaited_by) + return FutureCallGraph(future, tuple(st), tuple(awaited_by)) def capture_call_graph( - *, future: futures.Future | None = None, + /, + *, depth: int = 1, ) -> FutureCallGraph | None: """Capture async call graph for the current task or the provided Future. @@ -85,16 +86,16 @@ def capture_call_graph( Where 'future' is a reference to an asyncio.Future or asyncio.Task (or their subclasses.) - 'call_stack' is a list of FrameGraphEntry objects. + 'call_stack' is a tuple of FrameGraphEntry objects. - 'awaited_by' is a list of FutureCallGraph objects. + 'awaited_by' is a tuple of FutureCallGraph objects. * FrameCallGraphEntry(frame) Where 'frame' is a frame object of a regular Python function in the call stack. - Receives an optional keyword-only "future" argument. If not passed, + Receives an optional "future" argument. If not passed, the current task will be used. If there's no current task, the function returns None. @@ -154,12 +155,13 @@ def capture_call_graph( for parent in future._asyncio_awaited_by: awaited_by.append(_build_graph_for_future(parent)) - return FutureCallGraph(future, call_stack, awaited_by) + return FutureCallGraph(future, tuple(call_stack), tuple(awaited_by)) def format_call_graph( - *, future: futures.Future | None = None, + /, + *, depth: int = 1, ) -> str: """Return async call graph as a string for `future`. @@ -224,7 +226,7 @@ def add_line(line: str) -> None: for fut in st.awaited_by: render_level(fut, buf, level + 1) - graph = capture_call_graph(future=future, depth=depth + 1) + graph = capture_call_graph(future, depth=depth + 1) if graph is None: return @@ -238,10 +240,11 @@ def add_line(line: str) -> None: del graph def print_call_graph( - *, future: futures.Future | None = None, + /, + *, file: typing.TextIO | None = None, depth: int = 1, ) -> None: """Print async call graph for the current task or the provided Future.""" - print(format_call_graph(future=future, depth=depth), file=file) + print(format_call_graph(future, depth=depth), file=file) diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 9536451e0efa85..25046e4cc2c292 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -39,9 +39,9 @@ def walk(s): return ret buf = io.StringIO() - asyncio.print_call_graph(future=fut, file=buf, depth=depth+1) + asyncio.print_call_graph(fut, file=buf, depth=depth+1) - stack = asyncio.capture_call_graph(future=fut, depth=depth) + stack = asyncio.capture_call_graph(fut, depth=depth) return walk(stack), buf.getvalue() From 8f126f67ec1d21a8f4efed32b51ccccd04cb4a7c Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:06:06 +0530 Subject: [PATCH 61/84] doc fixes --- Doc/library/asyncio-graph.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 4ed5b2609b8b4d..2da58d2b3353eb 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -12,7 +12,7 @@ Stack Introspection ------------------------------------- asyncio has powerful runtime call graph introspection utilities -to trace the entire call graph of a running coroutine or task, or +to trace the entire call graph of a running *coroutine* or *task*, or a suspended *future*. These utilities and the underlying machinery can be used by users in their Python code or by external profilers and debuggers. @@ -20,12 +20,12 @@ and debuggers. .. versionadded:: next -.. function:: print_call_graph(*, future=None, file=None, depth=1) +.. function:: print_call_graph(future=None, /, *, file=None, depth=1) Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. - The function receives an optional keyword-only *future* argument. + The function receives an optional *future* argument. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. @@ -33,7 +33,7 @@ and debuggers. keyword-only *depth* argument can be used to skip the specified number of frames from top of the stack. - If *file* is not specified the function will print to :data:`sys.stdout`. + If *file* is omitted or ``None``, the function will print to :data:`sys.stdout`. **Example:** @@ -63,16 +63,16 @@ and debuggers. | File 'taskgroups.py', line 107, in async TaskGroup.__aexit__() | File 't2.py', line 7, in async main() -.. function:: format_call_graph(*, future=None, depth=1) +.. function:: format_call_graph(future=None, /, *, depth=1) Like :func:`print_call_graph`, but returns a string. -.. function:: capture_call_graph(*, future=None) +.. function:: capture_call_graph(future=None, /, *, depth=1) Capture the async call graph for the current task or the provided :class:`Task` or :class:`Future`. - The function receives an optional keyword-only *future* argument. + The function receives an optional *future* argument. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. @@ -87,9 +87,9 @@ and debuggers. Where *future* is a reference to a :class:`Future` or a :class:`Task` (or their subclasses.) - ``call_stack`` is a list of ``FrameCallGraphEntry`` objects. + ``call_stack`` is a tuple of ``FrameCallGraphEntry`` objects. - ``awaited_by`` is a list of ``FutureCallGraph`` objects. + ``awaited_by`` is a tuple of ``FutureCallGraph`` objects. * ``FrameCallGraphEntry(frame)`` From 966d84e75a5ec7fae16af718fb8997ee8218de5e Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:08:13 +0530 Subject: [PATCH 62/84] remove reduntant headers include and add my name to whatsnew --- Doc/whatsnew/3.14.rst | 4 ++-- Include/internal/pycore_runtime_init.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 594ac367a6c989..9af068c60c68db 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -541,8 +541,8 @@ asyncio * :mod:`asyncio` has new utility functions for introspecting and printing the program's call graph: :func:`asyncio.capture_call_graph` and :func:`asyncio.print_call_graph`. - (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa - in :gh:`91048`.) + (Contributed by Yury Selivanov, Pablo Galindo Salgado, Łukasz Langa + and Kumar Aditya in :gh:`91048`.) io --- diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index e4d9fa2e987eaa..8a8f47695fb8b0 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -23,7 +23,6 @@ extern "C" { #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT -#include "pycore_genobject.h" extern PyTypeObject _PyExc_MemoryError; From f56468af8b3789d0e20be30715c20ac1bb367641 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:19:14 +0530 Subject: [PATCH 63/84] improve comment --- Modules/_asynciomodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index aab57603a6a541..362d55ad97a4bb 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2155,8 +2155,8 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) // memory reads and logic, as Python's dict is a notoriously complex // and ever-changing data structure. // - // So the actual solution is to put a reference to the currently - // running asyncio Task to the interpreter thread state (we already + // So the easier solution is to put a strong reference of the currently + // running `asyncio.Task` to the interpreter thread state (we already // have some asyncio state there.) _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); if (ts->asyncio_running_loop == loop) { @@ -2172,7 +2172,7 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) // // That said, we still want to make sure we don't end up in // a broken state, so we check that we're in the correct thread - // by comparing the *loop* argument to the event loop set + // by comparing the *loop* argument to running event loop // in the current thread. If they match we know we're in the // right thread, as asyncio event loops don't change threads. assert(ts->asyncio_running_task == NULL); From 404b88aeb4c444dd77f3819c37a83d8396d1f07f Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:38:27 +0530 Subject: [PATCH 64/84] fix leave task --- Modules/_asynciomodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 362d55ad97a4bb..e227fd6dec76f1 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2218,7 +2218,7 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) Py_CLEAR(ts->asyncio_running_task); } - return 0; + return res; } static PyObject * From 911fed869bb7bd353c934cb48ef76d00499c3741 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:58:15 +0530 Subject: [PATCH 65/84] fix external inspection on linux --- Modules/_testexternalinspection.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index d0fb3ca2736c6d..b696d70716d592 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -359,7 +359,7 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) section_header_table[i].sh_name + 1 // "+1" accounts for the leading "." ); - this_sec_name += 1; + if (strcmp(secname, this_sec_name) == 0) { section = §ion_header_table[i]; break; From ab511a4f67ffd88f7709c1bc1990074e6e5b555d Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:59:10 +0530 Subject: [PATCH 66/84] minor format --- Modules/_asynciomodule.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index e227fd6dec76f1..287550f4b1662c 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -103,19 +103,19 @@ typedef struct { #endif typedef struct _Py_AsyncioModuleDebugOffsets { - struct _asyncio_task_object { - uint64_t size; - uint64_t task_name; - uint64_t task_awaited_by; - uint64_t task_is_task; - uint64_t task_awaited_by_is_set; - uint64_t task_coro; - } asyncio_task_object; - struct _asyncio_thread_state { - uint64_t size; - uint64_t asyncio_running_loop; - uint64_t asyncio_running_task; - } asyncio_thread_state; + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + } asyncio_thread_state; } Py_AsyncioModuleDebugOffsets; GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets AsyncioDebug) From c3c685ace22fe1c3839db254d76fcfc3de9c7e29 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 14:11:58 +0530 Subject: [PATCH 67/84] try to fix docs build --- Doc/whatsnew/3.14.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 9af068c60c68db..2ddb7e6d9cc995 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -542,7 +542,7 @@ asyncio the program's call graph: :func:`asyncio.capture_call_graph` and :func:`asyncio.print_call_graph`. (Contributed by Yury Selivanov, Pablo Galindo Salgado, Łukasz Langa - and Kumar Aditya in :gh:`91048`.) + and Kumar Aditya in :gh:`91048`.) io --- From 785adebc8d2e06f11a44f1c33aab2636a33dd8d1 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 16:15:16 +0530 Subject: [PATCH 68/84] Update Doc/whatsnew/3.14.rst --- Doc/whatsnew/3.14.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2ddb7e6d9cc995..11cc90c0160a5f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -541,8 +541,8 @@ asyncio * :mod:`asyncio` has new utility functions for introspecting and printing the program's call graph: :func:`asyncio.capture_call_graph` and :func:`asyncio.print_call_graph`. - (Contributed by Yury Selivanov, Pablo Galindo Salgado, Łukasz Langa - and Kumar Aditya in :gh:`91048`.) + (Contributed by Yury Selivanov, Pablo Galindo Salgado and Łukasz Langa + in :gh:`91048`.) io --- From a577328873f3db1785d2f367659100a86550a01b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 23 Nov 2024 11:09:54 +0100 Subject: [PATCH 69/84] Match indentation with _asynciomodule.c after ab511a4f67ffd88f7709c1bc1990074e6e5b555d --- Modules/_testexternalinspection.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index b696d70716d592..a7c3bf3cb7f65d 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -60,19 +60,19 @@ #endif struct _Py_AsyncioModuleDebugOffsets { - struct _asyncio_task_object { - uint64_t size; - uint64_t task_name; - uint64_t task_awaited_by; - uint64_t task_is_task; - uint64_t task_awaited_by_is_set; - uint64_t task_coro; - } asyncio_task_object; - struct _asyncio_thread_state { - uint64_t size; - uint64_t asyncio_running_loop; - uint64_t asyncio_running_task; - } asyncio_thread_state; + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + } asyncio_thread_state; }; #if defined(__APPLE__) && TARGET_OS_OSX From 064129a92ee045ac5b64ff817f7918052d92e112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 23 Nov 2024 11:10:40 +0100 Subject: [PATCH 70/84] Improve comments after f56468af8b3789d0e20be30715c20ac1bb367641 --- Modules/_asynciomodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 287550f4b1662c..0e7a6917e28c9e 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2155,8 +2155,8 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) // memory reads and logic, as Python's dict is a notoriously complex // and ever-changing data structure. // - // So the easier solution is to put a strong reference of the currently - // running `asyncio.Task` to the interpreter thread state (we already + // So the easier solution is to put a strong reference to the currently + // running `asyncio.Task` on the interpreter thread state (we already // have some asyncio state there.) _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); if (ts->asyncio_running_loop == loop) { @@ -2172,7 +2172,7 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) // // That said, we still want to make sure we don't end up in // a broken state, so we check that we're in the correct thread - // by comparing the *loop* argument to running event loop + // by comparing the *loop* argument to the event loop running // in the current thread. If they match we know we're in the // right thread, as asyncio event loops don't change threads. assert(ts->asyncio_running_task == NULL); From ce332d933d366a9519a78557bdc7fea9c68e3a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 23 Nov 2024 11:47:17 +0100 Subject: [PATCH 71/84] Restore the Oxford comma --- Doc/whatsnew/3.14.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 11cc90c0160a5f..594ac367a6c989 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -541,7 +541,7 @@ asyncio * :mod:`asyncio` has new utility functions for introspecting and printing the program's call graph: :func:`asyncio.capture_call_graph` and :func:`asyncio.print_call_graph`. - (Contributed by Yury Selivanov, Pablo Galindo Salgado and Łukasz Langa + (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa in :gh:`91048`.) io From d6d943f09290311e983ce83303a6f014b71647c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 23 Nov 2024 13:13:56 +0100 Subject: [PATCH 72/84] Address remaining suggestions of Andrew Svetlov --- Doc/library/asyncio-graph.rst | 32 ++++++++++++++------ Lib/asyncio/graph.py | 55 ++++++++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 2da58d2b3353eb..6fa7ac2029d88f 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -3,9 +3,9 @@ .. _asyncio-graph: -=================== -Stack Introspection -=================== +======================== +Call Graph Introspection +======================== **Source code:** :source:`Lib/asyncio/graph.py` @@ -20,20 +20,31 @@ and debuggers. .. versionadded:: next -.. function:: print_call_graph(future=None, /, *, file=None, depth=1) +.. function:: print_call_graph(future=None, /, *, file=None, depth=1, limit=None) Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. + This function prints entries starting from the currently executing frame, + i.e. the top frame, and going down towards the invocation point. + The function receives an optional *future* argument. - If not passed, the current running task will be used. If there's no - current task, the function returns ``None``. + If not passed, the current running task will be used. If the function is called on *the current task*, the optional keyword-only *depth* argument can be used to skip the specified number of frames from top of the stack. - If *file* is omitted or ``None``, the function will print to :data:`sys.stdout`. + If the optional keyword-only *limit* argument is provided, each call stack + in the resulting graph is truncated to include at most ``abs(limit)`` + entries. If *limit* is positive, the entries left are the closest to + the invocation point. If *limit* is negative, the topmost entries are + left. If *limit* is omitted or ``None``, all entries are present. + If *limit* is ``0``, the call stack is not printed at all, only + "awaited by" information is printed. + + If *file* is omitted or ``None``, the function will print + to :data:`sys.stdout`. **Example:** @@ -63,11 +74,14 @@ and debuggers. | File 'taskgroups.py', line 107, in async TaskGroup.__aexit__() | File 't2.py', line 7, in async main() -.. function:: format_call_graph(future=None, /, *, depth=1) +.. function:: format_call_graph(future=None, /, *, depth=1, limit=None) Like :func:`print_call_graph`, but returns a string. + If *future* is ``None`` and there's no current task, + the function returns an empty string. + -.. function:: capture_call_graph(future=None, /, *, depth=1) +.. function:: capture_call_graph(future=None, /, *, depth=1, limit=None) Capture the async call graph for the current task or the provided :class:`Task` or :class:`Future`. diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index fc84531818ddfa..95592ddadeeff0 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -37,7 +37,11 @@ class FutureCallGraph: awaited_by: tuple["FutureCallGraph", ...] -def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: +def _build_graph_for_future( + future: futures.Future, + *, + limit: int | None = None, +) -> FutureCallGraph: if not isinstance(future, futures.Future): raise TypeError( f"{future!r} object does not appear to be compatible " @@ -46,7 +50,7 @@ def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: coro = None if get_coro := getattr(future, 'get_coro', None): - coro = get_coro() + coro = get_coro() if limit != 0 else None st: list[FrameCallGraphEntry] = [] awaited_by: list[FutureCallGraph] = [] @@ -65,8 +69,13 @@ def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: if future._asyncio_awaited_by: for parent in future._asyncio_awaited_by: - awaited_by.append(_build_graph_for_future(parent)) + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + if limit is not None: + if limit > 0: + st = st[:limit] + elif limit < 0: + st = st[limit:] st.reverse() return FutureCallGraph(future, tuple(st), tuple(awaited_by)) @@ -76,8 +85,9 @@ def capture_call_graph( /, *, depth: int = 1, + limit: int | None = None, ) -> FutureCallGraph | None: - """Capture async call graph for the current task or the provided Future. + """Capture the async call graph for the current task or the provided Future. The graph is represented with three data structures: @@ -95,13 +105,21 @@ def capture_call_graph( Where 'frame' is a frame object of a regular Python function in the call stack. - Receives an optional "future" argument. If not passed, + Receives an optional 'future' argument. If not passed, the current task will be used. If there's no current task, the function returns None. If "capture_call_graph()" is introspecting *the current task*, the - optional keyword-only "depth" argument can be used to skip the specified + optional keyword-only 'depth' argument can be used to skip the specified number of frames from top of the stack. + + If the optional keyword-only 'limit' argument is provided, each call stack + in the resulting graph is truncated to include at most ``abs(limit)`` + entries. If 'limit' is positive, the entries left are the closest to + the invocation point. If 'limit' is negative, the topmost entries are + left. If 'limit' is omitted or None, all entries are present. + If 'limit' is 0, the call stack is not captured at all, only + "awaited by" information is present. """ loop = events._get_running_loop() @@ -111,7 +129,7 @@ def capture_call_graph( # if yes - check if the passed future is the currently # running task or not. if loop is None or future is not tasks.current_task(loop=loop): - return _build_graph_for_future(future) + return _build_graph_for_future(future, limit=limit) # else: future is the current task, move on. else: if loop is None: @@ -134,7 +152,7 @@ def capture_call_graph( call_stack: list[FrameCallGraphEntry] = [] - f = sys._getframe(depth) + f = sys._getframe(depth) if limit != 0 else None try: while f is not None: is_async = f.f_generator is not None @@ -153,7 +171,14 @@ def capture_call_graph( awaited_by = [] if future._asyncio_awaited_by: for parent in future._asyncio_awaited_by: - awaited_by.append(_build_graph_for_future(parent)) + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + limit *= -1 + if limit > 0: + call_stack = call_stack[:limit] + elif limit < 0: + call_stack = call_stack[limit:] return FutureCallGraph(future, tuple(call_stack), tuple(awaited_by)) @@ -163,8 +188,9 @@ def format_call_graph( /, *, depth: int = 1, + limit: int | None = None, ) -> str: - """Return async call graph as a string for `future`. + """Return the async call graph as a string for `future`. If `future` is not provided, format the call graph for the current task. """ @@ -226,9 +252,9 @@ def add_line(line: str) -> None: for fut in st.awaited_by: render_level(fut, buf, level + 1) - graph = capture_call_graph(future, depth=depth + 1) + graph = capture_call_graph(future, depth=depth + 1, limit=limit) if graph is None: - return + return "" try: buf: list[str] = [] @@ -245,6 +271,7 @@ def print_call_graph( *, file: typing.TextIO | None = None, depth: int = 1, + limit: int | None = None, ) -> None: - """Print async call graph for the current task or the provided Future.""" - print(format_call_graph(future, depth=depth), file=file) + """Print the async call graph for the current task or the provided Future.""" + print(format_call_graph(future, depth=depth, limit=limit), file=file) From 703ff4668e4b01b4ece27300c3e770301572db33 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sat, 23 Nov 2024 14:26:57 -0800 Subject: [PATCH 73/84] Fix gather() --- Lib/asyncio/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 4025163416cde3..d1587c59cc5728 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -887,13 +887,13 @@ def _done_callback(fut, cur_task): # can't control it, disable the "destroy pending task" # warning. fut._log_destroy_pending = False - if cur_task is not None: - futures.future_add_to_awaited_by(fut, cur_task) nfuts += 1 arg_to_fut[arg] = fut if fut.done(): done_futs.append(fut) else: + if cur_task is not None: + futures.future_add_to_awaited_by(fut, cur_task) fut.add_done_callback(lambda fut: _done_callback(fut, cur_task)) else: From e8678632077a75f9ad27edae3bc4ea4fcd72d8dd Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 26 Nov 2024 00:17:52 +0000 Subject: [PATCH 74/84] Replace lambda by closure --- Lib/asyncio/tasks.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index d1587c59cc5728..a25854cc4bd69e 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -807,7 +807,13 @@ def gather(*coros_or_futures, return_exceptions=False): outer.set_result([]) return outer - def _done_callback(fut, cur_task): + loop = events._get_running_loop() + if loop is not None: + cur_task = current_task(loop) + else: + cur_task = None + + def _done_callback(fut, cur_task=cur_task): nonlocal nfinished nfinished += 1 @@ -871,11 +877,6 @@ def _done_callback(fut, cur_task): nfinished = 0 done_futs = [] outer = None # bpo-46672 - loop = events._get_running_loop() - if loop is not None: - cur_task = current_task(loop) - else: - cur_task = None for arg in coros_or_futures: if arg not in arg_to_fut: fut = ensure_future(arg, loop=loop) @@ -894,7 +895,7 @@ def _done_callback(fut, cur_task): else: if cur_task is not None: futures.future_add_to_awaited_by(fut, cur_task) - fut.add_done_callback(lambda fut: _done_callback(fut, cur_task)) + fut.add_done_callback(_done_callback) else: # There's a duplicate Future object in coros_or_futures. @@ -909,7 +910,7 @@ def _done_callback(fut, cur_task): # this will effectively complete the gather eagerly, with the last # callback setting the result (or exception) on outer before returning it for fut in done_futs: - _done_callback(fut, cur_task) + _done_callback(fut) return outer From 9cb5b2900e46baff89afb696ce5482bc702cd0bc Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Mon, 25 Nov 2024 18:24:56 -0600 Subject: [PATCH 75/84] =?UTF-8?q?Let=E2=80=99s=20not=20emphasize=20*asynci?= =?UTF-8?q?o*?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jacob Coffee --- Doc/library/asyncio-graph.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 6fa7ac2029d88f..010d2061c8e2d5 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -118,7 +118,7 @@ To introspect an async call graph asyncio requires cooperation from control flow structures, such as :func:`shield` or :class:`TaskGroup`. Any time an intermediate :class:`Future` object with low-level APIs like :meth:`Future.add_done_callback() ` is -involved, the following two functions should be used to inform *asyncio* +involved, the following two functions should be used to inform asyncio about how exactly such intermediate future objects are connected with the tasks they wrap or control. From 61b2b7b87050ec5814716d7caa663867e44d9640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 26 Nov 2024 11:51:19 +0100 Subject: [PATCH 76/84] Apply suggestions from Irit's review Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Doc/library/asyncio-graph.rst | 2 +- Lib/asyncio/graph.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 010d2061c8e2d5..221438f9b85cf8 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -14,7 +14,7 @@ Call Graph Introspection asyncio has powerful runtime call graph introspection utilities to trace the entire call graph of a running *coroutine* or *task*, or a suspended *future*. These utilities and the underlying machinery -can be used by users in their Python code or by external profilers +can be used from within a Python program or by external profilers and debuggers. .. versionadded:: next diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index 95592ddadeeff0..5914a329945f70 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -17,7 +17,7 @@ 'FutureCallGraph', ) -# Sadly, we can't re-use the traceback's module datastructures as those +# Sadly, we can't re-use the traceback module's datastructures as those # are tailored for error reporting, whereas we need to represent an # async call graph. # @@ -93,8 +93,7 @@ def capture_call_graph( * FutureCallGraph(future, call_stack, awaited_by) - Where 'future' is a reference to an asyncio.Future or asyncio.Task - (or their subclasses.) + Where 'future' is an instance of asyncio.Future or asyncio.Task. 'call_stack' is a tuple of FrameGraphEntry objects. @@ -256,14 +255,14 @@ def add_line(line: str) -> None: if graph is None: return "" + buf: list[str] = [] try: - buf: list[str] = [] render_level(graph, buf, 0) - return '\n'.join(buf) finally: # 'graph' has references to frames so we should # make sure it's GC'ed as soon as we don't need it. del graph + return '\n'.join(buf) def print_call_graph( future: futures.Future | None = None, From 9533ab922ec3c2e81b9bba95d6b4f154512546a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 26 Nov 2024 14:00:06 +0100 Subject: [PATCH 77/84] Address remaining review by Irit --- Doc/library/asyncio-graph.rst | 3 +-- Doc/library/inspect.rst | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 221438f9b85cf8..b39f5d8a22455c 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -25,8 +25,7 @@ and debuggers. Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. - This function prints entries starting from the currently executing frame, - i.e. the top frame, and going down towards the invocation point. + This function prints entries starting from the top frame and going The function receives an optional *future* argument. If not passed, the current running task will be used. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 0902d64f9bd22a..4c755a9ecabf4b 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -150,6 +150,12 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | f_locals | local namespace seen by | | | | this frame | +-----------------+-------------------+---------------------------+ +| | f_generator | returns the generator or | +| | | coroutine object that | +| | | owns this frame, or | +| | | ``None`` if the frame is | +| | | of a regular function | ++-----------------+-------------------+---------------------------+ | | f_trace | tracing function for this | | | | frame, or ``None`` | +-----------------+-------------------+---------------------------+ @@ -162,12 +168,6 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | per-opcode events are | | | | requested | +-----------------+-------------------+---------------------------+ -| | f_generator | returns the generator or | -| | | coroutine object that | -| | | owns this frame, or | -| | | ``None`` if the frame is | -| | | of a regular function | -+-----------------+-------------------+---------------------------+ | | clear() | used to clear all | | | | references to local | | | | variables | From 596191d17220a5c047879914294ecdf85cea1f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 26 Nov 2024 14:38:38 +0100 Subject: [PATCH 78/84] Test pure-Python and C-accelerated versions of future_add_to/future_discard_from --- Lib/asyncio/futures.py | 13 ++-- Lib/test/test_asyncio/test_graph.py | 100 ++++++++++++++++++++++++++-- 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 71fd283acfa563..a1c97c5ae706b3 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -465,6 +465,9 @@ def future_discard_from_awaited_by(fut, waiter, /): fut._Future__asyncio_awaited_by.discard(waiter) +_py_future_add_to_awaited_by = future_add_to_awaited_by +_py_future_discard_from_awaited_by = future_discard_from_awaited_by + try: import _asyncio except ImportError: @@ -472,9 +475,7 @@ def future_discard_from_awaited_by(fut, waiter, /): else: # _CFuture is needed for tests. Future = _CFuture = _asyncio.Future - -try: - from _asyncio import future_add_to_awaited_by, \ - future_discard_from_awaited_by -except ImportError: - pass + future_add_to_awaited_by = _asyncio.future_add_to_awaited_by + future_discard_from_awaited_by = _asyncio.future_discard_from_awaited_by + _c_future_add_to_awaited_by = future_add_to_awaited_by + _c_future_discard_from_awaited_by = future_discard_from_awaited_by diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 25046e4cc2c292..9e984e2499bc1b 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -45,8 +45,7 @@ def walk(s): return walk(stack), buf.getvalue() -class TestCallStack(unittest.IsolatedAsyncioTestCase): - +class CallStackTestBase: async def test_stack_tgroup(self): @@ -107,7 +106,7 @@ async def main(): ]) self.assertIn( - ' async TestCallStack.test_stack_tgroup()', + ' async CallStackTestBase.test_stack_tgroup()', stack_for_c5[1]) @@ -143,8 +142,11 @@ async def main(): [] ]) + from pprint import pprint + pprint(stack_for_gen_nested_call[1]) + self.assertIn( - 'async generator TestCallStack.test_stack_async_gen..gen()', + 'async generator CallStackTestBase.test_stack_async_gen..gen()', stack_for_gen_nested_call[1]) async def test_stack_gather(self): @@ -345,3 +347,93 @@ async def main(): ) self.assertTrue(stack_for_fut[1].startswith('* Future(id=')) + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), + "C-accelerated asyncio call graph backend missing", +) +class TestCallStackC(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._CFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._CTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._c_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._c_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_py_future_add_to_awaited_by"), + "Pure Python asyncio call graph backend missing", +) +class TestCallStackPy(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._PyFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._PyTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._py_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._py_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future From ad9152eb0d0fcb91f96524bdb4647bc3b3d26c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 26 Nov 2024 14:44:17 +0100 Subject: [PATCH 79/84] fix blooper --- Lib/test/test_asyncio/test_graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 9e984e2499bc1b..97e1a48564d14c 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -386,7 +386,7 @@ def tearDown(self): del self._future_add_to_awaited_by asyncio.Task = self._Task - tasks = self._Task + tasks.Task = self._Task del self._Task asyncio.Future = self._Future @@ -431,7 +431,7 @@ def tearDown(self): del self._future_add_to_awaited_by asyncio.Task = self._Task - tasks = self._Task + tasks.Task = self._Task del self._Task asyncio.Future = self._Future From 066bf210ee388b0a7a046d0ee64015da05713485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 26 Nov 2024 14:45:45 +0100 Subject: [PATCH 80/84] Fix another blooper --- Doc/library/asyncio-graph.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index b39f5d8a22455c..fc8edeb426c567 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -26,6 +26,7 @@ and debuggers. :class:`Task` or :class:`Future`. This function prints entries starting from the top frame and going + down towards the invocation point. The function receives an optional *future* argument. If not passed, the current running task will be used. From 4caeec408cc085c378f01b5913bd4e526868c115 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 21 Jan 2025 20:39:03 +0000 Subject: [PATCH 81/84] Fix crash --- Modules/_asynciomodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 0e7a6917e28c9e..3ebc6b65052324 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2214,7 +2214,7 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) // See the comment in `enter_task` for the explanation of why // the following is needed. _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); - if (ts->asyncio_running_loop == loop) { + if (ts->asyncio_running_loop == NULL || ts->asyncio_running_loop == loop) { Py_CLEAR(ts->asyncio_running_task); } From a8dd667b26732f0d9af2993a604aaa88fed94603 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 22 Jan 2025 00:43:12 +0000 Subject: [PATCH 82/84] use private method for the policy --- Lib/test/test_asyncio/test_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 97e1a48564d14c..4b9d0bf5b470c5 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -5,7 +5,7 @@ # To prevent a warning "test altered the execution environment" def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def capture_test_stack(*, fut=None, depth=1): From cf8f5e569bdd0bdb4d6e6fc1875c040c7cfffeea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 22 Jan 2025 14:53:31 +0100 Subject: [PATCH 83/84] Avoid importing typing --- Lib/asyncio/graph.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index 5914a329945f70..d8df7c9919abbf 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -3,7 +3,6 @@ import dataclasses import sys import types -import typing from . import events from . import futures @@ -17,6 +16,9 @@ 'FutureCallGraph', ) +if False: # for type checkers + from typing import TextIO + # Sadly, we can't re-use the traceback module's datastructures as those # are tailored for error reporting, whereas we need to represent an # async call graph. @@ -268,7 +270,7 @@ def print_call_graph( future: futures.Future | None = None, /, *, - file: typing.TextIO | None = None, + file: TextIO | None = None, depth: int = 1, limit: int | None = None, ) -> None: From eda9c7cb2ef3003e24fa65913750ede09ef7590c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 22 Jan 2025 15:00:46 +0100 Subject: [PATCH 84/84] Remove debug printing from test_asyncio.test_graph --- Lib/test/test_asyncio/test_graph.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 4b9d0bf5b470c5..fd2160d4ca3137 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -142,9 +142,6 @@ async def main(): [] ]) - from pprint import pprint - pprint(stack_for_gen_nested_call[1]) - self.assertIn( 'async generator CallStackTestBase.test_stack_async_gen..gen()', stack_for_gen_nested_call[1])