From 2c1119db082903ccd215480dfdf7410d47d21ef4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 20 Aug 2024 17:55:58 +0100 Subject: [PATCH 1/9] Allow any fixed object to have inline values --- Include/cpython/object.h | 1 + Include/internal/pycore_object.h | 5 +++-- Objects/typeobject.c | 3 ++- Python/gc.c | 3 +++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index e1024ddbdf6062..8dddce421bd6f5 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -230,6 +230,7 @@ struct _typeobject { /* bitset of which type-watchers care about this type */ unsigned char tp_watched; uint16_t tp_versions_used; + uint16_t tp_inline_values_offset; }; /* This struct is used by the specializer diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index ee33da77f01f2d..28ef122187376a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -803,10 +803,11 @@ _PyObject_GetManagedDict(PyObject *obj) static inline PyDictValues * _PyObject_InlineValues(PyObject *obj) { + PyTypeObject *tp = Py_TYPE(obj); + assert(tp->tp_inline_values_offset > 0 && tp->tp_inline_values_offset % sizeof(PyObject *) == 0); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject)); - return (PyDictValues *)((char *)obj + sizeof(PyObject)); + return (PyDictValues *)((char *)obj + tp->tp_inline_values_offset); } extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0d7009ac57bd5f..c667870edb64bf 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8340,7 +8340,8 @@ type_ready_managed_dict(PyTypeObject *type) return -1; } } - if (type->tp_itemsize == 0 && type->tp_basicsize == sizeof(PyObject)) { + if (type->tp_itemsize == 0) { + type->tp_inline_values_offset = type->tp_basicsize; type->tp_flags |= Py_TPFLAGS_INLINE_VALUES; } return 0; diff --git a/Python/gc.c b/Python/gc.c index 923a79299cab03..f920743b9213de 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2055,6 +2055,9 @@ _PyObject_GC_New(PyTypeObject *tp) return NULL; } _PyObject_Init(op, tp); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(op, tp); + } return op; } From a3e64643f8a6cac1790e01459bb96e9b11f4a4a1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 20 Aug 2024 18:18:40 +0100 Subject: [PATCH 2/9] Use tp_basicsize --- Include/cpython/object.h | 1 - Include/internal/pycore_object.h | 4 ++-- Objects/typeobject.c | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 8dddce421bd6f5..e1024ddbdf6062 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -230,7 +230,6 @@ struct _typeobject { /* bitset of which type-watchers care about this type */ unsigned char tp_watched; uint16_t tp_versions_used; - uint16_t tp_inline_values_offset; }; /* This struct is used by the specializer diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 28ef122187376a..0f2de6fd28f56e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -804,10 +804,10 @@ static inline PyDictValues * _PyObject_InlineValues(PyObject *obj) { PyTypeObject *tp = Py_TYPE(obj); - assert(tp->tp_inline_values_offset > 0 && tp->tp_inline_values_offset % sizeof(PyObject *) == 0); + assert(tp->tp_basicsize > 0 && tp->tp_basicsize % sizeof(PyObject *) == 0); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - return (PyDictValues *)((char *)obj + tp->tp_inline_values_offset); + return (PyDictValues *)((char *)obj + tp->tp_basicsize); } extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index c667870edb64bf..f74d51222b7a65 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8341,7 +8341,6 @@ type_ready_managed_dict(PyTypeObject *type) } } if (type->tp_itemsize == 0) { - type->tp_inline_values_offset = type->tp_basicsize; type->tp_flags |= Py_TPFLAGS_INLINE_VALUES; } return 0; From 89655c994be185d2f1b28c581bea49abdcd9930f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 21 Aug 2024 08:51:19 +0100 Subject: [PATCH 3/9] Use offset, not index in LOAD/STORE_ATTR_INSTANCE_VALUE for speed. --- Python/bytecodes.c | 18 ++++++++++-------- Python/executor_cases.c.h | 22 +++++++++++++--------- Python/generated_cases.c.h | 15 +++++++++------ Python/optimizer_cases.c.h | 5 ++++- Python/specialize.c | 16 ++++++++++------ 5 files changed, 46 insertions(+), 30 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ec0f6ab8fde980..8b734d1b5ec60a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2012,9 +2012,10 @@ dummy_func( DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); } - split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); @@ -2187,7 +2188,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } - op(_GUARD_DORV_NO_DICT, (owner -- owner)) { + op(_GUARD_DORV_NO_DICT, (offset -- owner)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); @@ -2196,16 +2197,17 @@ dummy_func( EXIT_IF(_PyObject_InlineValues(owner_o)->valid == 0); } - op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { + op(_STORE_ATTR_INSTANCE_VALUE, (offset/1, value, owner --)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyDictValues *values = _PyObject_InlineValues(owner_o); - - PyObject *old_value = values->values[index]; - values->values[index] = PyStackRef_AsPyObjectSteal(value); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + *value_ptr = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + int index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } else { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9226b7a4e5d4bc..df3fbbb11b369b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2277,9 +2277,10 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2299,9 +2300,10 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2563,7 +2565,6 @@ case _GUARD_DORV_NO_DICT: { _PyStackRef owner; - owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); @@ -2575,6 +2576,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + stack_pointer[-1] = owner; break; } @@ -2583,14 +2585,16 @@ _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyDictValues *values = _PyObject_InlineValues(owner_o); - PyObject *old_value = values->values[index]; - values->values[index] = PyStackRef_AsPyObjectSteal(value); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + *value_ptr = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + int index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } else { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 27971ce75464f0..a58665c3a45fa5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4995,9 +4995,10 @@ } // _LOAD_ATTR_INSTANCE_VALUE { - uint16_t index = read_u16(&this_instr[4].cache); + uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); @@ -6825,14 +6826,16 @@ // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; { - uint16_t index = read_u16(&this_instr[4].cache); + uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyDictValues *values = _PyObject_InlineValues(owner_o); - PyObject *old_value = values->values[index]; - values->values[index] = PyStackRef_AsPyObjectSteal(value); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + *value_ptr = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + int index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } else { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index e5be9d0e3b5ee7..d4ff33d37cb914 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1064,7 +1064,7 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; + uint16_t offset = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)index; @@ -1204,6 +1204,9 @@ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ case _GUARD_DORV_NO_DICT: { + _Py_UopsSymbol *owner; + owner = sym_new_not_null(ctx); + stack_pointer[-1] = owner; break; } diff --git a/Python/specialize.c b/Python/specialize.c index b3a2e07c3bbcb8..db794bea0bee29 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -849,15 +849,19 @@ specialize_dict_access( assert(PyUnicode_CheckExact(name)); Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); assert (index != DKIX_ERROR); - if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, - index == DKIX_EMPTY ? - SPEC_FAIL_ATTR_NOT_IN_KEYS : - SPEC_FAIL_OUT_OF_RANGE); + if (index == DKIX_EMPTY) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); + return 0; + } + assert(index >= 0); + char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; + Py_ssize_t offset = value_addr - (char *)owner; + if (offset != (uint16_t)offset) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); return 0; } write_u32(cache->version, type->tp_version_tag); - cache->index = (uint16_t)index; + cache->index = (uint16_t)offset; instr->op.code = values_op; } else { From f26cf5327043d51577334f65df4f96f34077654c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 21 Aug 2024 08:53:40 +0100 Subject: [PATCH 4/9] Add news --- .../2024-08-21-08-53-00.gh-issue-115776.9A7Dv_.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-08-21-08-53-00.gh-issue-115776.9A7Dv_.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-21-08-53-00.gh-issue-115776.9A7Dv_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-21-08-53-00.gh-issue-115776.9A7Dv_.rst new file mode 100644 index 00000000000000..953ebd72382e1f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-21-08-53-00.gh-issue-115776.9A7Dv_.rst @@ -0,0 +1,2 @@ +Enables inline values (Python's equivalent of hidden classes) on any class +who's instances are of a fixed size. From 7cc899634f18d9ecf8a0c058c9413946bfd31c83 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 21 Aug 2024 10:24:37 +0100 Subject: [PATCH 5/9] Update gdb support --- Tools/gdb/libpython.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 8aa74635aedb4f..cf03788d037a81 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -69,9 +69,6 @@ def _type_unsigned_int_ptr(): def _sizeof_void_p(): return gdb.lookup_type('void').pointer().sizeof -def _sizeof_pyobject(): - return gdb.lookup_type('PyObject').sizeof - def _managed_dict_offset(): # See pycore_object.h pyobj = gdb.lookup_type("PyObject") @@ -505,7 +502,7 @@ def get_keys_values(self): dict_ptr = dict_ptr_ptr.cast(_type_char_ptr().pointer()).dereference() if int(dict_ptr): return None - char_ptr = obj_ptr + _sizeof_pyobject() + char_ptr = obj_ptr + typeobj.field('tp_basicsize') values_ptr = char_ptr.cast(gdb.lookup_type("PyDictValues").pointer()) values = values_ptr['values'] return PyKeysValuesPair(self.get_cached_keys(), values) From 2315e0a56195977b45ab9394a0cbec155d8a4ec8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 21 Aug 2024 10:33:52 +0100 Subject: [PATCH 6/9] Fix up tier 2 --- Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 2 +- Python/optimizer_bytecodes.c | 4 ++-- Python/optimizer_cases.c.h | 5 +---- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8b734d1b5ec60a..4d969c31155325 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2188,7 +2188,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } - op(_GUARD_DORV_NO_DICT, (offset -- owner)) { + op(_GUARD_DORV_NO_DICT, (owner -- owner)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index df3fbbb11b369b..5029fd85dbe2b9 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2565,6 +2565,7 @@ case _GUARD_DORV_NO_DICT: { _PyStackRef owner; + owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); @@ -2576,7 +2577,6 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - stack_pointer[-1] = owner; break; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6e46d9bed11fbc..9a1b9da52f4bb5 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -452,10 +452,10 @@ dummy_func(void) { top, unused[oparg-2], bottom)) { } - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { attr = sym_new_not_null(ctx); null = sym_new_null(ctx); - (void)index; + (void)offset; (void)owner; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index d4ff33d37cb914..672fec3946f2fb 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1067,7 +1067,7 @@ uint16_t offset = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); - (void)index; + (void)offset; (void)owner; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; @@ -1204,9 +1204,6 @@ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ case _GUARD_DORV_NO_DICT: { - _Py_UopsSymbol *owner; - owner = sym_new_not_null(ctx); - stack_pointer[-1] = owner; break; } From cb6b30a9ea19fad507b3273f90436f23e6d80c92 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 21 Aug 2024 10:48:16 +0100 Subject: [PATCH 7/9] Update free-threading version of the GC module --- Python/gc_free_threading.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index b95456519dca06..54de0c2671ae68 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1810,6 +1810,9 @@ _PyObject_GC_New(PyTypeObject *tp) return NULL; } _PyObject_Init(op, tp); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(op, tp); + } return op; } From ebad9340f7e47d6d33fcfc4d575b13dffc632404 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 21 Aug 2024 13:53:33 +0100 Subject: [PATCH 8/9] Update docs on inline values --- Objects/object_layout.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Objects/object_layout.md b/Objects/object_layout.md index 352409425ee802..9f534418d7a78e 100644 --- a/Objects/object_layout.md +++ b/Objects/object_layout.md @@ -28,6 +28,10 @@ So the pre-header is these two fields: If the object has no physical dictionary, then the ``dict_pointer`` is set to `NULL`. +In 3.13 only objects with no additional data could have inline values. +That is, instances of classes with `tp_basicsize == sizeof(PyObject)`. +In 3.14, any object who's class has `tp_itemsize == 0` can have inline values. +In both versions, the inline values starts `tp_basicsize` bytes after the object.
3.12 From 9523760069dc6e44b7fc127b7069c314eb03f462 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 21 Aug 2024 13:57:53 +0100 Subject: [PATCH 9/9] Update Objects/object_layout.md --- Objects/object_layout.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/object_layout.md b/Objects/object_layout.md index 9f534418d7a78e..4a781668636324 100644 --- a/Objects/object_layout.md +++ b/Objects/object_layout.md @@ -30,7 +30,7 @@ is set to `NULL`. In 3.13 only objects with no additional data could have inline values. That is, instances of classes with `tp_basicsize == sizeof(PyObject)`. -In 3.14, any object who's class has `tp_itemsize == 0` can have inline values. +In 3.14, any object whose class has `tp_itemsize == 0` can have inline values. In both versions, the inline values starts `tp_basicsize` bytes after the object.