From 23965ab27f78058c1d4e464cdccd070bf3250bfa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Sep 2025 15:46:08 +0100 Subject: [PATCH 001/183] [mypyc] Speed up unary "not" (#19774) Specialize "not" for common primitive types, optional types and native instance types. Also specialize variable-length tuple in a boolean context (while working on "not" I noticed that this wasn't specialized). This appears to speed up self check by 0.7%, but this is only barely above the noise floor. --- mypyc/irbuild/ll_builder.py | 49 +++++++- mypyc/test-data/irbuild-bool.test | 178 ++++++++++++++++++++++++++++++ mypyc/test-data/irbuild-i64.test | 8 ++ mypyc/test-data/run-bools.test | 112 +++++++++++++++++++ 4 files changed, 342 insertions(+), 5 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 475d490a48f2..4b85c13892c1 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -94,7 +94,6 @@ c_pyssize_t_rprimitive, c_size_t_rprimitive, check_native_int_range, - dict_rprimitive, float_rprimitive, int_rprimitive, is_bool_or_bit_rprimitive, @@ -110,13 +109,13 @@ is_list_rprimitive, is_none_rprimitive, is_object_rprimitive, + is_optional_type, is_set_rprimitive, is_short_int_rprimitive, is_str_rprimitive, is_tagged, is_tuple_rprimitive, is_uint8_rprimitive, - list_rprimitive, none_rprimitive, object_pointer_rprimitive, object_rprimitive, @@ -1684,6 +1683,44 @@ def unary_not(self, value: Value, line: int, *, likely_bool: bool = False) -> Va if is_bool_or_bit_rprimitive(typ): mask = Integer(1, typ, line) return self.int_op(typ, value, mask, IntOp.XOR, line) + if is_tagged(typ) or is_fixed_width_rtype(typ): + return self.binary_op(value, Integer(0), "==", line) + if ( + is_str_rprimitive(typ) + or is_list_rprimitive(typ) + or is_tuple_rprimitive(typ) + or is_dict_rprimitive(typ) + or isinstance(typ, RInstance) + ): + bool_val = self.bool_value(value) + return self.unary_not(bool_val, line) + if is_optional_type(typ): + value_typ = optional_value_type(typ) + assert value_typ + if ( + is_str_rprimitive(value_typ) + or is_list_rprimitive(value_typ) + or is_tuple_rprimitive(value_typ) + or is_dict_rprimitive(value_typ) + or isinstance(value_typ, RInstance) + ): + # 'X | None' type: Check for None first and then specialize for X. + res = Register(bit_rprimitive) + cmp = self.add(ComparisonOp(value, self.none_object(), ComparisonOp.EQ, line)) + none, not_none, out = BasicBlock(), BasicBlock(), BasicBlock() + self.add(Branch(cmp, none, not_none, Branch.BOOL)) + self.activate_block(none) + self.add(Assign(res, self.true())) + self.goto(out) + self.activate_block(not_none) + val = self.unary_not( + self.unbox_or_cast(value, value_typ, line, can_borrow=True, unchecked=True), + line, + ) + self.add(Assign(res, val)) + self.goto(out) + self.activate_block(out) + return res if likely_bool and is_object_rprimitive(typ): # First quickly check if it's a bool, and otherwise fall back to generic op. res = Register(bit_rprimitive) @@ -1882,10 +1919,12 @@ def bool_value(self, value: Value) -> Value: elif is_fixed_width_rtype(value.type): zero = Integer(0, value.type) result = self.add(ComparisonOp(value, zero, ComparisonOp.NEQ)) - elif is_same_type(value.type, str_rprimitive): + elif is_str_rprimitive(value.type): result = self.call_c(str_check_if_true, [value], value.line) - elif is_same_type(value.type, list_rprimitive) or is_same_type( - value.type, dict_rprimitive + elif ( + is_list_rprimitive(value.type) + or is_dict_rprimitive(value.type) + or is_tuple_rprimitive(value.type) ): length = self.builtin_len(value, value.line) zero = Integer(0) diff --git a/mypyc/test-data/irbuild-bool.test b/mypyc/test-data/irbuild-bool.test index 9810daf487fa..5eac6d8db24f 100644 --- a/mypyc/test-data/irbuild-bool.test +++ b/mypyc/test-data/irbuild-bool.test @@ -473,3 +473,181 @@ L0: r0 = x == y r1 = r0 ^ 1 return r1 + +[case testUnaryNotWithPrimitiveTypes] +def not_obj(x: object) -> bool: + return not x + +def not_int(x: int) -> bool: + return not x + +def not_str(x: str) -> bool: + return not x + +def not_list(x: list[int]) -> bool: + return not x + +def not_tuple(x: tuple[int, ...]) -> bool: + return not x + +def not_dict(x: dict[str, int]) -> bool: + return not x +[out] +def not_obj(x): + x :: object + r0 :: i32 + r1 :: bit + r2 :: bool +L0: + r0 = PyObject_Not(x) + r1 = r0 >= 0 :: signed + r2 = truncate r0: i32 to builtins.bool + return r2 +def not_int(x): + x :: int + r0 :: bit +L0: + r0 = int_eq x, 0 + return r0 +def not_str(x): + x :: str + r0, r1 :: bit +L0: + r0 = CPyStr_IsTrue(x) + r1 = r0 ^ 1 + return r1 +def not_list(x): + x :: list + r0 :: native_int + r1 :: short_int + r2, r3 :: bit +L0: + r0 = var_object_size x + r1 = r0 << 1 + r2 = int_ne r1, 0 + r3 = r2 ^ 1 + return r3 +def not_tuple(x): + x :: tuple + r0 :: native_int + r1 :: short_int + r2, r3 :: bit +L0: + r0 = var_object_size x + r1 = r0 << 1 + r2 = int_ne r1, 0 + r3 = r2 ^ 1 + return r3 +def not_dict(x): + x :: dict + r0 :: native_int + r1 :: short_int + r2, r3 :: bit +L0: + r0 = PyDict_Size(x) + r1 = r0 << 1 + r2 = int_ne r1, 0 + r3 = r2 ^ 1 + return r3 + +[case testUnaryNotWithNativeClass] +from __future__ import annotations + +class C: + def __bool__(self) -> bool: + return True + +def not_c(x: C) -> bool: + return not x + +def not_c_opt(x: C | None) -> bool: + return not x +[out] +def C.__bool__(self): + self :: __main__.C +L0: + return 1 +def not_c(x): + x :: __main__.C + r0, r1 :: bool +L0: + r0 = x.__bool__() + r1 = r0 ^ 1 + return r1 +def not_c_opt(x): + x :: union[__main__.C, None] + r0 :: object + r1, r2 :: bit + r3 :: __main__.C + r4, r5 :: bool +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = 1 + goto L3 +L2: + r3 = unchecked borrow cast(__main__.C, x) + r4 = r3.__bool__() + r5 = r4 ^ 1 + r2 = r5 +L3: + keep_alive x + return r2 + +[case testUnaryNotWithOptionalPrimitiveTypes] +from __future__ import annotations + +def not_str(x: str | None) -> bool: + return not x + +def not_list(x: list[int] | None) -> bool: + return not x +[out] +def not_str(x): + x :: union[str, None] + r0 :: object + r1, r2 :: bit + r3 :: str + r4, r5 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = 1 + goto L3 +L2: + r3 = unchecked borrow cast(str, x) + r4 = CPyStr_IsTrue(r3) + r5 = r4 ^ 1 + r2 = r5 +L3: + keep_alive x + return r2 +def not_list(x): + x :: union[list, None] + r0 :: object + r1, r2 :: bit + r3 :: list + r4 :: native_int + r5 :: short_int + r6, r7 :: bit +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = 1 + goto L3 +L2: + r3 = unchecked borrow cast(list, x) + r4 = var_object_size r3 + r5 = r4 << 1 + r6 = int_ne r5, 0 + r7 = r6 ^ 1 + r2 = r7 +L3: + keep_alive x + return r2 diff --git a/mypyc/test-data/irbuild-i64.test b/mypyc/test-data/irbuild-i64.test index e55c3bfe2acc..955f8b658f0e 100644 --- a/mypyc/test-data/irbuild-i64.test +++ b/mypyc/test-data/irbuild-i64.test @@ -947,6 +947,8 @@ def f(x: i64) -> i64: elif not x: return 6 return 3 +def unary_not(x: i64) -> bool: + return not x [out] def f(x): x :: i64 @@ -964,6 +966,12 @@ L3: L4: L5: return 3 +def unary_not(x): + x :: i64 + r0 :: bit +L0: + r0 = x == 0 + return r0 [case testI64AssignMixed_64bit] from mypy_extensions import i64 diff --git a/mypyc/test-data/run-bools.test b/mypyc/test-data/run-bools.test index b34fedebaa9f..45bf861e71e3 100644 --- a/mypyc/test-data/run-bools.test +++ b/mypyc/test-data/run-bools.test @@ -15,6 +15,8 @@ True False [case testBoolOps] +from __future__ import annotations + from typing import Optional, Any MYPY = False if MYPY: @@ -117,6 +119,29 @@ def test_optional_to_bool() -> None: assert not optional_to_bool3(F(False)) assert optional_to_bool3(F(True)) +def not_c(c: C) -> bool: + return not c + +def not_c_opt(c: C | None) -> bool: + return not c + +def not_d(d: D) -> bool: + return not d + +def not_d_opt(d: D | None) -> bool: + return not d + +def test_not_instance() -> None: + assert not not_c(C()) + assert not_c_opt(None) + assert not not_c_opt(C()) + + assert not_d(D(False)) + assert not not_d(D(True)) + assert not_d_opt(D(False)) + assert not_d_opt(None) + assert not not_d_opt(D(True)) + def test_any_to_bool() -> None: a: Any = int() b: Any = a + 1 @@ -222,6 +247,93 @@ def test_mixed_comparisons_i64() -> None: assert lt_mixed_i64(x, n) == (int(x) < n) assert gt_mixed_i64(n, x) == (n > int(x)) +def not_object(x: object) -> bool: + return not x + +def not_str(x: str) -> bool: + return not x + +def not_int(x: int) -> bool: + return not x + +def not_list(x: list[int]) -> bool: + return not x + +def not_tuple(x: tuple[int, ...]) -> bool: + return not x + +def not_dict(x: dict[str, int]) -> bool: + return not x + +def test_not_object() -> None: + assert not_object(None) + assert not_object([]) + assert not_object(0) + assert not not_object(1) + assert not not_object([1]) + +def test_not_str() -> None: + assert not_str(str()) + assert not not_str('x' + str()) + +def test_not_int() -> None: + assert not_int(int('0')) + assert not not_int(int('1')) + assert not not_int(int('-1')) + +def test_not_list() -> None: + assert not_list([]) + assert not not_list([1]) + +def test_not_tuple() -> None: + assert not_tuple(()) + assert not not_tuple((1,)) + +def test_not_dict() -> None: + assert not_dict({}) + assert not not_dict({'x': 1}) + +def not_str_opt(x: str | None) -> bool: + return not x + +def not_int_opt(x: int | None) -> bool: + return not x + +def not_list_opt(x: list[int] | None) -> bool: + return not x + +def not_tuple_opt(x: tuple[int, ...] | None) -> bool: + return not x + +def not_dict_opt(x: dict[str, int] | None) -> bool: + return not x + +def test_not_str_opt() -> None: + assert not_str_opt(str()) + assert not_str_opt(None) + assert not not_str_opt('x' + str()) + +def test_not_int_opt() -> None: + assert not_int_opt(int('0')) + assert not_int_opt(None) + assert not not_int_opt(int('1')) + assert not not_int_opt(int('-1')) + +def test_not_list_opt() -> None: + assert not_list_opt([]) + assert not_list_opt(None) + assert not not_list_opt([1]) + +def test_not_tuple_opt() -> None: + assert not_tuple_opt(()) + assert not_tuple_opt(None) + assert not not_tuple_opt((1,)) + +def test_not_dict_opt() -> None: + assert not_dict_opt({}) + assert not_dict_opt(None) + assert not not_dict_opt({'x': 1}) + [case testBoolMixInt] def test_mix() -> None: y = False From b8ee1f5d6c9c73b58cb6ea108f088eb001b4b4ec Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 1 Sep 2025 18:11:40 +0100 Subject: [PATCH 002/183] Use dedicated tags for most common instances (#19762) This uses dedicated secondary type tags to 5 most common instances. I also move the instance cache from `checker.py` to `types.py` so it is easier to share. The latter however requires couple tweaks to not break builtins fixtures in tests (see changes in `build.py` and `checkexpr.py`). This makes cache another ~20% smaller (so that with this PR FF is 4.5x smaller than JSON), and also this makes `mypy -c 'import torch'` almost 10% faster with warm cache (when one uses `--fixed-format-cache` obviously). I don't see any visible effect on cold cache runs. --- mypy/build.py | 5 ++- mypy/checker.py | 37 ++++++++++------------- mypy/checkexpr.py | 3 +- mypy/types.py | 77 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 23 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 39199b39b6ad..4ccc3dec408e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -91,7 +91,7 @@ from mypy.renaming import LimitedVariableRenameVisitor, VariableRenameVisitor from mypy.stats import dump_type_stats from mypy.stubinfo import is_module_from_legacy_bundled_package, stub_distribution_name -from mypy.types import Type +from mypy.types import Type, instance_cache from mypy.typestate import reset_global_state, type_state from mypy.util import json_dumps, json_loads from mypy.version import __version__ @@ -180,6 +180,9 @@ def build( # fields for callers that want the traditional API. messages = [] + # This is mostly for the benefit of tests that use builtins fixtures. + instance_cache.reset() + def default_flush_errors( filename: str | None, new_messages: list[str], is_serious: bool ) -> None: diff --git a/mypy/checker.py b/mypy/checker.py index 12a86fe6fba1..77822b7068ae 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -229,6 +229,7 @@ flatten_nested_unions, get_proper_type, get_proper_types, + instance_cache, is_literal_type, is_named_instance, ) @@ -467,12 +468,6 @@ def __init__( self, self.msg, self.plugin, per_line_checking_time_ns ) - self._str_type: Instance | None = None - self._function_type: Instance | None = None - self._int_type: Instance | None = None - self._bool_type: Instance | None = None - self._object_type: Instance | None = None - self.pattern_checker = PatternChecker(self, self.msg, self.plugin, options) self._unique_id = 0 @@ -7460,25 +7455,25 @@ def named_type(self, name: str) -> Instance: For example, named_type('builtins.object') produces the 'object' type. """ if name == "builtins.str": - if self._str_type is None: - self._str_type = self._named_type(name) - return self._str_type + if instance_cache.str_type is None: + instance_cache.str_type = self._named_type(name) + return instance_cache.str_type if name == "builtins.function": - if self._function_type is None: - self._function_type = self._named_type(name) - return self._function_type + if instance_cache.function_type is None: + instance_cache.function_type = self._named_type(name) + return instance_cache.function_type if name == "builtins.int": - if self._int_type is None: - self._int_type = self._named_type(name) - return self._int_type + if instance_cache.int_type is None: + instance_cache.int_type = self._named_type(name) + return instance_cache.int_type if name == "builtins.bool": - if self._bool_type is None: - self._bool_type = self._named_type(name) - return self._bool_type + if instance_cache.bool_type is None: + instance_cache.bool_type = self._named_type(name) + return instance_cache.bool_type if name == "builtins.object": - if self._object_type is None: - self._object_type = self._named_type(name) - return self._object_type + if instance_cache.object_type is None: + instance_cache.object_type = self._named_type(name) + return instance_cache.object_type return self._named_type(name) def _named_type(self, name: str) -> Instance: diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 250decd7567e..2e5cf6e544d5 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -494,7 +494,8 @@ def module_type(self, node: MypyFile) -> Instance: # In test cases might 'types' may not be available. # Fall back to a dummy 'object' type instead to # avoid a crash. - result = self.named_type("builtins.object") + # Make a copy so that we don't set extra_attrs (below) on a shared instance. + result = self.named_type("builtins.object").copy_modified() module_attrs: dict[str, Type] = {} immutable = set() for name, n in node.names.items(): diff --git a/mypy/types.py b/mypy/types.py index c23997d069d4..3f4bd94b5b24 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1708,6 +1708,23 @@ def deserialize(cls, data: JsonDict | str) -> Instance: def write(self, data: Buffer) -> None: write_tag(data, INSTANCE) + if not self.args and not self.last_known_value and not self.extra_attrs: + type_ref = self.type.fullname + if type_ref == "builtins.str": + write_tag(data, INSTANCE_STR) + elif type_ref == "builtins.function": + write_tag(data, INSTANCE_FUNCTION) + elif type_ref == "builtins.int": + write_tag(data, INSTANCE_INT) + elif type_ref == "builtins.bool": + write_tag(data, INSTANCE_BOOL) + elif type_ref == "builtins.object": + write_tag(data, INSTANCE_OBJECT) + else: + write_tag(data, INSTANCE_SIMPLE) + write_str(data, type_ref) + return + write_tag(data, INSTANCE_GENERIC) write_str(data, self.type.fullname) write_type_list(data, self.args) write_type_opt(data, self.last_known_value) @@ -1719,6 +1736,39 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> Instance: + tag = read_tag(data) + # This is quite verbose, but this is very hot code, so we are not + # using dictionary lookups here. + if tag == INSTANCE_STR: + if instance_cache.str_type is None: + instance_cache.str_type = Instance(NOT_READY, []) + instance_cache.str_type.type_ref = "builtins.str" + return instance_cache.str_type + if tag == INSTANCE_FUNCTION: + if instance_cache.function_type is None: + instance_cache.function_type = Instance(NOT_READY, []) + instance_cache.function_type.type_ref = "builtins.function" + return instance_cache.function_type + if tag == INSTANCE_INT: + if instance_cache.int_type is None: + instance_cache.int_type = Instance(NOT_READY, []) + instance_cache.int_type.type_ref = "builtins.int" + return instance_cache.int_type + if tag == INSTANCE_BOOL: + if instance_cache.bool_type is None: + instance_cache.bool_type = Instance(NOT_READY, []) + instance_cache.bool_type.type_ref = "builtins.bool" + return instance_cache.bool_type + if tag == INSTANCE_OBJECT: + if instance_cache.object_type is None: + instance_cache.object_type = Instance(NOT_READY, []) + instance_cache.object_type.type_ref = "builtins.object" + return instance_cache.object_type + if tag == INSTANCE_SIMPLE: + inst = Instance(NOT_READY, []) + inst.type_ref = read_str(data) + return inst + assert tag == INSTANCE_GENERIC type_ref = read_str(data) inst = Instance(NOT_READY, read_type_list(data)) inst.type_ref = type_ref @@ -1769,6 +1819,25 @@ def is_singleton_type(self) -> bool: ) +class InstanceCache: + def __init__(self) -> None: + self.str_type: Instance | None = None + self.function_type: Instance | None = None + self.int_type: Instance | None = None + self.bool_type: Instance | None = None + self.object_type: Instance | None = None + + def reset(self) -> None: + self.str_type = None + self.function_type = None + self.int_type = None + self.bool_type = None + self.object_type = None + + +instance_cache: Final = InstanceCache() + + class FunctionLike(ProperType): """Abstract base class for function types.""" @@ -4142,6 +4211,14 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: TYPE_TYPE: Final[Tag] = 18 PARAMETERS: Final[Tag] = 19 +INSTANCE_STR: Final[Tag] = 101 +INSTANCE_FUNCTION: Final[Tag] = 102 +INSTANCE_INT: Final[Tag] = 103 +INSTANCE_BOOL: Final[Tag] = 104 +INSTANCE_OBJECT: Final[Tag] = 105 +INSTANCE_SIMPLE: Final[Tag] = 106 +INSTANCE_GENERIC: Final[Tag] = 107 + def read_type(data: Buffer) -> Type: tag = read_tag(data) From e35e05f12fd6e682689abe0b39ec1ee07eefdf88 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 2 Sep 2025 13:53:24 +0100 Subject: [PATCH 003/183] [mypyc] Allow defining a single-item free "list" for a native class (#19785) It's quite common to have a class where we almost always have at most a single allocated instance (per thread). Now these instances can be allocated more quickly, by reusing the memory that was used for the most recently freed instance (a separate memory block is reused for each thread on free-threaded builds). It's used like this (only the value 1 is supported for now): ``` from mypy_extensions import mypyc_attr @mypyc_attr(free_list=1) class Foo: ... ``` This makes a microbenchmark that only allocates and immediately frees simple objects repeatedly around 3.8x faster. It's probably worth extending this to support larger free lists in the future. We can later look into enabling this automatically for certain native classes based on profile information. --- mypyc/irbuild/prepare.py | 12 +++++++++- mypyc/irbuild/util.py | 9 +++++-- mypyc/test-data/irbuild-classes.test | 20 ++++++++++++++++ mypyc/test-data/run-classes.test | 36 ++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 95c8c448d642..f47dcb52ceb7 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -351,13 +351,23 @@ def prepare_class_def( ir = mapper.type_to_ir[cdef.info] info = cdef.info - attrs = get_mypyc_attrs(cdef) + attrs, attrs_lines = get_mypyc_attrs(cdef) if attrs.get("allow_interpreted_subclasses") is True: ir.allow_interpreted_subclasses = True if attrs.get("serializable") is True: # Supports copy.copy and pickle (including subclasses) ir._serializable = True + free_list_len = attrs.get("free_list_len") + if free_list_len is not None: + line = attrs_lines["free_list_len"] + if ir.is_trait: + errors.error('"free_list_len" can\'t be used with traits', path, line) + if free_list_len == 1: + ir.reuse_freed_instance = True + else: + errors.error(f'Unsupported value for "free_list_len": {free_list_len}', path, line) + # Check for subclassing from builtin types for cls in info.mro: # Special case exceptions and dicts diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index 757b49c68c83..eca2cac7e9db 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -96,6 +96,8 @@ def get_mypyc_attr_literal(e: Expression) -> Any: return False elif isinstance(e, RefExpr) and e.fullname == "builtins.None": return None + elif isinstance(e, IntExpr): + return e.value return NotImplemented @@ -110,9 +112,10 @@ def get_mypyc_attr_call(d: Expression) -> CallExpr | None: return None -def get_mypyc_attrs(stmt: ClassDef | Decorator) -> dict[str, Any]: +def get_mypyc_attrs(stmt: ClassDef | Decorator) -> tuple[dict[str, Any], dict[str, int]]: """Collect all the mypyc_attr attributes on a class definition or a function.""" attrs: dict[str, Any] = {} + lines: dict[str, int] = {} for dec in stmt.decorators: d = get_mypyc_attr_call(dec) if d: @@ -120,10 +123,12 @@ def get_mypyc_attrs(stmt: ClassDef | Decorator) -> dict[str, Any]: if name is None: if isinstance(arg, StrExpr): attrs[arg.value] = True + lines[arg.value] = d.line else: attrs[name] = get_mypyc_attr_literal(arg) + lines[name] = d.line - return attrs + return attrs, lines def is_extension_class(path: str, cdef: ClassDef, errors: Errors) -> bool: diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index bb55958dc6dc..2f59bb000220 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1734,3 +1734,23 @@ class NonNative: class InheritsPython(dict): def __new__(cls) -> InheritsPython: return super().__new__(cls) # E: super().__new__() not supported for classes inheriting from non-native classes + +[case testClassWithFreeList] +from mypy_extensions import mypyc_attr, trait + +@mypyc_attr(free_list_len=1) +class UsesFreeList: + pass + +@mypyc_attr(free_list_len=None) +class NoFreeList: + pass + +@mypyc_attr(free_list_len=2) # E: Unsupported value for "free_list_len": 2 +class FreeListError: + pass + +@trait +@mypyc_attr(free_list_len=1) # E: "free_list_len" can't be used with traits +class NonNative: + pass diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index b25dc9458fd1..79ad2fa0a03b 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3794,3 +3794,39 @@ assert t.native == 43 assert t.generic == "{}" assert t.bitfield == 0x0C assert t.default == 10 + +[case testPerTypeFreeList] +from __future__ import annotations + +from mypy_extensions import mypyc_attr + +a = [] + +@mypyc_attr(free_list=1) +class Foo: + def __init__(self, x: int) -> None: + self.x = x + a.append(x) + +def test_alloc() -> None: + x: Foo | None + y: Foo | None + + x = Foo(1) + assert x.x == 1 + x = None + + x = Foo(2) + assert x.x == 2 + y = Foo(3) + assert x.x == 2 + assert y.x == 3 + x = None + y = None + assert a == [1, 2, 3] + + x = Foo(4) + assert x.x == 4 + y = Foo(5) + assert x.x == 4 + assert y.x == 5 From 341d3ccd07e591220e1c806bb775c22d9d0dbcef Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 2 Sep 2025 17:11:56 +0200 Subject: [PATCH 004/183] Consider non-empty enums assignable to Self (#19779) Fixes #18345. Fixes #16558. See the linked ticket for reasoning - enums with members are implicitly final and should be treated exactly as if they were decorated with `@final`. --- mypy/typeanal.py | 2 +- test-data/unit/check-selftype.test | 32 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index af70c52180aa..7429030573a3 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -779,7 +779,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ if self.api.type.has_base("builtins.type"): self.fail("Self type cannot be used in a metaclass", t) if self.api.type.self_type is not None: - if self.api.type.is_final: + if self.api.type.is_final or self.api.type.is_enum and self.api.type.enum_members: return fill_typevars(self.api.type) return self.api.type.self_type.copy_modified(line=t.line, column=t.column) # TODO: verify this is unreachable and replace with an assert? diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 6481a1766944..89603efafddd 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2346,3 +2346,35 @@ gc: G[D2] reveal_type(gb.test()) # N: Revealed type is "typing.Sequence[__main__.D1]" reveal_type(gc.test()) # N: Revealed type is "builtins.list[__main__.D2]" [builtins fixtures/list.pyi] + +[case testEnumImplicitlyFinalForSelfType] +from enum import Enum +from typing import Self + +# This enum has members and so is implicitly final. +# Foo and Self are interchangeable within the class. +class Foo(Enum): + A = 1 + + @classmethod + def foo(cls) -> Self: + return Foo.A + + @classmethod + def foo2(cls) -> Self: + return cls.bar() + + @classmethod + def bar(cls) -> Foo: + ... + +# This enum is empty and should not be assignable to Self +class Bar(Enum): + @classmethod + def foo(cls) -> Self: + return cls.bar() # E: Incompatible return value type (got "Bar", expected "Self") + + @classmethod + def bar(cls) -> Bar: + ... +[builtins fixtures/classmethod.pyi] From 7e7d7a7a3e09350f7e2beff0cde10d997539ef6e Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 2 Sep 2025 17:41:15 +0200 Subject: [PATCH 005/183] [mypyc] Fix object finalization (#19749) Fixes https://github.com/mypyc/mypyc/issues/1127 --- mypy/test/helpers.py | 3 +++ mypyc/codegen/emitclass.py | 35 +++++++++++++++----------------- mypyc/test-data/run-classes.test | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index ae432ff6981b..36ad5ad4ec1a 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -233,6 +233,9 @@ def clean_up(a: list[str]) -> list[str]: for p in prefix, prefix.replace(os.sep, "/"): if p != "/" and p != "//" and p != "\\" and p != "\\\\": ss = ss.replace(p, "") + # Replace memory address with zeros + if "at 0x" in ss: + ss = re.sub(r"(at 0x)\w+>", r"\g<1>000000000000>", ss) # Ignore spaces at end of line. ss = re.sub(" +$", "", ss) # Remove pwd from driver.py's path diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 0931c849131d..3103a0dcb5e4 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -889,8 +889,21 @@ def generate_dealloc_for_class( emitter.emit_line(f"{dealloc_func_name}({cl.struct_name(emitter.names)} *self)") emitter.emit_line("{") if has_tp_finalize: - emitter.emit_line("if (!PyObject_GC_IsFinalized((PyObject *)self)) {") - emitter.emit_line("Py_TYPE(self)->tp_finalize((PyObject *)self);") + emitter.emit_line("PyObject *type, *value, *traceback;") + emitter.emit_line("PyErr_Fetch(&type, &value, &traceback);") + emitter.emit_line("int res = PyObject_CallFinalizerFromDealloc((PyObject *)self);") + # CPython interpreter uses PyErr_WriteUnraisable: https://docs.python.org/3/c-api/exceptions.html#c.PyErr_WriteUnraisable + # However, the message is slightly different due to the way mypyc compiles classes. + # CPython interpreter prints: Exception ignored in: + # mypyc prints: Exception ignored in: + emitter.emit_line("if (PyErr_Occurred() != NULL) {") + # Don't untrack instance if error occurred + emitter.emit_line("PyErr_WriteUnraisable((PyObject *)self);") + emitter.emit_line("res = -1;") + emitter.emit_line("}") + emitter.emit_line("PyErr_Restore(type, value, traceback);") + emitter.emit_line("if (res < 0) {") + emitter.emit_line("goto done;") emitter.emit_line("}") emitter.emit_line("PyObject_GC_UnTrack(self);") if cl.reuse_freed_instance: @@ -900,6 +913,7 @@ def generate_dealloc_for_class( emitter.emit_line(f"{clear_func_name}(self);") emitter.emit_line("Py_TYPE(self)->tp_free((PyObject *)self);") emitter.emit_line("CPy_TRASHCAN_END(self)") + emitter.emit_line("done: ;") emitter.emit_line("}") @@ -930,8 +944,6 @@ def generate_finalize_for_class( emitter.emit_line("static void") emitter.emit_line(f"{finalize_func_name}(PyObject *self)") emitter.emit_line("{") - emitter.emit_line("PyObject *type, *value, *traceback;") - emitter.emit_line("PyErr_Fetch(&type, &value, &traceback);") emitter.emit_line( "{}{}{}(self);".format( emitter.get_group_prefix(del_method.decl), @@ -939,21 +951,6 @@ def generate_finalize_for_class( del_method.cname(emitter.names), ) ) - emitter.emit_line("if (PyErr_Occurred() != NULL) {") - emitter.emit_line('PyObject *del_str = PyUnicode_FromString("__del__");') - emitter.emit_line( - "PyObject *del_method = (del_str == NULL) ? NULL : _PyType_Lookup(Py_TYPE(self), del_str);" - ) - # CPython interpreter uses PyErr_WriteUnraisable: https://docs.python.org/3/c-api/exceptions.html#c.PyErr_WriteUnraisable - # However, the message is slightly different due to the way mypyc compiles classes. - # CPython interpreter prints: Exception ignored in: - # mypyc prints: Exception ignored in: - emitter.emit_line("PyErr_WriteUnraisable(del_method);") - emitter.emit_line("Py_XDECREF(del_method);") - emitter.emit_line("Py_XDECREF(del_str);") - emitter.emit_line("}") - # PyErr_Restore also clears exception raised in __del__. - emitter.emit_line("PyErr_Restore(type, value, traceback);") emitter.emit_line("}") diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 79ad2fa0a03b..46d5aaa0cbcb 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3110,7 +3110,7 @@ f = native.F() del f [out] -Exception ignored in: +Exception ignored in: Traceback (most recent call last): File "native.py", line 5, in __del__ raise Exception("e2") From 2ae06676f36d36a6ea573fd1e7679421aca3b1f5 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 2 Sep 2025 20:51:48 +0200 Subject: [PATCH 006/183] Add await to empty context hack (#19777) Fixes #19716. It is a follow-up to #19767 and was missed there due to malformed test stubs (the same testcase fails when run by full mypy against typeshed, I did not notice missing AwaitExpr because of the passing test...). I synced typevar variance with typeshed definitions and added AwaitExpr to the list of context-dependent exprs. Cc @ilevkivskyi --- mypy/checker.py | 7 +++- test-data/unit/check-inference-context.test | 2 + test-data/unit/fixtures/typing-async.pyi | 45 ++++++++++++--------- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 77822b7068ae..ba821df621e5 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -80,6 +80,7 @@ AssertStmt, AssignmentExpr, AssignmentStmt, + AwaitExpr, Block, BreakStmt, BytesExpr, @@ -4924,7 +4925,11 @@ def check_return_stmt(self, s: ReturnStmt) -> None: allow_none_func_call = is_lambda or declared_none_return or declared_any_return # Return with a value. - if isinstance(s.expr, (CallExpr, ListExpr, TupleExpr, DictExpr, SetExpr, OpExpr)): + if ( + isinstance(s.expr, (CallExpr, ListExpr, TupleExpr, DictExpr, SetExpr, OpExpr)) + or isinstance(s.expr, AwaitExpr) + and isinstance(s.expr.expr, CallExpr) + ): # For expressions that (strongly) depend on type context (i.e. those that # are handled like a function call), we allow fallback to empty type context # in case of errors, this improves user experience in some cases, diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index 7dbbd68c4215..a41ee5f59670 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -1582,3 +1582,5 @@ async def inner(c: Cls[T]) -> Optional[T]: async def outer(c: Cls[T]) -> Optional[T]: return await inner(c) +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] diff --git a/test-data/unit/fixtures/typing-async.pyi b/test-data/unit/fixtures/typing-async.pyi index 03728f822316..7ce2821d2916 100644 --- a/test-data/unit/fixtures/typing-async.pyi +++ b/test-data/unit/fixtures/typing-async.pyi @@ -28,7 +28,9 @@ Self = 0 T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) +R_co = TypeVar('R_co', covariant=True) T_contra = TypeVar('T_contra', contravariant=True) +S_contra = TypeVar('S_contra', contravariant=True) U = TypeVar('U') V = TypeVar('V') S = TypeVar('S') @@ -49,9 +51,9 @@ class Iterator(Iterable[T_co], Protocol): @abstractmethod def __next__(self) -> T_co: pass -class Generator(Iterator[T], Generic[T, U, V]): +class Generator(Iterator[T_co], Generic[T_co, S_contra, R_co]): @abstractmethod - def send(self, value: U) -> T: pass + def send(self, value: S_contra) -> T_co: pass @abstractmethod def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass @@ -60,34 +62,39 @@ class Generator(Iterator[T], Generic[T, U, V]): def close(self) -> None: pass @abstractmethod - def __iter__(self) -> 'Generator[T, U, V]': pass + def __iter__(self) -> 'Generator[T_co, S_contra, R_co]': pass -class AsyncGenerator(AsyncIterator[T], Generic[T, U]): +class AsyncGenerator(AsyncIterator[T_co], Generic[T_co, S_contra]): @abstractmethod - def __anext__(self) -> Awaitable[T]: pass + def __anext__(self) -> Awaitable[T_co]: pass @abstractmethod - def asend(self, value: U) -> Awaitable[T]: pass + def asend(self, value: S_contra) -> Awaitable[T_co]: pass @abstractmethod - def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T]: pass + def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T_co]: pass @abstractmethod - def aclose(self) -> Awaitable[T]: pass + def aclose(self) -> Awaitable[T_co]: pass @abstractmethod - def __aiter__(self) -> 'AsyncGenerator[T, U]': pass + def __aiter__(self) -> 'AsyncGenerator[T_co, S_contra]': pass -class Awaitable(Protocol[T]): +class Awaitable(Protocol[T_co]): @abstractmethod - def __await__(self) -> Generator[Any, Any, T]: pass + def __await__(self) -> Generator[Any, Any, T_co]: pass -class AwaitableGenerator(Generator[T, U, V], Awaitable[V], Generic[T, U, V, S], metaclass=ABCMeta): +class AwaitableGenerator( + Awaitable[R_co], + Generator[T_co, S_contra, R_co], + Generic[T_co, S_contra, R_co, S], + metaclass=ABCMeta +): pass -class Coroutine(Awaitable[V], Generic[T, U, V]): +class Coroutine(Awaitable[R_co], Generic[T_co, S_contra, R_co]): @abstractmethod - def send(self, value: U) -> T: pass + def send(self, value: S_contra) -> T_co: pass @abstractmethod def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass @@ -95,14 +102,14 @@ class Coroutine(Awaitable[V], Generic[T, U, V]): @abstractmethod def close(self) -> None: pass -class AsyncIterable(Protocol[T]): +class AsyncIterable(Protocol[T_co]): @abstractmethod - def __aiter__(self) -> 'AsyncIterator[T]': pass + def __aiter__(self) -> 'AsyncIterator[T_co]': pass -class AsyncIterator(AsyncIterable[T], Protocol): - def __aiter__(self) -> 'AsyncIterator[T]': return self +class AsyncIterator(AsyncIterable[T_co], Protocol): + def __aiter__(self) -> 'AsyncIterator[T_co]': return self @abstractmethod - def __anext__(self) -> Awaitable[T]: pass + def __anext__(self) -> Awaitable[T_co]: pass class Sequence(Iterable[T_co], Container[T_co]): @abstractmethod From d40f63cde42adeb0ac7e5fde70a12c26c6679053 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 3 Sep 2025 13:02:24 +0100 Subject: [PATCH 007/183] [mypyc] Allow per-class free list to be used with inheritance (#19790) It's still unsupported if interpreted subclasses are allowed. --- mypyc/codegen/emitclass.py | 4 -- mypyc/irbuild/prepare.py | 6 +++ mypyc/test-data/irbuild-classes.test | 4 ++ mypyc/test-data/run-classes.test | 63 +++++++++++++++++++++++++++- 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 3103a0dcb5e4..94f32b3224a9 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -205,10 +205,6 @@ def generate_class_reuse( TODO: Generalize to support a free list with up to N objects. """ assert cl.reuse_freed_instance - - # The free list implementation doesn't support class hierarchies - assert cl.is_final_class or cl.children == [] - context = c_emitter.context name = cl.name_prefix(c_emitter.names) + "_free_instance" struct_name = cl.struct_name(c_emitter.names) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index f47dcb52ceb7..61e3e5b95cf4 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -363,6 +363,12 @@ def prepare_class_def( line = attrs_lines["free_list_len"] if ir.is_trait: errors.error('"free_list_len" can\'t be used with traits', path, line) + if ir.allow_interpreted_subclasses: + errors.error( + '"free_list_len" can\'t be used in a class that allows interpreted subclasses', + path, + line, + ) if free_list_len == 1: ir.reuse_freed_instance = True else: diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 2f59bb000220..78ca7b68cefb 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1754,3 +1754,7 @@ class FreeListError: @mypyc_attr(free_list_len=1) # E: "free_list_len" can't be used with traits class NonNative: pass + +@mypyc_attr(free_list_len=1, allow_interpreted_subclasses=True) # E: "free_list_len" can't be used in a class that allows interpreted subclasses +class InterpSub: + pass diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 46d5aaa0cbcb..6c4ddc03887a 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3802,7 +3802,7 @@ from mypy_extensions import mypyc_attr a = [] -@mypyc_attr(free_list=1) +@mypyc_attr(free_list_len=1) class Foo: def __init__(self, x: int) -> None: self.x = x @@ -3830,3 +3830,64 @@ def test_alloc() -> None: y = Foo(5) assert x.x == 4 assert y.x == 5 + +@mypyc_attr(free_list_len=1) +class Base: + def __init__(self, x: str) -> None: + self.x = x + +class Deriv(Base): + def __init__(self, x: str, y: str) -> None: + super().__init__(x) + self.y = y + +@mypyc_attr(free_list_len=1) +class Deriv2(Base): + def __init__(self, x: str, y: str) -> None: + super().__init__(x) + self.y = y + +def test_inheritance() -> None: + x: Base | None + y: Base | None + x = Base('x' + str()) + y = Base('y' + str()) + y = None + d = Deriv('a' + str(), 'b' + str()) + assert type(d) is Deriv + assert d.x == 'a' + assert d.y == 'b' + assert x.x == 'x' + y = Base('z' + str()) + assert d.x == 'a' + assert d.y == 'b' + assert y.x == 'z' + x = None + y = None + +def test_inheritance_2() -> None: + x: Base | None + y: Base | None + d: Deriv2 | None + x = Base('x' + str()) + y = Base('y' + str()) + y = None + d = Deriv2('a' + str(), 'b' + str()) + assert type(d) is Deriv2 + assert d.x == 'a' + assert d.y == 'b' + assert x.x == 'x' + d = None + d = Deriv2('c' + str(), 'd' + str()) + assert type(d) is Deriv2 + assert d.x == 'c' + assert d.y == 'd' + assert x.x == 'x' + y = Base('z' + str()) + assert type(y) is Base + assert d.x == 'c' + assert d.y == 'd' + assert y.x == 'z' + x = None + y = None + d = None From 39427835ea4ab9e5735e7e5956063026d1bcc4bb Mon Sep 17 00:00:00 2001 From: Chainfire Date: Wed, 3 Sep 2025 17:58:12 +0200 Subject: [PATCH 008/183] [mypyc] Fix subclass processing in detect_undefined_bitmap (#19787) Incorrect processing in detect_undefined_bitmap could cause a ValueError exception in emit_undefined_attr_check. --- mypyc/analysis/attrdefined.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/analysis/attrdefined.py b/mypyc/analysis/attrdefined.py index 5be57d767e35..1dfd33630f1c 100644 --- a/mypyc/analysis/attrdefined.py +++ b/mypyc/analysis/attrdefined.py @@ -422,7 +422,7 @@ def detect_undefined_bitmap(cl: ClassIR, seen: set[ClassIR]) -> None: return seen.add(cl) for base in cl.base_mro[1:]: - detect_undefined_bitmap(cl, seen) + detect_undefined_bitmap(base, seen) if len(cl.base_mro) > 1: cl.bitmap_attrs.extend(cl.base_mro[1].bitmap_attrs) From 4007c7d5760cd87d6b309d358ffea414fbac975e Mon Sep 17 00:00:00 2001 From: Kevin Kannammalil Date: Thu, 4 Sep 2025 11:34:13 -0400 Subject: [PATCH 009/183] Bump version to 1.19.0+dev (#19793) The release branch has been cut: https://github.com/python/mypy/tree/release-1.18 So this PR increases the dev version --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index bb6a9582e74e..af216bddded1 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.18.0+dev" +__version__ = "1.19.0+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From 645b421fa23a3f0d63f651059fa1e8318a6f214a Mon Sep 17 00:00:00 2001 From: Joren Hammudoglu Date: Thu, 4 Sep 2025 19:18:10 +0200 Subject: [PATCH 010/183] [stubtest] temporary `--ignore-disjoint-bases` flag (#19740) closes #19737 ref: https://github.com/python/mypy/issues/19737#issuecomment-3224801978 --- It's not the prettiest code, but since it will be removed once PEP 800 gets accepted (or rejected), I figured it would be best to keep it simple, so that we can easily revert it. --- mypy/stubtest.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 31b3fd20b002..d4f96a3d9389 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -140,6 +140,11 @@ def is_positional_only_related(self) -> bool: # TODO: This is hacky, use error codes or something more resilient return "should be positional" in self.message + def is_disjoint_base_related(self) -> bool: + """Whether or not the error is related to @disjoint_base.""" + # TODO: This is hacky, use error codes or something more resilient + return "@disjoint_base" in self.message + def get_description(self, concise: bool = False) -> str: """Returns a description of the error. @@ -2181,6 +2186,7 @@ class _Arguments: concise: bool ignore_missing_stub: bool ignore_positional_only: bool + ignore_disjoint_bases: bool allowlist: list[str] generate_allowlist: bool ignore_unused_allowlist: bool @@ -2274,6 +2280,8 @@ def warning_callback(msg: str) -> None: continue if args.ignore_positional_only and error.is_positional_only_related(): continue + if args.ignore_disjoint_bases and error.is_disjoint_base_related(): + continue if error.object_desc in allowlist: allowlist[error.object_desc] = True continue @@ -2364,6 +2372,12 @@ def parse_options(args: list[str]) -> _Arguments: action="store_true", help="Ignore errors for whether an argument should or shouldn't be positional-only", ) + # TODO: Remove once PEP 800 is accepted + parser.add_argument( + "--ignore-disjoint-bases", + action="store_true", + help="Disable checks for PEP 800 @disjoint_base classes", + ) parser.add_argument( "--allowlist", "--whitelist", From d33c147138f56a78d3fca8e2f7b61be46677b13c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 5 Sep 2025 00:50:31 +0100 Subject: [PATCH 011/183] Make --allow-redefinition-new argument public (#19796) It is time to announce this (as still experimental obviously). --- mypy/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/main.py b/mypy/main.py index 706d1daef680..4ca1bde73d40 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -886,7 +886,7 @@ def add_invertible_flag( "--allow-redefinition-new", default=False, strict_flag=False, - help=argparse.SUPPRESS, # This is still very experimental + help="Allow more flexible variable redefinition semantics (experimental)", group=strictness_group, ) From 7e446b414917cbc32c0832236822611a4ff72993 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 5 Sep 2025 12:49:37 +0100 Subject: [PATCH 012/183] [mypyc] Add type annotations to tests (#19794) Missing type annotations can compromise test coverage. My eventual goal is to require annotations by default in all run tests. --- mypyc/test-data/fixtures/ir.py | 4 +- mypyc/test-data/fixtures/typing-full.pyi | 6 +- mypyc/test-data/run-dunders.test | 48 +++++++--- mypyc/test-data/run-singledispatch.test | 108 ++++++++++++++--------- 4 files changed, 106 insertions(+), 60 deletions(-) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index fb5512b77279..075a0eec28d2 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -243,7 +243,7 @@ def __add__(self, value: List[_S], /) -> List[_S | _T]: ... def __iadd__(self, value: Iterable[_T], /) -> List[_T]: ... # type: ignore[misc] def append(self, x: _T) -> None: pass def pop(self, i: int = -1) -> _T: pass - def count(self, _T) -> int: pass + def count(self, x: _T) -> int: pass def extend(self, l: Iterable[_T]) -> None: pass def insert(self, i: int, x: _T) -> None: pass def sort(self) -> None: pass @@ -365,7 +365,7 @@ def reversed(object: Sequence[_T]) -> Iterator[_T]: ... def id(o: object) -> int: pass # This type is obviously wrong but the test stubs don't have Sized anymore def len(o: object) -> int: pass -def print(*object) -> None: pass +def print(*args: object) -> None: pass def isinstance(x: object, t: object) -> bool: pass def iter(i: Iterable[_T]) -> Iterator[_T]: pass @overload diff --git a/mypyc/test-data/fixtures/typing-full.pyi b/mypyc/test-data/fixtures/typing-full.pyi index 8d89e4f93bc9..25aaaf700d05 100644 --- a/mypyc/test-data/fixtures/typing-full.pyi +++ b/mypyc/test-data/fixtures/typing-full.pyi @@ -11,10 +11,10 @@ from abc import abstractmethod, ABCMeta class GenericMeta(type): pass class _SpecialForm: - def __getitem__(self, index): ... + def __getitem__(self, index: Any) -> Any: ... class TypeVar: - def __init__(self, name, *args, bound=None): ... - def __or__(self, other): ... + def __init__(self, name: str, *args: Any, bound: Any = None): ... + def __or__(self, other: Any) -> Any: ... cast = 0 overload = 0 diff --git a/mypyc/test-data/run-dunders.test b/mypyc/test-data/run-dunders.test index ec992afbfbd1..a3ec06763d75 100644 --- a/mypyc/test-data/run-dunders.test +++ b/mypyc/test-data/run-dunders.test @@ -185,7 +185,7 @@ class SeqError: def __contains__(self, x: int) -> bool: raise RuntimeError() - def __len__(self): + def __len__(self) -> int: return -5 def any_seq_error() -> Any: @@ -545,6 +545,7 @@ def test_type_mismatch_fall_back_to_reverse() -> None: assert F()**G() == -6 [case testDundersBinaryNotImplemented] +# mypy: allow-untyped-defs from typing import Any, Union from testutil import assertRaises @@ -617,15 +618,28 @@ def test_unannotated_add() -> None: with assertRaises(TypeError, "unsupported operand type(s) for +: 'F' and 'str'"): o + 'x' + o2: Any = F(4) + assert o2 + 5 == 9 + with assertRaises(TypeError, "unsupported operand type(s) for +: 'F' and 'str'"): + o2 + 'x' + def test_unannotated_add_and_radd_1() -> None: o = F(4) assert o + G() == 5 + o2: Any = F(4) + assert o2 + G() == 5 + def test_unannotated_radd() -> None: assert 'x' + G() == 'a' with assertRaises(TypeError, "unsupported operand type(s) for +: 'int' and 'G'"): 1 + G() + o: Any = G() + assert 'x' + o == 'a' + with assertRaises(TypeError, "unsupported operand type(s) for +: 'int' and 'G'"): + 1 + o + class H: def __add__(self, x): if isinstance(x, int): @@ -644,40 +658,48 @@ def test_unannotated_add_and_radd_2() -> None: with assertRaises(TypeError, "unsupported operand type(s) for +: 'int' and 'H'"): 1 + h + h2: Any = H() + assert h + 5 == 6 + assert 'x' + h == 22 + with assertRaises(TypeError, "unsupported operand type(s) for +: 'int' and 'H'"): + 1 + h + # TODO: Inheritance [case testDifferentReverseDunders] +from typing import Any + class C: # __radd__ and __rsub__ are tested elsewhere - def __rmul__(self, x): + def __rmul__(self, x: Any) -> int: return 1 - def __rtruediv__(self, x): + def __rtruediv__(self, x: Any) -> int: return 2 - def __rmod__(self, x): + def __rmod__(self, x: Any) -> int: return 3 - def __rfloordiv__(self, x): + def __rfloordiv__(self, x: Any) -> int: return 4 - def __rlshift__(self, x): + def __rlshift__(self, x: Any) -> int: return 5 - def __rrshift__(self, x): + def __rrshift__(self, x: Any) -> int: return 6 - def __rand__(self, x): + def __rand__(self, x: Any) -> int: return 7 - def __ror__(self, x): + def __ror__(self, x: Any) -> int: return 8 - def __rxor__(self, x): + def __rxor__(self, x: Any) -> int: return 9 - def __rmatmul__(self, x): + def __rmatmul__(self, x: Any) -> int: return 10 def test_reverse_dunders() -> None: @@ -803,10 +825,10 @@ def test_error() -> None: c += 'x' class BadInplaceAdd: - def __init__(self): + def __init__(self) -> None: self.x = 0 - def __iadd__(self, x): + def __iadd__(self, x: int) -> Any: self.x += x def test_in_place_operator_returns_none() -> None: diff --git a/mypyc/test-data/run-singledispatch.test b/mypyc/test-data/run-singledispatch.test index a119c325984a..03b937261e22 100644 --- a/mypyc/test-data/run-singledispatch.test +++ b/mypyc/test-data/run-singledispatch.test @@ -4,9 +4,10 @@ [case testSpecializedImplementationUsed] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register @@ -19,11 +20,13 @@ def test_specialize() -> None: [case testSubclassesOfExpectedTypeUseSpecialized] from functools import singledispatch +from typing import Any + class A: pass class B(A): pass @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register @@ -36,11 +39,13 @@ def test_specialize() -> None: [case testSuperclassImplementationNotUsedWhenSubclassHasImplementation] from functools import singledispatch +from typing import Any + class A: pass class B(A): pass @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: # shouldn't be using this assert False @@ -58,9 +63,10 @@ def test_specialize() -> None: [case testMultipleUnderscoreFunctionsIsntError] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> str: +def fun(arg: Any) -> str: return 'default' @fun.register @@ -72,7 +78,7 @@ def _(arg: int) -> str: return 'int' # extra function to make sure all 3 underscore functions aren't treated as one OverloadedFuncDef -def a(b): pass +def a(b: Any) -> Any: pass @fun.register def _(arg: list) -> str: @@ -86,10 +92,12 @@ def test_singledispatch() -> None: [case testCanRegisterCompiledClasses] from functools import singledispatch +from typing import Any + class A: pass @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register def fun_specialized(arg: A) -> bool: @@ -101,13 +109,14 @@ def test_singledispatch() -> None: [case testTypeUsedAsArgumentToRegister] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register(int) -def fun_specialized(arg) -> bool: +def fun_specialized(arg: Any) -> bool: return True def test_singledispatch() -> None: @@ -116,12 +125,13 @@ def test_singledispatch() -> None: [case testUseRegisterAsAFunction] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False -def fun_specialized_impl(arg) -> bool: +def fun_specialized_impl(arg: Any) -> bool: return True fun.register(int, fun_specialized_impl) @@ -132,13 +142,14 @@ def test_singledispatch() -> None: [case testRegisterDoesntChangeFunction] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register(int) -def fun_specialized(arg) -> bool: +def fun_specialized(arg: Any) -> bool: return True def test_singledispatch() -> None: @@ -147,9 +158,10 @@ def test_singledispatch() -> None: # TODO: turn this into a mypy error [case testNoneIsntATypeWhenUsedAsArgumentToRegister] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False def test_argument() -> None: @@ -163,14 +175,15 @@ def test_argument() -> None: [case testRegisteringTheSameFunctionSeveralTimes] from functools import singledispatch +from typing import Any @singledispatch -def fun(arg) -> bool: +def fun(arg: Any) -> bool: return False @fun.register(int) @fun.register(str) -def fun_specialized(arg) -> bool: +def fun_specialized(arg: Any) -> bool: return True def test_singledispatch() -> None: @@ -179,11 +192,13 @@ def test_singledispatch() -> None: assert not fun([1, 2]) [case testTypeIsAnABC] +# mypy: allow-untyped-defs from functools import singledispatch from collections.abc import Mapping @singledispatch def fun(arg) -> bool: + # TODO: Adding an Any parameter annotation breaks the test case return False @fun.register @@ -241,6 +256,7 @@ def test_singledispatchmethod() -> None: [case testSingledispatchTreeSumAndEqual] from functools import singledispatch +from typing import cast class Tree: pass @@ -286,10 +302,10 @@ def build(n: int) -> Tree: return Leaf() return Node(n, build(n - 1), build(n - 1)) -def test_sum_and_equal(): +def test_sum_and_equal() -> None: tree = build(5) tree2 = build(5) - tree2.right.right.right.value = 10 + cast(Node, cast(Node, cast(Node, cast(Node, tree2).right).right).right).value = 10 assert calc_sum(tree) == 57 assert calc_sum(tree2) == 65 assert equal(tree, tree) @@ -354,7 +370,7 @@ def verify_typevarexpr(stub: TypeVarExpr, a: MaybeMissing[Any], b: List[str]) -> if False: yield None -def verify_list(stub, a, b) -> List[str]: +def verify_list(stub: Any, a: Any, b: Any) -> List[str]: """Helper function that converts iterator of errors to list of messages""" return list(err.msg for err in verify(stub, a, b)) @@ -368,31 +384,33 @@ def test_verify() -> None: [case testArgsInRegisteredImplNamedDifferentlyFromMainFunction] from functools import singledispatch +from typing import Any @singledispatch -def f(a) -> bool: +def f(a: Any) -> bool: return False @f.register def g(b: int) -> bool: return True -def test_singledispatch(): +def test_singledispatch() -> None: assert f(5) assert not f('a') [case testKeywordArguments] from functools import singledispatch +from typing import Any @singledispatch -def f(arg, *, kwarg: int = 0) -> int: +def f(arg: Any, *, kwarg: int = 0) -> int: return kwarg + 10 @f.register def g(arg: int, *, kwarg: int = 5) -> int: return kwarg - 10 -def test_keywords(): +def test_keywords() -> None: assert f('a') == 10 assert f('a', kwarg=3) == 13 assert f('a', kwarg=7) == 17 @@ -413,14 +431,14 @@ def f(arg: Any) -> Iterable[int]: def g(arg: str) -> Iterable[int]: return [0] -def test_iterables(): +def test_iterables() -> None: assert f(1) != [1] assert list(f(1)) == [1] assert f('a') == [0] [case testRegisterUsedAtSameTimeAsOtherDecorators] from functools import singledispatch -from typing import TypeVar +from typing import TypeVar, Any class A: pass class B: pass @@ -431,7 +449,7 @@ def decorator(f: T) -> T: return f @singledispatch -def f(arg) -> int: +def f(arg: Any) -> int: return 0 @f.register @@ -439,7 +457,7 @@ def f(arg) -> int: def h(arg: str) -> int: return 2 -def test_singledispatch(): +def test_singledispatch() -> None: assert f(1) == 0 assert f('a') == 2 @@ -450,12 +468,12 @@ from typing import Callable, Any class A: pass def decorator(f: Callable[[Any], int]) -> Callable[[Any], int]: - def wrapper(x) -> int: + def wrapper(x: Any) -> int: return f(x) * 7 return wrapper @singledispatch -def f(arg) -> int: +def f(arg: Any) -> int: return 10 @f.register @@ -464,17 +482,19 @@ def h(arg: str) -> int: return 5 -def test_singledispatch(): +def test_singledispatch() -> None: assert f('a') == 35 assert f(A()) == 10 [case testMoreSpecificTypeBeforeLessSpecificType] from functools import singledispatch +from typing import Any + class A: pass class B(A): pass @singledispatch -def f(arg) -> str: +def f(arg: Any) -> str: return 'default' @f.register @@ -485,20 +505,21 @@ def g(arg: B) -> str: def h(arg: A) -> str: return 'a' -def test_singledispatch(): +def test_singledispatch() -> None: assert f(B()) == 'b' assert f(A()) == 'a' assert f(5) == 'default' [case testMultipleRelatedClassesBeingRegistered] from functools import singledispatch +from typing import Any class A: pass class B(A): pass class C(B): pass @singledispatch -def f(arg) -> str: return 'default' +def f(arg: Any) -> str: return 'default' @f.register def _(arg: A) -> str: return 'a' @@ -509,7 +530,7 @@ def _(arg: C) -> str: return 'c' @f.register def _(arg: B) -> str: return 'b' -def test_singledispatch(): +def test_singledispatch() -> None: assert f(A()) == 'a' assert f(B()) == 'b' assert f(C()) == 'c' @@ -525,7 +546,7 @@ def a(arg: A) -> int: def _(arg: C) -> int: return 3 -def test_singledispatch(): +def test_singledispatch() -> None: assert f(B()) == 1 assert f(A()) == 2 assert f(C()) == 3 @@ -539,7 +560,7 @@ class B(A): pass class C(B): pass @singledispatch -def f(arg) -> int: +def f(arg: object) -> int: return 0 @f.register @@ -549,6 +570,7 @@ def g(arg: B) -> int: [case testOrderCanOnlyBeDeterminedFromMRONotIsinstanceChecks] from mypy_extensions import trait from functools import singledispatch +from typing import Any @trait class A: pass @@ -558,9 +580,8 @@ class AB(A, B): pass class BA(B, A): pass @singledispatch -def f(arg) -> str: +def f(arg: Any) -> str: return "default" - pass @f.register def fa(arg: A) -> str: @@ -570,18 +591,19 @@ def fa(arg: A) -> str: def fb(arg: B) -> str: return "b" -def test_singledispatch(): +def test_singledispatch() -> None: assert f(AB()) == "a" assert f(BA()) == "b" [case testCallingFunctionBeforeAllImplementationsRegistered] from functools import singledispatch +from typing import Any class A: pass class B(A): pass @singledispatch -def f(arg) -> str: +def f(arg: Any) -> str: return 'default' assert f(A()) == 'default' @@ -609,6 +631,7 @@ def test_final() -> None: [case testDynamicallyRegisteringFunctionFromInterpretedCode] from functools import singledispatch +from typing import Any class A: pass class B(A): pass @@ -616,7 +639,7 @@ class C(B): pass class D(C): pass @singledispatch -def f(arg) -> str: +def f(arg: Any) -> str: return "default" @f.register @@ -647,9 +670,10 @@ assert c(A()) == 'c' [case testMalformedDynamicRegisterCall] from functools import singledispatch +from typing import Any @singledispatch -def f(arg) -> None: +def f(arg: Any) -> None: pass [file register.py] from native import f @@ -667,7 +691,7 @@ import register from functools import singledispatch @singledispatch -def f(arg) -> str: +def f(arg: object) -> str: return 'default' [file register.py] From 8f2371a565eb9c29f03922b26da8eab054fbbcf8 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 5 Sep 2025 11:15:36 -0400 Subject: [PATCH 013/183] feat: new mypyc primitives for weakref.proxy (#19217) This PR adds 2 new weakref primitives for weakref.proxy (1 and 2 arg) --- mypyc/primitives/weakref_ops.py | 18 ++++++++++ mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-weakref.test | 52 ++++++++++++++++++++++++++++ mypyc/test-data/run-weakref.test | 44 +++++++++++++++++------ test-data/unit/lib-stub/_weakref.pyi | 11 ++++++ test-data/unit/lib-stub/weakref.pyi | 14 +++++++- 6 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 test-data/unit/lib-stub/_weakref.pyi diff --git a/mypyc/primitives/weakref_ops.py b/mypyc/primitives/weakref_ops.py index a7ac035b22a4..21379d3b2c82 100644 --- a/mypyc/primitives/weakref_ops.py +++ b/mypyc/primitives/weakref_ops.py @@ -20,3 +20,21 @@ c_function_name="PyWeakref_NewRef", error_kind=ERR_MAGIC, ) + +new_proxy_op = function_op( + name="_weakref.proxy", + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyWeakref_NewProxy", + extra_int_constants=[(0, pointer_rprimitive)], + error_kind=ERR_MAGIC, +) + +new_proxy_with_callback_op = function_op( + name="_weakref.proxy", + arg_types=[object_rprimitive, object_rprimitive], + # steals=[True, False], + return_type=object_rprimitive, + c_function_name="PyWeakref_NewProxy", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 075a0eec28d2..a4b4f3ce2b1f 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -343,6 +343,7 @@ class RuntimeError(Exception): pass class UnicodeEncodeError(RuntimeError): pass class UnicodeDecodeError(RuntimeError): pass class NotImplementedError(RuntimeError): pass +class ReferenceError(Exception): pass class StopIteration(Exception): value: Any diff --git a/mypyc/test-data/irbuild-weakref.test b/mypyc/test-data/irbuild-weakref.test index 58ac6417d297..2180b1e747aa 100644 --- a/mypyc/test-data/irbuild-weakref.test +++ b/mypyc/test-data/irbuild-weakref.test @@ -49,3 +49,55 @@ def f(x, cb): L0: r0 = PyWeakref_NewRef(x, cb) return r0 + +[case testWeakrefProxy] +import weakref +from typing import Any, Callable +def f(x: object) -> object: + return weakref.proxy(x) + +[out] +def f(x): + x, r0 :: object +L0: + r0 = PyWeakref_NewProxy(x, 0) + return r0 + +[case testWeakrefProxyCallback] +import weakref +from typing import Any, Callable +def f(x: object, cb: Callable[[object], Any]) -> object: + return weakref.proxy(x, cb) + +[out] +def f(x, cb): + x, cb, r0 :: object +L0: + r0 = PyWeakref_NewProxy(x, cb) + return r0 + +[case testFromWeakrefProxy] +from typing import Any, Callable +from weakref import proxy +def f(x: object) -> object: + return proxy(x) + +[out] +def f(x): + x, r0 :: object +L0: + r0 = PyWeakref_NewProxy(x, 0) + return r0 + +[case testFromWeakrefProxyCallback] +from typing import Any, Callable +from weakref import proxy +def f(x: object, cb: Callable[[object], Any]) -> object: + return proxy(x, cb) + +[out] +def f(x, cb): + x, cb, r0 :: object +L0: + r0 = PyWeakref_NewProxy(x, cb) + return r0 diff --git a/mypyc/test-data/run-weakref.test b/mypyc/test-data/run-weakref.test index 902c9e407ff4..0a0e180d635d 100644 --- a/mypyc/test-data/run-weakref.test +++ b/mypyc/test-data/run-weakref.test @@ -1,30 +1,52 @@ # Test cases for weakrefs (compile and run) [case testWeakrefRef] -from weakref import ref +# mypy: disable-error-code="union-attr" +from weakref import proxy, ref from mypy_extensions import mypyc_attr +from testutil import assertRaises +from typing import Optional @mypyc_attr(native_class=False) class Object: """some random weakreffable object""" - pass + def some_meth(self) -> int: + return 1 -def test_weakref_ref(): - obj = Object() +_callback_called_cache = {"ref": False, "proxy": False} + +def test_weakref_ref() -> None: + obj: Optional[Object] = Object() r = ref(obj) assert r() is obj obj = None assert r() is None, r() -def test_weakref_ref_with_callback(): - obj = Object() - r = ref(obj, lambda x: x) +def test_weakref_ref_with_callback() -> None: + obj: Optional[Object] = Object() + r = ref(obj, lambda x: _callback_called_cache.__setitem__("ref", True)) assert r() is obj obj = None assert r() is None, r() + assert _callback_called_cache["ref"] is True -[file driver.py] -from native import test_weakref_ref, test_weakref_ref_with_callback +def test_weakref_proxy() -> None: + obj: Optional[Object] = Object() + p = proxy(obj) + assert obj.some_meth() == 1 + assert p.some_meth() == 1 + obj.some_meth() + obj = None + with assertRaises(ReferenceError): + p.some_meth() -test_weakref_ref() -test_weakref_ref_with_callback() +def test_weakref_proxy_with_callback() -> None: + obj: Optional[Object] = Object() + p = proxy(obj, lambda x: _callback_called_cache.__setitem__("proxy", True)) + assert obj.some_meth() == 1 + assert p.some_meth() == 1 + obj.some_meth() + obj = None + with assertRaises(ReferenceError): + p.some_meth() + assert _callback_called_cache["proxy"] is True diff --git a/test-data/unit/lib-stub/_weakref.pyi b/test-data/unit/lib-stub/_weakref.pyi new file mode 100644 index 000000000000..50c59b65e267 --- /dev/null +++ b/test-data/unit/lib-stub/_weakref.pyi @@ -0,0 +1,11 @@ +from typing import Any, Callable, TypeVar, overload +from weakref import CallableProxyType, ProxyType + +_C = TypeVar("_C", bound=Callable[..., Any]) +_T = TypeVar("_T") + +# Return CallableProxyType if object is callable, ProxyType otherwise +@overload +def proxy(object: _C, callback: Callable[[CallableProxyType[_C]], Any] | None = None, /) -> CallableProxyType[_C]: ... +@overload +def proxy(object: _T, callback: Callable[[ProxyType[_T]], Any] | None = None, /) -> ProxyType[_T]: ... diff --git a/test-data/unit/lib-stub/weakref.pyi b/test-data/unit/lib-stub/weakref.pyi index 34e01f4d48f1..7d11b65d4548 100644 --- a/test-data/unit/lib-stub/weakref.pyi +++ b/test-data/unit/lib-stub/weakref.pyi @@ -1,11 +1,23 @@ +from _weakref import proxy from collections.abc import Callable -from typing import Any, Generic, TypeVar +from typing import Any, ClassVar, Generic, TypeVar, final from typing_extensions import Self +_C = TypeVar("_C", bound=Callable[..., Any]) _T = TypeVar("_T") class ReferenceType(Generic[_T]): # "weakref" __callback__: Callable[[Self], Any] def __new__(cls, o: _T, callback: Callable[[Self], Any] | None = ..., /) -> Self: ... + def __call__(self) -> _T | None: ... ref = ReferenceType + +@final +class CallableProxyType(Generic[_C]): # "weakcallableproxy" + def __eq__(self, value: object, /) -> bool: ... + def __getattr__(self, attr: str) -> Any: ... + __call__: _C + __hash__: ClassVar[None] # type: ignore[assignment] + +__all__ = ["proxy"] From 309b01e287e3afabeb888572455f9bf55d86acad Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Sep 2025 01:16:36 +0100 Subject: [PATCH 014/183] Make untyped_calls_exclude invalidate cache (#19801) Fixes https://github.com/python/mypy/issues/16652 This is minor and straightforward, so I am going to just merge it (assuming everything passes). --- mypy/options.py | 1 + test-data/unit/check-incremental.test | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/mypy/options.py b/mypy/options.py index b3dc9639a41d..5aced56c940f 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -74,6 +74,7 @@ class BuildType: "disable_memoryview_promotion", "strict_bytes", "fixed_format_cache", + "untyped_calls_exclude", } ) - {"debug_cache"} diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index defe7402730f..76532e6eba4a 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6865,6 +6865,24 @@ if int(): [out2] main:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") +[case testUntypedCallsExcludeAffectsCache] +# flags: --disallow-untyped-calls --untyped-calls-exclude=mod.Super +# flags2: --disallow-untyped-calls --untyped-calls-exclude=mod +# flags3: --disallow-untyped-calls --untyped-calls-exclude=mod.Super +import mod +[file mod.py] +class Super: + def draw(self): + ... +class Class(Super): + ... +Class().draw() +[out] +tmp/mod.py:6: error: Call to untyped function "draw" in typed context +[out2] +[out3] +tmp/mod.py:6: error: Call to untyped function "draw" in typed context + [case testMethodMakeBoundIncremental] from a import A a = A() From bf77aab8033ae1d8adcc3039c2923832a65da1df Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Sep 2025 12:15:24 +0100 Subject: [PATCH 015/183] Traverse ParamSpec prefix where we should (#19800) Fixes https://github.com/python/mypy/issues/18087 I started from fixing an incremental crash from `fixup.py`, but then noticed we don't traverse `ParamSpec` prefix in multiple places where we should, so I fixed most of those as well. --- mypy/checkexpr.py | 2 +- mypy/erasetype.py | 1 + mypy/fixup.py | 1 + mypy/indirection.py | 1 + mypy/server/astdiff.py | 1 + mypy/server/astmerge.py | 1 + mypy/server/deps.py | 17 ++----- mypy/type_visitor.py | 2 +- mypy/typeanal.py | 2 +- mypy/typetraverser.py | 1 + test-data/unit/check-incremental.test | 67 +++++++++++++++++++++++++++ test-data/unit/fine-grained.test | 33 +++++++++++++ 12 files changed, 114 insertions(+), 15 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 2e5cf6e544d5..835eeb725394 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -6435,7 +6435,7 @@ def visit_type_var(self, t: TypeVarType) -> bool: def visit_param_spec(self, t: ParamSpecType) -> bool: default = [t.default] if t.has_default() else [] - return self.query_types([t.upper_bound, *default]) + return self.query_types([t.upper_bound, *default, t.prefix]) def visit_type_var_tuple(self, t: TypeVarTupleType) -> bool: default = [t.default] if t.has_default() else [] diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 3f33ea1648f0..6645bcf916d9 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -222,6 +222,7 @@ def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: return t def visit_param_spec(self, t: ParamSpecType) -> Type: + # TODO: we should probably preserve prefix here. if self.erase_id is None or self.erase_id(t.id): return self.replacement return t diff --git a/mypy/fixup.py b/mypy/fixup.py index bec5929ad4b1..260c0f84cf1b 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -347,6 +347,7 @@ def visit_type_var(self, tvt: TypeVarType) -> None: def visit_param_spec(self, p: ParamSpecType) -> None: p.upper_bound.accept(self) p.default.accept(self) + p.prefix.accept(self) def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: t.tuple_fallback.accept(self) diff --git a/mypy/indirection.py b/mypy/indirection.py index 06a158818fbe..88258b94d94a 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -93,6 +93,7 @@ def visit_type_var(self, t: types.TypeVarType) -> None: def visit_param_spec(self, t: types.ParamSpecType) -> None: self._visit(t.upper_bound) self._visit(t.default) + self._visit(t.prefix) def visit_type_var_tuple(self, t: types.TypeVarTupleType) -> None: self._visit(t.upper_bound) diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 1df85a163e0f..25542ce37588 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -435,6 +435,7 @@ def visit_param_spec(self, typ: ParamSpecType) -> SnapshotItem: typ.flavor, snapshot_type(typ.upper_bound), snapshot_type(typ.default), + snapshot_type(typ.prefix), ) def visit_type_var_tuple(self, typ: TypeVarTupleType) -> SnapshotItem: diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 33e2d2b799cb..cda1d20fb8e4 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -489,6 +489,7 @@ def visit_type_var(self, typ: TypeVarType) -> None: def visit_param_spec(self, typ: ParamSpecType) -> None: typ.upper_bound.accept(self) typ.default.accept(self) + typ.prefix.accept(self) def visit_type_var_tuple(self, typ: TypeVarTupleType) -> None: typ.upper_bound.accept(self) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index b994a214f67a..9d4445a1f7e7 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -1037,10 +1037,8 @@ def visit_type_var(self, typ: TypeVarType) -> list[str]: triggers = [] if typ.fullname: triggers.append(make_trigger(typ.fullname)) - if typ.upper_bound: - triggers.extend(self.get_type_triggers(typ.upper_bound)) - if typ.default: - triggers.extend(self.get_type_triggers(typ.default)) + triggers.extend(self.get_type_triggers(typ.upper_bound)) + triggers.extend(self.get_type_triggers(typ.default)) for val in typ.values: triggers.extend(self.get_type_triggers(val)) return triggers @@ -1049,22 +1047,17 @@ def visit_param_spec(self, typ: ParamSpecType) -> list[str]: triggers = [] if typ.fullname: triggers.append(make_trigger(typ.fullname)) - if typ.upper_bound: - triggers.extend(self.get_type_triggers(typ.upper_bound)) - if typ.default: - triggers.extend(self.get_type_triggers(typ.default)) triggers.extend(self.get_type_triggers(typ.upper_bound)) + triggers.extend(self.get_type_triggers(typ.default)) + triggers.extend(self.get_type_triggers(typ.prefix)) return triggers def visit_type_var_tuple(self, typ: TypeVarTupleType) -> list[str]: triggers = [] if typ.fullname: triggers.append(make_trigger(typ.fullname)) - if typ.upper_bound: - triggers.extend(self.get_type_triggers(typ.upper_bound)) - if typ.default: - triggers.extend(self.get_type_triggers(typ.default)) triggers.extend(self.get_type_triggers(typ.upper_bound)) + triggers.extend(self.get_type_triggers(typ.default)) return triggers def visit_unpack_type(self, typ: UnpackType) -> list[str]: diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 65051ddbab67..15494393cae6 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -525,7 +525,7 @@ def visit_type_var(self, t: TypeVarType, /) -> bool: return self.query_types([t.upper_bound, t.default] + t.values) def visit_param_spec(self, t: ParamSpecType, /) -> bool: - return self.query_types([t.upper_bound, t.default]) + return self.query_types([t.upper_bound, t.default, t.prefix]) def visit_type_var_tuple(self, t: TypeVarTupleType, /) -> bool: return self.query_types([t.upper_bound, t.default]) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 7429030573a3..658730414763 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -2615,7 +2615,7 @@ def visit_type_var(self, t: TypeVarType) -> None: self.process_types([t.upper_bound, t.default] + t.values) def visit_param_spec(self, t: ParamSpecType) -> None: - self.process_types([t.upper_bound, t.default]) + self.process_types([t.upper_bound, t.default, t.prefix]) def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: self.process_types([t.upper_bound, t.default]) diff --git a/mypy/typetraverser.py b/mypy/typetraverser.py index 047c5caf6dae..abd0f6bf3bdf 100644 --- a/mypy/typetraverser.py +++ b/mypy/typetraverser.py @@ -64,6 +64,7 @@ def visit_type_var(self, t: TypeVarType, /) -> None: t.default.accept(self) def visit_param_spec(self, t: ParamSpecType, /) -> None: + # TODO: do we need to traverse prefix here? t.default.accept(self) def visit_parameters(self, t: Parameters, /) -> None: diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 76532e6eba4a..9f5c811dc0a1 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6915,3 +6915,70 @@ import does_not_exist [builtins fixtures/ops.pyi] [out] [out2] + +[case testIncrementalNoCrashOnParamSpecPrefixUpdateMethod] +import impl +[file impl.py] +from typing_extensions import ParamSpec +from lib import Sub + +P = ParamSpec("P") +class Impl(Sub[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.meth(1, *args, **kwargs) + +[file impl.py.2] +from typing_extensions import ParamSpec +from lib import Sub + +P = ParamSpec("P") +class Impl(Sub[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.meth("no", *args, **kwargs) + +[file lib.py] +from typing import Generic +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +class Base(Generic[P]): + def meth(self, *args: P.args, **kwargs: P.kwargs) -> None: ... +class Sub(Base[Concatenate[int, P]]): ... +[builtins fixtures/paramspec.pyi] +[out] +[out2] +tmp/impl.py:7: error: Argument 1 to "meth" of "Base" has incompatible type "str"; expected "int" + +[case testIncrementalNoCrashOnParamSpecPrefixUpdateMethodAlias] +import impl +[file impl.py] +from typing_extensions import ParamSpec +from lib import Sub + +P = ParamSpec("P") +class Impl(Sub[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.alias(1, *args, **kwargs) + +[file impl.py.2] +from typing_extensions import ParamSpec +from lib import Sub + +P = ParamSpec("P") +class Impl(Sub[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.alias("no", *args, **kwargs) + +[file lib.py] +from typing import Generic +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +class Base(Generic[P]): + def meth(self, *args: P.args, **kwargs: P.kwargs) -> None: ... + alias = meth +class Sub(Base[Concatenate[int, P]]): ... +[builtins fixtures/paramspec.pyi] +[out] +[out2] +tmp/impl.py:7: error: Argument 1 has incompatible type "str"; expected "int" diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 888b7bc7e97f..1bddee0e5ed2 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -11485,3 +11485,36 @@ class A: [out] == main:3: error: Too few arguments + +[case testFineGrainedParamSpecPrefixUpdateMethod] +import impl +[file impl.py] +from typing_extensions import ParamSpec +from lib import Sub + +P = ParamSpec("P") +class Impl(Sub[P]): + def test(self, *args: P.args, **kwargs: P.kwargs) -> None: + self.meth(1, *args, **kwargs) + +[file lib.py] +from typing import Generic +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +class Base(Generic[P]): + def meth(self, *args: P.args, **kwargs: P.kwargs) -> None: ... +class Sub(Base[Concatenate[int, P]]): ... + +[file lib.py.2] +from typing import Generic +from typing_extensions import ParamSpec, Concatenate + +P = ParamSpec("P") +class Base(Generic[P]): + def meth(self, *args: P.args, **kwargs: P.kwargs) -> None: ... +class Sub(Base[Concatenate[str, P]]): ... +[builtins fixtures/paramspec.pyi] +[out] +== +impl.py:7: error: Argument 1 to "meth" of "Base" has incompatible type "int"; expected "str" From 8437cf5d43cd099d10838f0bc9cadd9eed94425d Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sun, 7 Sep 2025 02:40:07 +0200 Subject: [PATCH 016/183] Do not report exhaustive-match after deferral (#19804) Fixes #19791 --- mypy/checker.py | 4 ++-- test-data/unit/check-python310.test | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index ba821df621e5..5843148d4b4e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5682,8 +5682,8 @@ def visit_match_stmt(self, s: MatchStmt) -> None: self.push_type_map(else_map, from_assignment=False) unmatched_types = else_map - if unmatched_types is not None: - for typ in list(unmatched_types.values()): + if unmatched_types is not None and not self.current_node_deferred: + for typ in unmatched_types.values(): self.msg.match_statement_inexhaustive_match(typ, s) # This is needed due to a quirk in frame_context. Without it types will stay narrowed diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 5c495d2ed863..7d76c09b6151 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -2975,3 +2975,18 @@ val: int = 8 match val: case FOO: # E: Cannot assign to final name "FOO" pass + +[case testMatchExhaustivenessWithDeferral] +# flags: --enable-error-code exhaustive-match +from typing import Literal +import unknown_module # E: Cannot find implementation or library stub for module named "unknown_module" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +def foo(e: Literal[0, 1]) -> None: + match e: + case 0: + defer + case 1: + ... + +defer = unknown_module.foo From 60639c5c07230c498c71d8d22bf16a4ee769777f Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 8 Sep 2025 03:41:35 +0700 Subject: [PATCH 017/183] Update options.py: typo `scrict_equality` and phrasing (#19806) Fixes a typo `scrict_equality` which meant `strict_equality`. While in there, I also changed a preposition to be more specific. --- mypy/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/options.py b/mypy/options.py index 5aced56c940f..b1456934c6c9 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -232,7 +232,7 @@ def __init__(self) -> None: # This makes 1 == '1', 1 in ['1'], and 1 is '1' errors. self.strict_equality = False - # Extend the logic of `scrict_equality` for comparisons with `None`. + # Extend the logic of `strict_equality` to comparisons with `None`. self.strict_equality_for_none = False # Disable treating bytearray and memoryview as subtypes of bytes From 9edd29ae2b8fe8411964a0dd91ac2d067f17c006 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 8 Sep 2025 10:46:07 +0100 Subject: [PATCH 018/183] Inverse interface freshness logic (#19809) Fixes https://github.com/python/mypy/issues/9554 This is another case where I am surprised id didn't work like this in the first place. Right now the freshness info originates from the dependency itself (like trust me, I am fresh, whatever it means). IMO this doesn't make much sense, instead a dependent should verify whether all dependencies are the same it last seen them. On the surface the idea is simple, but there are couple tricky parts: * This requires splitting `write_cache()` in two phases: first write all data files in an SCC (or at least serialize them), the write all meta files. I didn't find any elegant way to do the split, but it is probably fine, as we already have this untyped meta JSON in few places. * I am adding plugin data (used by mypyc separate compilation currently) as part of interface hash. It is not documented whether it should be this way or not, but I would say it should, and this is essentially how mypyc expects it (group name will appear in `#include <__native_group_name.h>`, so it _is_ a part of the interface). It used to work ~accidentally because we check plugin data in `find_cache_meta()` that is called before setting `interface_hash`, not in `validate_meta()`. --- mypy/build.py | 88 +++++++++++++++++---------- mypyc/test-data/run-multimodule.test | 2 +- test-data/unit/check-incremental.test | 18 +++++- 3 files changed, 73 insertions(+), 35 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 4ccc3dec408e..2271365f3aff 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -335,6 +335,7 @@ class CacheMeta(NamedTuple): # dep_prios and dep_lines are in parallel with dependencies + suppressed dep_prios: list[int] dep_lines: list[int] + dep_hashes: dict[str, str] interface_hash: str # hash representing the public interface version_id: str # mypy version for cache invalidation ignore_all: bool # if errors were ignored @@ -373,6 +374,7 @@ def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta: meta.get("options"), meta.get("dep_prios", []), meta.get("dep_lines", []), + meta.get("dep_hashes", {}), meta.get("interface_hash", ""), meta.get("version_id", sentinel), meta.get("ignore_all", True), @@ -890,8 +892,6 @@ def log(self, *message: str) -> None: self.stderr.flush() def log_fine_grained(self, *message: str) -> None: - import mypy.build - if self.verbosity() >= 1: self.log("fine-grained:", *message) elif mypy.build.DEBUG_FINE_GRAINED: @@ -1500,6 +1500,7 @@ def validate_meta( "options": (manager.options.clone_for_module(id).select_options_affecting_cache()), "dep_prios": meta.dep_prios, "dep_lines": meta.dep_lines, + "dep_hashes": meta.dep_hashes, "interface_hash": meta.interface_hash, "version_id": manager.version_id, "ignore_all": meta.ignore_all, @@ -1543,7 +1544,7 @@ def write_cache( source_hash: str, ignore_all: bool, manager: BuildManager, -) -> tuple[str, CacheMeta | None]: +) -> tuple[str, tuple[dict[str, Any], str, str] | None]: """Write cache files for a module. Note that this mypy's behavior is still correct when any given @@ -1564,9 +1565,9 @@ def write_cache( manager: the build manager (for pyversion, log/trace) Returns: - A tuple containing the interface hash and CacheMeta - corresponding to the metadata that was written (the latter may - be None if the cache could not be written). + A tuple containing the interface hash and inner tuple with cache meta JSON + that should be written and paths to cache files (inner tuple may be None, + if the cache data could not be written). """ metastore = manager.metastore # For Bazel we use relative paths and zero mtimes. @@ -1581,6 +1582,8 @@ def write_cache( if bazel: tree.path = path + plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False)) + # Serialize data and analyze interface if manager.options.fixed_format_cache: data_io = Buffer() @@ -1589,9 +1592,7 @@ def write_cache( else: data = tree.serialize() data_bytes = json_dumps(data, manager.options.debug_cache) - interface_hash = hash_digest(data_bytes) - - plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False)) + interface_hash = hash_digest(data_bytes + json_dumps(plugin_data)) # Obtain and set up metadata st = manager.get_stat(path) @@ -1659,8 +1660,14 @@ def write_cache( "ignore_all": ignore_all, "plugin_data": plugin_data, } + return interface_hash, (meta, meta_json, data_json) + +def write_cache_meta( + meta: dict[str, Any], manager: BuildManager, meta_json: str, data_json: str +) -> CacheMeta: # Write meta cache file + metastore = manager.metastore meta_str = json_dumps(meta, manager.options.debug_cache) if not metastore.write(meta_json, meta_str): # Most likely the error is the replace() call @@ -1668,7 +1675,7 @@ def write_cache( # The next run will simply find the cache entry out of date. manager.log(f"Error writing meta JSON file {meta_json}") - return interface_hash, cache_meta_from_dict(meta, data_json) + return cache_meta_from_dict(meta, data_json) def delete_cache(id: str, path: str, manager: BuildManager) -> None: @@ -1867,6 +1874,9 @@ class State: # Map each dependency to the line number where it is first imported dep_line_map: dict[str, int] + # Map from dependency id to its last observed interface hash + dep_hashes: dict[str, str] = {} + # Parent package, its parent, etc. ancestors: list[str] | None = None @@ -1879,9 +1889,6 @@ class State: # If caller_state is set, the line number in the caller where the import occurred caller_line = 0 - # If True, indicate that the public interface of this module is unchanged - externally_same = True - # Contains a hash of the public interface in incremental mode interface_hash: str = "" @@ -1994,6 +2001,7 @@ def __init__( self.priorities = {id: pri for id, pri in zip(all_deps, self.meta.dep_prios)} assert len(all_deps) == len(self.meta.dep_lines) self.dep_line_map = {id: line for id, line in zip(all_deps, self.meta.dep_lines)} + self.dep_hashes = self.meta.dep_hashes if temporary: self.load_tree(temporary=True) if not manager.use_fine_grained_cache(): @@ -2046,26 +2054,17 @@ def is_fresh(self) -> bool: """Return whether the cache data for this file is fresh.""" # NOTE: self.dependencies may differ from # self.meta.dependencies when a dependency is dropped due to - # suppression by silent mode. However when a suppressed + # suppression by silent mode. However, when a suppressed # dependency is added back we find out later in the process. - return ( - self.meta is not None - and self.is_interface_fresh() - and self.dependencies == self.meta.dependencies - ) - - def is_interface_fresh(self) -> bool: - return self.externally_same + return self.meta is not None and self.dependencies == self.meta.dependencies def mark_as_rechecked(self) -> None: """Marks this module as having been fully re-analyzed by the type-checker.""" self.manager.rechecked_modules.add(self.id) - def mark_interface_stale(self, *, on_errors: bool = False) -> None: + def mark_interface_stale(self) -> None: """Marks this module as having a stale public interface, and discards the cache data.""" - self.externally_same = False - if not on_errors: - self.manager.stale_modules.add(self.id) + self.manager.stale_modules.add(self.id) def check_blockers(self) -> None: """Raise CompileError if a blocking error is detected.""" @@ -2507,7 +2506,7 @@ def valid_references(self) -> set[str]: return valid_refs - def write_cache(self) -> None: + def write_cache(self) -> tuple[dict[str, Any], str, str] | None: assert self.tree is not None, "Internal error: method must be called on parsed file only" # We don't support writing cache files in fine-grained incremental mode. if ( @@ -2525,20 +2524,19 @@ def write_cache(self) -> None: except Exception: print(f"Error serializing {self.id}", file=self.manager.stdout) raise # Propagate to display traceback - return + return None is_errors = self.transitive_error if is_errors: delete_cache(self.id, self.path, self.manager) self.meta = None - self.mark_interface_stale(on_errors=True) - return + return None dep_prios = self.dependency_priorities() dep_lines = self.dependency_lines() assert self.source_hash is not None assert len(set(self.dependencies)) == len( self.dependencies ), f"Duplicates in dependencies list for {self.id} ({self.dependencies})" - new_interface_hash, self.meta = write_cache( + new_interface_hash, meta_tuple = write_cache( self.id, self.path, self.tree, @@ -2557,6 +2555,7 @@ def write_cache(self) -> None: self.manager.log(f"Cached module {self.id} has changed interface") self.mark_interface_stale() self.interface_hash = new_interface_hash + return meta_tuple def verify_dependencies(self, suppressed_only: bool = False) -> None: """Report errors for import targets in modules that don't exist. @@ -3287,7 +3286,19 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: for id in scc: deps.update(graph[id].dependencies) deps -= ascc - stale_deps = {id for id in deps if id in graph and not graph[id].is_interface_fresh()} + + # Verify that interfaces of dependencies still present in graph are up-to-date (fresh). + # Note: if a dependency is not in graph anymore, it should be considered interface-stale. + # This is important to trigger any relevant updates from indirect dependencies that were + # removed in load_graph(). + stale_deps = set() + for id in ascc: + for dep in graph[id].dep_hashes: + if dep not in graph: + stale_deps.add(dep) + continue + if graph[dep].interface_hash != graph[id].dep_hashes[dep]: + stale_deps.add(dep) fresh = fresh and not stale_deps undeps = set() if fresh: @@ -3518,14 +3529,25 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No if any(manager.errors.is_errors_for_file(graph[id].xpath) for id in stale): for id in stale: graph[id].transitive_error = True + meta_tuples = {} for id in stale: if graph[id].xpath not in manager.errors.ignored_files: errors = manager.errors.file_messages( graph[id].xpath, formatter=manager.error_formatter ) manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False) - graph[id].write_cache() + meta_tuples[id] = graph[id].write_cache() graph[id].mark_as_rechecked() + for id in stale: + meta_tuple = meta_tuples[id] + if meta_tuple is None: + graph[id].meta = None + continue + meta, meta_json, data_json = meta_tuple + meta["dep_hashes"] = { + dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph + } + graph[id].meta = write_cache_meta(meta, manager, meta_json, data_json) def sorted_components( diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 5112e126169f..4208af0f04c8 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -816,7 +816,7 @@ def foo() -> int: return 10 [file driver.py] import native -[rechecked native, other_a] +[rechecked other_a] [case testSeparateCompilationWithUndefinedAttribute] from other_a import A diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 9f5c811dc0a1..d1155f54a75d 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -205,7 +205,7 @@ def foo() -> int: return "foo" return inner2() -[rechecked mod1, mod2] +[rechecked mod2] [stale] [out2] tmp/mod2.py:4: error: Incompatible return value type (got "str", expected "int") @@ -6982,3 +6982,19 @@ class Sub(Base[Concatenate[int, P]]): ... [out] [out2] tmp/impl.py:7: error: Argument 1 has incompatible type "str"; expected "int" + +[case testIncrementalDifferentSourcesFreshnessCorrect] +# cmd: mypy -m foo bar +# cmd2: mypy -m foo +# cmd3: mypy -m foo bar +[file foo.py] +foo = 5 +[file foo.py.2] +foo = None +[file bar.py] +from foo import foo +bar: int = foo +[out] +[out2] +[out3] +tmp/bar.py:2: error: Incompatible types in assignment (expression has type "None", variable has type "int") From f09aa57e345eb3cf252e2922f0f832b86360faf0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 8 Sep 2025 23:46:51 +0100 Subject: [PATCH 019/183] Try some aliases speed-up (#19810) This makes self-check almost 2% faster on my desktop (Python 3.12, compiled -O2). Inspired by a slight regression in https://github.com/python/mypy/pull/19798 I decided to re-think how we detect/label the recursive types. The new algorithm is not 100% equivalent to old one, but should be much faster. The main semantic difference is this: ```python A = list[B1] B1 = list[B2] B2 = list[B1] ``` previously all three aliases where labeled as recursive, now only last two are. Which is kind of correct if you think about it for some time, there is nothing genuinely recursive in `A` by itself. As a result: * We have somewhat more verbose `reveal_type()` for recursive types after fine-grained increments. Excessive use of `get_proper_type()` in the daemon code is a known issue. I will take a look at it when I will have a chance. * I cleaned up some of relevant visitors to be more consistent with recursive aliases. * I also do couple cleanups/speedups in the type queries while I am at it. If there are no comments/objections, I will merge it later today. Then I will merge https://github.com/python/mypy/pull/19798, and then _maybe_ an equivalent optimization for recursive instances like `class str(Sequence[str]): ...` --- mypy/constraints.py | 6 +-- mypy/indirection.py | 3 +- mypy/mixedtraverser.py | 2 + mypy/semanal_typeargs.py | 20 ++++---- mypy/stats.py | 6 +-- mypy/test/testtypes.py | 12 ----- mypy/type_visitor.py | 33 ++++++------ mypy/typeanal.py | 22 +++----- mypy/typeops.py | 6 +-- mypy/types.py | 87 ++++++++++++-------------------- test-data/unit/fine-grained.test | 4 +- 11 files changed, 80 insertions(+), 121 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 6416791fa74a..96c0c7ccaf35 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -21,6 +21,7 @@ ArgKind, TypeInfo, ) +from mypy.type_visitor import ALL_STRATEGY, BoolTypeQuery from mypy.types import ( TUPLE_LIKE_INSTANCE_NAMES, AnyType, @@ -41,7 +42,6 @@ TypeAliasType, TypedDictType, TypeOfAny, - TypeQuery, TypeType, TypeVarId, TypeVarLikeType, @@ -670,9 +670,9 @@ def is_complete_type(typ: Type) -> bool: return typ.accept(CompleteTypeVisitor()) -class CompleteTypeVisitor(TypeQuery[bool]): +class CompleteTypeVisitor(BoolTypeQuery): def __init__(self) -> None: - super().__init__(all) + super().__init__(ALL_STRATEGY) def visit_uninhabited_type(self, t: UninhabitedType) -> bool: return False diff --git a/mypy/indirection.py b/mypy/indirection.py index 88258b94d94a..4e566194632b 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -39,8 +39,7 @@ def find_modules(self, typs: Iterable[types.Type]) -> set[str]: def _visit(self, typ: types.Type) -> None: if isinstance(typ, types.TypeAliasType): # Avoid infinite recursion for recursive type aliases. - if typ not in self.seen_aliases: - self.seen_aliases.add(typ) + self.seen_aliases.add(typ) typ.accept(self) def _visit_type_tuple(self, typs: tuple[types.Type, ...]) -> None: diff --git a/mypy/mixedtraverser.py b/mypy/mixedtraverser.py index 324e8a87c1bd..f47d762934bc 100644 --- a/mypy/mixedtraverser.py +++ b/mypy/mixedtraverser.py @@ -47,6 +47,8 @@ def visit_class_def(self, o: ClassDef, /) -> None: if info: for base in info.bases: base.accept(self) + if info.special_alias: + info.special_alias.accept(self) def visit_type_alias_expr(self, o: TypeAliasExpr, /) -> None: super().visit_type_alias_expr(o) diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index be39a8259c2e..686e7a57042d 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -83,12 +83,11 @@ def visit_block(self, o: Block) -> None: def visit_type_alias_type(self, t: TypeAliasType) -> None: super().visit_type_alias_type(t) - if t in self.seen_aliases: - # Avoid infinite recursion on recursive type aliases. - # Note: it is fine to skip the aliases we have already seen in non-recursive - # types, since errors there have already been reported. - return - self.seen_aliases.add(t) + if t.is_recursive: + if t in self.seen_aliases: + # Avoid infinite recursion on recursive type aliases. + return + self.seen_aliases.add(t) assert t.alias is not None, f"Unfixed type alias {t.type_ref}" is_error, is_invalid = self.validate_args( t.alias.name, tuple(t.args), t.alias.alias_tvars, t @@ -101,9 +100,12 @@ def visit_type_alias_type(self, t: TypeAliasType) -> None: if not is_error: # If there was already an error for the alias itself, there is no point in checking # the expansion, most likely it will result in the same kind of error. - get_proper_type(t).accept(self) - if t.alias is not None: - t.alias.accept(self) + if t.args: + # Since we always allow unbounded type variables in alias definitions, we need + # to verify the arguments satisfy the upper bounds of the expansion as well. + get_proper_type(t).accept(self) + if t.is_recursive: + self.seen_aliases.discard(t) def visit_tuple_type(self, t: TupleType) -> None: t.items = flatten_nested_tuples(t.items) diff --git a/mypy/stats.py b/mypy/stats.py index 6bad400ce5d5..e3499d234563 100644 --- a/mypy/stats.py +++ b/mypy/stats.py @@ -43,6 +43,7 @@ YieldFromExpr, ) from mypy.traverser import TraverserVisitor +from mypy.type_visitor import ANY_STRATEGY, BoolTypeQuery from mypy.typeanal import collect_all_inner_types from mypy.types import ( AnyType, @@ -52,7 +53,6 @@ TupleType, Type, TypeOfAny, - TypeQuery, TypeVarType, get_proper_type, get_proper_types, @@ -453,9 +453,9 @@ def is_imprecise(t: Type) -> bool: return t.accept(HasAnyQuery()) -class HasAnyQuery(TypeQuery[bool]): +class HasAnyQuery(BoolTypeQuery): def __init__(self) -> None: - super().__init__(any) + super().__init__(ANY_STRATEGY) def visit_any(self, t: AnyType) -> bool: return not is_special_form_any(t) diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index 0fe41bc28ecd..fc68d9aa6eac 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -201,18 +201,6 @@ def test_type_alias_expand_once(self) -> None: assert get_proper_type(A) == target assert get_proper_type(target) == target - def test_type_alias_expand_all(self) -> None: - A, _ = self.fx.def_alias_1(self.fx.a) - assert A.expand_all_if_possible() is None - A, _ = self.fx.def_alias_2(self.fx.a) - assert A.expand_all_if_possible() is None - - B = self.fx.non_rec_alias(self.fx.a) - C = self.fx.non_rec_alias(TupleType([B, B], Instance(self.fx.std_tuplei, [B]))) - assert C.expand_all_if_possible() == TupleType( - [self.fx.a, self.fx.a], Instance(self.fx.std_tuplei, [self.fx.a]) - ) - def test_recursive_nested_in_non_recursive(self) -> None: A, _ = self.fx.def_alias_1(self.fx.a) T = TypeVarType( diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 15494393cae6..86ef6ade8471 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -15,7 +15,7 @@ from abc import abstractmethod from collections.abc import Iterable, Sequence -from typing import Any, Callable, Final, Generic, TypeVar, cast +from typing import Any, Final, Generic, TypeVar, cast from mypy_extensions import mypyc_attr, trait @@ -353,16 +353,19 @@ class TypeQuery(SyntheticTypeVisitor[T]): # TODO: check that we don't have existing violations of this rule. """ - def __init__(self, strategy: Callable[[list[T]], T]) -> None: - self.strategy = strategy + def __init__(self) -> None: # Keep track of the type aliases already visited. This is needed to avoid # infinite recursion on types like A = Union[int, List[A]]. - self.seen_aliases: set[TypeAliasType] = set() + self.seen_aliases: set[TypeAliasType] | None = None # By default, we eagerly expand type aliases, and query also types in the # alias target. In most cases this is a desired behavior, but we may want # to skip targets in some cases (e.g. when collecting type variables). self.skip_alias_target = False + @abstractmethod + def strategy(self, items: list[T]) -> T: + raise NotImplementedError + def visit_unbound_type(self, t: UnboundType, /) -> T: return self.query_types(t.args) @@ -440,14 +443,15 @@ def visit_placeholder_type(self, t: PlaceholderType, /) -> T: return self.query_types(t.args) def visit_type_alias_type(self, t: TypeAliasType, /) -> T: - # Skip type aliases already visited types to avoid infinite recursion. - # TODO: Ideally we should fire subvisitors here (or use caching) if we care - # about duplicates. - if t in self.seen_aliases: - return self.strategy([]) - self.seen_aliases.add(t) if self.skip_alias_target: return self.query_types(t.args) + # Skip type aliases already visited types to avoid infinite recursion + # (also use this as a simple-minded cache). + if self.seen_aliases is None: + self.seen_aliases = set() + elif t in self.seen_aliases: + return self.strategy([]) + self.seen_aliases.add(t) return get_proper_type(t).accept(self) def query_types(self, types: Iterable[Type]) -> T: @@ -580,16 +584,15 @@ def visit_placeholder_type(self, t: PlaceholderType, /) -> bool: return self.query_types(t.args) def visit_type_alias_type(self, t: TypeAliasType, /) -> bool: - # Skip type aliases already visited types to avoid infinite recursion. - # TODO: Ideally we should fire subvisitors here (or use caching) if we care - # about duplicates. + if self.skip_alias_target: + return self.query_types(t.args) + # Skip type aliases already visited types to avoid infinite recursion + # (also use this as a simple-minded cache). if self.seen_aliases is None: self.seen_aliases = set() elif t in self.seen_aliases: return self.default self.seen_aliases.add(t) - if self.skip_alias_target: - return self.query_types(t.args) return get_proper_type(t).accept(self) def query_types(self, types: list[Type] | tuple[Type, ...]) -> bool: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 658730414763..81fb87fbf9ee 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -2377,9 +2377,9 @@ def has_explicit_any(t: Type) -> bool: return t.accept(HasExplicitAny()) -class HasExplicitAny(TypeQuery[bool]): +class HasExplicitAny(BoolTypeQuery): def __init__(self) -> None: - super().__init__(any) + super().__init__(ANY_STRATEGY) def visit_any(self, t: AnyType) -> bool: return t.type_of_any == TypeOfAny.explicit @@ -2418,15 +2418,11 @@ def collect_all_inner_types(t: Type) -> list[Type]: class CollectAllInnerTypesQuery(TypeQuery[list[Type]]): - def __init__(self) -> None: - super().__init__(self.combine_lists_strategy) - def query_types(self, types: Iterable[Type]) -> list[Type]: return self.strategy([t.accept(self) for t in types]) + list(types) - @classmethod - def combine_lists_strategy(cls, it: Iterable[list[Type]]) -> list[Type]: - return list(itertools.chain.from_iterable(it)) + def strategy(self, items: Iterable[list[Type]]) -> list[Type]: + return list(itertools.chain.from_iterable(items)) def make_optional_type(t: Type) -> Type: @@ -2556,7 +2552,6 @@ def __init__(self, api: SemanticAnalyzerCoreInterface, scope: TypeVarLikeScope) self.scope = scope self.type_var_likes: list[tuple[str, TypeVarLikeExpr]] = [] self.has_self_type = False - self.seen_aliases: set[TypeAliasType] | None = None self.include_callables = True def _seems_like_callable(self, type: UnboundType) -> bool: @@ -2653,7 +2648,8 @@ def visit_union_type(self, t: UnionType) -> None: self.process_types(t.items) def visit_overloaded(self, t: Overloaded) -> None: - self.process_types(t.items) # type: ignore[arg-type] + for it in t.items: + it.accept(self) def visit_type_type(self, t: TypeType) -> None: t.item.accept(self) @@ -2665,12 +2661,6 @@ def visit_placeholder_type(self, t: PlaceholderType) -> None: return self.process_types(t.args) def visit_type_alias_type(self, t: TypeAliasType) -> None: - # Skip type aliases in already visited types to avoid infinite recursion. - if self.seen_aliases is None: - self.seen_aliases = set() - elif t in self.seen_aliases: - return - self.seen_aliases.add(t) self.process_types(t.args) def process_types(self, types: list[Type] | tuple[Type, ...]) -> None: diff --git a/mypy/typeops.py b/mypy/typeops.py index 87a4d8cefd13..298ad4d16f8c 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -1114,12 +1114,12 @@ def get_all_type_vars(tp: Type) -> list[TypeVarLikeType]: class TypeVarExtractor(TypeQuery[list[TypeVarLikeType]]): def __init__(self, include_all: bool = False) -> None: - super().__init__(self._merge) + super().__init__() self.include_all = include_all - def _merge(self, iter: Iterable[list[TypeVarLikeType]]) -> list[TypeVarLikeType]: + def strategy(self, items: Iterable[list[TypeVarLikeType]]) -> list[TypeVarLikeType]: out = [] - for item in iter: + for item in items: out.extend(item) return out diff --git a/mypy/types.py b/mypy/types.py index 3f4bd94b5b24..e0e897e04cad 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -369,29 +369,6 @@ def _expand_once(self) -> Type: return self.alias.target.accept(InstantiateAliasVisitor(mapping)) - def _partial_expansion(self, nothing_args: bool = False) -> tuple[ProperType, bool]: - # Private method mostly for debugging and testing. - unroller = UnrollAliasVisitor(set(), {}) - if nothing_args: - alias = self.copy_modified(args=[UninhabitedType()] * len(self.args)) - else: - alias = self - unrolled = alias.accept(unroller) - assert isinstance(unrolled, ProperType) - return unrolled, unroller.recursed - - def expand_all_if_possible(self, nothing_args: bool = False) -> ProperType | None: - """Attempt a full expansion of the type alias (including nested aliases). - - If the expansion is not possible, i.e. the alias is (mutually-)recursive, - return None. If nothing_args is True, replace all type arguments with an - UninhabitedType() (used to detect recursively defined aliases). - """ - unrolled, recursed = self._partial_expansion(nothing_args=nothing_args) - if recursed: - return None - return unrolled - @property def is_recursive(self) -> bool: """Whether this type alias is recursive. @@ -404,7 +381,7 @@ def is_recursive(self) -> bool: assert self.alias is not None, "Unfixed type alias" is_recursive = self.alias._is_recursive if is_recursive is None: - is_recursive = self.expand_all_if_possible(nothing_args=True) is None + is_recursive = self.alias in self.alias.target.accept(CollectAliasesVisitor()) # We cache the value on the underlying TypeAlias node as an optimization, # since the value is the same for all instances of the same alias. self.alias._is_recursive = is_recursive @@ -3654,8 +3631,8 @@ class TypeStrVisitor(SyntheticTypeVisitor[str]): def __init__(self, id_mapper: IdMapper | None = None, *, options: Options) -> None: self.id_mapper = id_mapper - self.any_as_dots = False self.options = options + self.dotted_aliases: set[TypeAliasType] | None = None def visit_unbound_type(self, t: UnboundType, /) -> str: s = t.name + "?" @@ -3674,8 +3651,6 @@ def visit_callable_argument(self, t: CallableArgument, /) -> str: return f"{t.constructor}({typ}, {t.name})" def visit_any(self, t: AnyType, /) -> str: - if self.any_as_dots and t.type_of_any == TypeOfAny.special_form: - return "..." return "Any" def visit_none_type(self, t: NoneType, /) -> str: @@ -3902,13 +3877,18 @@ def visit_placeholder_type(self, t: PlaceholderType, /) -> str: return f"" def visit_type_alias_type(self, t: TypeAliasType, /) -> str: - if t.alias is not None: - unrolled, recursed = t._partial_expansion() - self.any_as_dots = recursed - type_str = unrolled.accept(self) - self.any_as_dots = False - return type_str - return "" + if t.alias is None: + return "" + if not t.is_recursive: + return get_proper_type(t).accept(self) + if self.dotted_aliases is None: + self.dotted_aliases = set() + elif t in self.dotted_aliases: + return "..." + self.dotted_aliases.add(t) + type_str = get_proper_type(t).accept(self) + self.dotted_aliases.discard(t) + return type_str def visit_unpack_type(self, t: UnpackType, /) -> str: return f"Unpack[{t.type.accept(self)}]" @@ -3943,28 +3923,23 @@ def visit_type_list(self, t: TypeList, /) -> Type: return t -class UnrollAliasVisitor(TrivialSyntheticTypeTranslator): - def __init__( - self, initial_aliases: set[TypeAliasType], cache: dict[Type, Type] | None - ) -> None: - assert cache is not None - super().__init__(cache) - self.recursed = False - self.initial_aliases = initial_aliases - - def visit_type_alias_type(self, t: TypeAliasType) -> Type: - if t in self.initial_aliases: - self.recursed = True - return AnyType(TypeOfAny.special_form) - # Create a new visitor on encountering a new type alias, so that an alias like - # A = Tuple[B, B] - # B = int - # will not be detected as recursive on the second encounter of B. - subvisitor = UnrollAliasVisitor(self.initial_aliases | {t}, self.cache) - result = get_proper_type(t).accept(subvisitor) - if subvisitor.recursed: - self.recursed = True - return result +class CollectAliasesVisitor(TypeQuery[list[mypy.nodes.TypeAlias]]): + def __init__(self) -> None: + super().__init__() + self.seen_alias_nodes: set[mypy.nodes.TypeAlias] = set() + + def strategy(self, items: list[list[mypy.nodes.TypeAlias]]) -> list[mypy.nodes.TypeAlias]: + out = [] + for item in items: + out.extend(item) + return out + + def visit_type_alias_type(self, t: TypeAliasType, /) -> list[mypy.nodes.TypeAlias]: + assert t.alias is not None + if t.alias not in self.seen_alias_nodes: + self.seen_alias_nodes.add(t.alias) + return [t.alias] + t.alias.target.accept(self) + return [] def is_named_instance(t: Type, fullnames: str | tuple[str, ...]) -> TypeGuard[Instance]: diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 1bddee0e5ed2..4a30c8a3828f 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3528,9 +3528,9 @@ reveal_type(a.n) [out] == == -c.py:4: note: Revealed type is "tuple[Union[tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +c.py:4: note: Revealed type is "tuple[Union[tuple[Union[tuple[Union[..., None], builtins.int, fallback=a.N], None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") -c.py:7: note: Revealed type is "tuple[Union[tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +c.py:7: note: Revealed type is "tuple[Union[tuple[Union[tuple[Union[..., None], builtins.int, fallback=a.N], None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" [case testTupleTypeUpdateNonRecursiveToRecursiveFine] import c From ccd290c5b3a752668623576a5e1aa29e78f53307 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Sep 2025 00:29:47 +0100 Subject: [PATCH 020/183] Re-work indirect dependencies (#19798) Wow, this was quite a ride. Indirect dependencies were always supported kind of on best effort. This PR puts them on some principled foundation. It fixes three crashes and three stale types reported. All tests are quite weird/obscure, they are designed to expose the flaws in current logic (plus one test that passes on master, but it covers important corner case, so I add it just in case ). A short summary of various fixes (in arbitrary order): * Update many outdated comments and docstrings * Missing transitive dependency is now considered stale * Handle transitive generic bases in indirection visitor * Handle chained alias targets in indirection visitor * Always record original aliases during semantic analysis * Replace ad-hoc `module_refs` logic in type checker with more principled one during semantic analysis * Delete `qualified_tvars` as a concept, they are not needed since long ago * Remove ad-hoc handling for `TypeInfo`s from `build.py` * Support symbols with setter type different from getter type In general the logic should be more simple/straightforward now: * Get all symbols we try to access in a module and record the modules they were defined in (note this automatically handles problem with possible excessive `get_proper_type()` calls). * Get all types in a type map, for each type _transitively_ find all named types in them (thus aggregating all interfaces the type depends on) Note since this makes the algorithm correct, it may also make it slower (most notably because we must visit generic bases). I tried to offset this by couple optimizations, hopefully performance impact will be minimal. On my machine slow down is ~0.6% --- mypy/build.py | 58 +++---- mypy/checker.py | 6 +- mypy/checkexpr.py | 33 ---- mypy/checkmember.py | 2 + mypy/fixup.py | 2 +- mypy/indirection.py | 101 ++++++----- mypy/nodes.py | 19 +- mypy/semanal.py | 77 +++++---- mypy/server/deps.py | 15 +- mypy/test/typefixture.py | 6 +- test-data/unit/check-incremental.test | 239 ++++++++++++++++++++++++++ 11 files changed, 402 insertions(+), 156 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 2271365f3aff..84dbf2b2df88 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -47,7 +47,7 @@ from mypy.graph_utils import prepare_sccs, strongly_connected_components, topsort from mypy.indirection import TypeIndirectionVisitor from mypy.messages import MessageBuilder -from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile, SymbolTable, TypeInfo +from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile, SymbolTable from mypy.partially_defined import PossiblyUndefinedVariableVisitor from mypy.semanal import SemanticAnalyzer from mypy.semanal_pass1 import SemanticAnalyzerPreAnalysis @@ -1765,26 +1765,24 @@ def delete_cache(id: str, path: str, manager: BuildManager) -> None: For single nodes, processing is simple. If the node was cached, we deserialize the cache data and fix up cross-references. Otherwise, we -do semantic analysis followed by type checking. We also handle (c) -above; if a module has valid cache data *but* any of its -dependencies was processed from source, then the module should be -processed from source. - -A relatively simple optimization (outside SCCs) we might do in the -future is as follows: if a node's cache data is valid, but one or more -of its dependencies are out of date so we have to re-parse the node -from source, once we have fully type-checked the node, we can decide -whether its symbol table actually changed compared to the cache data -(by reading the cache data and comparing it to the data we would be -writing). If there is no change we can declare the node up to date, -and any node that depends (and for which we have cached data, and -whose other dependencies are up to date) on it won't need to be -re-parsed from source. +do semantic analysis followed by type checking. Once we (re-)processed +an SCC we check whether its interface (symbol table) is still fresh +(matches previous cached value). If it is not, we consider dependent SCCs +stale so that they need to be re-parsed as well. + +Note on indirect dependencies: normally dependencies are determined from +imports, but since our interfaces are "opaque" (i.e. symbol tables can +contain cross-references as well as types identified by name), these are not +enough. We *must* also add "indirect" dependencies from symbols and types to +their definitions. For this purpose, we record all accessed symbols during +semantic analysis, and after we finished processing a module, we traverse its +type map, and for each type we find (transitively) on which named types it +depends. Import cycles ------------- -Finally we have to decide how to handle (c), import cycles. Here +Finally we have to decide how to handle (b), import cycles. Here we'll need a modified version of the original state machine (build.py), but we only need to do this per SCC, and we won't have to deal with changes to the list of nodes while we're processing it. @@ -2409,21 +2407,15 @@ def finish_passes(self) -> None: # We should always patch indirect dependencies, even in full (non-incremental) builds, # because the cache still may be written, and it must be correct. - # TODO: find a more robust way to traverse *all* relevant types? - all_types = list(self.type_map().values()) - for _, sym, _ in self.tree.local_definitions(): - if sym.type is not None: - all_types.append(sym.type) - if isinstance(sym.node, TypeInfo): - # TypeInfo symbols have some extra relevant types. - all_types.extend(sym.node.bases) - if sym.node.metaclass_type: - all_types.append(sym.node.metaclass_type) - if sym.node.typeddict_type: - all_types.append(sym.node.typeddict_type) - if sym.node.tuple_type: - all_types.append(sym.node.tuple_type) - self._patch_indirect_dependencies(self.type_checker().module_refs, all_types) + self._patch_indirect_dependencies( + # Two possible sources of indirect dependencies: + # * Symbols not directly imported in this module but accessed via an attribute + # or via a re-export (vast majority of these recorded in semantic analysis). + # * For each expression type we need to record definitions of type components + # since "meaning" of the type may be updated when definitions are updated. + self.tree.module_refs | self.type_checker().module_refs, + set(self.type_map().values()), + ) if self.options.dump_inference_stats: dump_type_stats( @@ -2452,7 +2444,7 @@ def free_state(self) -> None: self._type_checker.reset() self._type_checker = None - def _patch_indirect_dependencies(self, module_refs: set[str], types: list[Type]) -> None: + def _patch_indirect_dependencies(self, module_refs: set[str], types: set[Type]) -> None: assert None not in types valid = self.valid_references() diff --git a/mypy/checker.py b/mypy/checker.py index 5843148d4b4e..96b55f321a73 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -378,11 +378,9 @@ class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi): inferred_attribute_types: dict[Var, Type] | None = None # Don't infer partial None types if we are processing assignment from Union no_partial_types: bool = False - - # The set of all dependencies (suppressed or not) that this module accesses, either - # directly or indirectly. + # Extra module references not detected during semantic analysis (these are rare cases + # e.g. access to class-level import via instance). module_refs: set[str] - # A map from variable nodes to a snapshot of the frame ids of the # frames that were active when the variable was declared. This can # be used to determine nearest common ancestor frame of a variable's diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 835eeb725394..73282c94be4e 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -198,7 +198,6 @@ ) from mypy.typestate import type_state from mypy.typevars import fill_typevars -from mypy.util import split_module_names from mypy.visitor import ExpressionVisitor # Type of callback user for checking individual function arguments. See @@ -248,36 +247,6 @@ def allow_fast_container_literal(t: Type) -> bool: ) -def extract_refexpr_names(expr: RefExpr, output: set[str]) -> None: - """Recursively extracts all module references from a reference expression. - - Note that currently, the only two subclasses of RefExpr are NameExpr and - MemberExpr.""" - while isinstance(expr.node, MypyFile) or expr.fullname: - if isinstance(expr.node, MypyFile) and expr.fullname: - # If it's None, something's wrong (perhaps due to an - # import cycle or a suppressed error). For now we just - # skip it. - output.add(expr.fullname) - - if isinstance(expr, NameExpr): - is_suppressed_import = isinstance(expr.node, Var) and expr.node.is_suppressed_import - if isinstance(expr.node, TypeInfo): - # Reference to a class or a nested class - output.update(split_module_names(expr.node.module_name)) - elif "." in expr.fullname and not is_suppressed_import: - # Everything else (that is not a silenced import within a class) - output.add(expr.fullname.rsplit(".", 1)[0]) - break - elif isinstance(expr, MemberExpr): - if isinstance(expr.expr, RefExpr): - expr = expr.expr - else: - break - else: - raise AssertionError(f"Unknown RefExpr subclass: {type(expr)}") - - class Finished(Exception): """Raised if we can terminate overload argument check early (no match).""" @@ -370,7 +339,6 @@ def visit_name_expr(self, e: NameExpr) -> Type: It can be of any kind: local, member or global. """ - extract_refexpr_names(e, self.chk.module_refs) result = self.analyze_ref_expr(e) narrowed = self.narrow_type_from_binder(e, result) self.chk.check_deprecated(e.node, e) @@ -3344,7 +3312,6 @@ def check_union_call( def visit_member_expr(self, e: MemberExpr, is_lvalue: bool = False) -> Type: """Visit member expression (of form e.id).""" - extract_refexpr_names(e, self.chk.module_refs) result = self.analyze_ordinary_member_access(e, is_lvalue) narrowed = self.narrow_type_from_binder(e, result) self.chk.warn_deprecated(e.node, e) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index e7de1b7a304f..f19a76ec6a34 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -543,6 +543,8 @@ def analyze_member_var_access( if isinstance(v, FuncDef): assert False, "Did not expect a function" if isinstance(v, MypyFile): + # Special case: accessing module on instances is allowed, but will not + # be recorded by semantic analyzer. mx.chk.module_refs.add(v.fullname) if isinstance(vv, (TypeInfo, TypeAlias, MypyFile, TypeVarLikeExpr)): diff --git a/mypy/fixup.py b/mypy/fixup.py index 260c0f84cf1b..d0205f64b720 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -441,4 +441,4 @@ def missing_info(modules: dict[str, MypyFile]) -> TypeInfo: def missing_alias() -> TypeAlias: suggestion = _SUGGESTION.format("alias") - return TypeAlias(AnyType(TypeOfAny.special_form), suggestion, line=-1, column=-1) + return TypeAlias(AnyType(TypeOfAny.special_form), suggestion, "", line=-1, column=-1) diff --git a/mypy/indirection.py b/mypy/indirection.py index 4e566194632b..95023e303cbd 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -4,17 +4,6 @@ import mypy.types as types from mypy.types import TypeVisitor -from mypy.util import split_module_names - - -def extract_module_names(type_name: str | None) -> list[str]: - """Returns the module names of a fully qualified type name.""" - if type_name is not None: - # Discard the first one, which is just the qualified name of the type - possible_module_names = split_module_names(type_name) - return possible_module_names[1:] - else: - return [] class TypeIndirectionVisitor(TypeVisitor[None]): @@ -23,49 +12,57 @@ class TypeIndirectionVisitor(TypeVisitor[None]): def __init__(self) -> None: # Module references are collected here self.modules: set[str] = set() - # User to avoid infinite recursion with recursive type aliases - self.seen_aliases: set[types.TypeAliasType] = set() - # Used to avoid redundant work - self.seen_fullnames: set[str] = set() + # User to avoid infinite recursion with recursive types + self.seen_types: set[types.TypeAliasType | types.Instance] = set() def find_modules(self, typs: Iterable[types.Type]) -> set[str]: self.modules = set() - self.seen_fullnames = set() - self.seen_aliases = set() + self.seen_types = set() for typ in typs: self._visit(typ) return self.modules def _visit(self, typ: types.Type) -> None: - if isinstance(typ, types.TypeAliasType): - # Avoid infinite recursion for recursive type aliases. - self.seen_aliases.add(typ) + # Note: instances are needed for `class str(Sequence[str]): ...` + if ( + isinstance(typ, types.TypeAliasType) + or isinstance(typ, types.ProperType) + and isinstance(typ, types.Instance) + ): + # Avoid infinite recursion for recursive types. + if typ in self.seen_types: + return + self.seen_types.add(typ) typ.accept(self) def _visit_type_tuple(self, typs: tuple[types.Type, ...]) -> None: # Micro-optimization: Specialized version of _visit for lists for typ in typs: - if isinstance(typ, types.TypeAliasType): - # Avoid infinite recursion for recursive type aliases. - if typ in self.seen_aliases: + if ( + isinstance(typ, types.TypeAliasType) + or isinstance(typ, types.ProperType) + and isinstance(typ, types.Instance) + ): + # Avoid infinite recursion for recursive types. + if typ in self.seen_types: continue - self.seen_aliases.add(typ) + self.seen_types.add(typ) typ.accept(self) def _visit_type_list(self, typs: list[types.Type]) -> None: # Micro-optimization: Specialized version of _visit for tuples for typ in typs: - if isinstance(typ, types.TypeAliasType): - # Avoid infinite recursion for recursive type aliases. - if typ in self.seen_aliases: + if ( + isinstance(typ, types.TypeAliasType) + or isinstance(typ, types.ProperType) + and isinstance(typ, types.Instance) + ): + # Avoid infinite recursion for recursive types. + if typ in self.seen_types: continue - self.seen_aliases.add(typ) + self.seen_types.add(typ) typ.accept(self) - def _visit_module_name(self, module_name: str) -> None: - if module_name not in self.modules: - self.modules.update(split_module_names(module_name)) - def visit_unbound_type(self, t: types.UnboundType) -> None: self._visit_type_tuple(t.args) @@ -105,27 +102,36 @@ def visit_parameters(self, t: types.Parameters) -> None: self._visit_type_list(t.arg_types) def visit_instance(self, t: types.Instance) -> None: + # Instance is named, record its definition and continue digging into + # components that constitute semantic meaning of this type: bases, metaclass, + # tuple type, and typeddict type. + # Note: we cannot simply record the MRO, in case an intermediate base contains + # a reference to type alias, this affects meaning of map_instance_to_supertype(), + # see e.g. testDoubleReexportGenericUpdated. self._visit_type_tuple(t.args) if t.type: - # Uses of a class depend on everything in the MRO, - # as changes to classes in the MRO can add types to methods, - # change property types, change the MRO itself, etc. + # Important optimization: instead of simply recording the definition and + # recursing into bases, record the MRO and only traverse generic bases. for s in t.type.mro: - self._visit_module_name(s.module_name) - if t.type.metaclass_type is not None: - self._visit_module_name(t.type.metaclass_type.type.module_name) + self.modules.add(s.module_name) + for base in s.bases: + if base.args: + self._visit_type_tuple(base.args) + if t.type.metaclass_type: + self._visit(t.type.metaclass_type) + if t.type.typeddict_type: + self._visit(t.type.typeddict_type) + if t.type.tuple_type: + self._visit(t.type.tuple_type) def visit_callable_type(self, t: types.CallableType) -> None: self._visit_type_list(t.arg_types) self._visit(t.ret_type) - if t.definition is not None: - fullname = t.definition.fullname - if fullname not in self.seen_fullnames: - self.modules.update(extract_module_names(t.definition.fullname)) - self.seen_fullnames.add(fullname) + self._visit_type_tuple(t.variables) def visit_overloaded(self, t: types.Overloaded) -> None: - self._visit_type_list(list(t.items)) + for item in t.items: + self._visit(item) self._visit(t.fallback) def visit_tuple_type(self, t: types.TupleType) -> None: @@ -149,4 +155,9 @@ def visit_type_type(self, t: types.TypeType) -> None: self._visit(t.item) def visit_type_alias_type(self, t: types.TypeAliasType) -> None: - self._visit(types.get_proper_type(t)) + # Type alias is named, record its definition and continue digging into + # components that constitute semantic meaning of this type: target and args. + if t.alias: + self.modules.add(t.alias.module) + self._visit(t.alias.target) + self._visit_type_list(t.args) diff --git a/mypy/nodes.py b/mypy/nodes.py index 9cfc61c80b3e..7480745c6aa1 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -288,6 +288,7 @@ class MypyFile(SymbolNode): "path", "defs", "alias_deps", + "module_refs", "is_bom", "names", "imports", @@ -311,6 +312,9 @@ class MypyFile(SymbolNode): defs: list[Statement] # Type alias dependencies as mapping from target to set of alias full names alias_deps: defaultdict[str, set[str]] + # The set of all dependencies (suppressed or not) that this module accesses, either + # directly or indirectly. + module_refs: set[str] # Is there a UTF-8 BOM at the start? is_bom: bool names: SymbolTable @@ -351,6 +355,7 @@ def __init__( self.imports = imports self.is_bom = is_bom self.alias_deps = defaultdict(set) + self.module_refs = set() self.plugin_deps = {} if ignored_lines: self.ignored_lines = ignored_lines @@ -4121,7 +4126,8 @@ def f(x: B[T]) -> T: ... # without T, Any would be used here target: The target type. For generic aliases contains bound type variables as nested types (currently TypeVar and ParamSpec are supported). _fullname: Qualified name of this type alias. This is used in particular - to track fine grained dependencies from aliases. + to track fine-grained dependencies from aliases. + module: Module where the alias was defined. alias_tvars: Type variables used to define this alias. normalized: Used to distinguish between `A = List`, and `A = list`. Both are internally stored using `builtins.list` (because `typing.List` is @@ -4135,6 +4141,7 @@ def f(x: B[T]) -> T: ... # without T, Any would be used here __slots__ = ( "target", "_fullname", + "module", "alias_tvars", "no_args", "normalized", @@ -4150,6 +4157,7 @@ def __init__( self, target: mypy.types.Type, fullname: str, + module: str, line: int, column: int, *, @@ -4160,6 +4168,7 @@ def __init__( python_3_12_type_alias: bool = False, ) -> None: self._fullname = fullname + self.module = module self.target = target if alias_tvars is None: alias_tvars = [] @@ -4194,6 +4203,7 @@ def from_tuple_type(cls, info: TypeInfo) -> TypeAlias: ) ), info.fullname, + info.module_name, info.line, info.column, ) @@ -4215,6 +4225,7 @@ def from_typeddict_type(cls, info: TypeInfo) -> TypeAlias: ) ), info.fullname, + info.module_name, info.line, info.column, ) @@ -4238,6 +4249,7 @@ def serialize(self) -> JsonDict: data: JsonDict = { ".class": "TypeAlias", "fullname": self._fullname, + "module": self.module, "target": self.target.serialize(), "alias_tvars": [v.serialize() for v in self.alias_tvars], "no_args": self.no_args, @@ -4252,6 +4264,7 @@ def serialize(self) -> JsonDict: def deserialize(cls, data: JsonDict) -> TypeAlias: assert data[".class"] == "TypeAlias" fullname = data["fullname"] + module = data["module"] alias_tvars = [mypy.types.deserialize_type(v) for v in data["alias_tvars"]] assert all(isinstance(t, mypy.types.TypeVarLikeType) for t in alias_tvars) target = mypy.types.deserialize_type(data["target"]) @@ -4263,6 +4276,7 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: return cls( target, fullname, + module, line, column, alias_tvars=cast(list[mypy.types.TypeVarLikeType], alias_tvars), @@ -4274,6 +4288,7 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: def write(self, data: Buffer) -> None: write_tag(data, TYPE_ALIAS) write_str(data, self._fullname) + write_str(data, self.module) self.target.write(data) mypy.types.write_type_list(data, self.alias_tvars) write_int(data, self.line) @@ -4285,11 +4300,13 @@ def write(self, data: Buffer) -> None: @classmethod def read(cls, data: Buffer) -> TypeAlias: fullname = read_str(data) + module = read_str(data) target = mypy.types.read_type(data) alias_tvars = [mypy.types.read_type_var_like(data) for _ in range(read_int(data))] return TypeAlias( target, fullname, + module, read_int(data), read_int(data), alias_tvars=alias_tvars, diff --git a/mypy/semanal.py b/mypy/semanal.py index 50ee3b532463..b3fd1b98bfd2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -809,6 +809,7 @@ def create_alias(self, tree: MypyFile, target_name: str, alias: str, name: str) alias_node = TypeAlias( target, alias, + tree.fullname, line=-1, column=-1, # there is no context no_args=True, @@ -3885,16 +3886,15 @@ def analyze_alias( declared_type_vars: TypeVarLikeList | None = None, all_declared_type_params_names: list[str] | None = None, python_3_12_type_alias: bool = False, - ) -> tuple[Type | None, list[TypeVarLikeType], set[str], list[str], bool]: + ) -> tuple[Type | None, list[TypeVarLikeType], set[str], bool]: """Check if 'rvalue' is a valid type allowed for aliasing (e.g. not a type variable). - If yes, return the corresponding type, a list of - qualified type variable names for generic aliases, a set of names the alias depends on, - and a list of type variables if the alias is generic. - A schematic example for the dependencies: + If yes, return the corresponding type, a list of type variables for generic aliases, + a set of names the alias depends on, and True if the original type has empty tuple index. + An example for the dependencies: A = int B = str - analyze_alias(Dict[A, B])[2] == {'__main__.A', '__main__.B'} + analyze_alias(dict[A, B])[2] == {'__main__.A', '__main__.B'} """ dynamic = bool(self.function_stack and self.function_stack[-1].is_dynamic()) global_scope = not self.type and not self.function_stack @@ -3906,10 +3906,9 @@ def analyze_alias( self.fail( "Invalid type alias: expression is not a valid type", rvalue, code=codes.VALID_TYPE ) - return None, [], set(), [], False + return None, [], set(), False found_type_vars = self.find_type_var_likes(typ) - tvar_defs: list[TypeVarLikeType] = [] namespace = self.qualified_name(name) alias_type_vars = found_type_vars if declared_type_vars is None else declared_type_vars with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): @@ -3945,9 +3944,8 @@ def analyze_alias( variadic = True new_tvar_defs.append(td) - qualified_tvars = [node.fullname for _name, node in alias_type_vars] empty_tuple_index = typ.empty_tuple_index if isinstance(typ, UnboundType) else False - return analyzed, new_tvar_defs, depends_on, qualified_tvars, empty_tuple_index + return analyzed, new_tvar_defs, depends_on, empty_tuple_index def is_pep_613(self, s: AssignmentStmt) -> bool: if s.unanalyzed_type is not None and isinstance(s.unanalyzed_type, UnboundType): @@ -4042,11 +4040,10 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: res = NoneType() alias_tvars: list[TypeVarLikeType] = [] depends_on: set[str] = set() - qualified_tvars: list[str] = [] empty_tuple_index = False else: tag = self.track_incomplete_refs() - res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias( + res, alias_tvars, depends_on, empty_tuple_index = self.analyze_alias( lvalue.name, rvalue, allow_placeholder=True, @@ -4070,12 +4067,6 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: self.mark_incomplete(lvalue.name, rvalue, becomes_typeinfo=True) return True self.add_type_alias_deps(depends_on) - # In addition to the aliases used, we add deps on unbound - # type variables, since they are erased from target type. - self.add_type_alias_deps(qualified_tvars) - # The above are only direct deps on other aliases. - # For subscripted aliases, type deps from expansion are added in deps.py - # (because the type is stored). check_for_explicit_any(res, self.options, self.is_typeshed_stub_file, self.msg, context=s) # When this type alias gets "inlined", the Any is not explicit anymore, # so we need to replace it with non-explicit Anys. @@ -4106,6 +4097,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: alias_node = TypeAlias( res, self.qualified_name(lvalue.name), + self.cur_mod_id, s.line, s.column, alias_tvars=alias_tvars, @@ -5577,7 +5569,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: return tag = self.track_incomplete_refs() - res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias( + res, alias_tvars, depends_on, empty_tuple_index = self.analyze_alias( s.name.name, s.value.expr(), allow_placeholder=True, @@ -5606,12 +5598,6 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: return self.add_type_alias_deps(depends_on) - # In addition to the aliases used, we add deps on unbound - # type variables, since they are erased from target type. - self.add_type_alias_deps(qualified_tvars) - # The above are only direct deps on other aliases. - # For subscripted aliases, type deps from expansion are added in deps.py - # (because the type is stored). check_for_explicit_any( res, self.options, self.is_typeshed_stub_file, self.msg, context=s ) @@ -5627,6 +5613,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: alias_node = TypeAlias( res, self.qualified_name(s.name.name), + self.cur_mod_id, s.line, s.column, alias_tvars=alias_tvars, @@ -5941,6 +5928,8 @@ def visit_member_expr(self, expr: MemberExpr) -> None: if isinstance(sym.node, PlaceholderNode): self.process_placeholder(expr.name, "attribute", expr) return + if sym.node is not None: + self.record_imported_symbol(sym.node) expr.kind = sym.kind expr.fullname = sym.fullname or "" expr.node = sym.node @@ -5971,8 +5960,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None: if type_info: n = type_info.names.get(expr.name) if n is not None and isinstance(n.node, (MypyFile, TypeInfo, TypeAlias)): - if not n: - return + self.record_imported_symbol(n.node) expr.kind = n.kind expr.fullname = n.fullname or "" expr.node = n.node @@ -6292,6 +6280,37 @@ def visit_class_pattern(self, p: ClassPattern) -> None: def lookup( self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + node = self._lookup(name, ctx, suppress_errors) + if node is not None and node.node is not None: + # This call is unfortunate from performance point of view, but + # needed for rare cases like e.g. testIncrementalChangingAlias. + self.record_imported_symbol(node.node) + return node + + def record_imported_symbol(self, node: SymbolNode) -> None: + """If the symbol was not defined in current module, add its module to module_refs.""" + if not node.fullname: + return + if isinstance(node, MypyFile): + fullname = node.fullname + elif isinstance(node, TypeInfo): + fullname = node.module_name + elif isinstance(node, TypeAlias): + fullname = node.module + elif isinstance(node, (Var, FuncDef, OverloadedFuncDef)) and node.info: + fullname = node.info.module_name + else: + fullname = node.fullname.rsplit(".")[0] + if fullname == self.cur_mod_id: + return + while "." in fullname and fullname not in self.modules: + fullname = fullname.rsplit(".")[0] + if fullname != self.cur_mod_id: + self.cur_mod_node.module_refs.add(fullname) + + def _lookup( + self, name: str, ctx: Context, suppress_errors: bool = False ) -> SymbolTableNode | None: """Look up an unqualified (no dots) name in all active namespaces. @@ -6500,6 +6519,8 @@ def lookup_qualified( self.name_not_defined(name, ctx, namespace=namespace) return None sym = nextsym + if sym is not None and sym.node is not None: + self.record_imported_symbol(sym.node) return sym def lookup_type_node(self, expr: Expression) -> SymbolTableNode | None: @@ -7546,8 +7567,6 @@ def add_type_alias_deps( If `target` is None, then the target node used will be the current scope. """ if not aliases_used: - # A basic optimization to avoid adding targets with no dependencies to - # the `alias_deps` dict. return if target is None: target = self.scope.current_target() diff --git a/mypy/server/deps.py b/mypy/server/deps.py index 9d4445a1f7e7..076d95e2baf9 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -227,14 +227,17 @@ def __init__( self.scope = Scope() self.type_map = type_map # This attribute holds a mapping from target to names of type aliases - # it depends on. These need to be processed specially, since they are - # only present in expanded form in symbol tables. For example, after: - # A = List[int] + # it depends on. These need to be processed specially, since they may + # appear in expanded form in symbol tables, because of a get_proper_type() + # somewhere. For example, after: + # A = int # x: A - # The module symbol table will just have a Var `x` with type `List[int]`, - # and the dependency of `x` on `A` is lost. Therefore the alias dependencies + # the module symbol table will just have a Var `x` with type `int`, + # and the dependency of `x` on `A` is lost. Therefore, the alias dependencies # are preserved at alias expansion points in `semanal.py`, stored as an attribute # on MypyFile, and then passed here. + # TODO: fine-grained is more susceptible to this partially because we are reckless + # about get_proper_type() in *this specific file*. self.alias_deps = alias_deps self.map: dict[str, set[str]] = {} self.is_class = False @@ -979,8 +982,6 @@ def visit_type_alias_type(self, typ: TypeAliasType) -> list[str]: triggers = [trigger] for arg in typ.args: triggers.extend(self.get_type_triggers(arg)) - # TODO: Now that type aliases are its own kind of types we can simplify - # the logic to rely on intermediate dependencies (like for instance types). triggers.extend(self.get_type_triggers(typ.alias.target)) return triggers diff --git a/mypy/test/typefixture.py b/mypy/test/typefixture.py index 0defcdaebc99..f70c8b94f09c 100644 --- a/mypy/test/typefixture.py +++ b/mypy/test/typefixture.py @@ -374,7 +374,7 @@ def def_alias_1(self, base: Instance) -> tuple[TypeAliasType, Type]: target = Instance( self.std_tuplei, [UnionType([base, A])] ) # A = Tuple[Union[base, A], ...] - AN = TypeAlias(target, "__main__.A", -1, -1) + AN = TypeAlias(target, "__main__.A", "__main__", -1, -1) A.alias = AN return A, target @@ -383,7 +383,7 @@ def def_alias_2(self, base: Instance) -> tuple[TypeAliasType, Type]: target = UnionType( [base, Instance(self.std_tuplei, [A])] ) # A = Union[base, Tuple[A, ...]] - AN = TypeAlias(target, "__main__.A", -1, -1) + AN = TypeAlias(target, "__main__.A", "__main__", -1, -1) A.alias = AN return A, target @@ -393,7 +393,7 @@ def non_rec_alias( alias_tvars: list[TypeVarLikeType] | None = None, args: list[Type] | None = None, ) -> TypeAliasType: - AN = TypeAlias(target, "__main__.A", -1, -1, alias_tvars=alias_tvars) + AN = TypeAlias(target, "__main__.A", "__main__", -1, -1, alias_tvars=alias_tvars) if args is None: args = [] return TypeAliasType(AN, args) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index d1155f54a75d..06f228721a86 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6171,6 +6171,118 @@ class Base: [out2] main:5: error: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe +[case testLiteralCoarseGrainedChainedAliases] +from mod1 import Alias1 +from typing import Literal +x: Alias1 +def expect_int(x: int) -> None: pass +expect_int(x) +[file mod1.py] +from mod2 import Alias2 +Alias1 = Alias2 +[file mod2.py] +from mod3 import Alias3 +Alias2 = Alias3 +[file mod3.py] +from typing import Literal +Alias3 = int +[file mod3.py.2] +from typing import Literal +Alias3 = str +[builtins fixtures/tuple.pyi] +[out] +[out2] +main:5: error: Argument 1 to "expect_int" has incompatible type "str"; expected "int" + +[case testLiteralCoarseGrainedChainedAliases2] +from mod1 import Alias1 +from typing import Literal +x: Alias1 +def expect_3(x: Literal[3]) -> None: pass +expect_3(x) +[file mod1.py] +from mod2 import Alias2 +Alias1 = Alias2 +[file mod2.py] +from mod3 import Alias3 +Alias2 = Alias3 +[file mod3.py] +from typing import Literal +Alias3 = Literal[3] +[file mod3.py.2] +from typing import Literal +Alias3 = Literal[4] +[builtins fixtures/tuple.pyi] +[out] +[out2] +main:5: error: Argument 1 to "expect_3" has incompatible type "Literal[4]"; expected "Literal[3]" + +[case testDoubleReexportFunctionUpdated] +import m + +[file m.py] +import f +[file m.py.3] +import f +reveal_type(f.foo) + +[file f.py] +import c +def foo(arg: c.C) -> None: pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py] +class C: ... +[out] +[out2] +[out3] +tmp/m.py:2: note: Revealed type is "def (arg: pb2.C)" + +[case testDoubleReexportGenericUpdated] +import m + +[file m.py] +import f +[file m.py.3] +import f +x: f.F +reveal_type(x[0]) + +[file f.py] +import c +class FB(list[c.C]): ... +class F(FB): ... + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py] +class C: ... +[out] +[out2] +[out3] +tmp/m.py:3: note: Revealed type is "pb2.C" + [case testNoCrashDoubleReexportFunctionEmpty] import m @@ -6203,6 +6315,38 @@ class C: ... [out2] [out3] +[case testNoCrashDoubleReexportAliasEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +D = list[c.C] + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[out] +[out2] +[out3] + [case testNoCrashDoubleReexportBaseEmpty] import m @@ -6235,6 +6379,69 @@ class C: ... [out2] [out3] +[case testNoCrashDoubleReexportBaseEmpty2] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +class D(c.C): pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[out] +[out2] +[out3] + +[case testDoubleReexportMetaUpdated] +import m +class C(metaclass=m.M): ... + +[file m.py] +from types import M + +[file types.py] +class M(type): ... +[file types.py.2] +class M: ... +[out] +[out2] +main:2: error: Metaclasses not inheriting from "type" are not supported + +[case testIncrementalOkChangeWithSave2] +import mod1 +x: int = mod1.x + +[file mod1.py] +from mod2 import x + +[file mod2.py] +x = 1 + +[file mod2.py.2] +x = "no way" +[out] +[out2] +main:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") + [case testNoCrashDoubleReexportMetaEmpty] import m @@ -6267,6 +6474,38 @@ class C(type): ... [out2] [out3] +[case testNoCrashDoubleReexportMetaEmpty2] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +class D(metaclass=c.C): pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb2 +C = pb2.C + +[file pb1.py] +class C(type): ... +[file pb2.py.2] +class C(type): ... +[file pb1.py.2] +[out] +[out2] +[out3] + [case testNoCrashDoubleReexportTypedDictEmpty] import m From 4f872d7076c6d0803b6d12d824d33a477ce36449 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 9 Sep 2025 12:04:36 -0400 Subject: [PATCH 021/183] [mypyc] feat: cache len for container creation from expressions with length known at compile time (#19503) Currently, if a user uses an immutable type as the sequence input for a for loop, the length is checked once at each iteration which, while necessary for some container types such as list and dictionaries, is not necessary for iterating over immutable types tuple, str, and bytes. This PR modifies the codebase such that the length is only checked at the first iteration, and reused from there. Also, in cases where a simple genexp is the input argument for a tuple, the length is currently checked one additional time before entering the iteration (this is done to determine how to size the new tuple). In those cases, we don't even need a length check at the first iteration step, and can reuse the result of that first `len` call (or compile-time determined constant) instead. Lastly, in cases where a tuple is created from a genexp and the length of the genexp is knowable at compile time, this PR replaces PyList_AsTuple with the tuple constructor fast-path. --- mypyc/irbuild/for_helpers.py | 62 ++++- mypyc/test-data/irbuild-generics.test | 117 ++++---- mypyc/test-data/irbuild-lists.test | 342 +++++++++++++---------- mypyc/test-data/irbuild-tuple.test | 377 ++++++++++++++------------ mypyc/test-data/run-lists.test | 6 + mypyc/test-data/run-tuples.test | 7 + 6 files changed, 529 insertions(+), 382 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 762b41866a05..5edee6cb4df4 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -11,17 +11,22 @@ from mypy.nodes import ( ARG_POS, + BytesExpr, CallExpr, DictionaryComprehension, Expression, GeneratorExpr, + ListExpr, Lvalue, MemberExpr, NameExpr, RefExpr, SetExpr, + StarExpr, + StrExpr, TupleExpr, TypeAlias, + Var, ) from mypyc.ir.ops import ( ERR_NEVER, @@ -152,6 +157,7 @@ def for_loop_helper_with_index( expr_reg: Value, body_insts: Callable[[Value], None], line: int, + length: Value, ) -> None: """Generate IR for a sequence iteration. @@ -173,7 +179,7 @@ def for_loop_helper_with_index( condition_block = BasicBlock() for_gen = ForSequence(builder, index, body_block, exit_block, line, False) - for_gen.init(expr_reg, target_type, reverse=False) + for_gen.init(expr_reg, target_type, reverse=False, length=length) builder.push_loop_stack(step_block, exit_block) @@ -227,7 +233,9 @@ def sequence_from_generator_preallocate_helper( rtype = builder.node_type(gen.sequences[0]) if is_sequence_rprimitive(rtype): sequence = builder.accept(gen.sequences[0]) - length = builder.builder.builtin_len(sequence, gen.line, use_pyssize_t=True) + length = get_expr_length_value( + builder, gen.sequences[0], sequence, gen.line, use_pyssize_t=True + ) target_op = empty_op_llbuilder(length, gen.line) def set_item(item_index: Value) -> None: @@ -235,7 +243,7 @@ def set_item(item_index: Value) -> None: builder.call_c(set_item_op, [target_op, item_index, e], gen.line) for_loop_helper_with_index( - builder, gen.indices[0], gen.sequences[0], sequence, set_item, gen.line + builder, gen.indices[0], gen.sequences[0], sequence, set_item, gen.line, length ) return target_op @@ -788,9 +796,13 @@ class ForSequence(ForGenerator): length_reg: Value | AssignmentTarget | None - def init(self, expr_reg: Value, target_type: RType, reverse: bool) -> None: + def init( + self, expr_reg: Value, target_type: RType, reverse: bool, length: Value | None = None + ) -> None: assert is_sequence_rprimitive(expr_reg.type), expr_reg builder = self.builder + # Record a Value indicating the length of the sequence, if known at compile time. + self.length = length self.reverse = reverse # Define target to contain the expression, along with the index that will be used # for the for-loop. If we are inside of a generator function, spill these into the @@ -798,7 +810,7 @@ def init(self, expr_reg: Value, target_type: RType, reverse: bool) -> None: self.expr_target = builder.maybe_spill(expr_reg) if is_immutable_rprimitive(expr_reg.type): # If the expression is an immutable type, we can load the length just once. - self.length_reg = builder.maybe_spill(self.load_len(self.expr_target)) + self.length_reg = builder.maybe_spill(self.length or self.load_len(self.expr_target)) else: # Otherwise, even if the length is known, we must recalculate the length # at every iteration for compatibility with python semantics. @@ -1166,3 +1178,43 @@ def gen_step(self) -> None: def gen_cleanup(self) -> None: for gen in self.gens: gen.gen_cleanup() + + +def get_expr_length(expr: Expression) -> int | None: + if isinstance(expr, (StrExpr, BytesExpr)): + return len(expr.value) + elif isinstance(expr, (ListExpr, TupleExpr)): + # if there are no star expressions, or we know the length of them, + # we know the length of the expression + stars = [get_expr_length(i) for i in expr.items if isinstance(i, StarExpr)] + if None not in stars: + other = sum(not isinstance(i, StarExpr) for i in expr.items) + return other + sum(stars) # type: ignore [arg-type] + elif isinstance(expr, StarExpr): + return get_expr_length(expr.expr) + elif ( + isinstance(expr, RefExpr) + and isinstance(expr.node, Var) + and expr.node.is_final + and isinstance(expr.node.final_value, str) + and expr.node.has_explicit_value + ): + return len(expr.node.final_value) + # TODO: extend this, passing length of listcomp and genexp should have worthwhile + # performance boost and can be (sometimes) figured out pretty easily. set and dict + # comps *can* be done as well but will need special logic to consider the possibility + # of key conflicts. Range, enumerate, zip are all simple logic. + return None + + +def get_expr_length_value( + builder: IRBuilder, expr: Expression, expr_reg: Value, line: int, use_pyssize_t: bool +) -> Value: + rtype = builder.node_type(expr) + assert is_sequence_rprimitive(rtype), rtype + length = get_expr_length(expr) + if length is None: + # We cannot compute the length at compile time, so we will fetch it. + return builder.builder.builtin_len(expr_reg, line, use_pyssize_t=use_pyssize_t) + # The expression result is known at compile time, so we can use a constant. + return Integer(length, c_pyssize_t_rprimitive if use_pyssize_t else short_int_rprimitive) diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test index 96437a0079c9..9ec29182e89b 100644 --- a/mypyc/test-data/irbuild-generics.test +++ b/mypyc/test-data/irbuild-generics.test @@ -678,84 +678,83 @@ def inner_deco_obj.__call__(__mypyc_self__, args, kwargs): r0 :: __main__.deco_env r1 :: native_int r2 :: list - r3, r4 :: native_int - r5 :: bit - r6, x :: object - r7 :: native_int + r3 :: native_int + r4 :: bit + r5, x :: object + r6 :: native_int can_listcomp :: list - r8 :: dict - r9 :: short_int - r10 :: native_int - r11 :: object - r12 :: tuple[bool, short_int, object, object] - r13 :: short_int - r14 :: bool - r15, r16 :: object - r17, k :: str + r7 :: dict + r8 :: short_int + r9 :: native_int + r10 :: object + r11 :: tuple[bool, short_int, object, object] + r12 :: short_int + r13 :: bool + r14, r15 :: object + r16, k :: str v :: object - r18 :: i32 - r19, r20, r21 :: bit + r17 :: i32 + r18, r19, r20 :: bit can_dictcomp :: dict - r22, can_iter, r23, can_use_keys, r24, can_use_values :: list - r25 :: object - r26 :: dict - r27 :: object - r28 :: int + r21, can_iter, r22, can_use_keys, r23, can_use_values :: list + r24 :: object + r25 :: dict + r26 :: object + r27 :: int L0: r0 = __mypyc_self__.__mypyc_env__ r1 = var_object_size args r2 = PyList_New(r1) - r3 = var_object_size args - r4 = 0 + r3 = 0 L1: - r5 = r4 < r3 :: signed - if r5 goto L2 else goto L4 :: bool + r4 = r3 < r1 :: signed + if r4 goto L2 else goto L4 :: bool L2: - r6 = CPySequenceTuple_GetItemUnsafe(args, r4) - x = r6 - CPyList_SetItemUnsafe(r2, r4, x) + r5 = CPySequenceTuple_GetItemUnsafe(args, r3) + x = r5 + CPyList_SetItemUnsafe(r2, r3, x) L3: - r7 = r4 + 1 - r4 = r7 + r6 = r3 + 1 + r3 = r6 goto L1 L4: can_listcomp = r2 - r8 = PyDict_New() - r9 = 0 - r10 = PyDict_Size(kwargs) - r11 = CPyDict_GetItemsIter(kwargs) + r7 = PyDict_New() + r8 = 0 + r9 = PyDict_Size(kwargs) + r10 = CPyDict_GetItemsIter(kwargs) L5: - r12 = CPyDict_NextItem(r11, r9) - r13 = r12[1] - r9 = r13 - r14 = r12[0] - if r14 goto L6 else goto L8 :: bool + r11 = CPyDict_NextItem(r10, r8) + r12 = r11[1] + r8 = r12 + r13 = r11[0] + if r13 goto L6 else goto L8 :: bool L6: - r15 = r12[2] - r16 = r12[3] - r17 = cast(str, r15) - k = r17 - v = r16 - r18 = PyDict_SetItem(r8, k, v) - r19 = r18 >= 0 :: signed + r14 = r11[2] + r15 = r11[3] + r16 = cast(str, r14) + k = r16 + v = r15 + r17 = PyDict_SetItem(r7, k, v) + r18 = r17 >= 0 :: signed L7: - r20 = CPyDict_CheckSize(kwargs, r10) + r19 = CPyDict_CheckSize(kwargs, r9) goto L5 L8: - r21 = CPy_NoErrOccurred() + r20 = CPy_NoErrOccurred() L9: - can_dictcomp = r8 - r22 = PySequence_List(kwargs) - can_iter = r22 - r23 = CPyDict_Keys(kwargs) - can_use_keys = r23 - r24 = CPyDict_Values(kwargs) - can_use_values = r24 - r25 = r0.func - r26 = PyDict_Copy(kwargs) - r27 = PyObject_Call(r25, args, r26) - r28 = unbox(int, r27) - return r28 + can_dictcomp = r7 + r21 = PySequence_List(kwargs) + can_iter = r21 + r22 = CPyDict_Keys(kwargs) + can_use_keys = r22 + r23 = CPyDict_Values(kwargs) + can_use_values = r23 + r24 = r0.func + r25 = PyDict_Copy(kwargs) + r26 = PyObject_Call(r24, args, r25) + r27 = unbox(int, r26) + return r27 def deco(func): func :: object r0 :: __main__.deco_env diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index d83fb88390db..2f5b3b39319e 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -594,10 +594,8 @@ def test(): r3 :: list r4 :: native_int r5 :: bit - r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int + r6, x, r7 :: str + r8 :: native_int a :: list L0: r0 = 'abc' @@ -605,20 +603,18 @@ L0: r1 = CPyStr_Size_size_t(source) r2 = r1 >= 0 :: signed r3 = PyList_New(r1) - r4 = CPyStr_Size_size_t(source) - r5 = r4 >= 0 :: signed - r6 = 0 + r4 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r5 = r4 < r1 :: signed + if r5 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(source, r6) - x = r8 - r9 = f2(x) - CPyList_SetItemUnsafe(r3, r6, r9) + r6 = CPyStr_GetItemUnsafe(source, r4) + x = r6 + r7 = f2(x) + CPyList_SetItemUnsafe(r3, r4, r7) L3: - r10 = r6 + 1 - r6 = r10 + r8 = r4 + 1 + r4 = r8 goto L1 L4: a = r3 @@ -639,38 +635,30 @@ L0: return r1 def test(): r0 :: str - r1 :: native_int - r2 :: bit - r3 :: list - r4 :: native_int - r5 :: bit + r1 :: list + r2 :: native_int + r3 :: bit + r4, x, r5 :: str r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int a :: list L0: r0 = 'abc' - r1 = CPyStr_Size_size_t(r0) - r2 = r1 >= 0 :: signed - r3 = PyList_New(r1) - r4 = CPyStr_Size_size_t(r0) - r5 = r4 >= 0 :: signed - r6 = 0 + r1 = PyList_New(3) + r2 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(r0, r6) - x = r8 - r9 = f2(x) - CPyList_SetItemUnsafe(r3, r6, r9) + r4 = CPyStr_GetItemUnsafe(r0, r2) + x = r4 + r5 = f2(x) + CPyList_SetItemUnsafe(r1, r2, r5) L3: - r10 = r6 + 1 - r6 = r10 + r6 = r2 + 1 + r2 = r6 goto L1 L4: - a = r3 + a = r1 return 1 [case testListBuiltFromFinalStr] @@ -692,38 +680,30 @@ L0: return r1 def test(): r0 :: str - r1 :: native_int - r2 :: bit - r3 :: list - r4 :: native_int - r5 :: bit + r1 :: list + r2 :: native_int + r3 :: bit + r4, x, r5 :: str r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int a :: list L0: r0 = 'abc' - r1 = CPyStr_Size_size_t(r0) - r2 = r1 >= 0 :: signed - r3 = PyList_New(r1) - r4 = CPyStr_Size_size_t(r0) - r5 = r4 >= 0 :: signed - r6 = 0 + r1 = PyList_New(3) + r2 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(r0, r6) - x = r8 - r9 = f2(x) - CPyList_SetItemUnsafe(r3, r6, r9) + r4 = CPyStr_GetItemUnsafe(r0, r2) + x = r4 + r5 = f2(x) + CPyList_SetItemUnsafe(r1, r2, r5) L3: - r10 = r6 + 1 - r6 = r10 + r6 = r2 + 1 + r2 = r6 goto L1 L4: - a = r3 + a = r1 return 1 [case testListBuiltFromBytes_64bit] @@ -744,48 +724,47 @@ def test(): r0, source :: bytes r1 :: native_int r2 :: list - r3, r4 :: native_int - r5, r6, r7 :: bit - r8, r9, r10, r11 :: int - r12 :: object - r13, x, r14 :: int - r15 :: object - r16 :: native_int + r3 :: native_int + r4, r5, r6 :: bit + r7, r8, r9, r10 :: int + r11 :: object + r12, x, r13 :: int + r14 :: object + r15 :: native_int a :: list L0: r0 = b'abc' source = r0 r1 = var_object_size source r2 = PyList_New(r1) - r3 = var_object_size source - r4 = 0 + r3 = 0 L1: - r5 = r4 < r3 :: signed - if r5 goto L2 else goto L8 :: bool + r4 = r3 < r1 :: signed + if r4 goto L2 else goto L8 :: bool L2: - r6 = r4 <= 4611686018427387903 :: signed - if r6 goto L3 else goto L4 :: bool + r5 = r3 <= 4611686018427387903 :: signed + if r5 goto L3 else goto L4 :: bool L3: - r7 = r4 >= -4611686018427387904 :: signed - if r7 goto L5 else goto L4 :: bool + r6 = r3 >= -4611686018427387904 :: signed + if r6 goto L5 else goto L4 :: bool L4: - r8 = CPyTagged_FromInt64(r4) - r9 = r8 + r7 = CPyTagged_FromInt64(r3) + r8 = r7 goto L6 L5: - r10 = r4 << 1 - r9 = r10 + r9 = r3 << 1 + r8 = r9 L6: - r11 = CPyBytes_GetItem(source, r9) - r12 = box(int, r11) - r13 = unbox(int, r12) - x = r13 - r14 = f2(x) - r15 = box(int, r14) - CPyList_SetItemUnsafe(r2, r4, r15) + r10 = CPyBytes_GetItem(source, r8) + r11 = box(int, r10) + r12 = unbox(int, r11) + x = r12 + r13 = f2(x) + r14 = box(int, r13) + CPyList_SetItemUnsafe(r2, r3, r14) L7: - r16 = r4 + 1 - r4 = r16 + r15 = r3 + 1 + r3 = r15 goto L1 L8: a = r2 @@ -806,52 +785,49 @@ L0: return r0 def test(): r0 :: bytes - r1 :: native_int - r2 :: list - r3, r4 :: native_int - r5, r6, r7 :: bit - r8, r9, r10, r11 :: int - r12 :: object - r13, x, r14 :: int - r15 :: object - r16 :: native_int + r1 :: list + r2 :: native_int + r3, r4, r5 :: bit + r6, r7, r8, r9 :: int + r10 :: object + r11, x, r12 :: int + r13 :: object + r14 :: native_int a :: list L0: r0 = b'abc' - r1 = var_object_size r0 - r2 = PyList_New(r1) - r3 = var_object_size r0 - r4 = 0 + r1 = PyList_New(3) + r2 = 0 L1: - r5 = r4 < r3 :: signed - if r5 goto L2 else goto L8 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L8 :: bool L2: - r6 = r4 <= 4611686018427387903 :: signed - if r6 goto L3 else goto L4 :: bool + r4 = r2 <= 4611686018427387903 :: signed + if r4 goto L3 else goto L4 :: bool L3: - r7 = r4 >= -4611686018427387904 :: signed - if r7 goto L5 else goto L4 :: bool + r5 = r2 >= -4611686018427387904 :: signed + if r5 goto L5 else goto L4 :: bool L4: - r8 = CPyTagged_FromInt64(r4) - r9 = r8 + r6 = CPyTagged_FromInt64(r2) + r7 = r6 goto L6 L5: - r10 = r4 << 1 - r9 = r10 + r8 = r2 << 1 + r7 = r8 L6: - r11 = CPyBytes_GetItem(r0, r9) - r12 = box(int, r11) - r13 = unbox(int, r12) - x = r13 - r14 = f2(x) - r15 = box(int, r14) - CPyList_SetItemUnsafe(r2, r4, r15) + r9 = CPyBytes_GetItem(r0, r7) + r10 = box(int, r9) + r11 = unbox(int, r10) + x = r11 + r12 = f2(x) + r13 = box(int, r12) + CPyList_SetItemUnsafe(r1, r2, r13) L7: - r16 = r4 + 1 - r4 = r16 + r14 = r2 + 1 + r2 = r14 goto L1 L8: - a = r2 + a = r1 return 1 [case testListBuiltFromFinalBytes_64bit] @@ -876,13 +852,13 @@ def test(): r1 :: bool r2 :: native_int r3 :: list - r4, r5 :: native_int - r6, r7, r8 :: bit - r9, r10, r11, r12 :: int - r13 :: object - r14, x, r15 :: int - r16 :: object - r17 :: native_int + r4 :: native_int + r5, r6, r7 :: bit + r8, r9, r10, r11 :: int + r12 :: object + r13, x, r14 :: int + r15 :: object + r16 :: native_int a :: list L0: r0 = __main__.source :: static @@ -893,36 +869,102 @@ L1: L2: r2 = var_object_size r0 r3 = PyList_New(r2) - r4 = var_object_size r0 - r5 = 0 + r4 = 0 L3: - r6 = r5 < r4 :: signed - if r6 goto L4 else goto L10 :: bool + r5 = r4 < r2 :: signed + if r5 goto L4 else goto L10 :: bool L4: - r7 = r5 <= 4611686018427387903 :: signed - if r7 goto L5 else goto L6 :: bool + r6 = r4 <= 4611686018427387903 :: signed + if r6 goto L5 else goto L6 :: bool L5: - r8 = r5 >= -4611686018427387904 :: signed - if r8 goto L7 else goto L6 :: bool + r7 = r4 >= -4611686018427387904 :: signed + if r7 goto L7 else goto L6 :: bool L6: - r9 = CPyTagged_FromInt64(r5) - r10 = r9 + r8 = CPyTagged_FromInt64(r4) + r9 = r8 goto L8 L7: - r11 = r5 << 1 - r10 = r11 + r10 = r4 << 1 + r9 = r10 L8: - r12 = CPyBytes_GetItem(r0, r10) - r13 = box(int, r12) - r14 = unbox(int, r13) - x = r14 - r15 = f2(x) - r16 = box(int, r15) - CPyList_SetItemUnsafe(r3, r5, r16) + r11 = CPyBytes_GetItem(r0, r9) + r12 = box(int, r11) + r13 = unbox(int, r12) + x = r13 + r14 = f2(x) + r15 = box(int, r14) + CPyList_SetItemUnsafe(r3, r4, r15) L9: - r17 = r5 + 1 - r5 = r17 + r16 = r4 + 1 + r4 = r16 goto L3 L10: a = r3 return 1 + +[case testListBuiltFromStars] +from typing import Final + +abc: Final = "abc" + +def test() -> None: + a = [str(x) for x in [*abc, *"def", *b"ghi", ("j", "k"), *("l", "m", "n")]] + +[out] +def test(): + r0, r1 :: str + r2 :: bytes + r3, r4 :: str + r5 :: tuple[str, str] + r6, r7, r8 :: str + r9 :: tuple[str, str, str] + r10 :: list + r11, r12, r13, r14 :: object + r15 :: i32 + r16 :: bit + r17, r18 :: object + r19 :: list + r20, r21 :: native_int + r22 :: bit + r23, x :: object + r24 :: str + r25 :: native_int + a :: list +L0: + r0 = 'abc' + r1 = 'def' + r2 = b'ghi' + r3 = 'j' + r4 = 'k' + r5 = (r3, r4) + r6 = 'l' + r7 = 'm' + r8 = 'n' + r9 = (r6, r7, r8) + r10 = PyList_New(0) + r11 = CPyList_Extend(r10, r0) + r12 = CPyList_Extend(r10, r1) + r13 = CPyList_Extend(r10, r2) + r14 = box(tuple[str, str], r5) + r15 = PyList_Append(r10, r14) + r16 = r15 >= 0 :: signed + r17 = box(tuple[str, str, str], r9) + r18 = CPyList_Extend(r10, r17) + r19 = PyList_New(13) + r20 = 0 +L1: + r21 = var_object_size r10 + r22 = r20 < r21 :: signed + if r22 goto L2 else goto L4 :: bool +L2: + r23 = list_get_item_unsafe r10, r20 + x = r23 + r24 = PyObject_Str(x) + CPyList_SetItemUnsafe(r19, r20, r24) +L3: + r25 = r20 + 1 + r20 = r25 + goto L1 +L4: + a = r19 + return 1 diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 00ea7f074a5d..081cc1b174c9 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -365,10 +365,8 @@ def test(): r3 :: tuple r4 :: native_int r5 :: bit - r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int + r6, x, r7 :: str + r8 :: native_int a :: tuple L0: r0 = 'abc' @@ -376,20 +374,18 @@ L0: r1 = CPyStr_Size_size_t(source) r2 = r1 >= 0 :: signed r3 = PyTuple_New(r1) - r4 = CPyStr_Size_size_t(source) - r5 = r4 >= 0 :: signed - r6 = 0 + r4 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r5 = r4 < r1 :: signed + if r5 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(source, r6) - x = r8 - r9 = f2(x) - CPySequenceTuple_SetItemUnsafe(r3, r6, r9) + r6 = CPyStr_GetItemUnsafe(source, r4) + x = r6 + r7 = f2(x) + CPySequenceTuple_SetItemUnsafe(r3, r4, r7) L3: - r10 = r6 + 1 - r6 = r10 + r8 = r4 + 1 + r4 = r8 goto L1 L4: a = r3 @@ -411,38 +407,30 @@ L0: return r1 def test(): r0 :: str - r1 :: native_int - r2 :: bit - r3 :: tuple - r4 :: native_int - r5 :: bit + r1 :: tuple + r2 :: native_int + r3 :: bit + r4, x, r5 :: str r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int a :: tuple L0: r0 = 'abc' - r1 = CPyStr_Size_size_t(r0) - r2 = r1 >= 0 :: signed - r3 = PyTuple_New(r1) - r4 = CPyStr_Size_size_t(r0) - r5 = r4 >= 0 :: signed - r6 = 0 + r1 = PyTuple_New(3) + r2 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(r0, r6) - x = r8 - r9 = f2(x) - CPySequenceTuple_SetItemUnsafe(r3, r6, r9) + r4 = CPyStr_GetItemUnsafe(r0, r2) + x = r4 + r5 = f2(x) + CPySequenceTuple_SetItemUnsafe(r1, r2, r5) L3: - r10 = r6 + 1 - r6 = r10 + r6 = r2 + 1 + r2 = r6 goto L1 L4: - a = r3 + a = r1 return 1 [case testTupleBuiltFromFinalStr] @@ -464,38 +452,30 @@ L0: return r1 def test(): r0 :: str - r1 :: native_int - r2 :: bit - r3 :: tuple - r4 :: native_int - r5 :: bit + r1 :: tuple + r2 :: native_int + r3 :: bit + r4, x, r5 :: str r6 :: native_int - r7 :: bit - r8, x, r9 :: str - r10 :: native_int a :: tuple L0: r0 = 'abc' - r1 = CPyStr_Size_size_t(r0) - r2 = r1 >= 0 :: signed - r3 = PyTuple_New(r1) - r4 = CPyStr_Size_size_t(r0) - r5 = r4 >= 0 :: signed - r6 = 0 + r1 = PyTuple_New(3) + r2 = 0 L1: - r7 = r6 < r4 :: signed - if r7 goto L2 else goto L4 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r8 = CPyStr_GetItemUnsafe(r0, r6) - x = r8 - r9 = f2(x) - CPySequenceTuple_SetItemUnsafe(r3, r6, r9) + r4 = CPyStr_GetItemUnsafe(r0, r2) + x = r4 + r5 = f2(x) + CPySequenceTuple_SetItemUnsafe(r1, r2, r5) L3: - r10 = r6 + 1 - r6 = r10 + r6 = r2 + 1 + r2 = r6 goto L1 L4: - a = r3 + a = r1 return 1 [case testTupleBuiltFromBytes_64bit] @@ -516,48 +496,47 @@ def test(): r0, source :: bytes r1 :: native_int r2 :: tuple - r3, r4 :: native_int - r5, r6, r7 :: bit - r8, r9, r10, r11 :: int - r12 :: object - r13, x, r14 :: int - r15 :: object - r16 :: native_int + r3 :: native_int + r4, r5, r6 :: bit + r7, r8, r9, r10 :: int + r11 :: object + r12, x, r13 :: int + r14 :: object + r15 :: native_int a :: tuple L0: r0 = b'abc' source = r0 r1 = var_object_size source r2 = PyTuple_New(r1) - r3 = var_object_size source - r4 = 0 + r3 = 0 L1: - r5 = r4 < r3 :: signed - if r5 goto L2 else goto L8 :: bool + r4 = r3 < r1 :: signed + if r4 goto L2 else goto L8 :: bool L2: - r6 = r4 <= 4611686018427387903 :: signed - if r6 goto L3 else goto L4 :: bool + r5 = r3 <= 4611686018427387903 :: signed + if r5 goto L3 else goto L4 :: bool L3: - r7 = r4 >= -4611686018427387904 :: signed - if r7 goto L5 else goto L4 :: bool + r6 = r3 >= -4611686018427387904 :: signed + if r6 goto L5 else goto L4 :: bool L4: - r8 = CPyTagged_FromInt64(r4) - r9 = r8 + r7 = CPyTagged_FromInt64(r3) + r8 = r7 goto L6 L5: - r10 = r4 << 1 - r9 = r10 + r9 = r3 << 1 + r8 = r9 L6: - r11 = CPyBytes_GetItem(source, r9) - r12 = box(int, r11) - r13 = unbox(int, r12) - x = r13 - r14 = f2(x) - r15 = box(int, r14) - CPySequenceTuple_SetItemUnsafe(r2, r4, r15) + r10 = CPyBytes_GetItem(source, r8) + r11 = box(int, r10) + r12 = unbox(int, r11) + x = r12 + r13 = f2(x) + r14 = box(int, r13) + CPySequenceTuple_SetItemUnsafe(r2, r3, r14) L7: - r16 = r4 + 1 - r4 = r16 + r15 = r3 + 1 + r3 = r15 goto L1 L8: a = r2 @@ -578,52 +557,49 @@ L0: return r0 def test(): r0 :: bytes - r1 :: native_int - r2 :: tuple - r3, r4 :: native_int - r5, r6, r7 :: bit - r8, r9, r10, r11 :: int - r12 :: object - r13, x, r14 :: int - r15 :: object - r16 :: native_int + r1 :: tuple + r2 :: native_int + r3, r4, r5 :: bit + r6, r7, r8, r9 :: int + r10 :: object + r11, x, r12 :: int + r13 :: object + r14 :: native_int a :: tuple L0: r0 = b'abc' - r1 = var_object_size r0 - r2 = PyTuple_New(r1) - r3 = var_object_size r0 - r4 = 0 + r1 = PyTuple_New(3) + r2 = 0 L1: - r5 = r4 < r3 :: signed - if r5 goto L2 else goto L8 :: bool + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L8 :: bool L2: - r6 = r4 <= 4611686018427387903 :: signed - if r6 goto L3 else goto L4 :: bool + r4 = r2 <= 4611686018427387903 :: signed + if r4 goto L3 else goto L4 :: bool L3: - r7 = r4 >= -4611686018427387904 :: signed - if r7 goto L5 else goto L4 :: bool + r5 = r2 >= -4611686018427387904 :: signed + if r5 goto L5 else goto L4 :: bool L4: - r8 = CPyTagged_FromInt64(r4) - r9 = r8 + r6 = CPyTagged_FromInt64(r2) + r7 = r6 goto L6 L5: - r10 = r4 << 1 - r9 = r10 + r8 = r2 << 1 + r7 = r8 L6: - r11 = CPyBytes_GetItem(r0, r9) - r12 = box(int, r11) - r13 = unbox(int, r12) - x = r13 - r14 = f2(x) - r15 = box(int, r14) - CPySequenceTuple_SetItemUnsafe(r2, r4, r15) + r9 = CPyBytes_GetItem(r0, r7) + r10 = box(int, r9) + r11 = unbox(int, r10) + x = r11 + r12 = f2(x) + r13 = box(int, r12) + CPySequenceTuple_SetItemUnsafe(r1, r2, r13) L7: - r16 = r4 + 1 - r4 = r16 + r14 = r2 + 1 + r2 = r14 goto L1 L8: - a = r2 + a = r1 return 1 [case testTupleBuiltFromFinalBytes_64bit] @@ -648,13 +624,13 @@ def test(): r1 :: bool r2 :: native_int r3 :: tuple - r4, r5 :: native_int - r6, r7, r8 :: bit - r9, r10, r11, r12 :: int - r13 :: object - r14, x, r15 :: int - r16 :: object - r17 :: native_int + r4 :: native_int + r5, r6, r7 :: bit + r8, r9, r10, r11 :: int + r12 :: object + r13, x, r14 :: int + r15 :: object + r16 :: native_int a :: tuple L0: r0 = __main__.source :: static @@ -665,35 +641,34 @@ L1: L2: r2 = var_object_size r0 r3 = PyTuple_New(r2) - r4 = var_object_size r0 - r5 = 0 + r4 = 0 L3: - r6 = r5 < r4 :: signed - if r6 goto L4 else goto L10 :: bool + r5 = r4 < r2 :: signed + if r5 goto L4 else goto L10 :: bool L4: - r7 = r5 <= 4611686018427387903 :: signed - if r7 goto L5 else goto L6 :: bool + r6 = r4 <= 4611686018427387903 :: signed + if r6 goto L5 else goto L6 :: bool L5: - r8 = r5 >= -4611686018427387904 :: signed - if r8 goto L7 else goto L6 :: bool + r7 = r4 >= -4611686018427387904 :: signed + if r7 goto L7 else goto L6 :: bool L6: - r9 = CPyTagged_FromInt64(r5) - r10 = r9 + r8 = CPyTagged_FromInt64(r4) + r9 = r8 goto L8 L7: - r11 = r5 << 1 - r10 = r11 + r10 = r4 << 1 + r9 = r10 L8: - r12 = CPyBytes_GetItem(r0, r10) - r13 = box(int, r12) - r14 = unbox(int, r13) - x = r14 - r15 = f2(x) - r16 = box(int, r15) - CPySequenceTuple_SetItemUnsafe(r3, r5, r16) + r11 = CPyBytes_GetItem(r0, r9) + r12 = box(int, r11) + r13 = unbox(int, r12) + x = r13 + r14 = f2(x) + r15 = box(int, r14) + CPySequenceTuple_SetItemUnsafe(r3, r4, r15) L9: - r17 = r5 + 1 - r5 = r17 + r16 = r4 + 1 + r4 = r16 goto L3 L10: a = r3 @@ -825,36 +800,102 @@ def test(source): source :: tuple r0 :: native_int r1 :: tuple - r2, r3 :: native_int - r4 :: bit - r5 :: object - r6, x, r7 :: bool - r8 :: object - r9 :: native_int + r2 :: native_int + r3 :: bit + r4 :: object + r5, x, r6 :: bool + r7 :: object + r8 :: native_int a :: tuple L0: r0 = var_object_size source r1 = PyTuple_New(r0) - r2 = var_object_size source - r3 = 0 + r2 = 0 L1: - r4 = r3 < r2 :: signed - if r4 goto L2 else goto L4 :: bool + r3 = r2 < r0 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r5 = CPySequenceTuple_GetItemUnsafe(source, r3) - r6 = unbox(bool, r5) - x = r6 - r7 = f(x) - r8 = box(bool, r7) - CPySequenceTuple_SetItemUnsafe(r1, r3, r8) + r4 = CPySequenceTuple_GetItemUnsafe(source, r2) + r5 = unbox(bool, r4) + x = r5 + r6 = f(x) + r7 = box(bool, r6) + CPySequenceTuple_SetItemUnsafe(r1, r2, r7) L3: - r9 = r3 + 1 - r3 = r9 + r8 = r2 + 1 + r2 = r8 goto L1 L4: a = r1 return 1 +[case testTupleBuiltFromStars] +from typing import Final + +abc: Final = "abc" + +def test() -> None: + a = tuple(str(x) for x in [*abc, *"def", *b"ghi", ("j", "k"), *("l", "m", "n")]) + +[out] +def test(): + r0, r1 :: str + r2 :: bytes + r3, r4 :: str + r5 :: tuple[str, str] + r6, r7, r8 :: str + r9 :: tuple[str, str, str] + r10 :: list + r11, r12, r13, r14 :: object + r15 :: i32 + r16 :: bit + r17, r18 :: object + r19 :: tuple + r20, r21 :: native_int + r22 :: bit + r23, x :: object + r24 :: str + r25 :: native_int + a :: tuple +L0: + r0 = 'abc' + r1 = 'def' + r2 = b'ghi' + r3 = 'j' + r4 = 'k' + r5 = (r3, r4) + r6 = 'l' + r7 = 'm' + r8 = 'n' + r9 = (r6, r7, r8) + r10 = PyList_New(0) + r11 = CPyList_Extend(r10, r0) + r12 = CPyList_Extend(r10, r1) + r13 = CPyList_Extend(r10, r2) + r14 = box(tuple[str, str], r5) + r15 = PyList_Append(r10, r14) + r16 = r15 >= 0 :: signed + r17 = box(tuple[str, str, str], r9) + r18 = CPyList_Extend(r10, r17) + r19 = PyTuple_New(13) + r20 = 0 +L1: + r21 = var_object_size r10 + r22 = r20 < r21 :: signed + if r22 goto L2 else goto L4 :: bool +L2: + r23 = list_get_item_unsafe r10, r20 + x = r23 + r24 = PyObject_Str(x) + CPySequenceTuple_SetItemUnsafe(r19, r20, r24) +L3: + r25 = r20 + 1 + r20 = r25 + goto L1 +L4: + a = r19 + return 1 + [case testTupleAdd] from typing import Tuple def f(a: Tuple[int, ...], b: Tuple[int, ...]) -> None: diff --git a/mypyc/test-data/run-lists.test b/mypyc/test-data/run-lists.test index 1569579c1156..40ca1b6e005f 100644 --- a/mypyc/test-data/run-lists.test +++ b/mypyc/test-data/run-lists.test @@ -479,6 +479,8 @@ def test_in_operator_various_cases() -> None: assert list_in_mixed(type) [case testListBuiltFromGenerator] +from typing import Final +abc: Final = "abc" def test_from_gen() -> None: source_a = ["a", "b", "c"] a = list(x + "f2" for x in source_a) @@ -498,6 +500,10 @@ def test_from_gen() -> None: source_str = "abcd" f = list("str:" + x for x in source_str) assert f == ["str:a", "str:b", "str:c", "str:d"] +def test_known_length() -> None: + # not built from generator but doesnt need its own test either + built = [str(x) for x in [*abc, *"def", *b"ghi", ("j", "k"), *("l", "m", "n")]] + assert built == ['a', 'b', 'c', 'd', 'e', 'f', '103', '104', '105', "('j', 'k')", 'l', 'm', 'n'] [case testNext] from typing import List diff --git a/mypyc/test-data/run-tuples.test b/mypyc/test-data/run-tuples.test index f5e1733d429b..e2e8358bb43e 100644 --- a/mypyc/test-data/run-tuples.test +++ b/mypyc/test-data/run-tuples.test @@ -270,6 +270,11 @@ def test_slicing() -> None: def f8(val: int) -> bool: return val % 2 == 0 +abc: Final = "abc" + +def known_length() -> tuple[str, ...]: + return tuple(str(x) for x in [*abc, *"def", *b"ghi", ("j", "k"), *("l", "m", "n")]) + def test_sequence_generator() -> None: source_list = [1, 2, 3] a = tuple(f8(x) for x in source_list) @@ -287,6 +292,8 @@ def test_sequence_generator() -> None: b = tuple('s:' + x for x in source_str) assert b == ('s:a', 's:b', 's:b', 's:c') + assert known_length() == ('a', 'b', 'c', 'd', 'e', 'f', '103', '104', '105', "('j', 'k')", 'l', 'm', 'n') + TUPLE: Final[Tuple[str, ...]] = ('x', 'y') def test_final_boxed_tuple() -> None: From fd0526545419028090f064ba4c1fa6e576ccdd6b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Sep 2025 21:10:52 +0100 Subject: [PATCH 022/183] Expose --fixed-format-cache if compiled (#19815) --- mypy/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mypy/main.py b/mypy/main.py index 4ca1bde73d40..d5bbca704305 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1064,7 +1064,13 @@ def add_invertible_flag( help="Include fine-grained dependency information in the cache for the mypy daemon", ) incremental_group.add_argument( - "--fixed-format-cache", action="store_true", help=argparse.SUPPRESS + "--fixed-format-cache", + action="store_true", + help=( + "Use experimental fast and compact fixed format cache" + if compilation_status == "yes" + else argparse.SUPPRESS + ), ) incremental_group.add_argument( "--skip-version-check", From 33660dbb669ab5ce75d1ff7030c62b036e134588 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Sep 2025 14:34:00 +0100 Subject: [PATCH 023/183] Do not serialize line/column for type aliases (#19821) It looks like it is actually not used for anything now. Also it may unnecessarily invalidate cache and mask possible bugs. --- mypy/nodes.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 7480745c6aa1..040f3fc28dce 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -4254,8 +4254,6 @@ def serialize(self) -> JsonDict: "alias_tvars": [v.serialize() for v in self.alias_tvars], "no_args": self.no_args, "normalized": self.normalized, - "line": self.line, - "column": self.column, "python_3_12_type_alias": self.python_3_12_type_alias, } return data @@ -4270,15 +4268,13 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: target = mypy.types.deserialize_type(data["target"]) no_args = data["no_args"] normalized = data["normalized"] - line = data["line"] - column = data["column"] python_3_12_type_alias = data["python_3_12_type_alias"] return cls( target, fullname, module, - line, - column, + -1, + -1, alias_tvars=cast(list[mypy.types.TypeVarLikeType], alias_tvars), no_args=no_args, normalized=normalized, @@ -4291,8 +4287,6 @@ def write(self, data: Buffer) -> None: write_str(data, self.module) self.target.write(data) mypy.types.write_type_list(data, self.alias_tvars) - write_int(data, self.line) - write_int(data, self.column) write_bool(data, self.no_args) write_bool(data, self.normalized) write_bool(data, self.python_3_12_type_alias) @@ -4307,8 +4301,8 @@ def read(cls, data: Buffer) -> TypeAlias: target, fullname, module, - read_int(data), - read_int(data), + -1, + -1, alias_tvars=alias_tvars, no_args=read_bool(data), normalized=read_bool(data), From 9ae3e9aa160c11b99960f12eef111e4a3197b7d3 Mon Sep 17 00:00:00 2001 From: Kevin Kannammalil Date: Wed, 10 Sep 2025 13:04:02 -0400 Subject: [PATCH 024/183] Initial changelog for release 1.18 (#19818) --- CHANGELOG.md | 239 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bdb888ff9d6..76643a0b805c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,245 @@ ## Next Release +## Mypy 1.18 (Unreleased) + +We’ve just uploaded mypy 1.18 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features and bug fixes. +You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### `--allow-redefinition-new` + +TODO by Jukka + +This feature was contributed by Jukka Lehtosalo. + +### Fixed‑Format Cache (experimental) + +TODO by Jukka + +This feature was contributed by Ivan Levkivskyi (PR [19668](https://github.com/python/mypy/pull/19668), [19735](https://github.com/python/mypy/pull/19735), [19750](https://github.com/python/mypy/pull/19750), [19681](https://github.com/python/mypy/pull/19681), [19752](https://github.com/python/mypy/pull/19752), [19815](https://github.com/python/mypy/pull/19815)) + +### Disjoint Base Classes (@disjoint_base, PEP 800) + +Mypy now implements PEP 800 Disjoint bases: it understands the @disjoint_base marker, rejects class definitions that combine incompatible disjoint bases, and exploits the fact that such classes cannot exist in reachability and narrowing logic + +This feature was contributed by Jelle Zijlstra (PR [19678](https://github.com/python/mypy/pull/19678)). + +### Mypy Performance Improvements + +Mypy 1.18 includes numerous performance improvements, resulting in a 38% overall speedup compared to 1.17 + +- Improve self check performance by 1.8% (Jukka Lehtosalo, PR [19768](https://github.com/python/mypy/pull/19768), [19769](https://github.com/python/mypy/pull/19769), [19770](https://github.com/python/mypy/pull/19770)) +- Use fast Python wrappers in native_internal (Ivan Levkivskyi, PR [19765](https://github.com/python/mypy/pull/19765)) +- Use macros in native_internal hot paths (Ivan Levkivskyi, PR [19757](https://github.com/python/mypy/pull/19757)) +- Special‑case certain Enum method calls for speed (Ivan Levkivskyi, PR [19634](https://github.com/python/mypy/pull/19634)) +- Two additional micro‑optimizations (Ivan Levkivskyi, PR [19627](https://github.com/python/mypy/pull/19627)) +- Another set of micro‑optimizations (Ivan Levkivskyi, PR [19633](https://github.com/python/mypy/pull/19633)) +- Cache common instances (Ivan Levkivskyi, PR [19621](https://github.com/python/mypy/pull/19621)) +- Skip more method bodies in third‑party libraries for speed (Ivan Levkivskyi, PR [19586](https://github.com/python/mypy/pull/19586)) +- Avoid using a dict in CallableType (Ivan Levkivskyi, PR [19580](https://github.com/python/mypy/pull/19580)) +- Use cache for DictExpr (Ivan Levkivskyi, PR [19536](https://github.com/python/mypy/pull/19536)) +- Use cache for OpExpr (Ivan Levkivskyi, PR [19523](https://github.com/python/mypy/pull/19523)) +- Simple call‑expression cache (Ivan Levkivskyi, PR [19505](https://github.com/python/mypy/pull/19505)) +- Cache type_object_type() (Ivan Levkivskyi, PR [19514](https://github.com/python/mypy/pull/19514)) +- Avoid duplicate visits in boolean‑op checking (Ivan Levkivskyi, PR [19515](https://github.com/python/mypy/pull/19515)) +- Optimize generic inference passes (Ivan Levkivskyi, PR [19501](https://github.com/python/mypy/pull/19501)) +- Speed up default plugin (Jukka Lehtosalo, PR [19462](https://github.com/python/mypy/pull/19462)) +- Micro‑optimize ExpandTypeVisitor (Jukka Lehtosalo, PR [19461](https://github.com/python/mypy/pull/19461)) +- Micro‑optimize type indirection visitor (Jukka Lehtosalo, PR [19460](https://github.com/python/mypy/pull/19460)) +- Micro‑optimize chained plugin (Jukka Lehtosalo, PR [19464](https://github.com/python/mypy/pull/19464)) +- Avoid temporary set creation in is_proper_subtype (Jukka Lehtosalo, PR [19463](https://github.com/python/mypy/pull/19463)) +- Subtype checking micro‑optimization (Jukka Lehtosalo, PR [19384](https://github.com/python/mypy/pull/19384)) +- Speed up default plugin (earlier pass) (Jukka Lehtosalo, PR [19385](https://github.com/python/mypy/pull/19385)) +- Remove nested imports from default plugin (Ivan Levkivskyi, PR [19388](https://github.com/python/mypy/pull/19388)) +- is_subtype: return early where possible (Stanislav Terliakov, PR [19400](https://github.com/python/mypy/pull/19400)) +- Deduplicate fast_container_type / fast_dict_type items before join (Stanislav Terliakov, PR [19409](https://github.com/python/mypy/pull/19409)) +- Speed up type checking by caching argument inference context (Jukka Lehtosalo, PR [19323](https://github.com/python/mypy/pull/19323)) +- Optimize bind_self() and deprecation checks (Ivan Levkivskyi, PR [19556](https://github.com/python/mypy/pull/19556)) +- Keep trivial instances/aliases during expansion (Ivan Levkivskyi, PR [19543](https://github.com/python/mypy/pull/19543)) + +### Stubtest Improvements +- Add temporary --ignore-disjoint-bases flag to ease PEP 800 migration (Joren Hammudoglu, PR [19740](https://github.com/python/mypy/pull/19740)) +- Flag redundant uses of @disjoint_base (Jelle Zijlstra, PR [19715](https://github.com/python/mypy/pull/19715)) +- Improve signatures for `__init__` of C classes (Stephen Morton, PR [18259](https://github.com/python/mypy/pull/18259)) +- Handle overloads with mixed pos‑only parameters (Stephen Morton, PR [18287](https://github.com/python/mypy/pull/18287)) +- Use “parameter” (not “argument”) in error messages (PrinceNaroliya, PR [19707](https://github.com/python/mypy/pull/19707)) +- Don’t require @disjoint_base when `__slots__` imply finality (Jelle Zijlstra, PR [19701](https://github.com/python/mypy/pull/19701)) +- Allow runtime‑existing aliases of @type_check_only types (Brian Schubert, PR [19568](https://github.com/python/mypy/pull/19568)) +- More detailed checking of type objects in stubtest (Stephen Morton, PR [18251](https://github.com/python/mypy/pull/18251)) +- Support running stubtest in non-UTF8 terminals (Stanislav Terliakovm, PR [19085](https://github.com/python/mypy/pull/19085)) + +### Mypyc Improvements + +- Fix subclass processing in detect_undefined_bitmap (Chainfire, PR [19787](https://github.com/python/mypy/pull/19787)) +- Fix C function signature emission (Jukka Lehtosalo, PR [19773](https://github.com/python/mypy/pull/19773)) +- Use defined `__new__` in tp_new and constructor (Piotr Sawicki, PR [19739](https://github.com/python/mypy/pull/19739)) +- Speed up implicit `__ne__` (Jukka Lehtosalo, PR [19759](https://github.com/python/mypy/pull/19759)) +- Speed up equality with optional str/bytes (Jukka Lehtosalo, PR [19758](https://github.com/python/mypy/pull/19758)) +- Add `__mypyc_empty_tuple__` constant (BobTheBuidler, PR [19654](https://github.com/python/mypy/pull/19654)) +- Add PyObject_CallObject fast‑path op for fn(*args) (BobTheBuidler, PR [19631](https://github.com/python/mypy/pull/19631)) +- Add **kwargs star2 fast‑path (follow‑up to starargs) (BobTheBuidler, PR [19630](https://github.com/python/mypy/pull/19630)) +- Optimize type(x), x.`__class__`, and `.__name__` (Jukka Lehtosalo, PR [19691](https://github.com/python/mypy/pull/19691), [19683](https://github.com/python/mypy/pull/19683)) +- Specialize bytes.decode for common encodings (Jukka Lehtosalo, PR [19688](https://github.com/python/mypy/pull/19688)) +- Speed up in against final fixed‑length tuples (Jukka Lehtosalo, PR [19682](https://github.com/python/mypy/pull/19682)) +- Optimize f‑string building from Final values (BobTheBuidler, PR [19611](https://github.com/python/mypy/pull/19611)) +- Add exact_dict_set_item_op (BobTheBuidler, PR [19657](https://github.com/python/mypy/pull/19657)) +- Cache len() when iterating over immutable types (BobTheBuidler, PR [19656](https://github.com/python/mypy/pull/19656)) +- Add stararg fast‑path for tuple calls fn(*args) (BobTheBuidler, PR [19623](https://github.com/python/mypy/pull/19623)) +- Include more operations in the mypyc trace log (Jukka Lehtosalo, PR [19647](https://github.com/python/mypy/pull/19647)) +- Add prefix to attributes of generator classes (Piotr Sawicki, PR [19535](https://github.com/python/mypy/pull/19535)) +- Fix segfault from heap type objects with static tp_doc (Brian Schubert, PR [19636](https://github.com/python/mypy/pull/19636)) +- Unwrap NewType to its base type for optimized paths (BobTheBuidler, PR [19497](https://github.com/python/mypy/pull/19497)) +- Enable free‑threading when compiling multiple modules (Jukka Lehtosalo, PR [19541](https://github.com/python/mypy/pull/19541)) +- Make type objects immortal under free‑threading (Jukka Lehtosalo, PR [19538](https://github.com/python/mypy/pull/19538)) +- Fix list.pop primitive on free‑threaded builds (Jukka Lehtosalo, PR [19522](https://github.com/python/mypy/pull/19522)) +- Generate an export table only for separate compilation (Jukka Lehtosalo, PR [19521](https://github.com/python/mypy/pull/19521)) +- Add primitives for isinstance of built‑in types (Piotr Sawicki, PR [19435](https://github.com/python/mypy/pull/19435)) +- Add SetElement op to initialize struct values (Jukka Lehtosalo, PR [19437](https://github.com/python/mypy/pull/19437)) +- Simplify IR for for loops over strings (Jukka Lehtosalo, PR [19434](https://github.com/python/mypy/pull/19434)) +- Use native integers for some sequence indexing (Jukka Lehtosalo, PR [19426](https://github.com/python/mypy/pull/19426)) +- Remove unused CPyList_GetItemUnsafe primitive (Jukka Lehtosalo, PR [19424](https://github.com/python/mypy/pull/19424)) +- Add native‑int helper methods in IR builder (Jukka Lehtosalo, PR [19423](https://github.com/python/mypy/pull/19423)) +- Use PyList_Check for isinstance(obj, list) (Piotr Sawicki, PR [19416](https://github.com/python/mypy/pull/19416)) +- Speed up for loops over native generators (Jukka Lehtosalo, PR [19415](https://github.com/python/mypy/pull/19415)) +- Report error on reserved method names (Piotr Sawicki, PR [19407](https://github.com/python/mypy/pull/19407)) +- Add is_bool_or_bit_rprimitive (Piotr Sawicki, PR [19406](https://github.com/python/mypy/pull/19406)) +- Faster string equality primitive (Jukka Lehtosalo, PR [19402](https://github.com/python/mypy/pull/19402)) +- Speed up native‑to‑native calls using await (Jukka Lehtosalo, PR [19398](https://github.com/python/mypy/pull/19398)) +- Raise NameError on undefined names (Piotr Sawicki, PR [19395](https://github.com/python/mypy/pull/19395)) +- Simplify comparison of tuple elements (Piotr Sawicki, PR [19396](https://github.com/python/mypy/pull/19396)) +- Use per‑type freelists for nested functions (Jukka Lehtosalo, PR [19390](https://github.com/python/mypy/pull/19390)) +- Call generator helper directly in await expressions (Jukka Lehtosalo, PR [19376](https://github.com/python/mypy/pull/19376)) +- Generate introspection signatures for compiled functions (Brian Schubert, PR [19307](https://github.com/python/mypy/pull/19307)) +- Support C string literals in IR (Jukka Lehtosalo, PR [19383](https://github.com/python/mypy/pull/19383)) +- Fix error‑value check for GetAttr that allows nullable values (Jukka Lehtosalo, PR [19378](https://github.com/python/mypy/pull/19378)) +- Fix comparison of tuples with different lengths (Piotr Sawicki, PR [19372](https://github.com/python/mypy/pull/19372)) +- Speed up generator allocation with per‑type freelists (Jukka Lehtosalo, PR [19316](https://github.com/python/mypy/pull/19316)) +- Implement list.clear() primitive (Jahongir Qurbonov, PR [19344](https://github.com/python/mypy/pull/19344)) +- New primitives for weakref.proxy (BobTheBuidler, PR [19217](https://github.com/python/mypy/pull/19217)) +- New primitive for weakref.ref (BobTheBuidler, PR [19099](https://github.com/python/mypy/pull/19099)) +- New primitive for str.count (BobTheBuidler, PR [19264](https://github.com/python/mypy/pull/19264)) +- Tracing/tooling: optionally log sampled operation traces (Jukka Lehtosalo, PR [19457](https://github.com/python/mypy/pull/19457)) +- Tracing/tooling: script to compile with trace logging and run mypy (Jukka Lehtosalo, PR [19475](https://github.com/python/mypy/pull/19475)) + + +### Documentation Updates + +- Add idlemypyextension to IDE integrations (CoolCat467, PR [18615](https://github.com/python/mypy/pull/18615)) +- Document that object is often preferable to Any in APIs (wyattscarpenter, PR [19103](https://github.com/python/mypy/pull/19103)) +- Include a detailed listing of flags enabled by --strict (wyattscarpenter, PR [19062](https://github.com/python/mypy/pull/19062)) +- Update “common issues” (reveal_type/reveal_locals; note on orjson) (wyattscarpenter, PR [19059](https://github.com/python/mypy/pull/19059), [19058](https://github.com/python/mypy/pull/19058)) + +### Other Notable Improvements + +- Remove deprecated --new-type-inference flag (the new algorithm has long been default) (Ivan Levkivskyi, PR [19570](https://github.com/python/mypy/pull/19570)) +- Use empty context as a fallback for return expressions when outer context misleads inference (Ivan Levkivskyi, PR [19767](https://github.com/python/mypy/pull/19767)) +- Support --strict-equality checks involving None (Christoph Tyralla, PR [19718](https://github.com/python/mypy/pull/19718)) +- Don’t show import‑related errors after a module‑level assert False (Stanislav Terliakov, PR [19347](https://github.com/python/mypy/pull/19347)) +- Fix forward refs in type parameters of over‑parameterized PEP 695 aliases (Brian Schubert, PR [19725](https://github.com/python/mypy/pull/19725)) +- Don’t expand PEP 695 aliases when checking node fullnames (Brian Schubert, PR [19699](https://github.com/python/mypy/pull/19699)) +- Don’t use outer context for or expression inference when LHS is Any (Stanislav Terliakov, PR [19748](https://github.com/python/mypy/pull/19748)) +- Interpret bare ClassVar as inferred (not Any) (Ivan Levkivskyi, PR [19573](https://github.com/python/mypy/pull/19573)) +- Recognize buffer protocol special methods (Brian Schubert, PR [19581](https://github.com/python/mypy/pull/19581)) +- Add temporary named expressions for match subjects (Stanislav Terliakov, PR [18446](https://github.com/python/mypy/pull/18446)) +- Support attribute access on enum members correctly (Stanislav Terliakov, PR [19422](https://github.com/python/mypy/pull/19422)) +- Check `__slots__` assignments on self types (Stanislav Terliakov, PR [19332](https://github.com/python/mypy/pull/19332)) +- Move self‑argument checks after decorator application (Stanislav Terliakov, PR [19490](https://github.com/python/mypy/pull/19490)) +- Infer empty list for `__slots__` and module `__all__` (Stanislav Terliakov, PR [19348](https://github.com/python/mypy/pull/19348)) +- Use normalized tuples for fallback calculation (Stanislav Terliakov, PR [19111](https://github.com/python/mypy/pull/19111)) +- Preserve literals when joining Literal with Instance that has matching last_known_value (Stanislav Terliakov, PR [19279](https://github.com/python/mypy/pull/19279)) +- Allow adjacent conditionally‑defined overloads (Stanislav Terliakov, PR [19042](https://github.com/python/mypy/pull/19042)) +- Check property decorators more strictly (Stanislav Terliakov, PR [19313](https://github.com/python/mypy/pull/19313)) +- Support properties with generic setters (Ivan Levkivskyi, PR [19298](https://github.com/python/mypy/pull/19298)) +- Generalize class/static method and property alias support (Ivan Levkivskyi, PR [19297](https://github.com/python/mypy/pull/19297)) +- Re‑widen custom properties after narrowing (Ivan Levkivskyi, PR [19296](https://github.com/python/mypy/pull/19296)) +- Avoid erasing type objects when checking runtime cover (Shantanu, PR [19320](https://github.com/python/mypy/pull/19320)) +- Include tuple fallback in constraints built from tuple types (Stanislav Terliakov, PR [19100](https://github.com/python/mypy/pull/19100)) +- Somewhat better isinstance support on old‑style unions (Shantanu, PR [19714](https://github.com/python/mypy/pull/19714)) +- Improve promotions inside unions (Christoph Tyralla, PR [19245](https://github.com/python/mypy/pull/19245)) +- Uninhabited types should have all attributes (Ivan Levkivskyi, PR [19300](https://github.com/python/mypy/pull/19300)) +- Metaclass conflict checks improved (Robsdedude, PR [17682](https://github.com/python/mypy/pull/17682)) +- Metaclass resolution algorithm fixes (Robsdedude, PR [17713](https://github.com/python/mypy/pull/17713)) +- PEP 702 @deprecated: handle “combined” overloads (Christoph Tyralla, PR [19626](https://github.com/python/mypy/pull/19626)) +- PEP 702 @deprecated: include overloads in snapshot descriptions (Christoph Tyralla, PR [19613](https://github.com/python/mypy/pull/19613)) +- Ignore overload implementation when checking `__OP__` / `__rOP__` compatibility (Stanislav Terliakov, PR [18502](https://github.com/python/mypy/pull/18502)) +- Fix unwrapping of assignment expressions in match subject (Marc Mueller, PR [19742](https://github.com/python/mypy/pull/19742)) +- Omit errors for class patterns against object (Marc Mueller, PR [19709](https://github.com/python/mypy/pull/19709)) +- Remove unnecessary error for certain match class patterns (Marc Mueller, PR [19708](https://github.com/python/mypy/pull/19708)) +- Use union type for captured vars in or pattern (Marc Mueller, PR [19710](https://github.com/python/mypy/pull/19710)) +- Prevent final reassignment inside match case (Omer Hadari, PR [19496](https://github.com/python/mypy/pull/19496)) +- Support _value_ as a fallback for ellipsis Enum members (Stanislav Terliakov, PR [19352](https://github.com/python/mypy/pull/19352)) +- Sort arguments in TypedDict overlap messages (Marc Mueller, PR [19666](https://github.com/python/mypy/pull/19666)) +- Reset to previous statement on leaving return in semanal (Stanislav Terliakov, PR [19642](https://github.com/python/mypy/pull/19642)) +- Add ambiguous to UninhabitedType identity for better messaging (Stanislav Terliakov, PR [19648](https://github.com/python/mypy/pull/19648)) +- Further fix overload diagnostics for varargs/kwargs (Shantanu, PR [19619](https://github.com/python/mypy/pull/19619)) +- Fix overload diagnostics when vararg and varkwarg both match (Shantanu, PR [19614](https://github.com/python/mypy/pull/19614)) +- Show type variable name in “Cannot infer type argument” (Brian Schubert, PR [19290](https://github.com/python/mypy/pull/19290)) +- Fail gracefully on unsupported template strings (PEP 750) (Brian Schubert, PR [19700](https://github.com/python/mypy/pull/19700)) +- Revert colored argparse help for Python 3.14 (Marc Mueller, PR [19721](https://github.com/python/mypy/pull/19721)) +- Support type‑checking a code fragment in the profile script (Jukka Lehtosalo, PR [19379](https://github.com/python/mypy/pull/19379)) +- Fix C compiler flags in the profile self‑check script (Jukka Lehtosalo, PR [19326](https://github.com/python/mypy/pull/19326)) +- Add a script for profiling self‑check (Linux only) (Jukka Lehtosalo, PR [19322](https://github.com/python/mypy/pull/19322)) +- Retry PyPI upload script: skip existing files on retry (Jukka Lehtosalo, PR [19305](https://github.com/python/mypy/pull/19305)) +- Update stubinfo for latest typeshed (Shantanu, PR [19771](https://github.com/python/mypy/pull/19771)) +- Fix crash with variadic tuple arguments to a generic type (Randolf Scholz, PR [19705](https://github.com/python/mypy/pull/19705)) +- Fix crash when enable_error_code in pyproject.toml has wrong type (wyattscarpenter, PR [19494](https://github.com/python/mypy/pull/19494)) +- Fix dict assignment to a wider context when an incompatible same‑shape TypedDict exists (Stanislav Terliakov, PR [19592](https://github.com/python/mypy/pull/19592)) +- Prevent crash for dataclass with PEP 695 TypeVarTuple on Python 3.13+ (Stanislav Terliakov, PR [19565](https://github.com/python/mypy/pull/19565)) +- Fix constructor type for subclasses of Any (Ivan Levkivskyi, PR [19295](https://github.com/python/mypy/pull/19295)) +- Fix TypeGuard/TypeIs being forgotten when semanal defers (Brian Schubert, PR [19325](https://github.com/python/mypy/pull/19325)) +- Fix TypeIs negative narrowing for unions of generics (Brian Schubert, PR [18193](https://github.com/python/mypy/pull/18193)) +- dmypy suggest: fix incorrect signature suggestion when a type matches a module name (Brian Schubert, PR [18937](https://github.com/python/mypy/pull/18937)) +- dmypy suggest: fix interaction with `__new__` (Stanislav Terliakov, PR [18966](https://github.com/python/mypy/pull/18966)) +- dmypy suggest: support Callable / callable Protocols in decorator unwrapping (Anthony Sottile, PR [19072](https://github.com/python/mypy/pull/19072)) +- Fix missing error when redeclaring a type variable in a nested generic class (Brian Schubert, PR [18883](https://github.com/python/mypy/pull/18883)) +- Fix for overloaded type object erasure (Shantanu, PR [19338](https://github.com/python/mypy/pull/19338)) +- Fix TypeGuard with call on temporary object (Saul Shanabrook, PR [19577](https://github.com/python/mypy/pull/19577)) +- Fix crash on settable property alias (Ivan Levkivskyi, PR [19615](https://github.com/python/mypy/pull/19615)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=2480d7e7c74493a024eaf254c5d2c6f452c80ee2+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: + +- Ali Hamdan +- Anthony Sottile +- BobTheBuidler +- Brian Schubert +- Chainfire +- Charlie Denton +- Christoph Tyralla +- CoolCat467 +- Daniel Hnyk +- Emily +- Emma Smith +- Ethan Sarp +- Ivan Levkivskyi +- Jahongir Qurbonov +- Jelle Zijlstra +- Joren Hammudoglu +- Jukka Lehtosalo +- Marc Mueller +- Omer Hadari +- Piotr Sawicki +- PrinceNaroliya +- Randolf Scholz +- Robsdedude +- Saul Shanabrook +- Shantanu +- Stanislav Terliakov +- Stephen Morton +- wyattscarpenter + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 1.17 We’ve just uploaded mypy 1.17 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). From 73fa69ed3d7fe0f80d74874ec0d9c738e8674bd1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 11 Sep 2025 14:03:58 +0100 Subject: [PATCH 025/183] Updates to 1.18 changelog (#19826) Did various edits, added a few additional sections, and reordered some sections. --- CHANGELOG.md | 335 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 206 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76643a0b805c..5266a86c725e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,154 +5,244 @@ ## Mypy 1.18 (Unreleased) We’ve just uploaded mypy 1.18 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). -Mypy is a static type checker for Python. This release includes new features and bug fixes. -You can install it as follows: +Mypy is a static type checker for Python. This release includes new features, performance +improvements and bug fixes. You can install it as follows: python3 -m pip install -U mypy You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -### `--allow-redefinition-new` +### Mypy Performance Improvements + +Mypy 1.18 includes numerous performance improvements, resulting in about 40% speedup +compared to 1.17 when type checking mypy itself. In extreme cases, the improvement +can be 10x or higher. The list below is an overview of the various mypy optimizations. +Many mypyc improvements (discussed in a separate section below) also improve performance. -TODO by Jukka +Type caching optimizations have a small risk of causing regressions. When +reporting issues with unexpected inferred types, please also check if +`--disable-expression-cache` will work around the issue, as it turns off some of +these optimizations. + +- Improve self check performance by 1.8% (Jukka Lehtosalo, PR [19768](https://github.com/python/mypy/pull/19768), [19769](https://github.com/python/mypy/pull/19769), [19770](https://github.com/python/mypy/pull/19770)) +- Optimize fixed-format deserialization (Ivan Levkivskyi, PR [19765](https://github.com/python/mypy/pull/19765)) +- Use macros to optimize fixed-format deserialization (Ivan Levkivskyi, PR [19757](https://github.com/python/mypy/pull/19757)) +- Two additional micro‑optimizations (Ivan Levkivskyi, PR [19627](https://github.com/python/mypy/pull/19627)) +- Another set of micro‑optimizations (Ivan Levkivskyi, PR [19633](https://github.com/python/mypy/pull/19633)) +- Cache common types (Ivan Levkivskyi, PR [19621](https://github.com/python/mypy/pull/19621)) +- Skip more method bodies in third‑party libraries for speed (Ivan Levkivskyi, PR [19586](https://github.com/python/mypy/pull/19586)) +- Simplify the representation of callable types (Ivan Levkivskyi, PR [19580](https://github.com/python/mypy/pull/19580)) +- Add cache for types of some expressions (Ivan Levkivskyi, PR [19505](https://github.com/python/mypy/pull/19505)) +- Use cache for dictionary expressions (Ivan Levkivskyi, PR [19536](https://github.com/python/mypy/pull/19536)) +- Use cache for binary operations (Ivan Levkivskyi, PR [19523](https://github.com/python/mypy/pull/19523)) +- Cache types of type objects (Ivan Levkivskyi, PR [19514](https://github.com/python/mypy/pull/19514)) +- Avoid duplicate work when checking boolean operations (Ivan Levkivskyi, PR [19515](https://github.com/python/mypy/pull/19515)) +- Optimize generic inference passes (Ivan Levkivskyi, PR [19501](https://github.com/python/mypy/pull/19501)) +- Speed up the default plugin (Jukka Lehtosalo, PRs [19385](https://github.com/python/mypy/pull/19385) and [19462](https://github.com/python/mypy/pull/19462)) +- Remove nested imports from the default plugin (Ivan Levkivskyi, PR [19388](https://github.com/python/mypy/pull/19388)) +- Micro‑optimize type expansion (Jukka Lehtosalo, PR [19461](https://github.com/python/mypy/pull/19461)) +- Micro‑optimize type indirection (Jukka Lehtosalo, PR [19460](https://github.com/python/mypy/pull/19460)) +- Micro‑optimize the plugin framework (Jukka Lehtosalo, PR [19464](https://github.com/python/mypy/pull/19464)) +- Avoid temporary set creation in subtype checking (Jukka Lehtosalo, PR [19463](https://github.com/python/mypy/pull/19463)) +- Subtype checking micro‑optimization (Jukka Lehtosalo, PR [19384](https://github.com/python/mypy/pull/19384)) +- Return early where possible in subtype check (Stanislav Terliakov, PR [19400](https://github.com/python/mypy/pull/19400)) +- Deduplicate some types before joining (Stanislav Terliakov, PR [19409](https://github.com/python/mypy/pull/19409)) +- Speed up type checking by caching argument inference context (Jukka Lehtosalo, PR [19323](https://github.com/python/mypy/pull/19323)) +- Optimize binding method self argument type and deprecation checks (Ivan Levkivskyi, PR [19556](https://github.com/python/mypy/pull/19556)) +- Keep trivial instance types/aliases during expansion (Ivan Levkivskyi, PR [19543](https://github.com/python/mypy/pull/19543)) + +### Fixed‑Format Cache (Experimental) + +Mypy now supports a new cache format used for faster incremental builds. It makes +incremental builds up to twice as fast. The feature is experimental and +currently only supported when using a compiled version of mypy. Use `--fixed-format-cache` +to enable the new format, or `fixed_format_cache = True` in a configuration file. + +We plan to enable this by default in a future mypy release, and we'll eventually +deprecate and remove support for the original JSON-based format. + +Unlike the JSON-based cache format, the new binary format is currently +not easy to parse and inspect by mypy users. We are planning to provide a tool to +convert fixed-format cache files to JSON, but details of the output JSON may be +different from the current JSON format. If you rely on being able to inspect +mypy cache files, we recommend creating a GitHub issue and explaining your use +case, so that we can more likely provide support for it. (Using +`MypyFile.read(binary_data)` to inspect cache data may be sufficient to support +some use cases.) + +This feature was contributed by Ivan Levkivskyi (PR [19668](https://github.com/python/mypy/pull/19668), [19735](https://github.com/python/mypy/pull/19735), [19750](https://github.com/python/mypy/pull/19750), [19681](https://github.com/python/mypy/pull/19681), [19752](https://github.com/python/mypy/pull/19752), [19815](https://github.com/python/mypy/pull/19815)). + +### Flexible Variable Definitions: Update + +Mypy 1.16.0 introduced `--allow-redefinition-new`, which allows redefining variables +with different types, and inferring union types for variables from multiple assignments. +The feature is now documented in the `--help` output, but the feature is still experimental. + +We are planning to enable this by default in mypy 2.0, and we will also deprecate the +older `--allow-redefinition` flag. Since the new behavior differs significantly from +the older flag, we encourage users of `--allow-redefinition` to experiment with +`--allow-redefinition-new` and create a GitHub issue if the new functionality doesn't +support some important use cases. This feature was contributed by Jukka Lehtosalo. -### Fixed‑Format Cache (experimental) +### Inferred Type for Bare ClassVar + +A ClassVar without an explicit type annotation now causes the type of the variable +to be inferred from the initializer: + + +```python +from typing import ClassVar + +class Item: + # Type of 'next_id' is now 'int' (it was 'Any') + next_id: ClassVar = 1 -TODO by Jukka + ... +``` -This feature was contributed by Ivan Levkivskyi (PR [19668](https://github.com/python/mypy/pull/19668), [19735](https://github.com/python/mypy/pull/19735), [19750](https://github.com/python/mypy/pull/19750), [19681](https://github.com/python/mypy/pull/19681), [19752](https://github.com/python/mypy/pull/19752), [19815](https://github.com/python/mypy/pull/19815)) +This feature was contributed by Ivan Levkivskyi (PR [19573](https://github.com/python/mypy/pull/19573)). ### Disjoint Base Classes (@disjoint_base, PEP 800) -Mypy now implements PEP 800 Disjoint bases: it understands the @disjoint_base marker, rejects class definitions that combine incompatible disjoint bases, and exploits the fact that such classes cannot exist in reachability and narrowing logic +Mypy now understands disjoint bases (PEP 800): it recognizes the `@disjoint_base` +decorator, and rejects class definitions that combine mutually incompatible base classes, +and takes advantage of the fact that such classes cannot exist in reachability and +narrowing logic. + +This class definition will now generate an error: + +```python +# Error: Class "Bad" has incompatible disjoint bases +class Bad(str, Exception): + ... +``` This feature was contributed by Jelle Zijlstra (PR [19678](https://github.com/python/mypy/pull/19678)). -### Mypy Performance Improvements +### Miscellaneous New Mypy Features -Mypy 1.18 includes numerous performance improvements, resulting in a 38% overall speedup compared to 1.17 +- Add `--strict-equality-for-none` to flag non-overlapping comparisons involving None (Christoph Tyralla, PR [19718](https://github.com/python/mypy/pull/19718)) +- Don’t show import‑related errors after a module‑level assert such as `assert sys.platform == "linux"` that is always false (Stanislav Terliakov, PR [19347](https://github.com/python/mypy/pull/19347)) -- Improve self check performance by 1.8% (Jukka Lehtosalo, PR [19768](https://github.com/python/mypy/pull/19768), [19769](https://github.com/python/mypy/pull/19769), [19770](https://github.com/python/mypy/pull/19770)) -- Use fast Python wrappers in native_internal (Ivan Levkivskyi, PR [19765](https://github.com/python/mypy/pull/19765)) -- Use macros in native_internal hot paths (Ivan Levkivskyi, PR [19757](https://github.com/python/mypy/pull/19757)) -- Special‑case certain Enum method calls for speed (Ivan Levkivskyi, PR [19634](https://github.com/python/mypy/pull/19634)) -- Two additional micro‑optimizations (Ivan Levkivskyi, PR [19627](https://github.com/python/mypy/pull/19627)) -- Another set of micro‑optimizations (Ivan Levkivskyi, PR [19633](https://github.com/python/mypy/pull/19633)) -- Cache common instances (Ivan Levkivskyi, PR [19621](https://github.com/python/mypy/pull/19621)) -- Skip more method bodies in third‑party libraries for speed (Ivan Levkivskyi, PR [19586](https://github.com/python/mypy/pull/19586)) -- Avoid using a dict in CallableType (Ivan Levkivskyi, PR [19580](https://github.com/python/mypy/pull/19580)) -- Use cache for DictExpr (Ivan Levkivskyi, PR [19536](https://github.com/python/mypy/pull/19536)) -- Use cache for OpExpr (Ivan Levkivskyi, PR [19523](https://github.com/python/mypy/pull/19523)) -- Simple call‑expression cache (Ivan Levkivskyi, PR [19505](https://github.com/python/mypy/pull/19505)) -- Cache type_object_type() (Ivan Levkivskyi, PR [19514](https://github.com/python/mypy/pull/19514)) -- Avoid duplicate visits in boolean‑op checking (Ivan Levkivskyi, PR [19515](https://github.com/python/mypy/pull/19515)) -- Optimize generic inference passes (Ivan Levkivskyi, PR [19501](https://github.com/python/mypy/pull/19501)) -- Speed up default plugin (Jukka Lehtosalo, PR [19462](https://github.com/python/mypy/pull/19462)) -- Micro‑optimize ExpandTypeVisitor (Jukka Lehtosalo, PR [19461](https://github.com/python/mypy/pull/19461)) -- Micro‑optimize type indirection visitor (Jukka Lehtosalo, PR [19460](https://github.com/python/mypy/pull/19460)) -- Micro‑optimize chained plugin (Jukka Lehtosalo, PR [19464](https://github.com/python/mypy/pull/19464)) -- Avoid temporary set creation in is_proper_subtype (Jukka Lehtosalo, PR [19463](https://github.com/python/mypy/pull/19463)) -- Subtype checking micro‑optimization (Jukka Lehtosalo, PR [19384](https://github.com/python/mypy/pull/19384)) -- Speed up default plugin (earlier pass) (Jukka Lehtosalo, PR [19385](https://github.com/python/mypy/pull/19385)) -- Remove nested imports from default plugin (Ivan Levkivskyi, PR [19388](https://github.com/python/mypy/pull/19388)) -- is_subtype: return early where possible (Stanislav Terliakov, PR [19400](https://github.com/python/mypy/pull/19400)) -- Deduplicate fast_container_type / fast_dict_type items before join (Stanislav Terliakov, PR [19409](https://github.com/python/mypy/pull/19409)) -- Speed up type checking by caching argument inference context (Jukka Lehtosalo, PR [19323](https://github.com/python/mypy/pull/19323)) -- Optimize bind_self() and deprecation checks (Ivan Levkivskyi, PR [19556](https://github.com/python/mypy/pull/19556)) -- Keep trivial instances/aliases during expansion (Ivan Levkivskyi, PR [19543](https://github.com/python/mypy/pull/19543)) +### Improvements to Match Statements -### Stubtest Improvements -- Add temporary --ignore-disjoint-bases flag to ease PEP 800 migration (Joren Hammudoglu, PR [19740](https://github.com/python/mypy/pull/19740)) -- Flag redundant uses of @disjoint_base (Jelle Zijlstra, PR [19715](https://github.com/python/mypy/pull/19715)) -- Improve signatures for `__init__` of C classes (Stephen Morton, PR [18259](https://github.com/python/mypy/pull/18259)) -- Handle overloads with mixed pos‑only parameters (Stephen Morton, PR [18287](https://github.com/python/mypy/pull/18287)) -- Use “parameter” (not “argument”) in error messages (PrinceNaroliya, PR [19707](https://github.com/python/mypy/pull/19707)) -- Don’t require @disjoint_base when `__slots__` imply finality (Jelle Zijlstra, PR [19701](https://github.com/python/mypy/pull/19701)) -- Allow runtime‑existing aliases of @type_check_only types (Brian Schubert, PR [19568](https://github.com/python/mypy/pull/19568)) -- More detailed checking of type objects in stubtest (Stephen Morton, PR [18251](https://github.com/python/mypy/pull/18251)) -- Support running stubtest in non-UTF8 terminals (Stanislav Terliakovm, PR [19085](https://github.com/python/mypy/pull/19085)) +- Add temporary named expressions for match subjects (Stanislav Terliakov, PR [18446](https://github.com/python/mypy/pull/18446)) +- Fix unwrapping of assignment expressions in match subject (Marc Mueller, PR [19742](https://github.com/python/mypy/pull/19742)) +- Omit errors for class patterns against object (Marc Mueller, PR [19709](https://github.com/python/mypy/pull/19709)) +- Remove unnecessary error for certain match class patterns (Marc Mueller, PR [19708](https://github.com/python/mypy/pull/19708)) +- Use union type for captured vars in or pattern (Marc Mueller, PR [19710](https://github.com/python/mypy/pull/19710)) +- Prevent final reassignment inside match case (Omer Hadari, PR [19496](https://github.com/python/mypy/pull/19496)) -### Mypyc Improvements +### Fixes to Crashes -- Fix subclass processing in detect_undefined_bitmap (Chainfire, PR [19787](https://github.com/python/mypy/pull/19787)) -- Fix C function signature emission (Jukka Lehtosalo, PR [19773](https://github.com/python/mypy/pull/19773)) -- Use defined `__new__` in tp_new and constructor (Piotr Sawicki, PR [19739](https://github.com/python/mypy/pull/19739)) -- Speed up implicit `__ne__` (Jukka Lehtosalo, PR [19759](https://github.com/python/mypy/pull/19759)) -- Speed up equality with optional str/bytes (Jukka Lehtosalo, PR [19758](https://github.com/python/mypy/pull/19758)) -- Add `__mypyc_empty_tuple__` constant (BobTheBuidler, PR [19654](https://github.com/python/mypy/pull/19654)) -- Add PyObject_CallObject fast‑path op for fn(*args) (BobTheBuidler, PR [19631](https://github.com/python/mypy/pull/19631)) -- Add **kwargs star2 fast‑path (follow‑up to starargs) (BobTheBuidler, PR [19630](https://github.com/python/mypy/pull/19630)) -- Optimize type(x), x.`__class__`, and `.__name__` (Jukka Lehtosalo, PR [19691](https://github.com/python/mypy/pull/19691), [19683](https://github.com/python/mypy/pull/19683)) -- Specialize bytes.decode for common encodings (Jukka Lehtosalo, PR [19688](https://github.com/python/mypy/pull/19688)) -- Speed up in against final fixed‑length tuples (Jukka Lehtosalo, PR [19682](https://github.com/python/mypy/pull/19682)) -- Optimize f‑string building from Final values (BobTheBuidler, PR [19611](https://github.com/python/mypy/pull/19611)) -- Add exact_dict_set_item_op (BobTheBuidler, PR [19657](https://github.com/python/mypy/pull/19657)) -- Cache len() when iterating over immutable types (BobTheBuidler, PR [19656](https://github.com/python/mypy/pull/19656)) -- Add stararg fast‑path for tuple calls fn(*args) (BobTheBuidler, PR [19623](https://github.com/python/mypy/pull/19623)) -- Include more operations in the mypyc trace log (Jukka Lehtosalo, PR [19647](https://github.com/python/mypy/pull/19647)) -- Add prefix to attributes of generator classes (Piotr Sawicki, PR [19535](https://github.com/python/mypy/pull/19535)) -- Fix segfault from heap type objects with static tp_doc (Brian Schubert, PR [19636](https://github.com/python/mypy/pull/19636)) -- Unwrap NewType to its base type for optimized paths (BobTheBuidler, PR [19497](https://github.com/python/mypy/pull/19497)) +- Fix crash with variadic tuple arguments to a generic type (Randolf Scholz, PR [19705](https://github.com/python/mypy/pull/19705)) +- Fix crash when enable_error_code in pyproject.toml has wrong type (wyattscarpenter, PR [19494](https://github.com/python/mypy/pull/19494)) +- Prevent crash for dataclass with PEP 695 TypeVarTuple on Python 3.13+ (Stanislav Terliakov, PR [19565](https://github.com/python/mypy/pull/19565)) +- Fix crash on settable property alias (Ivan Levkivskyi, PR [19615](https://github.com/python/mypy/pull/19615)) + +### Experimental Free-threading Support for Mypyc + +All mypyc tests now pass on free-threading Python 3.14 release candidate builds. The performance +of various micro-benchmarks scale well across multiple threads. + +Free-threading support is still experimental. Note that native attribute access +(get and set), list item access and certain other operations are still +unsafe when there are race conditions. This will likely change in the future. +You can follow the +[area-free-threading label](https://github.com/mypyc/mypyc/issues?q=is%3Aissue%20state%3Aopen%20label%3Aarea-free-threading) +in the mypyc issues tracker to follow progress. + +Related PRs: - Enable free‑threading when compiling multiple modules (Jukka Lehtosalo, PR [19541](https://github.com/python/mypy/pull/19541)) +- Fix `list.pop` on free‑threaded builds (Jukka Lehtosalo, PR [19522](https://github.com/python/mypy/pull/19522)) - Make type objects immortal under free‑threading (Jukka Lehtosalo, PR [19538](https://github.com/python/mypy/pull/19538)) -- Fix list.pop primitive on free‑threaded builds (Jukka Lehtosalo, PR [19522](https://github.com/python/mypy/pull/19522)) + +### Mypyc: Support `__new__` + +Mypyc now has rudimentary support for user-defined `__new__` methods. + +This feature was contributed by Piotr Sawicki (PR [19739](https://github.com/python/mypy/pull/19739)). + +### Mypyc: Faster Generators and Async Functions + +Generators and calls of async functions are now faster, sometimes by 2x or more. + +Related PRs: +- Speed up for loops over native generators (Jukka Lehtosalo, PR [19415](https://github.com/python/mypy/pull/19415)) +- Speed up native‑to‑native calls using await (Jukka Lehtosalo, PR [19398](https://github.com/python/mypy/pull/19398)) +- Call generator helper directly in await expressions (Jukka Lehtosalo, PR [19376](https://github.com/python/mypy/pull/19376)) +- Speed up generator allocation with per‑type freelists (Jukka Lehtosalo, PR [19316](https://github.com/python/mypy/pull/19316)) + +### Miscellaneous Mypyc Improvements + +- Special‑case certain Enum method calls for speed (Ivan Levkivskyi, PR [19634](https://github.com/python/mypy/pull/19634)) +- Fix issues related to subclassing and undefined attribute tracking (Chainfire, PR [19787](https://github.com/python/mypy/pull/19787)) +- Fix invalid C function signature (Jukka Lehtosalo, PR [19773](https://github.com/python/mypy/pull/19773)) +- Speed up implicit `__ne__` (Jukka Lehtosalo, PR [19759](https://github.com/python/mypy/pull/19759)) +- Speed up equality with optional str/bytes types (Jukka Lehtosalo, PR [19758](https://github.com/python/mypy/pull/19758)) +- Speed up access to empty tuples (BobTheBuidler, PR [19654](https://github.com/python/mypy/pull/19654)) +- Speed up calls with `*args` (BobTheBuidler, PRs [19623](https://github.com/python/mypy/pull/19623) and [19631](https://github.com/python/mypy/pull/19631)) +- Speed up calls with `**kwargs` (BobTheBuidler, PR [19630](https://github.com/python/mypy/pull/19630)) +- Optimize `type(x)`, `x.__class__`, and `.__name__` (Jukka Lehtosalo, PR [19691](https://github.com/python/mypy/pull/19691), [19683](https://github.com/python/mypy/pull/19683)) +- Specialize `bytes.decode` for common encodings (Jukka Lehtosalo, PR [19688](https://github.com/python/mypy/pull/19688)) +- Speed up `in` operations using final fixed‑length tuples (Jukka Lehtosalo, PR [19682](https://github.com/python/mypy/pull/19682)) +- Optimize f‑string building from final values (BobTheBuidler, PR [19611](https://github.com/python/mypy/pull/19611)) +- Add dictionary set item for exact dict instances (BobTheBuidler, PR [19657](https://github.com/python/mypy/pull/19657)) +- Cache length when iterating over immutable types (BobTheBuidler, PR [19656](https://github.com/python/mypy/pull/19656)) +- Fix name conflict related to attributes of generator classes (Piotr Sawicki, PR [19535](https://github.com/python/mypy/pull/19535)) +- Fix segfault from heap type objects with a static docstring (Brian Schubert, PR [19636](https://github.com/python/mypy/pull/19636)) +- Unwrap NewType to its base type for additional optimizations (BobTheBuidler, PR [19497](https://github.com/python/mypy/pull/19497)) - Generate an export table only for separate compilation (Jukka Lehtosalo, PR [19521](https://github.com/python/mypy/pull/19521)) -- Add primitives for isinstance of built‑in types (Piotr Sawicki, PR [19435](https://github.com/python/mypy/pull/19435)) -- Add SetElement op to initialize struct values (Jukka Lehtosalo, PR [19437](https://github.com/python/mypy/pull/19437)) -- Simplify IR for for loops over strings (Jukka Lehtosalo, PR [19434](https://github.com/python/mypy/pull/19434)) +- Speed up `isinstance` with built‑in types (Piotr Sawicki, PR [19435](https://github.com/python/mypy/pull/19435)) - Use native integers for some sequence indexing (Jukka Lehtosalo, PR [19426](https://github.com/python/mypy/pull/19426)) -- Remove unused CPyList_GetItemUnsafe primitive (Jukka Lehtosalo, PR [19424](https://github.com/python/mypy/pull/19424)) -- Add native‑int helper methods in IR builder (Jukka Lehtosalo, PR [19423](https://github.com/python/mypy/pull/19423)) -- Use PyList_Check for isinstance(obj, list) (Piotr Sawicki, PR [19416](https://github.com/python/mypy/pull/19416)) -- Speed up for loops over native generators (Jukka Lehtosalo, PR [19415](https://github.com/python/mypy/pull/19415)) +- Speed up `isinstance(obj, list)` (Piotr Sawicki, PR [19416](https://github.com/python/mypy/pull/19416)) - Report error on reserved method names (Piotr Sawicki, PR [19407](https://github.com/python/mypy/pull/19407)) -- Add is_bool_or_bit_rprimitive (Piotr Sawicki, PR [19406](https://github.com/python/mypy/pull/19406)) -- Faster string equality primitive (Jukka Lehtosalo, PR [19402](https://github.com/python/mypy/pull/19402)) -- Speed up native‑to‑native calls using await (Jukka Lehtosalo, PR [19398](https://github.com/python/mypy/pull/19398)) -- Raise NameError on undefined names (Piotr Sawicki, PR [19395](https://github.com/python/mypy/pull/19395)) -- Simplify comparison of tuple elements (Piotr Sawicki, PR [19396](https://github.com/python/mypy/pull/19396)) +- Speed up string equality (Jukka Lehtosalo, PR [19402](https://github.com/python/mypy/pull/19402)) +- Raise `NameError` on undefined names (Piotr Sawicki, PR [19395](https://github.com/python/mypy/pull/19395)) - Use per‑type freelists for nested functions (Jukka Lehtosalo, PR [19390](https://github.com/python/mypy/pull/19390)) -- Call generator helper directly in await expressions (Jukka Lehtosalo, PR [19376](https://github.com/python/mypy/pull/19376)) +- Simplify comparison of tuple elements (Piotr Sawicki, PR [19396](https://github.com/python/mypy/pull/19396)) - Generate introspection signatures for compiled functions (Brian Schubert, PR [19307](https://github.com/python/mypy/pull/19307)) -- Support C string literals in IR (Jukka Lehtosalo, PR [19383](https://github.com/python/mypy/pull/19383)) -- Fix error‑value check for GetAttr that allows nullable values (Jukka Lehtosalo, PR [19378](https://github.com/python/mypy/pull/19378)) +- Fix undefined attribute checking special case (Jukka Lehtosalo, PR [19378](https://github.com/python/mypy/pull/19378)) - Fix comparison of tuples with different lengths (Piotr Sawicki, PR [19372](https://github.com/python/mypy/pull/19372)) -- Speed up generator allocation with per‑type freelists (Jukka Lehtosalo, PR [19316](https://github.com/python/mypy/pull/19316)) -- Implement list.clear() primitive (Jahongir Qurbonov, PR [19344](https://github.com/python/mypy/pull/19344)) -- New primitives for weakref.proxy (BobTheBuidler, PR [19217](https://github.com/python/mypy/pull/19217)) -- New primitive for weakref.ref (BobTheBuidler, PR [19099](https://github.com/python/mypy/pull/19099)) -- New primitive for str.count (BobTheBuidler, PR [19264](https://github.com/python/mypy/pull/19264)) -- Tracing/tooling: optionally log sampled operation traces (Jukka Lehtosalo, PR [19457](https://github.com/python/mypy/pull/19457)) -- Tracing/tooling: script to compile with trace logging and run mypy (Jukka Lehtosalo, PR [19475](https://github.com/python/mypy/pull/19475)) +- Speed up `list.clear` (Jahongir Qurbonov, PR [19344](https://github.com/python/mypy/pull/19344)) +- Speed up `weakref.proxy` (BobTheBuidler, PR [19217](https://github.com/python/mypy/pull/19217)) +- Speed up `weakref.ref` (BobTheBuidler, PR [19099](https://github.com/python/mypy/pull/19099)) +- Speed up `str.count` (BobTheBuidler, PR [19264](https://github.com/python/mypy/pull/19264)) +### Stubtest Improvements +- Add temporary `--ignore-disjoint-bases` flag to ease PEP 800 migration (Joren Hammudoglu, PR [19740](https://github.com/python/mypy/pull/19740)) +- Flag redundant uses of `@disjoint_base` (Jelle Zijlstra, PR [19715](https://github.com/python/mypy/pull/19715)) +- Improve signatures for `__init__` of C extension classes (Stephen Morton, PR [18259](https://github.com/python/mypy/pull/18259)) +- Handle overloads with mixed positional‑only parameters (Stephen Morton, PR [18287](https://github.com/python/mypy/pull/18287)) +- Use “parameter” (not “argument”) in error messages (PrinceNaroliya, PR [19707](https://github.com/python/mypy/pull/19707)) +- Don’t require `@disjoint_base` when `__slots__` imply finality (Jelle Zijlstra, PR [19701](https://github.com/python/mypy/pull/19701)) +- Allow runtime‑existing aliases of `@type_check_only` types (Brian Schubert, PR [19568](https://github.com/python/mypy/pull/19568)) +- More detailed checking of type objects in stubtest (Stephen Morton, PR [18251](https://github.com/python/mypy/pull/18251)) +- Support running stubtest in non-UTF8 terminals (Stanislav Terliakov, PR [19085](https://github.com/python/mypy/pull/19085)) ### Documentation Updates - Add idlemypyextension to IDE integrations (CoolCat467, PR [18615](https://github.com/python/mypy/pull/18615)) -- Document that object is often preferable to Any in APIs (wyattscarpenter, PR [19103](https://github.com/python/mypy/pull/19103)) -- Include a detailed listing of flags enabled by --strict (wyattscarpenter, PR [19062](https://github.com/python/mypy/pull/19062)) +- Document that `object` is often preferable to `Any` in APIs (wyattscarpenter, PR [19103](https://github.com/python/mypy/pull/19103)) +- Include a detailed listing of flags enabled by `--strict` (wyattscarpenter, PR [19062](https://github.com/python/mypy/pull/19062)) - Update “common issues” (reveal_type/reveal_locals; note on orjson) (wyattscarpenter, PR [19059](https://github.com/python/mypy/pull/19059), [19058](https://github.com/python/mypy/pull/19058)) -### Other Notable Improvements +### Other Notable Fixes and Improvements -- Remove deprecated --new-type-inference flag (the new algorithm has long been default) (Ivan Levkivskyi, PR [19570](https://github.com/python/mypy/pull/19570)) +- Remove deprecated `--new-type-inference` flag (the new algorithm has long been default) (Ivan Levkivskyi, PR [19570](https://github.com/python/mypy/pull/19570)) - Use empty context as a fallback for return expressions when outer context misleads inference (Ivan Levkivskyi, PR [19767](https://github.com/python/mypy/pull/19767)) -- Support --strict-equality checks involving None (Christoph Tyralla, PR [19718](https://github.com/python/mypy/pull/19718)) -- Don’t show import‑related errors after a module‑level assert False (Stanislav Terliakov, PR [19347](https://github.com/python/mypy/pull/19347)) -- Fix forward refs in type parameters of over‑parameterized PEP 695 aliases (Brian Schubert, PR [19725](https://github.com/python/mypy/pull/19725)) +- Fix forward references in type parameters of over‑parameterized PEP 695 aliases (Brian Schubert, PR [19725](https://github.com/python/mypy/pull/19725)) - Don’t expand PEP 695 aliases when checking node fullnames (Brian Schubert, PR [19699](https://github.com/python/mypy/pull/19699)) -- Don’t use outer context for or expression inference when LHS is Any (Stanislav Terliakov, PR [19748](https://github.com/python/mypy/pull/19748)) -- Interpret bare ClassVar as inferred (not Any) (Ivan Levkivskyi, PR [19573](https://github.com/python/mypy/pull/19573)) +- Don’t use outer context for 'or' expression inference when LHS is Any (Stanislav Terliakov, PR [19748](https://github.com/python/mypy/pull/19748)) - Recognize buffer protocol special methods (Brian Schubert, PR [19581](https://github.com/python/mypy/pull/19581)) -- Add temporary named expressions for match subjects (Stanislav Terliakov, PR [18446](https://github.com/python/mypy/pull/18446)) - Support attribute access on enum members correctly (Stanislav Terliakov, PR [19422](https://github.com/python/mypy/pull/19422)) - Check `__slots__` assignments on self types (Stanislav Terliakov, PR [19332](https://github.com/python/mypy/pull/19332)) - Move self‑argument checks after decorator application (Stanislav Terliakov, PR [19490](https://github.com/python/mypy/pull/19490)) - Infer empty list for `__slots__` and module `__all__` (Stanislav Terliakov, PR [19348](https://github.com/python/mypy/pull/19348)) - Use normalized tuples for fallback calculation (Stanislav Terliakov, PR [19111](https://github.com/python/mypy/pull/19111)) -- Preserve literals when joining Literal with Instance that has matching last_known_value (Stanislav Terliakov, PR [19279](https://github.com/python/mypy/pull/19279)) +- Preserve literals when joining similar types (Stanislav Terliakov, PR [19279](https://github.com/python/mypy/pull/19279)) - Allow adjacent conditionally‑defined overloads (Stanislav Terliakov, PR [19042](https://github.com/python/mypy/pull/19042)) - Check property decorators more strictly (Stanislav Terliakov, PR [19313](https://github.com/python/mypy/pull/19313)) - Support properties with generic setters (Ivan Levkivskyi, PR [19298](https://github.com/python/mypy/pull/19298)) @@ -162,45 +252,32 @@ Mypy 1.18 includes numerous performance improvements, resulting in a 38% overall - Include tuple fallback in constraints built from tuple types (Stanislav Terliakov, PR [19100](https://github.com/python/mypy/pull/19100)) - Somewhat better isinstance support on old‑style unions (Shantanu, PR [19714](https://github.com/python/mypy/pull/19714)) - Improve promotions inside unions (Christoph Tyralla, PR [19245](https://github.com/python/mypy/pull/19245)) -- Uninhabited types should have all attributes (Ivan Levkivskyi, PR [19300](https://github.com/python/mypy/pull/19300)) -- Metaclass conflict checks improved (Robsdedude, PR [17682](https://github.com/python/mypy/pull/17682)) -- Metaclass resolution algorithm fixes (Robsdedude, PR [17713](https://github.com/python/mypy/pull/17713)) +- Treat uninhabited types as having all attributes (Ivan Levkivskyi, PR [19300](https://github.com/python/mypy/pull/19300)) +- Improve metaclass conflict checks (Robsdedude, PR [17682](https://github.com/python/mypy/pull/17682)) +- Fixes to metaclass resolution algorithm (Robsdedude, PR [17713](https://github.com/python/mypy/pull/17713)) - PEP 702 @deprecated: handle “combined” overloads (Christoph Tyralla, PR [19626](https://github.com/python/mypy/pull/19626)) - PEP 702 @deprecated: include overloads in snapshot descriptions (Christoph Tyralla, PR [19613](https://github.com/python/mypy/pull/19613)) - Ignore overload implementation when checking `__OP__` / `__rOP__` compatibility (Stanislav Terliakov, PR [18502](https://github.com/python/mypy/pull/18502)) -- Fix unwrapping of assignment expressions in match subject (Marc Mueller, PR [19742](https://github.com/python/mypy/pull/19742)) -- Omit errors for class patterns against object (Marc Mueller, PR [19709](https://github.com/python/mypy/pull/19709)) -- Remove unnecessary error for certain match class patterns (Marc Mueller, PR [19708](https://github.com/python/mypy/pull/19708)) -- Use union type for captured vars in or pattern (Marc Mueller, PR [19710](https://github.com/python/mypy/pull/19710)) -- Prevent final reassignment inside match case (Omer Hadari, PR [19496](https://github.com/python/mypy/pull/19496)) -- Support _value_ as a fallback for ellipsis Enum members (Stanislav Terliakov, PR [19352](https://github.com/python/mypy/pull/19352)) +- Support `_value_` as a fallback for ellipsis Enum members (Stanislav Terliakov, PR [19352](https://github.com/python/mypy/pull/19352)) - Sort arguments in TypedDict overlap messages (Marc Mueller, PR [19666](https://github.com/python/mypy/pull/19666)) -- Reset to previous statement on leaving return in semanal (Stanislav Terliakov, PR [19642](https://github.com/python/mypy/pull/19642)) -- Add ambiguous to UninhabitedType identity for better messaging (Stanislav Terliakov, PR [19648](https://github.com/python/mypy/pull/19648)) -- Further fix overload diagnostics for varargs/kwargs (Shantanu, PR [19619](https://github.com/python/mypy/pull/19619)) -- Fix overload diagnostics when vararg and varkwarg both match (Shantanu, PR [19614](https://github.com/python/mypy/pull/19614)) -- Show type variable name in “Cannot infer type argument” (Brian Schubert, PR [19290](https://github.com/python/mypy/pull/19290)) +- Fix handling of implicit return in lambda (Stanislav Terliakov, PR [19642](https://github.com/python/mypy/pull/19642)) +- Improve behavior of uninhabited types (Stanislav Terliakov, PR [19648](https://github.com/python/mypy/pull/19648)) +- Fix overload diagnostics when `*args` and `**kwargs` both match (Shantanu, PR [19614](https://github.com/python/mypy/pull/19614)) +- Further fix overload diagnostics for `*args`/`**kwargs` (Shantanu, PR [19619](https://github.com/python/mypy/pull/19619)) +- Show type variable name in "Cannot infer type argument" (Brian Schubert, PR [19290](https://github.com/python/mypy/pull/19290)) - Fail gracefully on unsupported template strings (PEP 750) (Brian Schubert, PR [19700](https://github.com/python/mypy/pull/19700)) - Revert colored argparse help for Python 3.14 (Marc Mueller, PR [19721](https://github.com/python/mypy/pull/19721)) -- Support type‑checking a code fragment in the profile script (Jukka Lehtosalo, PR [19379](https://github.com/python/mypy/pull/19379)) -- Fix C compiler flags in the profile self‑check script (Jukka Lehtosalo, PR [19326](https://github.com/python/mypy/pull/19326)) -- Add a script for profiling self‑check (Linux only) (Jukka Lehtosalo, PR [19322](https://github.com/python/mypy/pull/19322)) -- Retry PyPI upload script: skip existing files on retry (Jukka Lehtosalo, PR [19305](https://github.com/python/mypy/pull/19305)) - Update stubinfo for latest typeshed (Shantanu, PR [19771](https://github.com/python/mypy/pull/19771)) -- Fix crash with variadic tuple arguments to a generic type (Randolf Scholz, PR [19705](https://github.com/python/mypy/pull/19705)) -- Fix crash when enable_error_code in pyproject.toml has wrong type (wyattscarpenter, PR [19494](https://github.com/python/mypy/pull/19494)) -- Fix dict assignment to a wider context when an incompatible same‑shape TypedDict exists (Stanislav Terliakov, PR [19592](https://github.com/python/mypy/pull/19592)) -- Prevent crash for dataclass with PEP 695 TypeVarTuple on Python 3.13+ (Stanislav Terliakov, PR [19565](https://github.com/python/mypy/pull/19565)) +- Fix dict assignment when an incompatible same‑shape TypedDict exists (Stanislav Terliakov, PR [19592](https://github.com/python/mypy/pull/19592)) - Fix constructor type for subclasses of Any (Ivan Levkivskyi, PR [19295](https://github.com/python/mypy/pull/19295)) -- Fix TypeGuard/TypeIs being forgotten when semanal defers (Brian Schubert, PR [19325](https://github.com/python/mypy/pull/19325)) +- Fix TypeGuard/TypeIs being forgotten in some cases (Brian Schubert, PR [19325](https://github.com/python/mypy/pull/19325)) - Fix TypeIs negative narrowing for unions of generics (Brian Schubert, PR [18193](https://github.com/python/mypy/pull/18193)) -- dmypy suggest: fix incorrect signature suggestion when a type matches a module name (Brian Schubert, PR [18937](https://github.com/python/mypy/pull/18937)) -- dmypy suggest: fix interaction with `__new__` (Stanislav Terliakov, PR [18966](https://github.com/python/mypy/pull/18966)) -- dmypy suggest: support Callable / callable Protocols in decorator unwrapping (Anthony Sottile, PR [19072](https://github.com/python/mypy/pull/19072)) +- dmypy suggest: Fix incorrect signature suggestion when a type matches a module name (Brian Schubert, PR [18937](https://github.com/python/mypy/pull/18937)) +- dmypy suggest: Fix interaction with `__new__` (Stanislav Terliakov, PR [18966](https://github.com/python/mypy/pull/18966)) +- dmypy suggest: Support Callable / callable Protocols in decorator unwrapping (Anthony Sottile, PR [19072](https://github.com/python/mypy/pull/19072)) - Fix missing error when redeclaring a type variable in a nested generic class (Brian Schubert, PR [18883](https://github.com/python/mypy/pull/18883)) - Fix for overloaded type object erasure (Shantanu, PR [19338](https://github.com/python/mypy/pull/19338)) - Fix TypeGuard with call on temporary object (Saul Shanabrook, PR [19577](https://github.com/python/mypy/pull/19577)) -- Fix crash on settable property alias (Ivan Levkivskyi, PR [19615](https://github.com/python/mypy/pull/19615)) ### Typeshed Updates From f0863a551ad1ee7f0116cf2580cdb19ffbbbf9c3 Mon Sep 17 00:00:00 2001 From: Kevin Kannammalil Date: Thu, 11 Sep 2025 11:19:13 -0400 Subject: [PATCH 026/183] Removed Unreleased in the Changelog for Release 1.18 (#19827) Remove Unreleased from section title --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5266a86c725e..3e6f8c2cac38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Next Release -## Mypy 1.18 (Unreleased) +## Mypy 1.18 We’ve just uploaded mypy 1.18 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance From 83d186a9cc76956534332b3618dea7f662237daf Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 12 Sep 2025 01:12:58 +0100 Subject: [PATCH 027/183] Remove some effectively dead code from build.py (#19833) We can't have `fresh` True when `stale_deps` is non-empty. I guess this part is a leftover from `--quick-and-dirty` times. --- mypy/build.py | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 84dbf2b2df88..5ccf1c86e7e3 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -3303,34 +3303,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: if undeps: fresh = False if fresh: - # All cache files are fresh. Check that no dependency's - # cache file is newer than any scc node's cache file. - oldest_in_scc = min(graph[id].xmeta.data_mtime for id in scc) - viable = {id for id in stale_deps if graph[id].meta is not None} - newest_in_deps = ( - 0 if not viable else max(graph[dep].xmeta.data_mtime for dep in viable) - ) - if manager.options.verbosity >= 3: # Dump all mtimes for extreme debugging. - all_ids = sorted(ascc | viable, key=lambda id: graph[id].xmeta.data_mtime) - for id in all_ids: - if id in scc: - if graph[id].xmeta.data_mtime < newest_in_deps: - key = "*id:" - else: - key = "id:" - else: - if graph[id].xmeta.data_mtime > oldest_in_scc: - key = "+dep:" - else: - key = "dep:" - manager.trace(" %5s %.0f %s" % (key, graph[id].xmeta.data_mtime, id)) - # If equal, give the benefit of the doubt, due to 1-sec time granularity - # (on some platforms). - if oldest_in_scc < newest_in_deps: - fresh = False - fresh_msg = f"out of date by {newest_in_deps - oldest_in_scc:.0f} seconds" - else: - fresh_msg = "fresh" + fresh_msg = "fresh" elif undeps: fresh_msg = f"stale due to changed suppression ({' '.join(sorted(undeps))})" elif stale_scc: From 4939b116adbd8550342ca79c87bb01a3c15c044f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 12 Sep 2025 12:47:05 +0100 Subject: [PATCH 028/183] [mypyc] Fix crash with NewType and other non-class types in incremental builds (#19837) Fixes https://github.com/mypyc/mypyc/issues/1138. Also fix similar issue with named tuples and TypedDicts. --- mypyc/irbuild/prepare.py | 8 ++++- mypyc/test-data/run-multimodule.test | 48 ++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 61e3e5b95cf4..20f2aeef8e6e 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -159,7 +159,13 @@ def load_type_map(mapper: Mapper, modules: list[MypyFile], deser_ctx: DeserMaps) """Populate a Mapper with deserialized IR from a list of modules.""" for module in modules: for node in module.names.values(): - if isinstance(node.node, TypeInfo) and is_from_module(node.node, module): + if ( + isinstance(node.node, TypeInfo) + and is_from_module(node.node, module) + and not node.node.is_newtype + and not node.node.is_named_tuple + and node.node.typeddict_type is None + ): ir = deser_ctx.classes[node.node.fullname] mapper.type_to_ir[node.node] = ir mapper.symbol_fullnames.add(node.node.fullname) diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 4208af0f04c8..9323612cb4fb 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -902,3 +902,51 @@ import native [out2] 0 None + +[case testIncrementalCompilationWithNonClassTypeDef] +import other_a +[file other_a.py] +from other_b import MyInt +[file other_a.py.2] +from other_b import MyInt, NT, TD +i = MyInt(42) + +def f(x: MyInt) -> int: + return x + 1 + +def g(x: int) -> MyInt: + return MyInt(x + 2) + +print(i) +print(f(i)) +print(g(13)) + +def make_nt(x: int) -> NT: + return NT(x=MyInt(x)) + +print(make_nt(4)) + +def make_td(x: int) -> TD: + return {"x": MyInt(x)} + +print(make_td(5)) + +[file other_b.py] +from typing import NewType, NamedTuple, TypedDict +from enum import Enum + +MyInt = NewType("MyInt", int) +NT = NamedTuple("NT", [("x", MyInt)]) +TD = TypedDict("TD", {"x": MyInt}) + +[file driver.py] +import native + +[typing fixtures/typing-full.pyi] +[out] +[out2] +42 +43 +15 +NT(x=4) +{'x': 5} From f8f618a79606d42bb0362361ec5d1d6c300f66e9 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Fri, 12 Sep 2025 19:26:54 +0700 Subject: [PATCH 029/183] Update main.py: remove superfluous `--experimental` flag (#19831) This commit removes the --experimental flag, completing a TODO from 2018-03-16. It seems like this flag is unused and undocumented and the task of removing it "after a short transition" simply slipped under everyone's radar. --- mypy/main.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index d5bbca704305..150d388af84c 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1204,13 +1204,6 @@ def add_invertible_flag( ) if server_options: - # TODO: This flag is superfluous; remove after a short transition (2018-03-16) - other_group.add_argument( - "--experimental", - action="store_true", - dest="fine_grained_incremental", - help="Enable fine-grained incremental mode", - ) other_group.add_argument( "--use-fine-grained-cache", action="store_true", From 8bfecd4e9fbbcb26390b4b851bf2c6f9e9e34e00 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 12 Sep 2025 15:22:15 +0100 Subject: [PATCH 030/183] Write cache for modules with errors (#19820) This is required for https://github.com/python/mypy/issues/933 Here I use a very simple-minded approach, errors are serialized simply as a list of strings. In near future I may switch to serializing `ErrorInfo`s (as this has some other benefits). Note that many tests have `[stale ...]` checks updated because previously modules with errors were not included in the list. I double-checked each test that the new values are correct. Note we still don't write cache if there were blockers in an SCC (like a syntax error). --- mypy/build.py | 62 +++++------------- mypy/test/testcheck.py | 44 ++++--------- test-data/unit/check-incremental.test | 94 ++++++++++++++++++++------- test-data/unit/check-serialize.test | 2 +- 4 files changed, 101 insertions(+), 101 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 5ccf1c86e7e3..2d3296a4713e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -15,7 +15,6 @@ import collections import contextlib -import errno import gc import json import os @@ -337,6 +336,7 @@ class CacheMeta(NamedTuple): dep_lines: list[int] dep_hashes: dict[str, str] interface_hash: str # hash representing the public interface + error_lines: list[str] version_id: str # mypy version for cache invalidation ignore_all: bool # if errors were ignored plugin_data: Any # config data from plugins @@ -376,6 +376,7 @@ def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta: meta.get("dep_lines", []), meta.get("dep_hashes", {}), meta.get("interface_hash", ""), + meta.get("error_lines", []), meta.get("version_id", sentinel), meta.get("ignore_all", True), meta.get("plugin_data", None), @@ -1502,6 +1503,7 @@ def validate_meta( "dep_lines": meta.dep_lines, "dep_hashes": meta.dep_hashes, "interface_hash": meta.interface_hash, + "error_lines": meta.error_lines, "version_id": manager.version_id, "ignore_all": meta.ignore_all, "plugin_data": meta.plugin_data, @@ -1678,28 +1680,6 @@ def write_cache_meta( return cache_meta_from_dict(meta, data_json) -def delete_cache(id: str, path: str, manager: BuildManager) -> None: - """Delete cache files for a module. - - The cache files for a module are deleted when mypy finds errors there. - This avoids inconsistent states with cache files from different mypy runs, - see #4043 for an example. - """ - # We don't delete .deps files on errors, since the dependencies - # are mostly generated from other files and the metadata is - # tracked separately. - meta_path, data_path, _ = get_cache_names(id, path, manager.options) - cache_paths = [meta_path, data_path] - manager.log(f"Deleting {id} {path} {' '.join(x for x in cache_paths if x)}") - - for filename in cache_paths: - try: - manager.metastore.remove(filename) - except OSError as e: - if e.errno != errno.ENOENT: - manager.log(f"Error deleting cache file {filename}: {e.strerror}") - - """Dependency manager. Design @@ -1875,6 +1855,9 @@ class State: # Map from dependency id to its last observed interface hash dep_hashes: dict[str, str] = {} + # List of errors reported for this file last time. + error_lines: list[str] = [] + # Parent package, its parent, etc. ancestors: list[str] | None = None @@ -1896,9 +1879,6 @@ class State: # Whether to ignore all errors ignore_all = False - # Whether the module has an error or any of its dependencies have one. - transitive_error = False - # Errors reported before semantic analysis, to allow fine-grained # mode to keep reporting them. early_errors: list[ErrorInfo] @@ -2000,6 +1980,7 @@ def __init__( assert len(all_deps) == len(self.meta.dep_lines) self.dep_line_map = {id: line for id, line in zip(all_deps, self.meta.dep_lines)} self.dep_hashes = self.meta.dep_hashes + self.error_lines = self.meta.error_lines if temporary: self.load_tree(temporary=True) if not manager.use_fine_grained_cache(): @@ -2517,11 +2498,6 @@ def write_cache(self) -> tuple[dict[str, Any], str, str] | None: print(f"Error serializing {self.id}", file=self.manager.stdout) raise # Propagate to display traceback return None - is_errors = self.transitive_error - if is_errors: - delete_cache(self.id, self.path, self.manager) - self.meta = None - return None dep_prios = self.dependency_priorities() dep_lines = self.dependency_lines() assert self.source_hash is not None @@ -3315,15 +3291,14 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: else: fresh_msg = f"stale due to deps ({' '.join(sorted(stale_deps))})" - # Initialize transitive_error for all SCC members from union - # of transitive_error of dependencies. - if any(graph[dep].transitive_error for dep in deps if dep in graph): - for id in scc: - graph[id].transitive_error = True - scc_str = " ".join(scc) if fresh: manager.trace(f"Queuing {fresh_msg} SCC ({scc_str})") + for id in scc: + if graph[id].error_lines: + manager.flush_errors( + manager.errors.simplify_path(graph[id].xpath), graph[id].error_lines, False + ) fresh_scc_queue.append(scc) else: if fresh_scc_queue: @@ -3335,11 +3310,6 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: # single fresh SCC. This is intentional -- we don't need those modules # loaded if there are no more stale SCCs to be rechecked. # - # Also note we shouldn't have to worry about transitive_error here, - # since modules with transitive errors aren't written to the cache, - # and if any dependencies were changed, this SCC would be stale. - # (Also, in quick_and_dirty mode we don't care about transitive errors.) - # # TODO: see if it's possible to determine if we need to process only a # _subset_ of the past SCCs instead of having to process them all. if ( @@ -3491,16 +3461,17 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No for id in stale: graph[id].generate_unused_ignore_notes() graph[id].generate_ignore_without_code_notes() - if any(manager.errors.is_errors_for_file(graph[id].xpath) for id in stale): - for id in stale: - graph[id].transitive_error = True + + # Flush errors, and write cache in two phases: first data files, then meta files. meta_tuples = {} + errors_by_id = {} for id in stale: if graph[id].xpath not in manager.errors.ignored_files: errors = manager.errors.file_messages( graph[id].xpath, formatter=manager.error_formatter ) manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False) + errors_by_id[id] = errors meta_tuples[id] = graph[id].write_cache() graph[id].mark_as_rechecked() for id in stale: @@ -3512,6 +3483,7 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No meta["dep_hashes"] = { dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph } + meta["error_lines"] = errors_by_id.get(id, []) graph[id].meta = write_cache_meta(meta, manager, meta_json, data_json) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 04ef5370d381..73f33c0323af 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -9,7 +9,6 @@ from pathlib import Path from mypy import build -from mypy.build import Graph from mypy.errors import CompileError from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths from mypy.test.config import test_data_prefix, test_temp_dir @@ -164,11 +163,13 @@ def run_case_once( sys.path.insert(0, plugin_dir) res = None + blocker = False try: res = build.build(sources=sources, options=options, alt_lib_path=test_temp_dir) a = res.errors except CompileError as e: a = e.messages + blocker = True finally: assert sys.path[0] == plugin_dir del sys.path[0] @@ -199,7 +200,7 @@ def run_case_once( if res: if options.cache_dir != os.devnull: - self.verify_cache(module_data, res.errors, res.manager, res.graph) + self.verify_cache(module_data, res.manager, blocker) name = "targets" if incremental_step: @@ -229,42 +230,23 @@ def run_case_once( check_test_output_files(testcase, incremental_step, strip_prefix="tmp/") def verify_cache( - self, - module_data: list[tuple[str, str, str]], - a: list[str], - manager: build.BuildManager, - graph: Graph, + self, module_data: list[tuple[str, str, str]], manager: build.BuildManager, blocker: bool ) -> None: - # There should be valid cache metadata for each module except - # for those that had an error in themselves or one of their - # dependencies. - error_paths = self.find_error_message_paths(a) - busted_paths = {m.path for id, m in manager.modules.items() if graph[id].transitive_error} - modules = self.find_module_files(manager) - modules.update({module_name: path for module_name, path, text in module_data}) - missing_paths = self.find_missing_cache_files(modules, manager) - # We would like to assert error_paths.issubset(busted_paths) - # but this runs into trouble because while some 'notes' are - # really errors that cause an error to be marked, many are - # just notes attached to other errors. - assert error_paths or not busted_paths, "Some modules reported error despite no errors" - if not missing_paths == busted_paths: - raise AssertionError(f"cache data discrepancy {missing_paths} != {busted_paths}") + if not blocker: + # There should be valid cache metadata for each module except + # in case of a blocking error in themselves or one of their + # dependencies. + modules = self.find_module_files(manager) + modules.update({module_name: path for module_name, path, text in module_data}) + missing_paths = self.find_missing_cache_files(modules, manager) + if missing_paths: + raise AssertionError(f"cache data missing for {missing_paths}") assert os.path.isfile(os.path.join(manager.options.cache_dir, ".gitignore")) cachedir_tag = os.path.join(manager.options.cache_dir, "CACHEDIR.TAG") assert os.path.isfile(cachedir_tag) with open(cachedir_tag) as f: assert f.read().startswith("Signature: 8a477f597d28d172789f06886806bc55") - def find_error_message_paths(self, a: list[str]) -> set[str]: - hits = set() - for line in a: - m = re.match(r"([^\s:]+):(\d+:)?(\d+:)? (error|warning|note):", line) - if m: - p = m.group(1) - hits.add(p) - return hits - def find_module_files(self, manager: build.BuildManager) -> dict[str, str]: return {id: module.path for id, module in manager.modules.items()} diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 06f228721a86..8e05f922be17 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -124,7 +124,7 @@ class A: pass def func1() -> A: pass [rechecked mod1] -[stale] +[stale mod1] [out2] tmp/mod1.py:1: error: Name "A" is not defined @@ -428,7 +428,7 @@ class CustomType: def foo(self) -> str: return "a" [rechecked mod1, mod2, mod2.mod3] -[stale mod2, mod2.mod3] +[stale mod1, mod2, mod2.mod3] [builtins fixtures/module.pyi] [out1] [out2] @@ -466,7 +466,7 @@ class CustomType: def foo(self) -> str: return "a" [rechecked mod1, mod2, mod2.mod3] -[stale mod2.mod3] +[stale mod1, mod2.mod3] [builtins fixtures/module.pyi] [out1] [out2] @@ -541,7 +541,7 @@ def func2() -> str: return "foo" [rechecked mod0, mod1, mod2] -[stale mod2] +[stale mod0, mod2] [out2] tmp/mod1.py:4: error: Incompatible return value type (got "str", expected "int") @@ -952,7 +952,7 @@ reveal_type(b.x) [file parent/b.py.2] x = 10 -[stale parent.b] +[stale parent.a, parent.b] [rechecked parent.a, parent.b] [out2] tmp/parent/a.py:2: note: Revealed type is "builtins.int" @@ -1080,7 +1080,7 @@ class Class: pass [builtins fixtures/args.pyi] [rechecked collections, main, package.subpackage.mod1] -[stale collections, package.subpackage.mod1] +[stale collections, main, package.subpackage.mod1] [out2] tmp/main.py:4: error: "Class" has no attribute "some_attribute" @@ -1120,7 +1120,7 @@ if int(): [builtins fixtures/module_all.pyi] [rechecked main, c, c.submodule] -[stale c] +[stale main, c, c.submodule] [out2] tmp/c/submodule.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") tmp/main.py:7: error: "C" has no attribute "foo" @@ -1174,7 +1174,7 @@ reveal_type(foo) foo = 3.14 reveal_type(foo) [rechecked m, n] -[stale] +[stale n] [out1] tmp/n.py:2: note: Revealed type is "builtins.str" tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" @@ -1204,7 +1204,7 @@ from bad import foo foo(3) [rechecked client] -[stale] +[stale client] [out2] tmp/client.py:4: error: Argument 1 to "foo" has incompatible type "int"; expected "str" @@ -1316,7 +1316,7 @@ reveal_type(bar) bar = "str" [rechecked main] -[stale] +[stale main] [out1] tmp/main.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" tmp/main.py:4: note: Revealed type is "builtins.str" @@ -1354,8 +1354,8 @@ class B: class C: def foo(self) -> int: return 1 -[rechecked mod3, mod2, mod1] -[stale mod3, mod2] +[rechecked mod3] +[stale] [out1] tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") tmp/mod1.py:3: note: Revealed type is "builtins.int" @@ -1393,8 +1393,8 @@ class C: class C: def foo(self) -> str: return 'a' -[rechecked mod4, mod3, mod2, mod1] -[stale mod4] +[rechecked mod4, mod3, mod1] +[stale mod1, mod4] [out1] tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") tmp/mod1.py:3: note: Revealed type is "builtins.int" @@ -1438,8 +1438,8 @@ class C: class C: def foo(self) -> str: return 'a' -[rechecked mod4, mod3, mod2, mod1] -[stale mod4, mod3, mod2] +[rechecked mod4, mod3, mod1] +[stale mod1, mod4] [out1] tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") tmp/mod1.py:3: note: Revealed type is "builtins.int" @@ -2173,7 +2173,7 @@ import m x = 1 [delete m.py.2] [rechecked n] -[stale] +[stale n] [out2] tmp/n.py:1: error: Cannot find implementation or library stub for module named "m" tmp/n.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports @@ -2224,7 +2224,7 @@ def foo() -> int: [rechecked m] [stale m] [rechecked2 m] -[stale2] +[stale2 m] [out3] tmp/m.py:2: error: Incompatible return value type (got "str", expected "int") @@ -2243,7 +2243,7 @@ def foo() -> str: def foo(x) -> int: pass [rechecked m, n] -[stale m] +[stale m, n] [rechecked2 m, n] [stale2 m, n] [out2] @@ -2894,7 +2894,7 @@ extra = 1 import m.a # Depends on module with error [file m/c.py] import m # No error here -[rechecked m.a, m.b] +[rechecked] [out1] tmp/m/a.py:1: error: Unsupported operand types for + ("int" and "str") [out2] @@ -3091,10 +3091,10 @@ class A: [stale] [out2] main:2: note: Revealed type is "def (a: builtins.int) -> a.A" -main:3: note: Revealed type is "def [_AT] (self: _AT`1, other: _AT`1) -> builtins.bool" -main:4: note: Revealed type is "def [_AT] (self: _AT`2, other: _AT`2) -> builtins.bool" -main:5: note: Revealed type is "def [_AT] (self: _AT`3, other: _AT`3) -> builtins.bool" -main:6: note: Revealed type is "def [_AT] (self: _AT`4, other: _AT`4) -> builtins.bool" +main:3: note: Revealed type is "def [_AT] (self: _AT`3, other: _AT`3) -> builtins.bool" +main:4: note: Revealed type is "def [_AT] (self: _AT`4, other: _AT`4) -> builtins.bool" +main:5: note: Revealed type is "def [_AT] (self: _AT`5, other: _AT`5) -> builtins.bool" +main:6: note: Revealed type is "def [_AT] (self: _AT`6, other: _AT`6) -> builtins.bool" main:15: error: Unsupported operand types for < ("A" and "int") main:16: error: Unsupported operand types for <= ("A" and "int") main:17: error: Unsupported operand types for > ("A" and "int") @@ -7237,3 +7237,49 @@ bar: int = foo [out2] [out3] tmp/bar.py:2: error: Incompatible types in assignment (expression has type "None", variable has type "int") + +[case testIncrementalBlockingErrorRepeatAndUndo] +import m +[file m.py] +import f +reveal_type(f.x) +[file m.py.3] +import f +reveal_type(f.x) +# touch +[file f.py] +x = 1 +[file f.py.2] +no way +[file f.py.4] +x = 1 +[out] +tmp/m.py:2: note: Revealed type is "builtins.int" +[out2] +tmp/f.py:1: error: Invalid syntax +[out3] +tmp/f.py:1: error: Invalid syntax +[out4] +tmp/m.py:2: note: Revealed type is "builtins.int" + +[case testIncrementalSameErrorOrder] +import m +[file m.py] +import n +def accept_int(x: int) -> None: pass +accept_int(n.foo) +[file n.py] +import other +foo = "hello" +reveal_type(foo) +[file other.py] +[file other.py.2] +# touch +[rechecked other] +[stale] +[out] +tmp/n.py:3: note: Revealed type is "builtins.str" +tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" +[out2] +tmp/n.py:3: note: Revealed type is "builtins.str" +tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" diff --git a/test-data/unit/check-serialize.test b/test-data/unit/check-serialize.test index 03c185a5694b..1498c8d82826 100644 --- a/test-data/unit/check-serialize.test +++ b/test-data/unit/check-serialize.test @@ -31,7 +31,7 @@ x = '' -- We only do the following two sections once here to avoid repetition. -- Most other test cases are similar. [rechecked a] -[stale] +[stale a] [out2] tmp/a.py:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") From 530bdc5063f2309702ec08797388d635cad4b634 Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Sat, 13 Sep 2025 10:56:02 -0700 Subject: [PATCH 031/183] stubtest: additional guidance on errors when runtime is object.__init__ (#19733) Fixes #19732 This is a simple check to point users in the right direction when they get errors because their class uses `__new__` but they wrote stubs for `__init__`. I don't feel strongly about the exact wording used here. I also considered "Maybe you meant to define `__new__` instead of `__init__`?". --- mypy/stubtest.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index d4f96a3d9389..4126f3959ee1 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1053,7 +1053,10 @@ def get_kind(arg_name: str) -> nodes.ArgKind: def _verify_signature( - stub: Signature[nodes.Argument], runtime: Signature[inspect.Parameter], function_name: str + stub: Signature[nodes.Argument], + runtime: Signature[inspect.Parameter], + function_name: str, + warn_runtime_is_object_init: bool = False, ) -> Iterator[str]: # Check positional arguments match up for stub_arg, runtime_arg in zip(stub.pos, runtime.pos): @@ -1098,6 +1101,8 @@ def _verify_signature( msg = f'runtime does not have parameter "{stub_arg.variable.name}"' if runtime.varkw is not None: msg += ". Maybe you forgot to make it keyword-only in the stub?" + elif warn_runtime_is_object_init: + msg += ". You may need to write stubs for __new__ instead of __init__." yield msg else: yield f'stub parameter "{stub_arg.variable.name}" is not keyword-only' @@ -1137,7 +1142,11 @@ def _verify_signature( if arg not in {runtime_arg.name for runtime_arg in runtime.pos[len(stub.pos) :]}: yield f'runtime parameter "{arg}" is not keyword-only' else: - yield f'runtime does not have parameter "{arg}"' + msg = f'runtime does not have parameter "{arg}"' + if warn_runtime_is_object_init: + msg += ". You may need to write stubs for __new__ instead of __init__." + yield msg + for arg in sorted(set(runtime.kwonly) - set(stub.kwonly)): if arg in {stub_arg.variable.name for stub_arg in stub.pos}: # Don't report this if we've reported it before @@ -1223,7 +1232,12 @@ def verify_funcitem( if not signature: return - for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name): + for message in _verify_signature( + stub_sig, + runtime_sig, + function_name=stub.name, + warn_runtime_is_object_init=runtime is object.__init__, + ): yield Error( object_path, "is inconsistent, " + message, @@ -1333,7 +1347,12 @@ def verify_overloadedfuncdef( stub_sig = Signature.from_overloadedfuncdef(stub) runtime_sig = Signature.from_inspect_signature(signature) - for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name): + for message in _verify_signature( + stub_sig, + runtime_sig, + function_name=stub.name, + warn_runtime_is_object_init=runtime is object.__init__, + ): # TODO: This is a little hacky, but the addition here is super useful if "has a default value of type" in message: message += ( From 6cc96f48ab6a8250598012062fe572a2a9e46838 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 14 Sep 2025 06:05:07 +0700 Subject: [PATCH 032/183] Update docs.yml: add mypy/main.py (#19829) Part of the documentation is automatically generated from the options definitions in mypy/main.py, so we need to run the docs CI when that file is modified. This follows up on https://github.com/python/mypy/pull/19727, which itself follows up on https://github.com/python/mypy/pull/19062 --- .github/workflows/docs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3e78bf51913e..66e7c997f4fa 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,6 +12,9 @@ on: # so it's important to do the docs build on all PRs touching mypy/errorcodes.py # in case somebody's adding a new error code without any docs - 'mypy/errorcodes.py' + # Part of the documentation is automatically generated from the options + # definitions in mypy/main.py + - 'mypy/main.py' - 'mypyc/doc/**' - '**/*.rst' - '**/*.md' From 8412d1dd19c45628159ae37ce1822b7d49e66567 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 14 Sep 2025 06:14:50 +0700 Subject: [PATCH 033/183] Refactor/nit main.py: rename the variable other_group to misc_group (#19832) This better reflects its external name, "Miscellaneous". The current CI suffices to check that this code is correct. --- mypy/main.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 150d388af84c..b543cd33fe44 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1163,22 +1163,22 @@ def add_invertible_flag( "--skip-c-gen", dest="mypyc_skip_c_generation", action="store_true", help=argparse.SUPPRESS ) - other_group = parser.add_argument_group(title="Miscellaneous") - other_group.add_argument("--quickstart-file", help=argparse.SUPPRESS) - other_group.add_argument("--junit-xml", help="Write junit.xml to the given file") + misc_group = parser.add_argument_group(title="Miscellaneous") + misc_group.add_argument("--quickstart-file", help=argparse.SUPPRESS) + misc_group.add_argument("--junit-xml", help="Write junit.xml to the given file") imports_group.add_argument( "--junit-format", choices=["global", "per_file"], default="global", help="If --junit-xml is set, specifies format. global: single test with all errors; per_file: one test entry per file with failures", ) - other_group.add_argument( + misc_group.add_argument( "--find-occurrences", metavar="CLASS.MEMBER", dest="special-opts:find_occurrences", help="Print out all usages of a class member (experimental)", ) - other_group.add_argument( + misc_group.add_argument( "--scripts-are-modules", action="store_true", help="Script x becomes module x instead of __main__", @@ -1189,7 +1189,7 @@ def add_invertible_flag( default=False, strict_flag=False, help="Install detected missing library stub packages using pip", - group=other_group, + group=misc_group, ) add_invertible_flag( "--non-interactive", @@ -1199,12 +1199,12 @@ def add_invertible_flag( "Install stubs without asking for confirmation and hide " + "errors, with --install-types" ), - group=other_group, + group=misc_group, inverse="--interactive", ) if server_options: - other_group.add_argument( + misc_group.add_argument( "--use-fine-grained-cache", action="store_true", help="Use the cache in fine-grained incremental mode", From 647ea8cf07b93c0ffa8c480143dfedc449e6f2e2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 14 Sep 2025 17:39:09 -0700 Subject: [PATCH 034/183] Sync typeshed (#19848) Sync typeshed Source commit: https://github.com/python/typeshed/commit/0d100b9110f1b30529ba4d1be26d1eb09ae5d42c Note that you will need to close and re-open the PR in order to trigger CI. --------- Co-authored-by: mypybot <> Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: AlexWaygood --- mypy/typeshed/stdlib/asyncio/events.pyi | 32 +++++++++++++++---------- mypy/typeshed/stdlib/builtins.pyi | 7 ++---- mypy/typeshed/stdlib/turtle.pyi | 14 +++++++++-- mypy/typeshed/stdlib/unittest/mock.pyi | 3 ++- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/mypy/typeshed/stdlib/asyncio/events.pyi b/mypy/typeshed/stdlib/asyncio/events.pyi index 14c4c0bf3d5a..5dc698bc5e15 100644 --- a/mypy/typeshed/stdlib/asyncio/events.pyi +++ b/mypy/typeshed/stdlib/asyncio/events.pyi @@ -602,18 +602,25 @@ class AbstractEventLoop: @abstractmethod async def shutdown_default_executor(self) -> None: ... -# This class does not exist at runtime, but stubtest complains if it's marked as -# @type_check_only because it has an alias that does exist at runtime. See mypy#19568. -# @type_check_only -class _AbstractEventLoopPolicy: - @abstractmethod - def get_event_loop(self) -> AbstractEventLoop: ... - @abstractmethod - def set_event_loop(self, loop: AbstractEventLoop | None) -> None: ... - @abstractmethod - def new_event_loop(self) -> AbstractEventLoop: ... - # Child processes handling (Unix only). - if sys.version_info < (3, 14): +if sys.version_info >= (3, 14): + class _AbstractEventLoopPolicy: + @abstractmethod + def get_event_loop(self) -> AbstractEventLoop: ... + @abstractmethod + def set_event_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def new_event_loop(self) -> AbstractEventLoop: ... + +else: + @type_check_only + class _AbstractEventLoopPolicy: + @abstractmethod + def get_event_loop(self) -> AbstractEventLoop: ... + @abstractmethod + def set_event_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def new_event_loop(self) -> AbstractEventLoop: ... + # Child processes handling (Unix only). if sys.version_info >= (3, 12): @abstractmethod @deprecated("Deprecated since Python 3.12; removed in Python 3.14.") @@ -627,7 +634,6 @@ class _AbstractEventLoopPolicy: @abstractmethod def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... -if sys.version_info < (3, 14): AbstractEventLoopPolicy = _AbstractEventLoopPolicy if sys.version_info >= (3, 14): diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index ca8d56cb4297..ef6c712e0005 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -923,8 +923,7 @@ class slice(Generic[_StartT_co, _StopT_co, _StepT_co]): def indices(self, len: SupportsIndex, /) -> tuple[int, int, int]: ... -# Making this a disjoint_base upsets pyright -# @disjoint_base +@disjoint_base class tuple(Sequence[_T_co]): def __new__(cls, iterable: Iterable[_T_co] = ..., /) -> Self: ... def __len__(self) -> int: ... @@ -1266,10 +1265,8 @@ class property: def __set__(self, instance: Any, value: Any, /) -> None: ... def __delete__(self, instance: Any, /) -> None: ... -# This class does not exist at runtime, but stubtest complains if it's marked as -# @type_check_only because it has an alias that does exist at runtime. See mypy#19568. -# @type_check_only @final +@type_check_only class _NotImplementedType(Any): __call__: None diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index 0b93429904c5..39a995de2612 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -463,7 +463,12 @@ class RawTurtle(TPen, TNavigator): # type: ignore[misc] # Conflicting methods def begin_fill(self) -> None: ... def end_fill(self) -> None: ... - def dot(self, size: int | None = None, *color: _Color) -> None: ... + @overload + def dot(self, size: int | _Color | None = None) -> None: ... + @overload + def dot(self, size: int | None, color: _Color, /) -> None: ... + @overload + def dot(self, size: int | None, r: float, g: float, b: float, /) -> None: ... def write( self, arg: object, move: bool = False, align: str = "left", font: tuple[str, int, str] = ("Arial", 8, "normal") ) -> None: ... @@ -747,7 +752,12 @@ if sys.version_info >= (3, 14): def begin_fill() -> None: ... def end_fill() -> None: ... -def dot(size: int | None = None, *color: _Color) -> None: ... +@overload +def dot(size: int | _Color | None = None) -> None: ... +@overload +def dot(size: int | None, color: _Color, /) -> None: ... +@overload +def dot(size: int | None, r: float, g: float, b: float, /) -> None: ... def write(arg: object, move: bool = False, align: str = "left", font: tuple[str, int, str] = ("Arial", 8, "normal")) -> None: ... if sys.version_info >= (3, 14): diff --git a/mypy/typeshed/stdlib/unittest/mock.pyi b/mypy/typeshed/stdlib/unittest/mock.pyi index f4b59e7cab90..f3e58bcd1c00 100644 --- a/mypy/typeshed/stdlib/unittest/mock.pyi +++ b/mypy/typeshed/stdlib/unittest/mock.pyi @@ -508,7 +508,8 @@ class MagicProxy(Base): def create_mock(self) -> Any: ... def __get__(self, obj: Any, _type: Any | None = None) -> Any: ... -class _ANY: +# See https://github.com/python/typeshed/issues/14701 +class _ANY(Any): def __eq__(self, other: object) -> Literal[True]: ... def __ne__(self, other: object) -> Literal[False]: ... __hash__: ClassVar[None] # type: ignore[assignment] From 73affc0c60aa8d9a7fdc43c8d57fd65c9ea870f1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 15 Sep 2025 17:41:36 +0100 Subject: [PATCH 035/183] Incremental regression test for recursive aliases (#19853) See original PR https://github.com/python/mypy/pull/19845 --- test-data/unit/check-incremental.test | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 8e05f922be17..d9d78715b396 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -2577,6 +2577,13 @@ C(1)[0] [builtins fixtures/list.pyi] [out] +[case testSerializeRecursiveAlias] +from typing import Callable, Union + +Node = Union[str, int, Callable[[], "Node"]] +n: Node +[out] + [case testSerializeRecursiveAliases1] from typing import Type, Callable, Union From dce8e1c407ccaa9effebbb1ed09fbf0e7070636d Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 15 Sep 2025 21:27:43 -0400 Subject: [PATCH 036/183] [mypyc] fix: inappropriate `None`s in f-strings (#19846) if a variable is Final but the value is not yet known at compile-time, and that variable is used as an input to an f-string, the f-string will incorrectly contain "None" Fixes [mypyc#1140](https://github.com/mypyc/mypyc/issues/1140) --- mypyc/irbuild/specialize.py | 4 +++- mypyc/test-data/run-strings.test | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 0880c62bc7a5..576b7a7ebffd 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -719,7 +719,9 @@ def get_literal_str(expr: Expression) -> str | None: if isinstance(expr, StrExpr): return expr.value elif isinstance(expr, RefExpr) and isinstance(expr.node, Var) and expr.node.is_final: - return str(expr.node.final_value) + final_value = expr.node.final_value + if final_value is not None: + return str(final_value) return None for i in range(len(exprs) - 1): diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index 6960b0a04303..6a62db6ee3ee 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -412,9 +412,16 @@ def test_basics() -> None: [case testFStrings] import decimal from datetime import datetime +from typing import Final var = 'mypyc' num = 20 +final_known_at_compile_time: Final = 'hello' + +def final_value_setter() -> str: + return 'goodbye' + +final_unknown_at_compile_time: Final = final_value_setter() def test_fstring_basics() -> None: assert f'Hello {var}, this is a test' == "Hello mypyc, this is a test" @@ -451,6 +458,8 @@ def test_fstring_basics() -> None: inf_num = float('inf') assert f'{nan_num}, {inf_num}' == 'nan, inf' + assert f'{final_known_at_compile_time} {final_unknown_at_compile_time}' == 'hello goodbye' + # F-strings would be translated into ''.join[string literals, format method call, ...] in mypy AST. # Currently we are using a str.join specializer for f-string speed up. We might not cover all cases # and the rest ones should fall back to a normal str.join method call. From d27b43b0cb622a8a8a894fbb989032f92ea68eab Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 17 Sep 2025 20:28:44 +0200 Subject: [PATCH 037/183] Sync variance of typing classes in fixtures with typeshed (#19778) Originally discovered in #19777. Our test fixtures use definitions that are very far from their real counterparts, but at least generics should match if possible. Let's see if it kills more than one testcase (expected from #19777). --- test-data/unit/fixtures/typing-async.pyi | 8 ++-- test-data/unit/fixtures/typing-full.pyi | 46 +++++++++++-------- test-data/unit/fixtures/typing-medium.pyi | 14 +++--- test-data/unit/fixtures/typing-namedtuple.pyi | 4 +- test-data/unit/fixtures/typing-override.pyi | 4 +- test-data/unit/fixtures/typing-typeddict.pyi | 2 +- test-data/unit/lib-stub/typing.pyi | 14 +++--- 7 files changed, 48 insertions(+), 44 deletions(-) diff --git a/test-data/unit/fixtures/typing-async.pyi b/test-data/unit/fixtures/typing-async.pyi index 7ce2821d2916..66509a91b82b 100644 --- a/test-data/unit/fixtures/typing-async.pyi +++ b/test-data/unit/fixtures/typing-async.pyi @@ -123,13 +123,13 @@ class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): @overload def get(self, k: T, default: Union[T_co, V]) -> Union[T_co, V]: pass -class ContextManager(Generic[T]): - def __enter__(self) -> T: pass +class ContextManager(Generic[T_co]): + def __enter__(self) -> T_co: pass # Use Any because not all the precise types are in the fixtures. def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: pass -class AsyncContextManager(Generic[T]): - def __aenter__(self) -> Awaitable[T]: pass +class AsyncContextManager(Generic[T_co]): + def __aenter__(self) -> Awaitable[T_co]: pass # Use Any because not all the precise types are in the fixtures. def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Awaitable[Any]: pass diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index 8e0116aab1c2..3757e868552e 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -44,7 +44,8 @@ Literal: _SpecialForm T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) -T_contra = TypeVar('T_contra', contravariant=True) +R_co = TypeVar('R_co', covariant=True) +S_contra = TypeVar('S_contra', contravariant=True) U = TypeVar('U') V = TypeVar('V') S = TypeVar('S') @@ -82,9 +83,9 @@ class Iterator(Iterable[T_co], Protocol): @abstractmethod def __next__(self) -> T_co: pass -class Generator(Iterator[T], Generic[T, U, V]): +class Generator(Iterator[T_co], Generic[T_co, S_contra, R_co]): @abstractmethod - def send(self, value: U) -> T: pass + def send(self, value: S_contra) -> T_co: pass @abstractmethod def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass @@ -93,35 +94,40 @@ class Generator(Iterator[T], Generic[T, U, V]): def close(self) -> None: pass @abstractmethod - def __iter__(self) -> 'Generator[T, U, V]': pass + def __iter__(self) -> 'Generator[T_co, S_contra, R_co]': pass -class AsyncGenerator(AsyncIterator[T], Generic[T, U]): +class AsyncGenerator(AsyncIterator[T_co], Generic[T_co, S_contra]): @abstractmethod - def __anext__(self) -> Awaitable[T]: pass + def __anext__(self) -> Awaitable[T_co]: pass @abstractmethod - def asend(self, value: U) -> Awaitable[T]: pass + def asend(self, value: S_contra) -> Awaitable[T_co]: pass @abstractmethod - def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T]: pass + def athrow(self, typ: Any, val: Any=None, tb: Any=None) -> Awaitable[T_co]: pass @abstractmethod - def aclose(self) -> Awaitable[T]: pass + def aclose(self) -> Awaitable[T_co]: pass @abstractmethod - def __aiter__(self) -> 'AsyncGenerator[T, U]': pass + def __aiter__(self) -> 'AsyncGenerator[T_co, S_contra]': pass @runtime_checkable -class Awaitable(Protocol[T]): +class Awaitable(Protocol[T_co]): @abstractmethod - def __await__(self) -> Generator[Any, Any, T]: pass + def __await__(self) -> Generator[Any, Any, T_co]: pass -class AwaitableGenerator(Generator[T, U, V], Awaitable[V], Generic[T, U, V, S], metaclass=ABCMeta): +class AwaitableGenerator( + Awaitable[R_co], + Generator[T_co, S_contra, R_co], + Generic[T_co, S_contra, R_co, S], + metaclass=ABCMeta +): pass -class Coroutine(Awaitable[V], Generic[T, U, V]): +class Coroutine(Awaitable[R_co], Generic[T_co, S_contra, R_co]): @abstractmethod - def send(self, value: U) -> T: pass + def send(self, value: S_contra) -> T_co: pass @abstractmethod def throw(self, typ: Any, val: Any=None, tb: Any=None) -> None: pass @@ -130,15 +136,15 @@ class Coroutine(Awaitable[V], Generic[T, U, V]): def close(self) -> None: pass @runtime_checkable -class AsyncIterable(Protocol[T]): +class AsyncIterable(Protocol[T_co]): @abstractmethod - def __aiter__(self) -> 'AsyncIterator[T]': pass + def __aiter__(self) -> 'AsyncIterator[T_co]': pass @runtime_checkable -class AsyncIterator(AsyncIterable[T], Protocol): - def __aiter__(self) -> 'AsyncIterator[T]': return self +class AsyncIterator(AsyncIterable[T_co], Protocol): + def __aiter__(self) -> 'AsyncIterator[T_co]': return self @abstractmethod - def __anext__(self) -> Awaitable[T]: pass + def __anext__(self) -> Awaitable[T_co]: pass class Sequence(Iterable[T_co], Container[T_co]): @abstractmethod diff --git a/test-data/unit/fixtures/typing-medium.pyi b/test-data/unit/fixtures/typing-medium.pyi index c722a9ddb12c..077d4eebf7d3 100644 --- a/test-data/unit/fixtures/typing-medium.pyi +++ b/test-data/unit/fixtures/typing-medium.pyi @@ -32,10 +32,8 @@ Self = 0 T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) -T_contra = TypeVar('T_contra', contravariant=True) -U = TypeVar('U') -V = TypeVar('V') -S = TypeVar('S') +R_co = TypeVar('R_co', covariant=True) +S_contra = TypeVar('S_contra', contravariant=True) # Note: definitions below are different from typeshed, variances are declared # to silence the protocol variance checks. Maybe it is better to use type: ignore? @@ -49,8 +47,8 @@ class Iterable(Protocol[T_co]): class Iterator(Iterable[T_co], Protocol): def __next__(self) -> T_co: pass -class Generator(Iterator[T], Generic[T, U, V]): - def __iter__(self) -> 'Generator[T, U, V]': pass +class Generator(Iterator[T_co], Generic[T_co, S_contra, R_co]): + def __iter__(self) -> 'Generator[T_co, S_contra, R_co]': pass class Sequence(Iterable[T_co]): def __getitem__(self, n: Any) -> T_co: pass @@ -65,8 +63,8 @@ class SupportsInt(Protocol): class SupportsFloat(Protocol): def __float__(self) -> float: pass -class ContextManager(Generic[T]): - def __enter__(self) -> T: pass +class ContextManager(Generic[T_co]): + def __enter__(self) -> T_co: pass # Use Any because not all the precise types are in the fixtures. def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: pass diff --git a/test-data/unit/fixtures/typing-namedtuple.pyi b/test-data/unit/fixtures/typing-namedtuple.pyi index fbb4e43b62e6..5b0ef0845dad 100644 --- a/test-data/unit/fixtures/typing-namedtuple.pyi +++ b/test-data/unit/fixtures/typing-namedtuple.pyi @@ -18,8 +18,8 @@ class Iterable(Generic[T_co]): pass class Iterator(Iterable[T_co]): pass class Sequence(Iterable[T_co]): pass class Mapping(Iterable[KT], Generic[KT, T_co]): - def keys(self) -> Iterable[T]: pass # Approximate return type - def __getitem__(self, key: T) -> T_co: pass + def keys(self) -> Iterable[KT]: pass # Approximate return type + def __getitem__(self, key: KT) -> T_co: pass class NamedTuple(tuple[Any, ...]): _fields: ClassVar[tuple[str, ...]] diff --git a/test-data/unit/fixtures/typing-override.pyi b/test-data/unit/fixtures/typing-override.pyi index e9d2dfcf55c4..a0287524c84a 100644 --- a/test-data/unit/fixtures/typing-override.pyi +++ b/test-data/unit/fixtures/typing-override.pyi @@ -18,8 +18,8 @@ class Iterable(Generic[T_co]): pass class Iterator(Iterable[T_co]): pass class Sequence(Iterable[T_co]): pass class Mapping(Iterable[KT], Generic[KT, T_co]): - def keys(self) -> Iterable[T]: pass # Approximate return type - def __getitem__(self, key: T) -> T_co: pass + def keys(self) -> Iterable[KT]: pass # Approximate return type + def __getitem__(self, key: KT) -> T_co: pass def override(__arg: T) -> T: ... diff --git a/test-data/unit/fixtures/typing-typeddict.pyi b/test-data/unit/fixtures/typing-typeddict.pyi index f841a9aae6e7..16658c82528b 100644 --- a/test-data/unit/fixtures/typing-typeddict.pyi +++ b/test-data/unit/fixtures/typing-typeddict.pyi @@ -61,7 +61,7 @@ class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): def __len__(self) -> int: ... def __contains__(self, arg: object) -> int: pass -class MutableMapping(Mapping[T, T_co], Generic[T, T_co], metaclass=ABCMeta): +class MutableMapping(Mapping[T, V], Generic[T, V], metaclass=ABCMeta): # Other methods are not used in tests. def clear(self) -> None: ... diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 86d542a918ee..00fce56920b7 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -35,8 +35,8 @@ TYPE_CHECKING = 0 T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) -U = TypeVar('U') -V = TypeVar('V') +S_contra = TypeVar('S_contra', contravariant=True) +R_co = TypeVar('R_co', covariant=True) class Iterable(Protocol[T_co]): def __iter__(self) -> Iterator[T_co]: pass @@ -44,8 +44,8 @@ class Iterable(Protocol[T_co]): class Iterator(Iterable[T_co], Protocol): def __next__(self) -> T_co: pass -class Generator(Iterator[T], Generic[T, U, V]): - def __iter__(self) -> Generator[T, U, V]: pass +class Generator(Iterator[T_co], Generic[T_co, S_contra, R_co]): + def __iter__(self) -> Generator[T_co, S_contra, R_co]: pass class Sequence(Iterable[T_co]): def __getitem__(self, n: Any) -> T_co: pass @@ -56,10 +56,10 @@ class Mapping(Iterable[T], Generic[T, T_co]): def keys(self) -> Iterable[T]: pass # Approximate return type def __getitem__(self, key: T) -> T_co: pass -class Awaitable(Protocol[T]): - def __await__(self) -> Generator[Any, Any, T]: pass +class Awaitable(Protocol[T_co]): + def __await__(self) -> Generator[Any, Any, T_co]: pass -class Coroutine(Awaitable[V], Generic[T, U, V]): pass +class Coroutine(Awaitable[R_co], Generic[T_co, S_contra, R_co]): pass def final(meth: T) -> T: pass From 4301be16747910ad00b4360dcc20152a7e377e3a Mon Sep 17 00:00:00 2001 From: Kevin Kannammalil Date: Thu, 18 Sep 2025 13:02:52 -0400 Subject: [PATCH 038/183] Update changelog for 1.18.2 (#19873) Changelog update for 1.18.2 Also updated the changelog to reflect the initial release being 1.18.1, since we had to bump the version due to wheels failing. This adds the cherry picked PRs mentioned in https://github.com/python/mypy/issues/19764#issuecomment-3293411266 --- CHANGELOG.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e6f8c2cac38..134d251d90b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ ## Next Release -## Mypy 1.18 +## Mypy 1.18.1 -We’ve just uploaded mypy 1.18 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +We’ve just uploaded mypy 1.18.1 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: @@ -14,7 +14,7 @@ You can read the full documentation for this release on [Read the Docs](http://m ### Mypy Performance Improvements -Mypy 1.18 includes numerous performance improvements, resulting in about 40% speedup +Mypy 1.18.1 includes numerous performance improvements, resulting in about 40% speedup compared to 1.17 when type checking mypy itself. In extreme cases, the improvement can be 10x or higher. The list below is an overview of the various mypy optimizations. Many mypyc improvements (discussed in a separate section below) also improve performance. @@ -283,6 +283,12 @@ Related PRs: Please see [git log](https://github.com/python/typeshed/commits/main?after=2480d7e7c74493a024eaf254c5d2c6f452c80ee2+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. +### Mypy 1.18.2 + +- Fix crash on recursive alias (Ivan Levkivskyi, PR [19845](https://github.com/python/mypy/pull/19845)) +- Add additional guidance for stubtest errors when runtime is `object.__init__` (Stephen Morton, PR [19733](https://github.com/python/mypy/pull/19733)) +- Fix handling of None values in f-string expressions in mypyc (BobTheBuidler, PR [19846](https://github.com/python/mypy/pull/19846)) + ### Acknowledgements Thanks to all mypy contributors who contributed to this release: From f955623ad845ef8f066fe9822c0bc458ced49271 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 21 Sep 2025 03:46:46 +0700 Subject: [PATCH 039/183] [refactor] Update errorcodes.py: harmonize all of the type annotations (#19880) This completes an in-line todo from 3 years ago relating to a weakness that mypy no longer has (edit: I did not read carefully enough. But I did eventually stumble upon a sufficient workaround for https://github.com/mypyc/mypyc/issues/1142), and removes code with no downside. --- mypy/errorcodes.py | 55 ++++++++++++++++++---------------------- mypy/message_registry.py | 3 ++- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index bcfdbf6edc2b..a96f5f723a7d 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -52,32 +52,28 @@ def __hash__(self) -> int: ATTR_DEFINED: Final = ErrorCode("attr-defined", "Check that attribute exists", "General") NAME_DEFINED: Final = ErrorCode("name-defined", "Check that name is defined", "General") -CALL_ARG: Final[ErrorCode] = ErrorCode( +CALL_ARG: Final = ErrorCode( "call-arg", "Check number, names and kinds of arguments in calls", "General" ) ARG_TYPE: Final = ErrorCode("arg-type", "Check argument types in calls", "General") CALL_OVERLOAD: Final = ErrorCode( "call-overload", "Check that an overload variant matches arguments", "General" ) -VALID_TYPE: Final[ErrorCode] = ErrorCode( - "valid-type", "Check that type (annotation) is valid", "General" -) +VALID_TYPE: Final = ErrorCode("valid-type", "Check that type (annotation) is valid", "General") VAR_ANNOTATED: Final = ErrorCode( "var-annotated", "Require variable annotation if type can't be inferred", "General" ) OVERRIDE: Final = ErrorCode( "override", "Check that method override is compatible with base class", "General" ) -RETURN: Final[ErrorCode] = ErrorCode( - "return", "Check that function always returns a value", "General" -) -RETURN_VALUE: Final[ErrorCode] = ErrorCode( +RETURN: Final = ErrorCode("return", "Check that function always returns a value", "General") +RETURN_VALUE: Final = ErrorCode( "return-value", "Check that return value is compatible with signature", "General" ) -ASSIGNMENT: Final[ErrorCode] = ErrorCode( +ASSIGNMENT: Final = ErrorCode( "assignment", "Check that assigned value is compatible with target", "General" ) -METHOD_ASSIGN: Final[ErrorCode] = ErrorCode( +METHOD_ASSIGN: Final = ErrorCode( "method-assign", "Check that assignment target is not a method", "General", @@ -143,9 +139,7 @@ def __hash__(self) -> int: UNUSED_COROUTINE: Final = ErrorCode( "unused-coroutine", "Ensure that all coroutines are used", "General" ) -# TODO: why do we need the explicit type here? Without it mypyc CI builds fail with -# mypy/message_registry.py:37: error: Cannot determine type of "EMPTY_BODY" [has-type] -EMPTY_BODY: Final[ErrorCode] = ErrorCode( +EMPTY_BODY: Final = ErrorCode( "empty-body", "A dedicated error code to opt out return errors for empty/trivial bodies", "General", @@ -160,7 +154,7 @@ def __hash__(self) -> int: "await-not-async", 'Warn about "await" outside coroutine ("async def")', "General" ) # These error codes aren't enabled by default. -NO_UNTYPED_DEF: Final[ErrorCode] = ErrorCode( +NO_UNTYPED_DEF: Final = ErrorCode( "no-untyped-def", "Check that every function has an annotation", "General" ) NO_UNTYPED_CALL: Final = ErrorCode( @@ -186,13 +180,13 @@ def __hash__(self) -> int: UNREACHABLE: Final = ErrorCode( "unreachable", "Warn about unreachable statements or expressions", "General" ) -ANNOTATION_UNCHECKED = ErrorCode( +ANNOTATION_UNCHECKED: Final = ErrorCode( "annotation-unchecked", "Notify about type annotations in unchecked functions", "General" ) -TYPEDDICT_READONLY_MUTATED = ErrorCode( +TYPEDDICT_READONLY_MUTATED: Final = ErrorCode( "typeddict-readonly-mutated", "TypedDict's ReadOnly key is mutated", "General" ) -POSSIBLY_UNDEFINED: Final[ErrorCode] = ErrorCode( +POSSIBLY_UNDEFINED: Final = ErrorCode( "possibly-undefined", "Warn about variables that are defined only in some execution paths", "General", @@ -201,18 +195,18 @@ def __hash__(self) -> int: REDUNDANT_EXPR: Final = ErrorCode( "redundant-expr", "Warn about redundant expressions", "General", default_enabled=False ) -TRUTHY_BOOL: Final[ErrorCode] = ErrorCode( +TRUTHY_BOOL: Final = ErrorCode( "truthy-bool", "Warn about expressions that could always evaluate to true in boolean contexts", "General", default_enabled=False, ) -TRUTHY_FUNCTION: Final[ErrorCode] = ErrorCode( +TRUTHY_FUNCTION: Final = ErrorCode( "truthy-function", "Warn about function that always evaluate to true in boolean contexts", "General", ) -TRUTHY_ITERABLE: Final[ErrorCode] = ErrorCode( +TRUTHY_ITERABLE: Final = ErrorCode( "truthy-iterable", "Warn about Iterable expressions that could always evaluate to true in boolean contexts", "General", @@ -238,13 +232,13 @@ def __hash__(self) -> int: "General", default_enabled=False, ) -REDUNDANT_SELF_TYPE = ErrorCode( +REDUNDANT_SELF_TYPE: Final = ErrorCode( "redundant-self", "Warn about redundant Self type annotations on method first argument", "General", default_enabled=False, ) -USED_BEFORE_DEF: Final[ErrorCode] = ErrorCode( +USED_BEFORE_DEF: Final = ErrorCode( "used-before-def", "Warn about variables that are used before they are defined", "General" ) UNUSED_IGNORE: Final = ErrorCode( @@ -262,7 +256,7 @@ def __hash__(self) -> int: "General", default_enabled=False, ) -MUTABLE_OVERRIDE: Final[ErrorCode] = ErrorCode( +MUTABLE_OVERRIDE: Final = ErrorCode( "mutable-override", "Reject covariant overrides for mutable attributes", "General", @@ -274,10 +268,10 @@ def __hash__(self) -> int: "General", default_enabled=False, ) -METACLASS: Final[ErrorCode] = ErrorCode("metaclass", "Ensure that metaclass is valid", "General") +METACLASS: Final = ErrorCode("metaclass", "Ensure that metaclass is valid", "General") # Syntax errors are often blocking. -SYNTAX: Final[ErrorCode] = ErrorCode("syntax", "Report syntax errors", "General") +SYNTAX: Final = ErrorCode("syntax", "Report syntax errors", "General") # This is an internal marker code for a whole-file ignore. It is not intended to # be user-visible. @@ -285,31 +279,30 @@ def __hash__(self) -> int: del error_codes[FILE.code] # This is a catch-all for remaining uncategorized errors. -MISC: Final[ErrorCode] = ErrorCode("misc", "Miscellaneous other checks", "General") +MISC: Final = ErrorCode("misc", "Miscellaneous other checks", "General") -OVERLOAD_CANNOT_MATCH: Final[ErrorCode] = ErrorCode( +OVERLOAD_CANNOT_MATCH: Final = ErrorCode( "overload-cannot-match", "Warn if an @overload signature can never be matched", "General", sub_code_of=MISC, ) - -OVERLOAD_OVERLAP: Final[ErrorCode] = ErrorCode( +OVERLOAD_OVERLAP: Final = ErrorCode( "overload-overlap", "Warn if multiple @overload variants overlap in unsafe ways", "General", sub_code_of=MISC, ) -PROPERTY_DECORATOR = ErrorCode( +PROPERTY_DECORATOR: Final = ErrorCode( "prop-decorator", "Decorators on top of @property are not supported", "General", sub_code_of=MISC, ) -NARROWED_TYPE_NOT_SUBTYPE: Final[ErrorCode] = ErrorCode( +NARROWED_TYPE_NOT_SUBTYPE: Final = ErrorCode( "narrowed-type-not-subtype", "Warn if a TypeIs function's narrowed type is not a subtype of the original type", "General", diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 09004322aee9..b0f9ed1b0dfe 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -11,11 +11,12 @@ from typing import Final, NamedTuple from mypy import errorcodes as codes +from mypy.errorcodes import ErrorCode class ErrorMessage(NamedTuple): value: str - code: codes.ErrorCode | None = None + code: ErrorCode | None = None def format(self, *args: object, **kwargs: object) -> ErrorMessage: return ErrorMessage(self.value.format(*args, **kwargs), code=self.code) From e1aada828c2dc41e448382434bf04b0cb091cc42 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 21 Sep 2025 03:48:39 +0700 Subject: [PATCH 040/183] Enable warn_unreachable = True for `mypyc` and all other files as well (#19050) Achieves a todo, enabling warn_unreachable = True for `mypyc` and all other files as well --- misc/analyze_cache.py | 6 +++--- misc/profile_check.py | 32 ++++++++++++++++---------------- mypy_self_check.ini | 3 --- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/misc/analyze_cache.py b/misc/analyze_cache.py index 0a05493b77a3..f911522f5c64 100644 --- a/misc/analyze_cache.py +++ b/misc/analyze_cache.py @@ -41,7 +41,7 @@ def extract(chunks: Iterable[JsonDict]) -> Iterable[JsonDict]: if isinstance(chunk, dict): yield chunk yield from extract(chunk.values()) - elif isinstance(chunk, list): + elif isinstance(chunk, list): # type: ignore[unreachable] #TODO: is this actually unreachable, or are our types wrong? yield from extract(chunk) yield from extract([chunk.data for chunk in chunks]) @@ -93,7 +93,7 @@ def compress(chunk: JsonDict) -> JsonDict: def helper(chunk: JsonDict) -> JsonDict: nonlocal counter if not isinstance(chunk, dict): - return chunk + return chunk # type: ignore[unreachable] #TODO: is this actually unreachable, or are our types wrong? if len(chunk) <= 2: return chunk @@ -124,7 +124,7 @@ def decompress(chunk: JsonDict) -> JsonDict: def helper(chunk: JsonDict) -> JsonDict: if not isinstance(chunk, dict): - return chunk + return chunk # type: ignore[unreachable] #TODO: is this actually unreachable, or are our types wrong? if ".id" in chunk: return cache[chunk[".id"]] diff --git a/misc/profile_check.py b/misc/profile_check.py index b29535020f0a..6bd23b09b2d5 100644 --- a/misc/profile_check.py +++ b/misc/profile_check.py @@ -78,22 +78,22 @@ def check_requirements() -> None: if sys.platform != "linux": # TODO: How to make this work on other platforms? sys.exit("error: Only Linux is supported") - - try: - subprocess.run(["perf", "-h"], capture_output=True) - except (subprocess.CalledProcessError, FileNotFoundError): - print("error: The 'perf' profiler is not installed") - sys.exit(1) - - try: - subprocess.run(["clang", "--version"], capture_output=True) - except (subprocess.CalledProcessError, FileNotFoundError): - print("error: The clang compiler is not installed") - sys.exit(1) - - if not os.path.isfile("mypy_self_check.ini"): - print("error: Run this in the mypy repository root") - sys.exit(1) + else: # fun fact/todo: we have to use else here, because of https://github.com/python/mypy/issues/10773 + try: + subprocess.run(["perf", "-h"], capture_output=True) + except (subprocess.CalledProcessError, FileNotFoundError): + print("error: The 'perf' profiler is not installed") + sys.exit(1) + + try: + subprocess.run(["clang", "--version"], capture_output=True) + except (subprocess.CalledProcessError, FileNotFoundError): + print("error: The clang compiler is not installed") + sys.exit(1) + + if not os.path.isfile("mypy_self_check.ini"): + print("error: Run this in the mypy repository root") + sys.exit(1) def main() -> None: diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 8bf7a514f481..67b65381cfd0 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -12,7 +12,4 @@ exclude = mypy/typeshed/|mypyc/test-data/|mypyc/lib-rt/ enable_error_code = ignore-without-code,redundant-expr enable_incomplete_feature = PreciseTupleTypes show_error_code_links = True - -[mypy-mypy.*] -# TODO: enable for `mypyc` and other files as well warn_unreachable = True From feeb3f00a63d31cf5c7369885c4dd4a294133f59 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 21 Sep 2025 03:50:05 +0700 Subject: [PATCH 041/183] [docs] main.py: junit documentation elaboration (#19867) Elaborate on the information given in --help and command_line.rst for junit, to make it more correct and comprehensive. I manually examined the generated results and found them satisfactory. Note that this also puts --junit-format into misc group not import group; the inclusion into imports group seems to have been a mistake in #16388 although one could perhaps argue it is tangentially related to imports in some way. But it does not influence import discovery, unlike the other options. Putting it into import group also makes it display in a completely different place, which is not as helpful as right next to its related option. --- docs/source/command_line.rst | 8 +++++++- docs/source/config_file.rst | 9 +++++++++ mypy/main.py | 10 +++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index c1b757a00ef2..270125e96cb6 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1255,12 +1255,18 @@ Miscellaneous stub packages were found, they are installed and then another run is performed. -.. option:: --junit-xml JUNIT_XML +.. option:: --junit-xml JUNIT_XML_OUTPUT_FILE Causes mypy to generate a JUnit XML test result document with type checking results. This can make it easier to integrate mypy with continuous integration (CI) tools. +.. option:: --junit-format {global,per_file} + + If --junit-xml is set, specifies format. + global (default): single test with all errors; + per_file: one test entry per file with failures. + .. option:: --find-occurrences CLASS.MEMBER This flag will make mypy print out all usages of a class member diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index 934e465a7c23..7abd1f02db68 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -1153,6 +1153,15 @@ These options may only be set in the global section (``[mypy]``). type checking results. This can make it easier to integrate mypy with continuous integration (CI) tools. +.. confval:: junit_format + + :type: string + :default: ``global`` + + If junit_xml is set, specifies format. + global (default): single test with all errors; + per_file: one test entry per file with failures. + .. confval:: scripts_are_modules :type: boolean diff --git a/mypy/main.py b/mypy/main.py index b543cd33fe44..9ebbf78ded09 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1165,12 +1165,16 @@ def add_invertible_flag( misc_group = parser.add_argument_group(title="Miscellaneous") misc_group.add_argument("--quickstart-file", help=argparse.SUPPRESS) - misc_group.add_argument("--junit-xml", help="Write junit.xml to the given file") - imports_group.add_argument( + misc_group.add_argument( + "--junit-xml", + metavar="JUNIT_XML_OUTPUT_FILE", + help="Write a JUnit XML test result document with type checking results to the given file", + ) + misc_group.add_argument( "--junit-format", choices=["global", "per_file"], default="global", - help="If --junit-xml is set, specifies format. global: single test with all errors; per_file: one test entry per file with failures", + help="If --junit-xml is set, specifies format. global (default): single test with all errors; per_file: one test entry per file with failures", ) misc_group.add_argument( "--find-occurrences", From fa3566a87d3466e8465d5bafea5b0f0bde3b4eaf Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Mon, 22 Sep 2025 21:13:45 +0200 Subject: [PATCH 042/183] [mypyc] Transform object.__new__ inside __new__ (#19866) #19739 introduced support for compiling `__new__` methods in native classes. Native classes differ internally from regular python types which results in `TypeError`s being raised when `object.__new__(cls)` is called when `cls` is a native class. To avoid making this call, `super().__new__(cls)` is transformed into a call to an internal setup function. I forgot to replicate this for calls to equivalent `object.__new__(cls)` calls so this PR fixes that. This introduced a regression because before my changes, `__new__` methods with `object.__new__(cls)` were effectively ignored at runtime, and after my changes they started raising `TypeError`s. Note that these calls are left as-is outside of `__new__` methods so it's still possible to trigger the `TypeError` but that is not a regression as this was the case before. For example this code: ``` class Test: pass t = object.__new__(Test) ``` results in `TypeError: object.__new__(Test) is not safe, use Test.__new__()`. This differs from interpreted python but the error message is actually correct in that using `Test.__new__(Test)` instead works. --- mypyc/irbuild/builder.py | 4 + mypyc/irbuild/expression.py | 35 +-- mypyc/irbuild/specialize.py | 44 ++++ mypyc/irbuild/statement.py | 34 +++ mypyc/test-data/irbuild-classes.test | 334 ++++++++++++++++++++++++++- mypyc/test-data/run-classes.test | 197 ++++++++++++++++ 6 files changed, 620 insertions(+), 28 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 4f2f539118d7..12b5bc7f8f82 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1437,6 +1437,10 @@ def add_function(self, func_ir: FuncIR, line: int) -> None: self.function_names.add(name) self.functions.append(func_ir) + def get_current_class_ir(self) -> ClassIR | None: + type_info = self.fn_info.fitem.info + return self.mapper.type_to_ir.get(type_info) + def gen_arg_defaults(builder: IRBuilder) -> None: """Generate blocks for arguments that have default values. diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 4409b1acff26..1f39b09c0995 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -57,7 +57,6 @@ from mypyc.ir.ops import ( Assign, BasicBlock, - Call, ComparisonOp, Integer, LoadAddress, @@ -98,7 +97,11 @@ join_formatted_strings, tokenizer_printf_style, ) -from mypyc.irbuild.specialize import apply_function_specialization, apply_method_specialization +from mypyc.irbuild.specialize import ( + apply_function_specialization, + apply_method_specialization, + translate_object_new, +) from mypyc.primitives.bytes_ops import bytes_slice_op from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, exact_dict_set_item_op from mypyc.primitives.generic_ops import iter_op, name_op @@ -473,35 +476,15 @@ def translate_super_method_call(builder: IRBuilder, expr: CallExpr, callee: Supe if callee.name in base.method_decls: break else: + if callee.name == "__new__": + result = translate_object_new(builder, expr, MemberExpr(callee.call, "__new__")) + if result: + return result if ir.is_ext_class and ir.builtin_base is None and not ir.inherits_python: if callee.name == "__init__" and len(expr.args) == 0: # Call translates to object.__init__(self), which is a # no-op, so omit the call. return builder.none() - elif callee.name == "__new__": - # object.__new__(cls) - assert ( - len(expr.args) == 1 - ), f"Expected object.__new__() call to have exactly 1 argument, got {len(expr.args)}" - typ_arg = expr.args[0] - method_args = builder.fn_info.fitem.arg_names - if ( - isinstance(typ_arg, NameExpr) - and len(method_args) > 0 - and method_args[0] == typ_arg.name - ): - subtype = builder.accept(expr.args[0]) - return builder.add(Call(ir.setup, [subtype], expr.line)) - - if callee.name == "__new__": - call = "super().__new__()" - if not ir.is_ext_class: - builder.error(f"{call} not supported for non-extension classes", expr.line) - if ir.inherits_python: - builder.error( - f"{call} not supported for classes inheriting from non-native classes", - expr.line, - ) return translate_call(builder, expr, callee) decl = base.method_decl(callee.name) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 576b7a7ebffd..29820787d10c 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -30,12 +30,14 @@ NameExpr, RefExpr, StrExpr, + SuperExpr, TupleExpr, Var, ) from mypy.types import AnyType, TypeOfAny from mypyc.ir.ops import ( BasicBlock, + Call, Extend, Integer, RaiseStandardError, @@ -68,6 +70,7 @@ is_list_rprimitive, is_uint8_rprimitive, list_rprimitive, + object_rprimitive, set_rprimitive, str_rprimitive, uint8_rprimitive, @@ -1002,3 +1005,44 @@ def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value if isinstance(arg, (StrExpr, BytesExpr)) and len(arg.value) == 1: return Integer(ord(arg.value)) return None + + +@specialize_function("__new__", object_rprimitive) +def translate_object_new(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + fn = builder.fn_info + if fn.name != "__new__": + return None + + is_super_new = isinstance(expr.callee, SuperExpr) + is_object_new = ( + isinstance(callee, MemberExpr) + and isinstance(callee.expr, NameExpr) + and callee.expr.fullname == "builtins.object" + ) + if not (is_super_new or is_object_new): + return None + + ir = builder.get_current_class_ir() + if ir is None: + return None + + call = '"object.__new__()"' + if not ir.is_ext_class: + builder.error(f"{call} not supported for non-extension classes", expr.line) + return None + if ir.inherits_python: + builder.error( + f"{call} not supported for classes inheriting from non-native classes", expr.line + ) + return None + if len(expr.args) != 1: + builder.error(f"{call} supported only with 1 argument, got {len(expr.args)}", expr.line) + return None + + typ_arg = expr.args[0] + method_args = fn.fitem.arg_names + if isinstance(typ_arg, NameExpr) and len(method_args) > 0 and method_args[0] == typ_arg.name: + subtype = builder.accept(expr.args[0]) + return builder.add(Call(ir.setup, [subtype], expr.line)) + + return None diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index eeeb40ac672f..c83c5550d059 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -33,6 +33,7 @@ ListExpr, Lvalue, MatchStmt, + NameExpr, OperatorAssignmentStmt, RaiseStmt, ReturnStmt, @@ -170,10 +171,43 @@ def transform_return_stmt(builder: IRBuilder, stmt: ReturnStmt) -> None: builder.nonlocal_control[-1].gen_return(builder, retval, stmt.line) +def check_unsupported_cls_assignment(builder: IRBuilder, stmt: AssignmentStmt) -> None: + fn = builder.fn_info + method_args = fn.fitem.arg_names + if fn.name != "__new__" or len(method_args) == 0: + return + + ir = builder.get_current_class_ir() + if ir is None or ir.inherits_python or not ir.is_ext_class: + return + + cls_arg = method_args[0] + + def flatten(lvalues: list[Expression]) -> list[Expression]: + flat = [] + for lvalue in lvalues: + if isinstance(lvalue, (TupleExpr, ListExpr)): + flat += flatten(lvalue.items) + else: + flat.append(lvalue) + return flat + + lvalues = flatten(stmt.lvalues) + + for lvalue in lvalues: + if isinstance(lvalue, NameExpr) and lvalue.name == cls_arg: + # Disallowed because it could break the transformation of object.__new__ calls + # inside __new__ methods. + builder.error( + f'Assignment to argument "{cls_arg}" in "__new__" method unsupported', stmt.line + ) + + def transform_assignment_stmt(builder: IRBuilder, stmt: AssignmentStmt) -> None: lvalues = stmt.lvalues assert lvalues builder.disallow_class_assignments(lvalues, stmt.line) + check_unsupported_cls_assignment(builder, stmt) first_lvalue = lvalues[0] if stmt.type and isinstance(stmt.rvalue, TempNode): # This is actually a variable annotation without initializer. Don't generate diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 78ca7b68cefb..92857f525cca 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1662,6 +1662,7 @@ L0: [case testDunderNew] from __future__ import annotations +from typing import Any class Test: val: int @@ -1686,6 +1687,169 @@ class NewClassMethod: def fn2() -> NewClassMethod: return NewClassMethod.__new__(42) +class NotTransformed: + def __new__(cls, val: int) -> Any: + return super().__new__(str) + + def factory(cls: Any, val: int) -> Any: + cls = str + return super().__new__(cls) + +[out] +def Test.__new__(cls, val): + cls :: object + val :: int + r0, obj :: __main__.Test + r1 :: bool +L0: + r0 = __mypyc__Test_setup(cls) + obj = r0 + obj.val = val; r1 = is_error + return obj +def fn(): + r0 :: object + r1 :: __main__.Test +L0: + r0 = __main__.Test :: type + r1 = Test.__new__(r0, 84) + return r1 +def NewClassMethod.__new__(cls, val): + cls :: object + val :: int + r0, obj :: __main__.NewClassMethod + r1 :: bool +L0: + r0 = __mypyc__NewClassMethod_setup(cls) + obj = r0 + obj.val = val; r1 = is_error + return obj +def fn2(): + r0 :: object + r1 :: __main__.NewClassMethod +L0: + r0 = __main__.NewClassMethod :: type + r1 = NewClassMethod.__new__(r0, 84) + return r1 +def NotTransformed.__new__(cls, val): + cls :: object + val :: int + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: object[2] + r5 :: object_ptr + r6 :: object + r7 :: str + r8, r9 :: object + r10 :: object[1] + r11 :: object_ptr + r12 :: object + r13 :: str +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.NotTransformed :: type + r4 = [r3, cls] + r5 = load_address r4 + r6 = PyObject_Vectorcall(r2, r5, 2, 0) + keep_alive r3, cls + r7 = '__new__' + r8 = CPyObject_GetAttr(r6, r7) + r9 = load_address PyUnicode_Type + r10 = [r9] + r11 = load_address r10 + r12 = PyObject_Vectorcall(r8, r11, 1, 0) + keep_alive r9 + r13 = cast(str, r12) + return r13 +def NotTransformed.factory(cls, val): + cls :: object + val :: int + r0, r1 :: object + r2 :: str + r3, r4 :: object + r5 :: object[2] + r6 :: object_ptr + r7 :: object + r8 :: str + r9 :: object + r10 :: object[1] + r11 :: object_ptr + r12 :: object +L0: + r0 = load_address PyUnicode_Type + cls = r0 + r1 = builtins :: module + r2 = 'super' + r3 = CPyObject_GetAttr(r1, r2) + r4 = __main__.NotTransformed :: type + r5 = [r4, cls] + r6 = load_address r5 + r7 = PyObject_Vectorcall(r3, r6, 2, 0) + keep_alive r4, cls + r8 = '__new__' + r9 = CPyObject_GetAttr(r7, r8) + r10 = [cls] + r11 = load_address r10 + r12 = PyObject_Vectorcall(r9, r11, 1, 0) + keep_alive cls + return r12 + +[case testObjectDunderNew_64bit] +from __future__ import annotations +from mypy_extensions import mypyc_attr +from typing import Any + +class Test: + val: int + + def __new__(cls, val: int) -> Test: + obj = object.__new__(cls) + obj.val = val + return obj + +def fn() -> Test: + return Test.__new__(Test, 42) + +class NewClassMethod: + val: int + + @classmethod + def __new__(cls, val: int) -> NewClassMethod: + obj = object.__new__(cls) + obj.val = val + return obj + +def fn2() -> NewClassMethod: + return NewClassMethod.__new__(42) + +class NotTransformed: + def __new__(cls, val: int) -> Any: + return object.__new__(str) + + def factory(cls: Any, val: int) -> Any: + cls = str + return object.__new__(cls) + +@mypyc_attr(native_class=False) +class NonNative: + def __new__(cls: Any) -> Any: + cls = str + return cls("str") + +class InheritsPython(dict): + def __new__(cls: Any) -> Any: + cls = dict + return cls({}) + +class ObjectNewOutsideDunderNew: + def __init__(self) -> None: + object.__new__(ObjectNewOutsideDunderNew) + +def object_new_outside_class() -> None: + object.__new__(Test) + [out] def Test.__new__(cls, val): cls :: object @@ -1721,19 +1885,185 @@ L0: r0 = __main__.NewClassMethod :: type r1 = NewClassMethod.__new__(r0, 84) return r1 +def NotTransformed.__new__(cls, val): + cls :: object + val :: int + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: str + r5 :: object[2] + r6 :: object_ptr + r7 :: object + r8 :: str +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = load_address PyUnicode_Type + r4 = '__new__' + r5 = [r2, r3] + r6 = load_address r5 + r7 = PyObject_VectorcallMethod(r4, r6, 9223372036854775810, 0) + keep_alive r2, r3 + r8 = cast(str, r7) + return r8 +def NotTransformed.factory(cls, val): + cls :: object + val :: int + r0, r1 :: object + r2 :: str + r3 :: object + r4 :: str + r5 :: object[2] + r6 :: object_ptr + r7 :: object +L0: + r0 = load_address PyUnicode_Type + cls = r0 + r1 = builtins :: module + r2 = 'object' + r3 = CPyObject_GetAttr(r1, r2) + r4 = '__new__' + r5 = [r3, cls] + r6 = load_address r5 + r7 = PyObject_VectorcallMethod(r4, r6, 9223372036854775810, 0) + keep_alive r3, cls + return r7 +def __new___NonNative_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def __new___NonNative_obj.__call__(__mypyc_self__, cls): + __mypyc_self__ :: __main__.__new___NonNative_obj + cls, r0 :: object + r1 :: str + r2 :: object[1] + r3 :: object_ptr + r4 :: object +L0: + r0 = load_address PyUnicode_Type + cls = r0 + r1 = 'str' + r2 = [r1] + r3 = load_address r2 + r4 = PyObject_Vectorcall(cls, r3, 1, 0) + keep_alive r1 + return r4 +def InheritsPython.__new__(cls): + cls, r0 :: object + r1 :: dict + r2 :: object[1] + r3 :: object_ptr + r4 :: object +L0: + r0 = load_address PyDict_Type + cls = r0 + r1 = PyDict_New() + r2 = [r1] + r3 = load_address r2 + r4 = PyObject_Vectorcall(cls, r3, 1, 0) + keep_alive r1 + return r4 +def ObjectNewOutsideDunderNew.__init__(self): + self :: __main__.ObjectNewOutsideDunderNew + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: str + r5 :: object[2] + r6 :: object_ptr + r7 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.ObjectNewOutsideDunderNew :: type + r4 = '__new__' + r5 = [r2, r3] + r6 = load_address r5 + r7 = PyObject_VectorcallMethod(r4, r6, 9223372036854775810, 0) + keep_alive r2, r3 + return 1 +def object_new_outside_class(): + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: str + r5 :: object[2] + r6 :: object_ptr + r7 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.Test :: type + r4 = '__new__' + r5 = [r2, r3] + r6 = load_address r5 + r7 = PyObject_VectorcallMethod(r4, r6, 9223372036854775810, 0) + keep_alive r2, r3 + return 1 [case testUnsupportedDunderNew] from __future__ import annotations from mypy_extensions import mypyc_attr +from typing import Any @mypyc_attr(native_class=False) class NonNative: def __new__(cls) -> NonNative: - return super().__new__(cls) # E: super().__new__() not supported for non-extension classes + return super().__new__(cls) # E: "object.__new__()" not supported for non-extension classes class InheritsPython(dict): def __new__(cls) -> InheritsPython: - return super().__new__(cls) # E: super().__new__() not supported for classes inheriting from non-native classes + return super().__new__(cls) # E: "object.__new__()" not supported for classes inheriting from non-native classes + +@mypyc_attr(native_class=False) +class NonNativeObjectNew: + def __new__(cls) -> NonNativeObjectNew: + return object.__new__(cls) # E: "object.__new__()" not supported for non-extension classes + +class InheritsPythonObjectNew(dict): + def __new__(cls) -> InheritsPythonObjectNew: + return object.__new__(cls) # E: "object.__new__()" not supported for classes inheriting from non-native classes + +class ClsAssignment: + def __new__(cls: Any) -> Any: + cls = str # E: Assignment to argument "cls" in "__new__" method unsupported + return super().__new__(cls) + +class ClsTupleAssignment: + def __new__(class_i_want: Any, val: int) -> Any: + class_i_want, val = dict, 1 # E: Assignment to argument "class_i_want" in "__new__" method unsupported + return object.__new__(class_i_want) + +class ClsListAssignment: + def __new__(cls: Any, val: str) -> Any: + [cls, val] = [object, "object"] # E: Assignment to argument "cls" in "__new__" method unsupported + return object.__new__(cls) + +class ClsNestedAssignment: + def __new__(cls: Any, val1: str, val2: int) -> Any: + [val1, [val2, cls]] = ["val1", [2, int]] # E: Assignment to argument "cls" in "__new__" method unsupported + return object.__new__(cls) + +class WrongNumberOfArgs: + def __new__(cls): + return super().__new__() # E: "object.__new__()" supported only with 1 argument, got 0 + +class WrongNumberOfArgsObjectNew: + def __new__(cls): + return object.__new__(cls, 1) # E: "object.__new__()" supported only with 1 argument, got 2 [case testClassWithFreeList] from mypy_extensions import mypyc_attr, trait diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 6c4ddc03887a..3d0250cd24ee 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3493,6 +3493,156 @@ Add(0, 5)=5 running __new__ with 1 and 0 Add(1, 0)=1 +[case testObjectDunderNew] +from __future__ import annotations +from typing import Any, Union + +from testutil import assertRaises + +class Add: + l: IntLike + r: IntLike + + def __new__(cls, l: IntLike, r: IntLike) -> Any: + return ( + l if r == 0 else + r if l == 0 else + object.__new__(cls) + ) + + def __init__(self, l: IntLike, r: IntLike): + self.l = l + self.r = r + +IntLike = Union[int, Add] + +class RaisesException: + def __new__(cls, val: int) -> RaisesException: + if val == 0: + raise RuntimeError("Invalid value!") + return object.__new__(cls) + + def __init__(self, val: int) -> None: + self.val = val + +class ClsArgNotPassed: + def __new__(cls) -> Any: + return object.__new__(str) + +class SkipsBase(Add): + def __new__(cls) -> Any: + obj = object.__new__(cls) + obj.l = 0 + obj.r = 0 + return obj + +def test_dunder_new() -> None: + add_instance: Any = Add(1, 5) + assert type(add_instance) == Add + assert add_instance.l == 1 + assert add_instance.r == 5 + + # TODO: explicit types should not be needed but mypy does not use + # the return type of __new__ which makes mypyc add casts to Add. + right_int: Any = Add(0, 5) + assert type(right_int) == int + assert right_int == 5 + + left_int: Any = Add(1, 0) + assert type(left_int) == int + assert left_int == 1 + + with assertRaises(RuntimeError, "Invalid value!"): + _ = RaisesException(0) + + not_raised = RaisesException(1) + assert not_raised.val == 1 + + with assertRaises(TypeError, "object.__new__(str) is not safe, use str.__new__()"): + _ = ClsArgNotPassed() + + skip = SkipsBase.__new__(SkipsBase) + assert type(skip) == SkipsBase + assert skip.l == 0 + assert skip.r == 0 + +[case testObjectDunderNewInInterpreted] +from __future__ import annotations +from typing import Any, Union + +class Add: + l: IntLike + r: IntLike + + def __new__(cls, l: IntLike, r: IntLike) -> Any: + print(f'running __new__ with {l} and {r}') + + return ( + l if r == 0 else + r if l == 0 else + object.__new__(cls) + ) + + def __init__(self, l: IntLike, r: IntLike): + self.l = l + self.r = r + + def __repr__(self) -> str: + return f'({self.l} + {self.r})' + +IntLike = Union[int, Add] + +class RaisesException: + def __new__(cls, val: int) -> RaisesException: + if val == 0: + raise RuntimeError("Invalid value!") + return object.__new__(cls) + + def __init__(self, val: int) -> None: + self.val = val + +class ClsArgNotPassed: + def __new__(cls) -> Any: + return object.__new__(str) + +class SkipsBase(Add): + def __new__(cls) -> Any: + obj = object.__new__(cls) + obj.l = 0 + obj.r = 0 + return obj + +[file driver.py] +from native import Add, ClsArgNotPassed, RaisesException, SkipsBase + +from testutil import assertRaises + +print(f'{Add(1, 5)=}') +print(f'{Add(0, 5)=}') +print(f'{Add(1, 0)=}') + +with assertRaises(RuntimeError, "Invalid value!"): + raised = RaisesException(0) + +not_raised = RaisesException(1) +assert not_raised.val == 1 + +with assertRaises(TypeError, "object.__new__(str) is not safe, use str.__new__()"): + str_as_cls = ClsArgNotPassed() + +skip = SkipsBase.__new__(SkipsBase) +assert type(skip) == SkipsBase +assert skip.l == 0 +assert skip.r == 0 + +[out] +running __new__ with 1 and 5 +Add(1, 5)=(1 + 5) +running __new__ with 0 and 5 +Add(0, 5)=5 +running __new__ with 1 and 0 +Add(1, 0)=1 + [case testInheritedDunderNew] from __future__ import annotations from mypy_extensions import mypyc_attr @@ -3795,6 +3945,53 @@ assert t.generic == "{}" assert t.bitfield == 0x0C assert t.default == 10 +[case testUntransformedDunderNewCalls] +from testutil import assertRaises +from typing import Any + +class TestStrCls: + def __new__(cls): + return str.__new__(cls) + + @classmethod + def factory(cls): + return str.__new__(cls) + +class TestStrStr: + def __new__(cls): + return str.__new__(str) + + @classmethod + def factory(cls): + return str.__new__(str) + +class TestStrInt: + def __new__(cls): + return str.__new__(int) + + @classmethod + def factory(cls): + return str.__new__(int) + +def test_untransformed_dunder_new() -> None: + with assertRaises(TypeError, "str.__new__(TestStrCls): TestStrCls is not a subtype of str"): + i = TestStrCls() + + j: Any = TestStrStr() + assert j == "" + + with assertRaises(TypeError, "str.__new__(int): int is not a subtype of str"): + k = TestStrInt() + + with assertRaises(TypeError, "str.__new__(TestStrCls): TestStrCls is not a subtype of str"): + i = TestStrCls.factory() + + j = TestStrStr.factory() + assert j == "" + + with assertRaises(TypeError, "str.__new__(int): int is not a subtype of str"): + k = TestStrInt.factory() + [case testPerTypeFreeList] from __future__ import annotations From d96b9dcb1dcd0c642e920e46e9f077ca9c7e88c2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 23 Sep 2025 14:12:14 +0100 Subject: [PATCH 043/183] Fix generation of incorrect indirect deps from locals (#19906) Don't generate an indirect dependency to module `bar` if a local variable has name `bar`. This aims to fix the root cause of the issue #19903 tries to solve. --- mypy/semanal.py | 18 ++++++++++-------- test-data/unit/check-incremental.test | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index b3fd1b98bfd2..d78df2e199b8 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5928,8 +5928,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None: if isinstance(sym.node, PlaceholderNode): self.process_placeholder(expr.name, "attribute", expr) return - if sym.node is not None: - self.record_imported_symbol(sym.node) + self.record_imported_symbol(sym) expr.kind = sym.kind expr.fullname = sym.fullname or "" expr.node = sym.node @@ -5960,7 +5959,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None: if type_info: n = type_info.names.get(expr.name) if n is not None and isinstance(n.node, (MypyFile, TypeInfo, TypeAlias)): - self.record_imported_symbol(n.node) + self.record_imported_symbol(n) expr.kind = n.kind expr.fullname = n.fullname or "" expr.node = n.node @@ -6282,14 +6281,17 @@ def lookup( self, name: str, ctx: Context, suppress_errors: bool = False ) -> SymbolTableNode | None: node = self._lookup(name, ctx, suppress_errors) - if node is not None and node.node is not None: + if node is not None: # This call is unfortunate from performance point of view, but # needed for rare cases like e.g. testIncrementalChangingAlias. - self.record_imported_symbol(node.node) + self.record_imported_symbol(node) return node - def record_imported_symbol(self, node: SymbolNode) -> None: + def record_imported_symbol(self, sym: SymbolTableNode) -> None: """If the symbol was not defined in current module, add its module to module_refs.""" + if sym.kind == LDEF or sym.node is None: + return + node = sym.node if not node.fullname: return if isinstance(node, MypyFile): @@ -6519,8 +6521,8 @@ def lookup_qualified( self.name_not_defined(name, ctx, namespace=namespace) return None sym = nextsym - if sym is not None and sym.node is not None: - self.record_imported_symbol(sym.node) + if sym is not None: + self.record_imported_symbol(sym) return sym def lookup_type_node(self, expr: Expression) -> SymbolTableNode | None: diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index d9d78715b396..e91b8778e986 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -7290,3 +7290,24 @@ tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expec [out2] tmp/n.py:3: note: Revealed type is "builtins.str" tmp/m.py:3: error: Argument 1 to "accept_int" has incompatible type "str"; expected "int" + +[case testIncrementalNoIndirectDepFromLocal] +import foo +import bar + +[file foo.py] +# Having a local named 'bar' shouldn't generate a dependency on module 'bar' +def f(bar: int) -> int: + return bar + +[file bar.py] +import foo +x = 1 + +[file bar.py.2] +import foo +x = 2 + +[out] +[rechecked bar] +[stale] From 354bea6352ee7a38b05e2f42c874e7d1f7bf557a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 23 Sep 2025 14:15:08 +0100 Subject: [PATCH 044/183] Don't consider indirect dependencies when calculating SCCs (#19903) SCC construction should only need to consider import dependencies, since indirect dependencies are not available during non-incremental runs, and we want SCCs to be identical in incremental and non-incremental runs. This may improve performance slightly and will make mypy more robust in case there are extra indirect dependencies (see #19906). --- mypy/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index 2d3296a4713e..ad25b811ff7c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -3488,7 +3488,7 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No def sorted_components( - graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_ALL + graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_INDIRECT ) -> list[AbstractSet[str]]: """Return the graph's SCCs, topologically sorted by dependencies. From 00f2b29ecceaaad0d69700f651adf9b7678f9907 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 25 Sep 2025 17:00:15 +0100 Subject: [PATCH 045/183] Improve record_imported_symbol() (#19924) This has four improvements: * Skip placeholders, they are not guaranteed to have a correct `fullname`. * Skip nodes from `builtins`/`typing`, these don't add anything. * Don't use `rsplit(".")` on variables/functions recursively, always use enclosing class. * (Most importantly) add missing `maxsplit=1`. --- mypy/semanal.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d78df2e199b8..17dc9bfadc1f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -6292,22 +6292,37 @@ def record_imported_symbol(self, sym: SymbolTableNode) -> None: if sym.kind == LDEF or sym.node is None: return node = sym.node - if not node.fullname: + if isinstance(node, PlaceholderNode) or not node.fullname: + # This node is not ready yet. return + if node.fullname.startswith(("builtins.", "typing.")): + # Skip dependencies on builtins/typing. + return + # Modules, classes, and type aliases store defining module directly. if isinstance(node, MypyFile): fullname = node.fullname elif isinstance(node, TypeInfo): fullname = node.module_name elif isinstance(node, TypeAlias): fullname = node.module - elif isinstance(node, (Var, FuncDef, OverloadedFuncDef)) and node.info: - fullname = node.info.module_name + elif isinstance(node, (Var, FuncDef, OverloadedFuncDef, Decorator)): + # For functions/variables infer defining module from enclosing class. + info = node.var.info if isinstance(node, Decorator) else node.info + if info: + fullname = info.module_name + else: + # global function/variable + fullname = node.fullname.rsplit(".", maxsplit=1)[0] else: - fullname = node.fullname.rsplit(".")[0] + # Some nodes (currently only TypeVarLikeExpr subclasses) don't store + # module fullname explicitly, infer it from the node fullname iteratively. + # TODO: this is not 100% robust for type variables nested within a class + # with a name that matches name of a submodule. + fullname = node.fullname.rsplit(".", maxsplit=1)[0] if fullname == self.cur_mod_id: return while "." in fullname and fullname not in self.modules: - fullname = fullname.rsplit(".")[0] + fullname = fullname.rsplit(".", maxsplit=1)[0] if fullname != self.cur_mod_id: self.cur_mod_node.module_refs.add(fullname) From c058b09f544271c582416e873348c63868eeae50 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 25 Sep 2025 17:41:13 +0100 Subject: [PATCH 046/183] Install librt in mypy_primer (#19925) Fixes https://github.com/python/mypy/issues/19844 --- .github/workflows/mypy_primer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index 1ff984247fb6..8a04c75b0e2e 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -67,6 +67,7 @@ jobs: --debug \ --additional-flags="--debug-serialize" \ --output concise \ + --mypy-install-librt \ | tee diff_${{ matrix.shard-index }}.txt ) || [ $? -eq 1 ] - if: ${{ matrix.shard-index == 0 }} From a936e3008d86b12b575978da1069e76fa1e89c3b Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Thu, 25 Sep 2025 18:55:26 +0200 Subject: [PATCH 047/183] [mypyc] Generate __getattr__ wrapper (#19909) Fixes https://github.com/mypyc/mypyc/issues/887 Generate a wrapper function for `__getattr__` in classes where it's defined and put it into the `tp_getattro` slot. At runtime, this wrapper function is called for every attribute access, so to match behavior of interpreted python it needs to first check if the given name is present in the type dictionary and only call the user-defined `__getattr__` when it's not present. Checking the type dictionary is implemented using a cpython function `_PyObject_GenericGetAttrWithDict` which is also used in the default attribute access handler in [cpython](https://github.com/python/cpython/blob/dd45179fa0f5ad2fd169cdd35065df2c3bce85bc/Objects/typeobject.c#L10676). In compiled code, the wrapper will only be called when the attribute name cannot be statically resolved. When it can be resolved, the attribute will be accessed directly in the underlying C struct, or the generated function will be directly called in case of resolving method names. No change from existing behavior. When the name cannot be statically resolved, mypyc generates calls to `PyObject_GetAttr` which internally calls `tp_getattro`. In interpreted code that uses compiled classes, attribute access will always result in calls to `PyObject_GetAttr` so there's always a dict look-up in the wrapper to find the attribute. But that's the case already, the dict look-up happens in the default attribute access handler in cpython. So the wrapper should not bring any negative performance impact. --- mypyc/codegen/emitclass.py | 7 + mypyc/ir/ops.py | 6 +- mypyc/irbuild/builder.py | 3 +- mypyc/irbuild/function.py | 56 +++- mypyc/irbuild/ll_builder.py | 2 + mypyc/lib-rt/CPy.h | 4 + mypyc/primitives/generic_ops.py | 9 + mypyc/primitives/registry.py | 3 + mypyc/test-data/irbuild-classes.test | 125 ++++++++ mypyc/test-data/run-classes.test | 432 +++++++++++++++++++++++++++ mypyc/transform/refcount.py | 5 +- 11 files changed, 648 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 94f32b3224a9..9e8f9c74bc6d 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -39,6 +39,12 @@ def native_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: return f"{NATIVE_PREFIX}{fn.cname(emitter.names)}" +def dunder_attr_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + wrapper_fn = cl.get_method(fn.name + "__wrapper") + assert wrapper_fn + return f"{NATIVE_PREFIX}{wrapper_fn.cname(emitter.names)}" + + # We maintain a table from dunder function names to struct slots they # correspond to and functions that generate a wrapper (if necessary) # and return the function name to stick in the slot. @@ -55,6 +61,7 @@ def native_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: "__iter__": ("tp_iter", native_slot), "__hash__": ("tp_hash", generate_hash_wrapper), "__get__": ("tp_descr_get", generate_get_wrapper), + "__getattr__": ("tp_getattro", dunder_attr_slot), } AS_MAPPING_SLOT_DEFS: SlotTable = { diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 4b3b5eb3c8ca..76c1e07a79d5 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -1221,6 +1221,7 @@ def __init__( var_arg_idx: int = -1, *, is_pure: bool = False, + returns_null: bool = False, ) -> None: self.error_kind = error_kind super().__init__(line) @@ -1235,7 +1236,10 @@ def __init__( # and all the arguments are immutable. Pure functions support # additional optimizations. Pure functions never fail. self.is_pure = is_pure - if is_pure: + # The function might return a null value that does not indicate + # an error. + self.returns_null = returns_null + if is_pure or returns_null: assert error_kind == ERR_NEVER def sources(self) -> list[Value]: diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 12b5bc7f8f82..f4ee4371b9bf 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1241,6 +1241,7 @@ def enter_method( ret_type: RType, fn_info: FuncInfo | str = "", self_type: RType | None = None, + internal: bool = False, ) -> Iterator[None]: """Generate IR for a method. @@ -1268,7 +1269,7 @@ def enter_method( sig = FuncSignature(args, ret_type) name = self.function_name_stack.pop() class_ir = self.class_ir_stack.pop() - decl = FuncDecl(name, class_ir.name, self.module_name, sig) + decl = FuncDecl(name, class_ir.name, self.module_name, sig, internal=internal) ir = FuncIR(decl, arg_regs, blocks) class_ir.methods[name] = ir class_ir.method_decls[name] = ir.decl diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index f0fc424aea54..a9a098d25dde 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -42,6 +42,7 @@ ) from mypyc.ir.ops import ( BasicBlock, + ComparisonOp, GetAttr, Integer, LoadAddress, @@ -81,7 +82,7 @@ dict_new_op, exact_dict_set_item_op, ) -from mypyc.primitives.generic_ops import py_setattr_op +from mypyc.primitives.generic_ops import generic_getattr, py_setattr_op from mypyc.primitives.misc_ops import register_function from mypyc.primitives.registry import builtin_names from mypyc.sametype import is_same_method_signature, is_same_type @@ -364,6 +365,56 @@ def gen_func_ir( return (func_ir, func_reg) +def generate_getattr_wrapper(builder: IRBuilder, cdef: ClassDef, getattr: FuncDef) -> None: + """ + Generate a wrapper function for __getattr__ that can be put into the tp_getattro slot. + The wrapper takes one argument besides self which is the attribute name. + It first checks if the name matches any of the attributes of this class. + If it does, it returns that attribute. If none match, it calls __getattr__. + + __getattr__ is not supported in classes that allow interpreted subclasses because the + tp_getattro slot is inherited by subclasses and if the subclass overrides __getattr__, + the override would be ignored in our wrapper. TODO: To support this, the wrapper would + have to check type of self and if it's not the compiled class, resolve "__getattr__" against + the type at runtime and call the returned method, like _Py_slot_tp_getattr_hook in cpython. + + __getattr__ is not supported in classes which inherit from non-native classes because those + have __dict__ which currently has some strange interactions when class attributes and + variables are assigned through __dict__ vs. through regular attribute access. Allowing + __getattr__ on top of that could be problematic. + """ + name = getattr.name + "__wrapper" + ir = builder.mapper.type_to_ir[cdef.info] + line = getattr.line + + error_base = f'"__getattr__" not supported in class "{cdef.name}" because ' + if ir.allow_interpreted_subclasses: + builder.error(error_base + "it allows interpreted subclasses", line) + if ir.inherits_python: + builder.error(error_base + "it inherits from a non-native class", line) + + with builder.enter_method(ir, name, object_rprimitive, internal=True): + attr_arg = builder.add_argument("attr", object_rprimitive) + generic_getattr_result = builder.call_c(generic_getattr, [builder.self(), attr_arg], line) + + return_generic, call_getattr = BasicBlock(), BasicBlock() + null = Integer(0, object_rprimitive, line) + got_generic = builder.add( + ComparisonOp(generic_getattr_result, null, ComparisonOp.NEQ, line) + ) + builder.add_bool_branch(got_generic, return_generic, call_getattr) + + builder.activate_block(return_generic) + builder.add(Return(generic_getattr_result, line)) + + builder.activate_block(call_getattr) + # No attribute matched so call user-provided __getattr__. + getattr_result = builder.gen_method_call( + builder.self(), getattr.name, [attr_arg], object_rprimitive, line + ) + builder.add(Return(getattr_result, line)) + + def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None: # Perform the function of visit_method for methods inside extension classes. name = fdef.name @@ -430,6 +481,9 @@ def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None class_ir.glue_methods[(class_ir, name)] = f builder.functions.append(f) + if fdef.name == "__getattr__": + generate_getattr_wrapper(builder, cdef, fdef) + def handle_non_ext_method( builder: IRBuilder, non_ext: NonExtClassInfo, cdef: ClassDef, fdef: FuncDef diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 4b85c13892c1..37f2add4abbd 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -2048,6 +2048,7 @@ def call_c( line, var_arg_idx, is_pure=desc.is_pure, + returns_null=desc.returns_null, ) ) if desc.is_borrowed: @@ -2131,6 +2132,7 @@ def primitive_op( desc.extra_int_constants, desc.priority, is_pure=desc.is_pure, + returns_null=False, ) return self.call_c(c_desc, args, line, result_type=result_type) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 5dec7509ac7b..b9cecb9280f3 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -949,6 +949,10 @@ PyObject *CPy_GetANext(PyObject *aiter); void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_value); void CPyTrace_LogEvent(const char *location, const char *line, const char *op, const char *details); +static inline PyObject *CPyObject_GenericGetAttr(PyObject *self, PyObject *name) { + return _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); +} + #if CPY_3_11_FEATURES PyObject *CPy_GetName(PyObject *obj); #endif diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index 8a4ddc370280..ff978b7c8c3b 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -401,3 +401,12 @@ c_function_name="CPy_GetName", error_kind=ERR_MAGIC, ) + +# look-up name in tp_dict but don't raise AttributeError on failure +generic_getattr = custom_op( + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyObject_GenericGetAttr", + error_kind=ERR_NEVER, + returns_null=True, +) diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index 07546663d08e..3188bc322809 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -61,6 +61,7 @@ class CFunctionDescription(NamedTuple): extra_int_constants: list[tuple[int, RType]] priority: int is_pure: bool + returns_null: bool # A description for C load operations including LoadGlobal and LoadAddress @@ -253,6 +254,7 @@ def custom_op( is_borrowed: bool = False, *, is_pure: bool = False, + returns_null: bool = False, ) -> CFunctionDescription: """Create a one-off CallC op that can't be automatically generated from the AST. @@ -274,6 +276,7 @@ def custom_op( extra_int_constants, 0, is_pure=is_pure, + returns_null=returns_null, ) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 92857f525cca..76e28711c5e3 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2088,3 +2088,128 @@ class NonNative: @mypyc_attr(free_list_len=1, allow_interpreted_subclasses=True) # E: "free_list_len" can't be used in a class that allows interpreted subclasses class InterpSub: pass + +[case testUnsupportedGetAttr] +from mypy_extensions import mypyc_attr + +@mypyc_attr(allow_interpreted_subclasses=True) +class AllowsInterpreted: + def __getattr__(self, attr: str) -> object: # E: "__getattr__" not supported in class "AllowsInterpreted" because it allows interpreted subclasses + return 0 + +class InheritsInterpreted(dict): + def __getattr__(self, attr: str) -> object: # E: "__getattr__" not supported in class "InheritsInterpreted" because it inherits from a non-native class + return 0 + +@mypyc_attr(native_class=False) +class NonNative: + pass + +class InheritsNonNative(NonNative): + def __getattr__(self, attr: str) -> object: # E: "__getattr__" not supported in class "InheritsNonNative" because it inherits from a non-native class + return 0 + +[case testGetAttr] +from typing import ClassVar + +class GetAttr: + class_var = "x" + class_var_annotated: ClassVar[int] = 99 + + def __init__(self, regular_attr: int): + self.regular_attr = regular_attr + + def __getattr__(self, attr: str) -> object: + return attr + + def method(self) -> int: + return 0 + +def test_getattr() -> list[object]: + i = GetAttr(42) + one = i.one + two = i.regular_attr + three = i.class_var + four = i.class_var_annotated + five = i.method() + return [one, two, three, four, five] + +[typing fixtures/typing-full.pyi] +[out] +def GetAttr.__init__(self, regular_attr): + self :: __main__.GetAttr + regular_attr :: int +L0: + self.regular_attr = regular_attr + return 1 +def GetAttr.__getattr__(self, attr): + self :: __main__.GetAttr + attr :: str +L0: + return attr +def GetAttr.__getattr____wrapper(__mypyc_self__, attr): + __mypyc_self__ :: __main__.GetAttr + attr, r0 :: object + r1 :: bit + r2 :: str + r3 :: object +L0: + r0 = CPyObject_GenericGetAttr(__mypyc_self__, attr) + r1 = r0 != 0 + if r1 goto L1 else goto L2 :: bool +L1: + return r0 +L2: + r2 = cast(str, attr) + r3 = __mypyc_self__.__getattr__(r2) + return r3 +def GetAttr.method(self): + self :: __main__.GetAttr +L0: + return 0 +def GetAttr.__mypyc_defaults_setup(__mypyc_self__): + __mypyc_self__ :: __main__.GetAttr + r0 :: str +L0: + r0 = 'x' + __mypyc_self__.class_var = r0 + return 1 +def test_getattr(): + r0, i :: __main__.GetAttr + r1 :: str + r2, one :: object + r3, two :: int + r4, three, r5 :: str + r6 :: object + r7, four, r8, five :: int + r9 :: list + r10, r11, r12 :: object + r13 :: ptr +L0: + r0 = GetAttr(84) + i = r0 + r1 = 'one' + r2 = CPyObject_GetAttr(i, r1) + one = r2 + r3 = i.regular_attr + two = r3 + r4 = i.class_var + three = r4 + r5 = 'class_var_annotated' + r6 = CPyObject_GetAttr(i, r5) + r7 = unbox(int, r6) + four = r7 + r8 = i.method() + five = r8 + r9 = PyList_New(5) + r10 = box(int, two) + r11 = box(int, four) + r12 = box(int, five) + r13 = list_items r9 + buf_init_item r13, 0, one + buf_init_item r13, 1, r10 + buf_init_item r13, 2, three + buf_init_item r13, 3, r11 + buf_init_item r13, 4, r12 + keep_alive r9 + return r9 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 3d0250cd24ee..b2f1a088585d 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -4088,3 +4088,435 @@ def test_inheritance_2() -> None: x = None y = None d = None + +[case testDunderGetAttr] +from mypy_extensions import mypyc_attr +from typing import ClassVar + +class GetAttr: + class_var = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str) -> object: + return self.extra_attrs.get(attr) + +class GetAttrDefault: + class_var: ClassVar[str] = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str, default: int = 8, mult: int = 1) -> object: + return self.extra_attrs.get(attr, default * mult) + +class GetAttrInherited(GetAttr): + subclass_var = "y" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int, sub_attr: int): + super().__init__(extra_attrs, regular_attr) + self.sub_attr = sub_attr + +class GetAttrOverridden(GetAttr): + subclass_var: ClassVar[str] = "y" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int, sub_attr: int): + super().__init__(extra_attrs, regular_attr) + self.sub_attr = sub_attr + + def __getattr__(self, attr: str) -> str: + return attr + +@mypyc_attr(native_class=False) +class GetAttrNonNative: + class_var = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str) -> object: + return self.extra_attrs.get(attr) + +def test_getattr() -> None: + i = GetAttr({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == None + + assert i.__class__ == GetAttr + + i.extra_attrs["regular_attr"] = (4, 4, 4) + assert i.__getattr__("regular_attr") == (4, 4, 4) + assert getattr(i, "regular_attr") == 42 + assert i.regular_attr == 42 + +def test_getattr_default() -> None: + i = GetAttrDefault({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == 8 + assert i.__getattr__("class_var") == 8 + assert i.__getattr__("four", 4, 3) == 12 + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == 8 + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == 8 + + assert i.__class__ == GetAttrDefault + + i.extra_attrs["class_var"] = (4, 4, 4) + assert i.__getattr__("class_var") == (4, 4, 4) + assert getattr(i, "class_var") == "x" + assert i.class_var == "x" + +def test_getattr_inherited() -> None: + i = GetAttrInherited({"one": 1, "two": "two", "three": 3.14}, 42, 24) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("sub_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("subclass_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "sub_attr") == 24 + assert getattr(i, "class_var") == "x" + assert getattr(i, "subclass_var") == "y" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.sub_attr == 24 + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.four == None + + assert i.__class__ == GetAttrInherited + + i.extra_attrs["sub_attr"] = (4, 4, 4) + assert i.__getattr__("sub_attr") == (4, 4, 4) + assert getattr(i, "sub_attr") == 24 + assert i.sub_attr == 24 + + base_ref: GetAttr = i + assert getattr(base_ref, "sub_attr") == 24 + assert base_ref.sub_attr == 24 + + assert getattr(base_ref, "subclass_var") == "y" + assert base_ref.subclass_var == "y" + + assert getattr(base_ref, "new") == None + assert base_ref.new == None + + assert base_ref.__class__ == GetAttrInherited + + +def test_getattr_overridden() -> None: + i = GetAttrOverridden({"one": 1, "two": "two", "three": 3.14}, 42, 24) + assert i.__getattr__("one") == "one" + assert i.__getattr__("regular_attr") == "regular_attr" + assert i.__getattr__("sub_attr") == "sub_attr" + assert i.__getattr__("class_var") == "class_var" + assert i.__getattr__("subclass_var") == "subclass_var" + assert i.__getattr__("four") == "four" + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "sub_attr") == 24 + assert getattr(i, "class_var") == "x" + assert getattr(i, "subclass_var") == "y" + assert getattr(i, "four") == "four" + + assert i.three == "three" + assert i.regular_attr == 42 + assert i.sub_attr == 24 + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.four == "four" + + assert i.__class__ == GetAttrOverridden + + i.extra_attrs["subclass_var"] = (4, 4, 4) + assert i.__getattr__("subclass_var") == "subclass_var" + assert getattr(i, "subclass_var") == "y" + assert i.subclass_var == "y" + + base_ref: GetAttr = i + assert getattr(base_ref, "sub_attr") == 24 + assert base_ref.sub_attr == 24 + + assert getattr(base_ref, "subclass_var") == "y" + assert base_ref.subclass_var == "y" + + assert getattr(base_ref, "new") == "new" + assert base_ref.new == "new" + + assert base_ref.__class__ == GetAttrOverridden + +def test_getattr_nonnative() -> None: + i = GetAttr({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == None + + assert i.__class__ == GetAttr + + i.extra_attrs["regular_attr"] = (4, 4, 4) + assert i.__getattr__("regular_attr") == (4, 4, 4) + assert getattr(i, "regular_attr") == 42 + assert i.regular_attr == 42 + +[typing fixtures/typing-full.pyi] + +[case testDunderGetAttrInterpreted] +from mypy_extensions import mypyc_attr +from typing import ClassVar + +class GetAttr: + class_var = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str) -> object: + return self.extra_attrs.get(attr) + +class GetAttrDefault: + class_var: ClassVar[str] = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str, default: int = 8, mult: int = 1) -> object: + return self.extra_attrs.get(attr, default * mult) + +class GetAttrInherited(GetAttr): + subclass_var = "y" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int, sub_attr: int): + super().__init__(extra_attrs, regular_attr) + self.sub_attr = sub_attr + +class GetAttrOverridden(GetAttr): + subclass_var: ClassVar[str] = "y" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int, sub_attr: int): + super().__init__(extra_attrs, regular_attr) + self.sub_attr = sub_attr + + def __getattr__(self, attr: str) -> str: + return attr + +@mypyc_attr(native_class=False) +class GetAttrNonNative: + class_var = "x" + + def __init__(self, extra_attrs: dict[str, object], regular_attr: int): + self.extra_attrs = extra_attrs + self.regular_attr = regular_attr + + def __getattr__(self, attr: str) -> object: + return self.extra_attrs.get(attr) + +[file driver.py] +from native import GetAttr, GetAttrDefault, GetAttrInherited, GetAttrOverridden, GetAttrNonNative + +def test_getattr() -> None: + i = GetAttr({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == None + + assert i.__class__ == GetAttr + + i.extra_attrs["regular_attr"] = (4, 4, 4) + assert i.__getattr__("regular_attr") == (4, 4, 4) + assert getattr(i, "regular_attr") == 42 + assert i.regular_attr == 42 + +def test_getattr_default() -> None: + i = GetAttrDefault({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == 8 + assert i.__getattr__("class_var") == 8 + assert i.__getattr__("four", 4, 3) == 12 + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == 8 + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == 8 + + assert i.__class__ == GetAttrDefault + + i.extra_attrs["class_var"] = (4, 4, 4) + assert i.__getattr__("class_var") == (4, 4, 4) + assert getattr(i, "class_var") == "x" + assert i.class_var == "x" + +def test_getattr_inherited() -> None: + i = GetAttrInherited({"one": 1, "two": "two", "three": 3.14}, 42, 24) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("sub_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("subclass_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "sub_attr") == 24 + assert getattr(i, "class_var") == "x" + assert getattr(i, "subclass_var") == "y" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.sub_attr == 24 + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.four == None + + assert i.__class__ == GetAttrInherited + + i.extra_attrs["sub_attr"] = (4, 4, 4) + assert i.__getattr__("sub_attr") == (4, 4, 4) + assert getattr(i, "sub_attr") == 24 + assert i.sub_attr == 24 + + base_ref: GetAttr = i + assert getattr(base_ref, "sub_attr") == 24 + assert base_ref.sub_attr == 24 + + assert getattr(base_ref, "subclass_var") == "y" + assert base_ref.subclass_var == "y" + + assert getattr(base_ref, "new") == None + assert base_ref.new == None + + assert base_ref.__class__ == GetAttrInherited + + +def test_getattr_overridden() -> None: + i = GetAttrOverridden({"one": 1, "two": "two", "three": 3.14}, 42, 24) + assert i.__getattr__("one") == "one" + assert i.__getattr__("regular_attr") == "regular_attr" + assert i.__getattr__("sub_attr") == "sub_attr" + assert i.__getattr__("class_var") == "class_var" + assert i.__getattr__("subclass_var") == "subclass_var" + assert i.__getattr__("four") == "four" + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "sub_attr") == 24 + assert getattr(i, "class_var") == "x" + assert getattr(i, "subclass_var") == "y" + assert getattr(i, "four") == "four" + + assert i.three == "three" + assert i.regular_attr == 42 + assert i.sub_attr == 24 + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.four == "four" + + assert i.__class__ == GetAttrOverridden + + i.extra_attrs["subclass_var"] = (4, 4, 4) + assert i.__getattr__("subclass_var") == "subclass_var" + assert getattr(i, "subclass_var") == "y" + assert i.subclass_var == "y" + + base_ref: GetAttr = i + assert getattr(base_ref, "sub_attr") == 24 + assert base_ref.sub_attr == 24 + + assert getattr(base_ref, "subclass_var") == "y" + assert base_ref.subclass_var == "y" + + assert getattr(base_ref, "new") == "new" + assert base_ref.new == "new" + + assert base_ref.__class__ == GetAttrOverridden + +def test_getattr_nonnative() -> None: + i = GetAttr({"one": 1, "two": "two", "three": 3.14}, 42) + assert i.__getattr__("one") == 1 + assert i.__getattr__("regular_attr") == None + assert i.__getattr__("class_var") == None + assert i.__getattr__("four") == None + + assert getattr(i, "two") == "two" + assert getattr(i, "regular_attr") == 42 + assert getattr(i, "class_var") == "x" + assert getattr(i, "four") == None + + assert i.three == 3.14 + assert i.regular_attr == 42 + assert i.class_var == "x" + assert i.four == None + + assert i.__class__ == GetAttr + + i.extra_attrs["regular_attr"] = (4, 4, 4) + assert i.__getattr__("regular_attr") == (4, 4, 4) + assert getattr(i, "regular_attr") == 42 + assert i.regular_attr == 42 + +test_getattr() +test_getattr_default() +test_getattr_inherited() +test_getattr_overridden() +test_getattr_nonnative() + + +[typing fixtures/typing-full.pyi] diff --git a/mypyc/transform/refcount.py b/mypyc/transform/refcount.py index 60daebc415fd..beacb409edfb 100644 --- a/mypyc/transform/refcount.py +++ b/mypyc/transform/refcount.py @@ -33,6 +33,7 @@ Assign, BasicBlock, Branch, + CallC, ControlOp, DecRef, Goto, @@ -89,7 +90,9 @@ def insert_ref_count_opcodes(ir: FuncIR) -> None: def is_maybe_undefined(post_must_defined: set[Value], src: Value) -> bool: - return isinstance(src, Register) and src not in post_must_defined + return (isinstance(src, Register) and src not in post_must_defined) or ( + isinstance(src, CallC) and src.returns_null + ) def maybe_append_dec_ref( From 19697af9051707b4db55bc2d5301436d872f452c Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 25 Sep 2025 15:06:30 -0400 Subject: [PATCH 048/183] fix: something Shantanu found (#19859) to be honest I have no idea what this code does, but I know we need to fix it per advice from @hauntsaninja in #19846 --- mypy/strconv.py | 4 +++- test-data/unit/semanal-basic.test | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/strconv.py b/mypy/strconv.py index 3e9d37586f72..d1595139572a 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -388,7 +388,9 @@ def visit_name_expr(self, o: mypy.nodes.NameExpr) -> str: o.name, o.kind, o.fullname, o.is_inferred_def or o.is_special_form, o.node ) if isinstance(o.node, mypy.nodes.Var) and o.node.is_final: - pretty += f" = {o.node.final_value}" + final_value = o.node.final_value + if final_value is not None: + pretty += f" = {o.node.final_value}" return short_type(o) + "(" + pretty + ")" def pretty_name( diff --git a/test-data/unit/semanal-basic.test b/test-data/unit/semanal-basic.test index 1f03ed22648d..773092cab623 100644 --- a/test-data/unit/semanal-basic.test +++ b/test-data/unit/semanal-basic.test @@ -521,7 +521,7 @@ MypyFile:1( NameExpr(True [builtins.True]) Literal[True]?) AssignmentStmt:8( - NameExpr(n* [__main__.n] = None) + NameExpr(n* [__main__.n]) CallExpr:8( NameExpr(func [__main__.func]) Args()))) From 89f7223b57db2a7a6188767988481dd81b58e14c Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 27 Sep 2025 02:19:41 -0400 Subject: [PATCH 049/183] [docs] Fix a common and harmless grammatical error, 'allows to' (#19900) This is a grammatical error (at least in American English, which I think this project is written in) because in the relevant sense the verb "allows" is transitive and needs to apply to a noun, not a verb. More info, if desired: https://english.stackexchange.com/questions/60271/grammatical-complements-for-allow https://english.stackexchange.com/questions/85069/is-the-construction-it-allows-to-proper-english (I don't know of any weighty publications who have taken this question on, so it's just a bunch of speakers, such as myself, opining.) --- docs/source/command_line.rst | 4 ++-- docs/source/error_code_list.rst | 4 ++-- docs/source/mypy_daemon.rst | 6 +++--- mypy/traverser.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 270125e96cb6..d667fa0ff727 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -372,7 +372,7 @@ definitions or calls. .. option:: --untyped-calls-exclude - This flag allows to selectively disable :option:`--disallow-untyped-calls` + This flag allows one to selectively disable :option:`--disallow-untyped-calls` for functions and methods defined in specific packages, modules, or classes. Note that each exclude entry acts as a prefix. For example (assuming there are no type annotations for ``third_party_lib`` available): @@ -562,7 +562,7 @@ potentially problematic or redundant in some way. .. option:: --deprecated-calls-exclude - This flag allows to selectively disable :ref:`deprecated` warnings + This flag allows one to selectively disable :ref:`deprecated` warnings for functions and methods defined in specific packages, modules, or classes. Note that each exclude entry acts as a prefix. For example (assuming ``foo.A.func`` is deprecated): diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index 6deed549c2f1..229230eda4ca 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -1032,8 +1032,8 @@ Warn about top level await expressions [top-level-await] This error code is separate from the general ``[syntax]`` errors, because in some environments (e.g. IPython) a top level ``await`` is allowed. In such environments a user may want to use ``--disable-error-code=top-level-await``, -that allows to still have errors for other improper uses of ``await``, for -example: +which allows one to still have errors for other improper uses of ``await``, +for example: .. code-block:: python diff --git a/docs/source/mypy_daemon.rst b/docs/source/mypy_daemon.rst index 6c511e14eb95..e0fc8129a0b8 100644 --- a/docs/source/mypy_daemon.rst +++ b/docs/source/mypy_daemon.rst @@ -252,16 +252,16 @@ command. Statically inspect expressions ****************************** -The daemon allows to get declared or inferred type of an expression (or other +The daemon allows one to get the declared or inferred type of an expression (or other information about an expression, such as known attributes or definition location) -using ``dmypy inspect LOCATION`` command. The location of the expression should be +using the ``dmypy inspect LOCATION`` command. The location of the expression should be specified in the format ``path/to/file.py:line:column[:end_line:end_column]``. Both line and column are 1-based. Both start and end position are inclusive. These rules match how mypy prints the error location in error messages. If a span is given (i.e. all 4 numbers), then only an exactly matching expression is inspected. If only a position is given (i.e. 2 numbers, line and column), mypy -will inspect all *expressions*, that include this position, starting from the +will inspect all expressions that include this position, starting from the innermost one. Consider this Python code snippet: diff --git a/mypy/traverser.py b/mypy/traverser.py index 7d7794822396..3c249391bdf8 100644 --- a/mypy/traverser.py +++ b/mypy/traverser.py @@ -504,7 +504,7 @@ class ExtendedTraverserVisitor(TraverserVisitor): In addition to the base traverser it: * has visit_ methods for leaf nodes * has common method that is called for all nodes - * allows to skip recursing into a node + * allows skipping recursing into a node Note that this traverser still doesn't visit some internal mypy constructs like _promote expression and Var. From 16cd4c5215aa2301b50341c22048ffb8a4737a63 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Sep 2025 11:48:05 +0100 Subject: [PATCH 050/183] Traverse type alias args in collect visitor (#19943) Fixes https://github.com/python/mypy/issues/19941 Fix is trivial (and a bit embarrassing, LOL). --- mypy/type_visitor.py | 2 +- mypy/types.py | 8 ++++++-- test-data/unit/check-recursive-types.test | 9 +++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 86ef6ade8471..35846e7c3ddd 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -51,7 +51,7 @@ get_proper_type, ) -T = TypeVar("T") +T = TypeVar("T", covariant=True) @trait diff --git a/mypy/types.py b/mypy/types.py index e0e897e04cad..38c17e240ccf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3938,8 +3938,12 @@ def visit_type_alias_type(self, t: TypeAliasType, /) -> list[mypy.nodes.TypeAlia assert t.alias is not None if t.alias not in self.seen_alias_nodes: self.seen_alias_nodes.add(t.alias) - return [t.alias] + t.alias.target.accept(self) - return [] + res = [t.alias] + t.alias.target.accept(self) + else: + res = [] + for arg in t.args: + res.extend(arg.accept(self)) + return res def is_named_instance(t: Type, fullnames: str | tuple[str, ...]) -> TypeGuard[Instance]: diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 86e9f02b5263..4f451aa062d6 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -1014,3 +1014,12 @@ from bogus import Foo # type: ignore A = Callable[[Foo, "B"], Foo] # E: Type alias target becomes "Callable[[Any, B], Any]" due to an unfollowed import B = Callable[[Foo, A], Foo] # E: Type alias target becomes "Callable[[Any, A], Any]" due to an unfollowed import + +[case testRecursiveAliasOnArgumentDetected] +from typing import TypeVar + +T = TypeVar("T") +L = list[T] + +A = L[A] +a: A = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "A") From df4ae6a6b64d49f9f930bbdaf82ed5294dd0f2bc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 Sep 2025 14:56:54 +0100 Subject: [PATCH 051/183] [mypyc] Fix broken exception/cancellation handling in async def (#19951) Fix an unexpected undefined attribute error in a generator/async def. The intent was to explicitly check if an attribute was undefined, but the attribute read implicitly raised `AttributeError`, so the check was never reached. Fixed by allowing error values to be read without raising `AttributeError`. The error this fixes would look like this (with no line number associated with it): ``` AttributeError("attribute '__mypyc_temp__12' of 'wait_Condition_gen' undefined") ``` --- mypyc/irbuild/builder.py | 17 ++++++++- mypyc/irbuild/statement.py | 9 ++++- mypyc/test-data/fixtures/testutil.py | 2 +- mypyc/test-data/run-async.test | 57 ++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index f4ee4371b9bf..fa0e605d6776 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -700,7 +700,12 @@ def get_assignment_target( assert False, "Unsupported lvalue: %r" % lvalue def read( - self, target: Value | AssignmentTarget, line: int = -1, can_borrow: bool = False + self, + target: Value | AssignmentTarget, + line: int = -1, + *, + can_borrow: bool = False, + allow_error_value: bool = False, ) -> Value: if isinstance(target, Value): return target @@ -716,7 +721,15 @@ def read( if isinstance(target, AssignmentTargetAttr): if isinstance(target.obj.type, RInstance) and target.obj.type.class_ir.is_ext_class: borrow = can_borrow and target.can_borrow - return self.add(GetAttr(target.obj, target.attr, line, borrow=borrow)) + return self.add( + GetAttr( + target.obj, + target.attr, + line, + borrow=borrow, + allow_error_value=allow_error_value, + ) + ) else: return self.py_get_attr(target.obj, target.attr, line) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index c83c5550d059..fdcf4f777153 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -829,7 +829,14 @@ def transform_try_finally_stmt_async( # Check if we have a return value if ret_reg: return_block, check_old_exc = BasicBlock(), BasicBlock() - builder.add(Branch(builder.read(ret_reg), check_old_exc, return_block, Branch.IS_ERROR)) + builder.add( + Branch( + builder.read(ret_reg, allow_error_value=True), + check_old_exc, + return_block, + Branch.IS_ERROR, + ) + ) builder.activate_block(return_block) builder.nonlocal_control[-1].gen_return(builder, builder.read(ret_reg), -1) diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index 36ec41c8f38b..b2700f731ddd 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -44,7 +44,7 @@ def assertRaises(typ: type, msg: str = '') -> Iterator[None]: try: yield - except Exception as e: + except BaseException as e: assert type(e) is typ, f"{e!r} is not a {typ.__name__}" assert msg in str(e), f'Message "{e}" does not match "{msg}"' else: diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 55cde4ab44f1..94a1cd2e97c5 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1234,3 +1234,60 @@ def test_callable_arg_same_name_as_helper() -> None: [file asyncio/__init__.pyi] def run(x: object) -> object: ... + +[case testRunAsyncCancelFinallySpecialCase] +import asyncio + +from testutil import assertRaises + +# Greatly simplified from asyncio.Condition +class Condition: + async def acquire(self) -> None: pass + + async def wait(self) -> bool: + l = asyncio.get_running_loop() + fut = l.create_future() + a = [] + try: + try: + a.append(fut) + try: + await fut + return True + finally: + a.pop() + finally: + err = None + while True: + try: + await self.acquire() + break + except asyncio.CancelledError as e: + err = e + + if err is not None: + try: + raise err + finally: + err = None + except BaseException: + raise + +async def do_cancel() -> None: + cond = Condition() + wait = asyncio.create_task(cond.wait()) + asyncio.get_running_loop().call_soon(wait.cancel) + with assertRaises(asyncio.CancelledError): + await wait + +def test_cancel_special_case() -> None: + asyncio.run(do_cancel()) + +[file asyncio/__init__.pyi] +from typing import Any + +class CancelledError(Exception): ... + +def run(x: object) -> object: ... +def get_running_loop() -> Any: ... +def create_task(x: object) -> Any: ... From 8392e1a74a8af9a0d8e4c85e0c1a8247c08b8e0b Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Mon, 29 Sep 2025 16:18:52 +0200 Subject: [PATCH 052/183] [mypyc] Generate __setattr__ wrapper (#19937) Generate wrapper function for `__setattr__` and set it as the `tp_setattro` slot. The wrapper doesn't have to do anything because interpreted python runs the defined `__setattr__` on every attribute assignment. So the wrapper only reports errors on unsupported uses of `__setattr__` and makes the function signature match with the slot. Since `__setattr__` should run on every attribute assignment, native classes with `__setattr__` generate calls to this function on attribute assignment instead of direct assignment to the underlying C struct. Native classes generally don't have `__dict__` which makes implementing dynamic attributes more challenging. The native class has to manage its own dictionary of names to values and handle assignment to regular attributes specially. With `__dict__`, assigning values to regular and dynamic attributes can be done by simply assigning the value in `__dict__`, ie. `self.__dict__[name] = value` or `object.__setattr__(self, name, value)`. With a custom attribute dictionary, assigning with `name` being a regular attribute doesn't work because it would only update the value in the custom dictionary, not the actual attribute. On the other hand, the `object.__setattr__` call doesn't work for dynamic attributes and raises an `AttributeError` without `__dict__`. So something like this has to be implemented as a work-around: ``` def __setattr__(self, name: str, val: object) -> None: if name == "regular_attribute": object.__setattr__(self, "regular_attribute", val) else: self._attribute_dict[name] = val ``` To make this efficient in native classes, calls to `object.__setattr__` or equivalent `super().__setattr__` are transformed to direct C struct assignments when the name literal matches an attribute name. --- mypyc/codegen/emitclass.py | 1 + mypyc/irbuild/builder.py | 12 +- mypyc/irbuild/expression.py | 7 + mypyc/irbuild/function.py | 31 ++ mypyc/irbuild/specialize.py | 52 ++- mypyc/lib-rt/CPy.h | 3 + mypyc/primitives/generic_ops.py | 7 + mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-classes.test | 473 ++++++++++++++++++++++ mypyc/test-data/run-classes.test | 585 +++++++++++++++++++++++++++ 10 files changed, 1161 insertions(+), 11 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 9e8f9c74bc6d..122f62a0d582 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -62,6 +62,7 @@ def dunder_attr_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: "__hash__": ("tp_hash", generate_hash_wrapper), "__get__": ("tp_descr_get", generate_get_wrapper), "__getattr__": ("tp_getattro", dunder_attr_slot), + "__setattr__": ("tp_setattro", dunder_attr_slot), } AS_MAPPING_SLOT_DEFS: SlotTable = { diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index fa0e605d6776..125382145991 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -76,6 +76,7 @@ Integer, IntOp, LoadStatic, + MethodCall, Op, PrimitiveDescription, RaiseStandardError, @@ -748,8 +749,15 @@ def assign(self, target: Register | AssignmentTarget, rvalue_reg: Value, line: i self.add(Assign(target.register, rvalue_reg)) elif isinstance(target, AssignmentTargetAttr): if isinstance(target.obj_type, RInstance): - rvalue_reg = self.coerce_rvalue(rvalue_reg, target.type, line) - self.add(SetAttr(target.obj, target.attr, rvalue_reg, line)) + setattr = target.obj_type.class_ir.get_method("__setattr__") + if setattr: + key = self.load_str(target.attr) + boxed_reg = self.builder.box(rvalue_reg) + call = MethodCall(target.obj, setattr.name, [key, boxed_reg], line) + self.add(call) + else: + rvalue_reg = self.coerce_rvalue(rvalue_reg, target.type, line) + self.add(SetAttr(target.obj, target.attr, rvalue_reg, line)) else: key = self.load_str(target.attr) boxed_reg = self.builder.box(rvalue_reg) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 1f39b09c0995..54a101bc4961 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -101,6 +101,7 @@ apply_function_specialization, apply_method_specialization, translate_object_new, + translate_object_setattr, ) from mypyc.primitives.bytes_ops import bytes_slice_op from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, exact_dict_set_item_op @@ -480,6 +481,12 @@ def translate_super_method_call(builder: IRBuilder, expr: CallExpr, callee: Supe result = translate_object_new(builder, expr, MemberExpr(callee.call, "__new__")) if result: return result + elif callee.name == "__setattr__": + result = translate_object_setattr( + builder, expr, MemberExpr(callee.call, "__setattr__") + ) + if result: + return result if ir.is_ext_class and ir.builtin_base is None and not ir.inherits_python: if callee.name == "__init__" and len(expr.args) == 0: # Call translates to object.__init__(self), which is a diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index a9a098d25dde..51bdc76495f2 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -57,6 +57,7 @@ from mypyc.ir.rtypes import ( RInstance, bool_rprimitive, + c_int_rprimitive, dict_rprimitive, int_rprimitive, object_rprimitive, @@ -415,6 +416,34 @@ def generate_getattr_wrapper(builder: IRBuilder, cdef: ClassDef, getattr: FuncDe builder.add(Return(getattr_result, line)) +def generate_setattr_wrapper(builder: IRBuilder, cdef: ClassDef, setattr: FuncDef) -> None: + """ + Generate a wrapper function for __setattr__ that can be put into the tp_setattro slot. + The wrapper takes two arguments besides self - attribute name and the new value. + Returns 0 on success and -1 on failure. Restrictions are similar to the __getattr__ + wrapper above. + + This one is simpler because to match interpreted python semantics it's enough to always + call the user-provided function, including for names matching regular attributes. + """ + name = setattr.name + "__wrapper" + ir = builder.mapper.type_to_ir[cdef.info] + line = setattr.line + + error_base = f'"__setattr__" not supported in class "{cdef.name}" because ' + if ir.allow_interpreted_subclasses: + builder.error(error_base + "it allows interpreted subclasses", line) + if ir.inherits_python: + builder.error(error_base + "it inherits from a non-native class", line) + + with builder.enter_method(ir, name, c_int_rprimitive, internal=True): + attr_arg = builder.add_argument("attr", object_rprimitive) + value_arg = builder.add_argument("value", object_rprimitive) + + builder.gen_method_call(builder.self(), setattr.name, [attr_arg, value_arg], None, line) + builder.add(Return(Integer(0, c_int_rprimitive), line)) + + def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None: # Perform the function of visit_method for methods inside extension classes. name = fdef.name @@ -483,6 +512,8 @@ def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None if fdef.name == "__getattr__": generate_getattr_wrapper(builder, cdef, fdef) + elif fdef.name == "__setattr__": + generate_setattr_wrapper(builder, cdef, fdef) def handle_non_ext_method( diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 29820787d10c..42b7710c98da 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -42,6 +42,7 @@ Integer, RaiseStandardError, Register, + SetAttr, Truncate, Unreachable, Value, @@ -97,6 +98,7 @@ isinstance_dict, ) from mypyc.primitives.float_ops import isinstance_float +from mypyc.primitives.generic_ops import generic_setattr from mypyc.primitives.int_ops import isinstance_int from mypyc.primitives.list_ops import isinstance_list, new_list_set_item_op from mypyc.primitives.misc_ops import isinstance_bool @@ -1007,19 +1009,24 @@ def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value return None -@specialize_function("__new__", object_rprimitive) -def translate_object_new(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: - fn = builder.fn_info - if fn.name != "__new__": - return None - - is_super_new = isinstance(expr.callee, SuperExpr) - is_object_new = ( +def is_object(callee: RefExpr) -> bool: + """Returns True for object. calls.""" + return ( isinstance(callee, MemberExpr) and isinstance(callee.expr, NameExpr) and callee.expr.fullname == "builtins.object" ) - if not (is_super_new or is_object_new): + + +def is_super_or_object(expr: CallExpr, callee: RefExpr) -> bool: + """Returns True for super(). or object. calls.""" + return isinstance(expr.callee, SuperExpr) or is_object(callee) + + +@specialize_function("__new__", object_rprimitive) +def translate_object_new(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + fn = builder.fn_info + if fn.name != "__new__" or not is_super_or_object(expr, callee): return None ir = builder.get_current_class_ir() @@ -1046,3 +1053,30 @@ def translate_object_new(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> return builder.add(Call(ir.setup, [subtype], expr.line)) return None + + +@specialize_function("__setattr__", object_rprimitive) +def translate_object_setattr(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + is_super = isinstance(expr.callee, SuperExpr) + is_object_callee = is_object(callee) + if not ((is_super and len(expr.args) >= 2) or (is_object_callee and len(expr.args) >= 3)): + return None + + self_reg = builder.accept(expr.args[0]) if is_object_callee else builder.self() + ir = builder.get_current_class_ir() + if ir and (not ir.is_ext_class or ir.builtin_base or ir.inherits_python): + return None + # Need to offset by 1 for super().__setattr__ calls because there is no self arg in this case. + name_idx = 0 if is_super else 1 + value_idx = 1 if is_super else 2 + attr_name = expr.args[name_idx] + attr_value = expr.args[value_idx] + value = builder.accept(attr_value) + + if isinstance(attr_name, StrExpr) and ir and ir.has_attr(attr_name.value): + name = attr_name.value + value = builder.coerce(value, ir.attributes[name], expr.line) + return builder.add(SetAttr(self_reg, name, value, expr.line)) + + name_reg = builder.accept(attr_name) + return builder.call_c(generic_setattr, [self_reg, name_reg, value], expr.line) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index b9cecb9280f3..e9dfd8de3683 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -952,6 +952,9 @@ void CPyTrace_LogEvent(const char *location, const char *line, const char *op, c static inline PyObject *CPyObject_GenericGetAttr(PyObject *self, PyObject *name) { return _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); } +static inline int CPyObject_GenericSetAttr(PyObject *self, PyObject *name, PyObject *value) { + return _PyObject_GenericSetAttrWithDict(self, name, value, NULL); +} #if CPY_3_11_FEATURES PyObject *CPy_GetName(PyObject *obj); diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index ff978b7c8c3b..16bd074396d2 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -410,3 +410,10 @@ error_kind=ERR_NEVER, returns_null=True, ) + +generic_setattr = custom_op( + arg_types=[object_rprimitive, object_rprimitive, object_rprimitive], + return_type=c_int_rprimitive, + c_function_name="CPyObject_GenericSetAttr", + error_kind=ERR_NEG_INT, +) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index a4b4f3ce2b1f..22a6a5986cbd 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -45,6 +45,7 @@ def __init__(self) -> None: pass def __eq__(self, x: object) -> bool: pass def __ne__(self, x: object) -> bool: pass def __str__(self) -> str: pass + def __setattr__(self, k: str, v: object) -> None: pass class type: def __init__(self, o: object) -> None: ... diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 76e28711c5e3..a98b3a7d3dcf 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2213,3 +2213,476 @@ L0: buf_init_item r13, 4, r12 keep_alive r9 return r9 + +[case testUnsupportedSetAttr] +from mypy_extensions import mypyc_attr + +@mypyc_attr(allow_interpreted_subclasses=True) +class AllowsInterpreted: + def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "AllowsInterpreted" because it allows interpreted subclasses + pass + +class InheritsInterpreted(dict): + def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "InheritsInterpreted" because it inherits from a non-native class + pass + +@mypyc_attr(native_class=False) +class NonNative: + pass + +class InheritsNonNative(NonNative): + def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "InheritsNonNative" because it inherits from a non-native class + pass + +[case testSetAttr] +from typing import ClassVar +class SetAttr: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object], new_attr: str, new_val: object) -> None: + super().__setattr__("_attributes", extra_attrs) + object.__setattr__(self, "regular_attr", regular_attr) + + super().__setattr__(new_attr, new_val) + object.__setattr__(self, new_attr, new_val) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var": + raise AttributeError() + else: + self._attributes[key] = val + +def test(attr: str, val: object) -> None: + i = SetAttr(99, {}, attr, val) + i.regular_attr = 100 + i.new_attr = 101 + + object.__setattr__(i, "regular_attr", 11) + object.__setattr__(i, attr, val) + +[typing fixtures/typing-full.pyi] +[out] +def SetAttr.__init__(self, regular_attr, extra_attrs, new_attr, new_val): + self :: __main__.SetAttr + regular_attr :: int + extra_attrs :: dict + new_attr :: str + new_val :: object + r0 :: i32 + r1 :: bit + r2 :: i32 + r3 :: bit +L0: + self._attributes = extra_attrs + self.regular_attr = regular_attr + r0 = CPyObject_GenericSetAttr(self, new_attr, new_val) + r1 = r0 >= 0 :: signed + r2 = CPyObject_GenericSetAttr(self, new_attr, new_val) + r3 = r2 >= 0 :: signed + return 1 +def SetAttr.__setattr__(self, key, val): + self :: __main__.SetAttr + key :: str + val :: object + r0 :: str + r1 :: bool + r2 :: int + r3 :: bool + r4 :: str + r5 :: bool + r6 :: object + r7 :: str + r8, r9 :: object + r10 :: dict + r11 :: i32 + r12 :: bit +L0: + r0 = 'regular_attr' + r1 = CPyStr_Equal(key, r0) + if r1 goto L1 else goto L2 :: bool +L1: + r2 = unbox(int, val) + self.regular_attr = r2; r3 = is_error + goto L6 +L2: + r4 = 'class_var' + r5 = CPyStr_Equal(key, r4) + if r5 goto L3 else goto L4 :: bool +L3: + r6 = builtins :: module + r7 = 'AttributeError' + r8 = CPyObject_GetAttr(r6, r7) + r9 = PyObject_Vectorcall(r8, 0, 0, 0) + CPy_Raise(r9) + unreachable +L4: + r10 = self._attributes + r11 = CPyDict_SetItem(r10, key, val) + r12 = r11 >= 0 :: signed +L5: +L6: + return 1 +def SetAttr.__setattr____wrapper(__mypyc_self__, attr, value): + __mypyc_self__ :: __main__.SetAttr + attr, value :: object + r0 :: str + r1 :: None +L0: + r0 = cast(str, attr) + r1 = __mypyc_self__.__setattr__(r0, value) + return 0 +def test(attr, val): + attr :: str + val :: object + r0 :: dict + r1, i :: __main__.SetAttr + r2 :: str + r3 :: object + r4 :: None + r5 :: str + r6 :: object + r7 :: i32 + r8 :: bit + r9 :: str + r10 :: object + r11 :: i32 + r12 :: bit + r13 :: i32 + r14 :: bit +L0: + r0 = PyDict_New() + r1 = SetAttr(198, r0, attr, val) + i = r1 + r2 = 'regular_attr' + r3 = object 100 + r4 = i.__setattr__(r2, r3) + r5 = 'new_attr' + r6 = object 101 + r7 = PyObject_SetAttr(i, r5, r6) + r8 = r7 >= 0 :: signed + r9 = 'regular_attr' + r10 = object 11 + r11 = CPyObject_GenericSetAttr(i, r9, r10) + r12 = r11 >= 0 :: signed + r13 = CPyObject_GenericSetAttr(i, attr, val) + r14 = r13 >= 0 :: signed + return 1 + +[case testUntransformedSetAttr_64bit] +from mypy_extensions import mypyc_attr + +class SetAttr: + def super_missing_args(self): + super().__setattr__() + super().__setattr__("attr") + + def object_missing_args(self): + object.__setattr__() + object.__setattr__(self) + object.__setattr__(self, "attr") + +@mypyc_attr(native_class=False) +class NonNative: + def super_setattr(self, key: str, val: object) -> None: + super().__setattr__(key, val) + + def object_setattr(self, key: str, val: object) -> None: + object.__setattr__(self, key, val) + +class InheritsPython(NonNative): + def super_setattr(self, key: str, val: object) -> None: + super().__setattr__(key, val) + + def object_setattr(self, key: str, val: object) -> None: + object.__setattr__(self, key, val) + +class BuiltInBase(dict): + def super_setattr(self, key: str, val: object) -> None: + super().__setattr__(key, val) + + def object_setattr(self, key: str, val: object) -> None: + object.__setattr__(self, key, val) + +[typing fixtures/typing-full.pyi] +[out] +def SetAttr.super_missing_args(self): + self :: __main__.SetAttr + r0 :: object + r1 :: str + r2, r3 :: object + r4 :: object[2] + r5 :: object_ptr + r6 :: object + r7 :: str + r8, r9, r10 :: object + r11 :: str + r12, r13 :: object + r14 :: object[2] + r15 :: object_ptr + r16 :: object + r17 :: str + r18 :: object + r19 :: str + r20 :: object[1] + r21 :: object_ptr + r22, r23 :: object +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.SetAttr :: type + r4 = [r3, self] + r5 = load_address r4 + r6 = PyObject_Vectorcall(r2, r5, 2, 0) + keep_alive r3, self + r7 = '__setattr__' + r8 = CPyObject_GetAttr(r6, r7) + r9 = PyObject_Vectorcall(r8, 0, 0, 0) + r10 = builtins :: module + r11 = 'super' + r12 = CPyObject_GetAttr(r10, r11) + r13 = __main__.SetAttr :: type + r14 = [r13, self] + r15 = load_address r14 + r16 = PyObject_Vectorcall(r12, r15, 2, 0) + keep_alive r13, self + r17 = '__setattr__' + r18 = CPyObject_GetAttr(r16, r17) + r19 = 'attr' + r20 = [r19] + r21 = load_address r20 + r22 = PyObject_Vectorcall(r18, r21, 1, 0) + keep_alive r19 + r23 = box(None, 1) + return r23 +def SetAttr.object_missing_args(self): + self :: __main__.SetAttr + r0 :: object + r1 :: str + r2 :: object + r3 :: str + r4 :: object[1] + r5 :: object_ptr + r6, r7 :: object + r8 :: str + r9 :: object + r10 :: str + r11 :: object[2] + r12 :: object_ptr + r13, r14 :: object + r15 :: str + r16 :: object + r17, r18 :: str + r19 :: object[3] + r20 :: object_ptr + r21, r22 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = '__setattr__' + r4 = [r2] + r5 = load_address r4 + r6 = PyObject_VectorcallMethod(r3, r5, 9223372036854775809, 0) + keep_alive r2 + r7 = builtins :: module + r8 = 'object' + r9 = CPyObject_GetAttr(r7, r8) + r10 = '__setattr__' + r11 = [r9, self] + r12 = load_address r11 + r13 = PyObject_VectorcallMethod(r10, r12, 9223372036854775810, 0) + keep_alive r9, self + r14 = builtins :: module + r15 = 'object' + r16 = CPyObject_GetAttr(r14, r15) + r17 = 'attr' + r18 = '__setattr__' + r19 = [r16, self, r17] + r20 = load_address r19 + r21 = PyObject_VectorcallMethod(r18, r20, 9223372036854775811, 0) + keep_alive r16, self, r17 + r22 = box(None, 1) + return r22 +def super_setattr_NonNative_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def super_setattr_NonNative_obj.__call__(__mypyc_self__, self, key, val): + __mypyc_self__ :: __main__.super_setattr_NonNative_obj + self :: __main__.NonNative + key :: str + val, r0 :: object + r1 :: str + r2, r3 :: object + r4 :: object[2] + r5 :: object_ptr + r6 :: object + r7 :: str + r8 :: object + r9 :: object[2] + r10 :: object_ptr + r11 :: object +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.NonNative :: type + r4 = [r3, self] + r5 = load_address r4 + r6 = PyObject_Vectorcall(r2, r5, 2, 0) + keep_alive r3, self + r7 = '__setattr__' + r8 = CPyObject_GetAttr(r6, r7) + r9 = [key, val] + r10 = load_address r9 + r11 = PyObject_Vectorcall(r8, r10, 2, 0) + keep_alive key, val + return 1 +def object_setattr_NonNative_obj.__get__(__mypyc_self__, instance, owner): + __mypyc_self__, instance, owner, r0 :: object + r1 :: bit + r2 :: object +L0: + r0 = load_address _Py_NoneStruct + r1 = instance == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return __mypyc_self__ +L2: + r2 = PyMethod_New(__mypyc_self__, instance) + return r2 +def object_setattr_NonNative_obj.__call__(__mypyc_self__, self, key, val): + __mypyc_self__ :: __main__.object_setattr_NonNative_obj + self :: __main__.NonNative + key :: str + val, r0 :: object + r1 :: str + r2 :: object + r3 :: str + r4 :: object[4] + r5 :: object_ptr + r6 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = '__setattr__' + r4 = [r2, self, key, val] + r5 = load_address r4 + r6 = PyObject_VectorcallMethod(r3, r5, 9223372036854775812, 0) + keep_alive r2, self, key, val + return 1 +def InheritsPython.super_setattr(self, key, val): + self :: __main__.InheritsPython + key :: str + val, r0 :: object + r1 :: str + r2, r3 :: object + r4 :: object[2] + r5 :: object_ptr + r6 :: object + r7 :: str + r8 :: object + r9 :: object[2] + r10 :: object_ptr + r11 :: object +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.InheritsPython :: type + r4 = [r3, self] + r5 = load_address r4 + r6 = PyObject_Vectorcall(r2, r5, 2, 0) + keep_alive r3, self + r7 = '__setattr__' + r8 = CPyObject_GetAttr(r6, r7) + r9 = [key, val] + r10 = load_address r9 + r11 = PyObject_Vectorcall(r8, r10, 2, 0) + keep_alive key, val + return 1 +def InheritsPython.object_setattr(self, key, val): + self :: __main__.InheritsPython + key :: str + val, r0 :: object + r1 :: str + r2 :: object + r3 :: str + r4 :: object[4] + r5 :: object_ptr + r6 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = '__setattr__' + r4 = [r2, self, key, val] + r5 = load_address r4 + r6 = PyObject_VectorcallMethod(r3, r5, 9223372036854775812, 0) + keep_alive r2, self, key, val + return 1 +def BuiltInBase.super_setattr(self, key, val): + self :: dict + key :: str + val, r0 :: object + r1 :: str + r2, r3 :: object + r4 :: object[2] + r5 :: object_ptr + r6 :: object + r7 :: str + r8 :: object + r9 :: object[2] + r10 :: object_ptr + r11 :: object +L0: + r0 = builtins :: module + r1 = 'super' + r2 = CPyObject_GetAttr(r0, r1) + r3 = __main__.BuiltInBase :: type + r4 = [r3, self] + r5 = load_address r4 + r6 = PyObject_Vectorcall(r2, r5, 2, 0) + keep_alive r3, self + r7 = '__setattr__' + r8 = CPyObject_GetAttr(r6, r7) + r9 = [key, val] + r10 = load_address r9 + r11 = PyObject_Vectorcall(r8, r10, 2, 0) + keep_alive key, val + return 1 +def BuiltInBase.object_setattr(self, key, val): + self :: dict + key :: str + val, r0 :: object + r1 :: str + r2 :: object + r3 :: str + r4 :: object[4] + r5 :: object_ptr + r6 :: object +L0: + r0 = builtins :: module + r1 = 'object' + r2 = CPyObject_GetAttr(r0, r1) + r3 = '__setattr__' + r4 = [r2, self, key, val] + r5 = load_address r4 + r6 = PyObject_VectorcallMethod(r3, r5, 9223372036854775812, 0) + keep_alive r2, self, key, val + return 1 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index b2f1a088585d..d10f7b19067c 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -4518,5 +4518,590 @@ test_getattr_inherited() test_getattr_overridden() test_getattr_nonnative() +[typing fixtures/typing-full.pyi] + +[case testDunderSetAttr] +from mypy_extensions import mypyc_attr +from testutil import assertRaises +from typing import ClassVar + +class SetAttr: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + const: int = 42 + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__setattr__("_attributes", extra_attrs) + super().__setattr__("regular_attr", regular_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var" or key == "const": + raise AttributeError() + else: + self._attributes[key] = val + + def __getattr__(self, key: str) -> object: + return self._attributes.get(key) + +class SetAttrInherited(SetAttr): + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__init__(regular_attr, extra_attrs) + +class SetAttrOverridden(SetAttr): + sub_attr: int + subclass_var: ClassVar[str] = "y" + + def __init__(self, regular_attr: int, sub_attr: int, extra_attrs: dict[str, object]) -> None: + super().__init__(regular_attr, extra_attrs) + object.__setattr__(self, "sub_attr", sub_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "sub_attr": + object.__setattr__(self, "sub_attr", val) + elif key == "subclass_var": + raise AttributeError() + else: + super().__setattr__(key, val) + +@mypyc_attr(native_class=False) +class SetAttrNonNative: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + const: int = 42 + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__setattr__("_attributes", extra_attrs) + super().__setattr__("regular_attr", regular_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var" or key == "const": + raise AttributeError() + else: + self._attributes[key] = val + + def __getattr__(self, key: str) -> object: + return self._attributes.get(key) + +class NoSetAttr: + def __init__(self, attr: int) -> None: + self.attr = attr + + def object_setattr(self, attr: str, val: object) -> None: + object.__setattr__(self, attr, val) + + def super_setattr(self, attr: str, val: object) -> None: + super().__setattr__(attr, val) + +@mypyc_attr(native_class=False) +class NoSetAttrNonNative: + def __init__(self, attr: int) -> None: + self.attr = attr + + def object_setattr(self, attr: str, val: object) -> None: + object.__setattr__(self, attr, val) + + def super_setattr(self, attr: str, val: object) -> None: + super().__setattr__(attr, val) + + def __getattr__(self, attr: str) -> object: + pass + +def test_setattr() -> None: + i = SetAttr(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_setattr_inherited() -> None: + i = SetAttrInherited(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_setattr_overridden() -> None: + i = SetAttrOverridden(99, 1, {"one": 1}) + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.regular_attr == 99 + assert i.sub_attr == 1 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + i.__setattr__("sub_attr", 2) + assert i.sub_attr == 2 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("subclass_var", "a") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + setattr(i, "sub_attr", 3) + assert i.sub_attr == 3 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "subclass_var", "b") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + i.sub_attr = 4 + assert i.sub_attr == 4 + with assertRaises(AttributeError): + i.const = 45 + + base_ref: SetAttr = i + setattr(base_ref, "sub_attr", 5) + assert base_ref.sub_attr == 5 + + base_ref.sub_attr = 6 + assert base_ref.sub_attr == 6 + + with assertRaises(AttributeError): + setattr(base_ref, "subclass_var", "c") + +def test_setattr_nonnative() -> None: + i = SetAttrNonNative(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_no_setattr() -> None: + i = NoSetAttr(99) + i.super_setattr("attr", 100) + assert i.attr == 100 + + i.object_setattr("attr", 101) + assert i.attr == 101 + + object.__setattr__(i, "attr", 102) + assert i.attr == 102 + + with assertRaises(AttributeError): + i.super_setattr("not_attr", 100) + + with assertRaises(AttributeError): + i.object_setattr("not_attr", 101) + + with assertRaises(AttributeError): + object.__setattr__(i, "not_attr", 102) + +def test_no_setattr_nonnative() -> None: + i = NoSetAttrNonNative(99) + i.super_setattr("attr", 100) + assert i.attr == 100 + + i.object_setattr("attr", 101) + assert i.attr == 101 + + object.__setattr__(i, "attr", 102) + assert i.attr == 102 + + i.super_setattr("one", 100) + assert i.one == 100 + + i.object_setattr("two", 101) + assert i.two == 101 + + object.__setattr__(i, "three", 102) + assert i.three == 102 + +[typing fixtures/typing-full.pyi] + +[case testDunderSetAttrInterpreted] +from mypy_extensions import mypyc_attr +from typing import ClassVar + +class SetAttr: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + const: int = 42 + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__setattr__("_attributes", extra_attrs) + super().__setattr__("regular_attr", regular_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var" or key == "const": + raise AttributeError() + else: + self._attributes[key] = val + + def __getattr__(self, key: str) -> object: + return self._attributes.get(key) + +class SetAttrInherited(SetAttr): + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__init__(regular_attr, extra_attrs) + +class SetAttrOverridden(SetAttr): + sub_attr: int + subclass_var: ClassVar[str] = "y" + + def __init__(self, regular_attr: int, sub_attr: int, extra_attrs: dict[str, object]) -> None: + super().__init__(regular_attr, extra_attrs) + object.__setattr__(self, "sub_attr", sub_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "sub_attr": + object.__setattr__(self, "sub_attr", val) + elif key == "subclass_var": + raise AttributeError() + else: + super().__setattr__(key, val) + +@mypyc_attr(native_class=False) +class SetAttrNonNative: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + const: int = 42 + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object]) -> None: + super().__setattr__("_attributes", extra_attrs) + super().__setattr__("regular_attr", regular_attr) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var" or key == "const": + raise AttributeError() + else: + self._attributes[key] = val + + def __getattr__(self, key: str) -> object: + return self._attributes.get(key) + +class NoSetAttr: + def __init__(self, attr: int) -> None: + self.attr = attr + + def object_setattr(self, attr: str, val: object) -> None: + object.__setattr__(self, attr, val) + + def super_setattr(self, attr: str, val: object) -> None: + super().__setattr__(attr, val) + +@mypyc_attr(native_class=False) +class NoSetAttrNonNative: + def __init__(self, attr: int) -> None: + self.attr = attr + + def object_setattr(self, attr: str, val: object) -> None: + object.__setattr__(self, attr, val) + + def super_setattr(self, attr: str, val: object) -> None: + super().__setattr__(attr, val) + + def __getattr__(self, attr: str) -> object: + pass + +[file driver.py] +from native import SetAttr, SetAttrInherited, SetAttrOverridden, SetAttrNonNative, NoSetAttr, NoSetAttrNonNative +from testutil import assertRaises + +def test_setattr() -> None: + i = SetAttr(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_setattr_inherited() -> None: + i = SetAttrInherited(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_setattr_overridden() -> None: + i = SetAttrOverridden(99, 1, {"one": 1}) + assert i.class_var == "x" + assert i.subclass_var == "y" + assert i.regular_attr == 99 + assert i.sub_attr == 1 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + i.__setattr__("sub_attr", 2) + assert i.sub_attr == 2 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("subclass_var", "a") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + setattr(i, "sub_attr", 3) + assert i.sub_attr == 3 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "subclass_var", "b") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + i.sub_attr = 4 + assert i.sub_attr == 4 + with assertRaises(AttributeError): + i.const = 45 + + base_ref: SetAttr = i + setattr(base_ref, "sub_attr", 5) + assert base_ref.sub_attr == 5 + + base_ref.sub_attr = 6 + assert base_ref.sub_attr == 6 + + with assertRaises(AttributeError): + setattr(base_ref, "subclass_var", "c") + +def test_setattr_nonnative() -> None: + i = SetAttrNonNative(99, {"one": 1}) + assert i.class_var == "x" + assert i.regular_attr == 99 + assert i.one == 1 + assert i.two == None + assert i.const == 42 + + i.__setattr__("two", "2") + assert i.two == "2" + i.__setattr__("regular_attr", 101) + assert i.regular_attr == 101 + with assertRaises(AttributeError): + i.__setattr__("class_var", "y") + with assertRaises(AttributeError): + i.__setattr__("const", 43) + + setattr(i, "three", (3,3,3)) + assert i.three == (3,3,3) + setattr(i, "regular_attr", 102) + assert i.regular_attr == 102 + with assertRaises(AttributeError): + setattr(i, "class_var", "z") + with assertRaises(AttributeError): + setattr(i, "const", 44) + + i.four = [4,4] + assert i.four == [4,4] + i.regular_attr = 103 + assert i.regular_attr == 103 + with assertRaises(AttributeError): + i.const = 45 + +def test_no_setattr() -> None: + i = NoSetAttr(99) + i.super_setattr("attr", 100) + assert i.attr == 100 + + i.object_setattr("attr", 101) + assert i.attr == 101 + + object.__setattr__(i, "attr", 102) + assert i.attr == 102 + + with assertRaises(AttributeError): + i.super_setattr("not_attr", 100) + + with assertRaises(AttributeError): + i.object_setattr("not_attr", 101) + + with assertRaises(AttributeError): + object.__setattr__(i, "not_attr", 102) + +def test_no_setattr_nonnative() -> None: + i = NoSetAttrNonNative(99) + i.super_setattr("attr", 100) + assert i.attr == 100 + + i.object_setattr("attr", 101) + assert i.attr == 101 + + object.__setattr__(i, "attr", 102) + assert i.attr == 102 + + i.super_setattr("one", 100) + assert i.one == 100 + + i.object_setattr("two", 101) + assert i.two == 101 + + object.__setattr__(i, "three", 102) + assert i.three == 102 + +test_setattr() +test_setattr_inherited() +test_setattr_overridden() +test_setattr_nonnative() +test_no_setattr() +test_no_setattr_nonnative() [typing fixtures/typing-full.pyi] From 44bbb18db02efd86446e039fbbfa0353f7db537c Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 29 Sep 2025 10:25:02 -0400 Subject: [PATCH 053/183] [mypyc] feat: optimize away first index check in for loops if length > 1 (#19933) It is not necessary to enter the gen_condition block on the first iteration if the length is known to be positive, we can safely skip it. This won't be particularly impactful but its a low-hanging fruit. --- mypyc/irbuild/for_helpers.py | 7 ++++++- mypyc/test-data/irbuild-lists.test | 4 ++++ mypyc/test-data/irbuild-tuple.test | 4 ++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 5edee6cb4df4..db986f3fd9a7 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -183,7 +183,12 @@ def for_loop_helper_with_index( builder.push_loop_stack(step_block, exit_block) - builder.goto_and_activate(condition_block) + if isinstance(length, Integer) and length.value > 0: + builder.goto(body_block) + builder.activate_block(condition_block) + else: + builder.goto_and_activate(condition_block) + for_gen.gen_condition() builder.activate_block(body_block) diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index 2f5b3b39319e..d864bfd19df2 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -645,6 +645,7 @@ L0: r0 = 'abc' r1 = PyList_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L4 :: bool @@ -690,6 +691,7 @@ L0: r0 = 'abc' r1 = PyList_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L4 :: bool @@ -798,6 +800,7 @@ L0: r0 = b'abc' r1 = PyList_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L8 :: bool @@ -952,6 +955,7 @@ L0: r18 = CPyList_Extend(r10, r17) r19 = PyList_New(13) r20 = 0 + goto L2 L1: r21 = var_object_size r10 r22 = r20 < r21 :: signed diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 081cc1b174c9..7507b6255740 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -417,6 +417,7 @@ L0: r0 = 'abc' r1 = PyTuple_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L4 :: bool @@ -462,6 +463,7 @@ L0: r0 = 'abc' r1 = PyTuple_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L4 :: bool @@ -570,6 +572,7 @@ L0: r0 = b'abc' r1 = PyTuple_New(3) r2 = 0 + goto L2 L1: r3 = r2 < 3 :: signed if r3 goto L2 else goto L8 :: bool @@ -879,6 +882,7 @@ L0: r18 = CPyList_Extend(r10, r17) r19 = PyTuple_New(13) r20 = 0 + goto L2 L1: r21 = var_object_size r10 r22 = r20 < r21 :: signed From e0090d0ab74b4933088e94abaad0b2409e9cd50a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 30 Sep 2025 09:48:31 +0100 Subject: [PATCH 054/183] Use published version of librt (#19945) Ref https://github.com/mypyc/mypyc/issues/1128 Another step towards making fixed format cache the default one. --- mypy-requirements.txt | 1 + mypy_self_check.ini | 1 + mypyc/lib-rt/setup.py | 4 +--- pyproject.toml | 1 + setup.py | 2 -- test-requirements.txt | 2 ++ 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 8965a70c13b7..6927ddd25d81 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,3 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' +librt>=0.1.0 diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 67b65381cfd0..0b49b3de862b 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -13,3 +13,4 @@ enable_error_code = ignore-without-code,redundant-expr enable_incomplete_feature = PreciseTupleTypes show_error_code_links = True warn_unreachable = True +fixed_format_cache = True diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 3a5976cf88b2..36b55e44dcd1 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -73,8 +73,6 @@ def run(self) -> None: # TODO: we need a way to share our preferred C flags and get_extension() logic with # mypyc/build.py without code duplication. setup( - name="mypy-native", - version="0.0.1", ext_modules=[ Extension( "native_internal", @@ -88,5 +86,5 @@ def run(self) -> None: ], include_dirs=["."], ) - ], + ] ) diff --git a/pyproject.toml b/pyproject.toml index 032bfcb609e7..46593af0ab72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", + "librt>=0.1.0", # the following is from build-requirements.txt "types-psutil", "types-setuptools", diff --git a/setup.py b/setup.py index 798ff4f6c710..3f53d96dbd85 100644 --- a/setup.py +++ b/setup.py @@ -156,8 +156,6 @@ def run(self) -> None: log_trace=log_trace, # Mypy itself is allowed to use native_internal extension. depends_on_native_internal=True, - # TODO: temporary, remove this after we publish mypy-native on PyPI. - install_native_libs=True, ) else: diff --git a/test-requirements.txt b/test-requirements.txt index 521208c5aa27..5402adcb682c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,6 +22,8 @@ identify==2.6.13 # via pre-commit iniconfig==2.1.0 # via pytest +librt==0.1.0 + # via -r mypy-requirements.txt lxml==6.0.1 ; python_version < "3.15" # via -r test-requirements.in mypy-extensions==1.1.0 From a12565c8253f6b0996a9b11162776c9083aaceb8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 30 Sep 2025 12:16:23 +0100 Subject: [PATCH 055/183] Do not update states after writing cache (#19936) This is a little preparation for https://github.com/python/mypy/issues/933. We need to minimize the number of actions after writing cache, since these are things workers will need to communicate back to coordinator. Previously we updated `State.meta` after writing cache, but this looks not needed anymore, and would complicate parallel checking. Also delete couple unused attributes on `State`. --- mypy/build.py | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index ad25b811ff7c..234dd6843292 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1546,7 +1546,7 @@ def write_cache( source_hash: str, ignore_all: bool, manager: BuildManager, -) -> tuple[str, tuple[dict[str, Any], str, str] | None]: +) -> tuple[str, tuple[dict[str, Any], str] | None]: """Write cache files for a module. Note that this mypy's behavior is still correct when any given @@ -1568,7 +1568,7 @@ def write_cache( Returns: A tuple containing the interface hash and inner tuple with cache meta JSON - that should be written and paths to cache files (inner tuple may be None, + that should be written and path to cache file (inner tuple may be None, if the cache data could not be written). """ metastore = manager.metastore @@ -1662,12 +1662,10 @@ def write_cache( "ignore_all": ignore_all, "plugin_data": plugin_data, } - return interface_hash, (meta, meta_json, data_json) + return interface_hash, (meta, meta_json) -def write_cache_meta( - meta: dict[str, Any], manager: BuildManager, meta_json: str, data_json: str -) -> CacheMeta: +def write_cache_meta(meta: dict[str, Any], manager: BuildManager, meta_json: str) -> None: # Write meta cache file metastore = manager.metastore meta_str = json_dumps(meta, manager.options.debug_cache) @@ -1677,8 +1675,6 @@ def write_cache_meta( # The next run will simply find the cache entry out of date. manager.log(f"Error writing meta JSON file {meta_json}") - return cache_meta_from_dict(meta, data_json) - """Dependency manager. @@ -1864,9 +1860,6 @@ class State: # List of (path, line number) tuples giving context for import import_context: list[tuple[str, int]] - # The State from which this module was imported, if any - caller_state: State | None = None - # If caller_state is set, the line number in the caller where the import occurred caller_line = 0 @@ -1917,7 +1910,6 @@ def __init__( self.manager = manager State.order_counter += 1 self.order = State.order_counter - self.caller_state = caller_state self.caller_line = caller_line if caller_state: self.import_context = caller_state.import_context.copy() @@ -2008,11 +2000,6 @@ def __init__( self.parse_file(temporary=temporary) self.compute_dependencies() - @property - def xmeta(self) -> CacheMeta: - assert self.meta, "missing meta on allegedly fresh module" - return self.meta - def add_ancestors(self) -> None: if self.path is not None: _, name = os.path.split(self.path) @@ -2479,7 +2466,7 @@ def valid_references(self) -> set[str]: return valid_refs - def write_cache(self) -> tuple[dict[str, Any], str, str] | None: + def write_cache(self) -> tuple[dict[str, Any], str] | None: assert self.tree is not None, "Internal error: method must be called on parsed file only" # We don't support writing cache files in fine-grained incremental mode. if ( @@ -3477,14 +3464,13 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No for id in stale: meta_tuple = meta_tuples[id] if meta_tuple is None: - graph[id].meta = None continue - meta, meta_json, data_json = meta_tuple + meta, meta_json = meta_tuple meta["dep_hashes"] = { dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph } meta["error_lines"] = errors_by_id.get(id, []) - graph[id].meta = write_cache_meta(meta, manager, meta_json, data_json) + write_cache_meta(meta, manager, meta_json) def sorted_components( From bcef9f4c96b2a41283ff20963ced3b9ecb119435 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 30 Sep 2025 09:42:54 -0400 Subject: [PATCH 056/183] [mypyc] feat: support RTuple in `sequence_from_generator_preallocate_helper` (#19931) This PR extends `sequence_from_generator_preallocate_helper` to work with RTuple types. This is ready for review. --- mypyc/irbuild/for_helpers.py | 66 +++++++++------ mypyc/test-data/irbuild-tuple.test | 128 ++++++++++++++++++----------- 2 files changed, 120 insertions(+), 74 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index db986f3fd9a7..20440d4a26f4 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -28,6 +28,7 @@ TypeAlias, Var, ) +from mypy.types import LiteralType, TupleType, get_proper_type, get_proper_types from mypyc.ir.ops import ( ERR_NEVER, BasicBlock, @@ -36,6 +37,7 @@ IntOp, LoadAddress, LoadErrorValue, + LoadLiteral, LoadMem, MethodCall, RaiseStandardError, @@ -170,7 +172,7 @@ def for_loop_helper_with_index( body_insts: a function that generates the body of the loop. It needs a index as parameter. """ - assert is_sequence_rprimitive(expr_reg.type) + assert is_sequence_rprimitive(expr_reg.type), (expr_reg, expr_reg.type) target_type = builder.get_sequence_type(expr) body_block = BasicBlock() @@ -217,10 +219,9 @@ def sequence_from_generator_preallocate_helper( there is no condition list in the generator and only one original sequence with one index is allowed. - e.g. (1) tuple(f(x) for x in a_list/a_tuple/a_str/a_bytes) - (2) list(f(x) for x in a_list/a_tuple/a_str/a_bytes) - (3) [f(x) for x in a_list/a_tuple/a_str/a_bytes] - RTuple as an original sequence is not supported yet. + e.g. (1) tuple(f(x) for x in a_list/a_tuple/a_str/a_bytes/an_rtuple) + (2) list(f(x) for x in a_list/a_tuple/a_str/a_bytes/an_rtuple) + (3) [f(x) for x in a_list/a_tuple/a_str/a_bytes/an_rtuple] Args: empty_op_llbuilder: A function that can generate an empty sequence op when @@ -235,23 +236,41 @@ def sequence_from_generator_preallocate_helper( implementation. """ if len(gen.sequences) == 1 and len(gen.indices) == 1 and len(gen.condlists[0]) == 0: - rtype = builder.node_type(gen.sequences[0]) - if is_sequence_rprimitive(rtype): - sequence = builder.accept(gen.sequences[0]) - length = get_expr_length_value( - builder, gen.sequences[0], sequence, gen.line, use_pyssize_t=True - ) - target_op = empty_op_llbuilder(length, gen.line) - - def set_item(item_index: Value) -> None: - e = builder.accept(gen.left_expr) - builder.call_c(set_item_op, [target_op, item_index, e], gen.line) - - for_loop_helper_with_index( - builder, gen.indices[0], gen.sequences[0], sequence, set_item, gen.line, length - ) + line = gen.line + sequence_expr = gen.sequences[0] + rtype = builder.node_type(sequence_expr) + if not (is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple)): + return None + sequence = builder.accept(sequence_expr) + length = get_expr_length_value(builder, sequence_expr, sequence, line, use_pyssize_t=True) + if isinstance(rtype, RTuple): + # If input is RTuple, box it to tuple_rprimitive for generic iteration + # TODO: this can be optimized a bit better with an unrolled ForRTuple helper + proper_type = get_proper_type(builder.types[sequence_expr]) + assert isinstance(proper_type, TupleType), proper_type + + get_item_ops = [ + ( + LoadLiteral(typ.value, object_rprimitive) + if isinstance(typ, LiteralType) + else TupleGet(sequence, i, line) + ) + for i, typ in enumerate(get_proper_types(proper_type.items)) + ] + items = list(map(builder.add, get_item_ops)) + sequence = builder.new_tuple(items, line) + + target_op = empty_op_llbuilder(length, line) + + def set_item(item_index: Value) -> None: + e = builder.accept(gen.left_expr) + builder.call_c(set_item_op, [target_op, item_index, e], line) + + for_loop_helper_with_index( + builder, gen.indices[0], sequence_expr, sequence, set_item, line, length + ) - return target_op + return target_op return None @@ -804,7 +823,7 @@ class ForSequence(ForGenerator): def init( self, expr_reg: Value, target_type: RType, reverse: bool, length: Value | None = None ) -> None: - assert is_sequence_rprimitive(expr_reg.type), expr_reg + assert is_sequence_rprimitive(expr_reg.type), (expr_reg, expr_reg.type) builder = self.builder # Record a Value indicating the length of the sequence, if known at compile time. self.length = length @@ -834,7 +853,6 @@ def init( def gen_condition(self) -> None: builder = self.builder line = self.line - # TODO: Don't reload the length each time when iterating an immutable sequence? if self.reverse: # If we are iterating in reverse order, we obviously need # to check that the index is still positive. Somewhat less @@ -1216,7 +1234,7 @@ def get_expr_length_value( builder: IRBuilder, expr: Expression, expr_reg: Value, line: int, use_pyssize_t: bool ) -> Value: rtype = builder.node_type(expr) - assert is_sequence_rprimitive(rtype), rtype + assert is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple), rtype length = get_expr_length(expr) if length is None: # We cannot compute the length at compile time, so we will fetch it. diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 7507b6255740..3613c5f0101d 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -694,36 +694,50 @@ L0: return r1 def test(): r0, source :: tuple[int, int, int] - r1 :: list - r2, r3, r4 :: object - r5, x :: int - r6 :: bool - r7 :: object - r8 :: i32 - r9, r10 :: bit - r11, a :: tuple + r1 :: object + r2 :: native_int + r3 :: bit + r4, r5, r6 :: int + r7, r8, r9 :: object + r10, r11 :: tuple + r12 :: native_int + r13 :: bit + r14 :: object + r15, x :: int + r16 :: bool + r17 :: object + r18 :: native_int + a :: tuple L0: r0 = (2, 4, 6) source = r0 - r1 = PyList_New(0) - r2 = box(tuple[int, int, int], source) - r3 = PyObject_GetIter(r2) + r1 = box(tuple[int, int, int], source) + r2 = PyObject_Size(r1) + r3 = r2 >= 0 :: signed + r4 = source[0] + r5 = source[1] + r6 = source[2] + r7 = box(int, r4) + r8 = box(int, r5) + r9 = box(int, r6) + r10 = PyTuple_Pack(3, r7, r8, r9) + r11 = PyTuple_New(r2) + r12 = 0 L1: - r4 = PyIter_Next(r3) - if is_error(r4) goto L4 else goto L2 + r13 = r12 < r2 :: signed + if r13 goto L2 else goto L4 :: bool L2: - r5 = unbox(int, r4) - x = r5 - r6 = f(x) - r7 = box(bool, r6) - r8 = PyList_Append(r1, r7) - r9 = r8 >= 0 :: signed + r14 = CPySequenceTuple_GetItemUnsafe(r10, r12) + r15 = unbox(int, r14) + x = r15 + r16 = f(x) + r17 = box(bool, r16) + CPySequenceTuple_SetItemUnsafe(r11, r12, r17) L3: + r18 = r12 + 1 + r12 = r18 goto L1 L4: - r10 = CPy_NoErrOccurred() -L5: - r11 = PyList_AsTuple(r1) a = r11 return 1 @@ -746,42 +760,56 @@ L0: r1 = int_eq r0, 0 return r1 def test(): - r0 :: list - r1 :: tuple[int, int, int] - r2 :: bool - r3, r4, r5 :: object - r6, x :: int - r7 :: bool - r8 :: object - r9 :: i32 - r10, r11 :: bit - r12, a :: tuple -L0: - r0 = PyList_New(0) - r1 = __main__.source :: static - if is_error(r1) goto L1 else goto L2 + r0 :: tuple[int, int, int] + r1 :: bool + r2 :: object + r3 :: native_int + r4 :: bit + r5, r6, r7 :: int + r8, r9, r10 :: object + r11, r12 :: tuple + r13 :: native_int + r14 :: bit + r15 :: object + r16, x :: int + r17 :: bool + r18 :: object + r19 :: native_int + a :: tuple +L0: + r0 = __main__.source :: static + if is_error(r0) goto L1 else goto L2 L1: - r2 = raise NameError('value for final name "source" was not set') + r1 = raise NameError('value for final name "source" was not set') unreachable L2: - r3 = box(tuple[int, int, int], r1) - r4 = PyObject_GetIter(r3) + r2 = box(tuple[int, int, int], r0) + r3 = PyObject_Size(r2) + r4 = r3 >= 0 :: signed + r5 = r0[0] + r6 = r0[1] + r7 = r0[2] + r8 = box(int, r5) + r9 = box(int, r6) + r10 = box(int, r7) + r11 = PyTuple_Pack(3, r8, r9, r10) + r12 = PyTuple_New(r3) + r13 = 0 L3: - r5 = PyIter_Next(r4) - if is_error(r5) goto L6 else goto L4 + r14 = r13 < r3 :: signed + if r14 goto L4 else goto L6 :: bool L4: - r6 = unbox(int, r5) - x = r6 - r7 = f(x) - r8 = box(bool, r7) - r9 = PyList_Append(r0, r8) - r10 = r9 >= 0 :: signed + r15 = CPySequenceTuple_GetItemUnsafe(r11, r13) + r16 = unbox(int, r15) + x = r16 + r17 = f(x) + r18 = box(bool, r17) + CPySequenceTuple_SetItemUnsafe(r12, r13, r18) L5: + r19 = r13 + 1 + r13 = r19 goto L3 L6: - r11 = CPy_NoErrOccurred() -L7: - r12 = PyList_AsTuple(r0) a = r12 return 1 From 6feecce803fc7ffcb6b68405205703e04874cb99 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 30 Sep 2025 09:47:20 -0400 Subject: [PATCH 057/183] [mypyc] feat: specialize isinstance for tuple of primitive types (#19949) This PR specializes isinstance calls where the type argument is a tuple of primitive types. We can skip tuple creation and the associated refcounting, and daisy-chain the primitive checks with an early exit option at each step. --- mypyc/irbuild/specialize.py | 72 ++++++++++++++++++++++--- mypyc/test-data/irbuild-isinstance.test | 28 ++++++++++ mypyc/test-data/run-misc.test | 23 ++++++++ 3 files changed, 115 insertions(+), 8 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 42b7710c98da..84807a7fdb53 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Callable, Final, Optional +from typing import Callable, Final, Optional, cast from mypy.nodes import ( ARG_NAMED, @@ -40,6 +40,7 @@ Call, Extend, Integer, + PrimitiveDescription, RaiseStandardError, Register, SetAttr, @@ -589,26 +590,81 @@ def translate_isinstance(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> if not (len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]): return None - if isinstance(expr.args[1], (RefExpr, TupleExpr)): - builder.types[expr.args[0]] = AnyType(TypeOfAny.from_error) + obj_expr = expr.args[0] + type_expr = expr.args[1] - irs = builder.flatten_classes(expr.args[1]) + if isinstance(type_expr, TupleExpr) and not type_expr.items: + # we can compile this case to a noop + return builder.false() + + if isinstance(type_expr, (RefExpr, TupleExpr)): + builder.types[obj_expr] = AnyType(TypeOfAny.from_error) + + irs = builder.flatten_classes(type_expr) if irs is not None: can_borrow = all( ir.is_ext_class and not ir.inherits_python and not ir.allow_interpreted_subclasses for ir in irs ) - obj = builder.accept(expr.args[0], can_borrow=can_borrow) + obj = builder.accept(obj_expr, can_borrow=can_borrow) return builder.builder.isinstance_helper(obj, irs, expr.line) - if isinstance(expr.args[1], RefExpr): - node = expr.args[1].node + if isinstance(type_expr, RefExpr): + node = type_expr.node if node: desc = isinstance_primitives.get(node.fullname) if desc: - obj = builder.accept(expr.args[0]) + obj = builder.accept(obj_expr) return builder.primitive_op(desc, [obj], expr.line) + elif isinstance(type_expr, TupleExpr): + node_names: list[str] = [] + for item in type_expr.items: + if not isinstance(item, RefExpr): + return None + if item.node is None: + return None + if item.node.fullname not in node_names: + node_names.append(item.node.fullname) + + descs = [isinstance_primitives.get(fullname) for fullname in node_names] + if None in descs: + # not all types are primitive types, abort + return None + + obj = builder.accept(obj_expr) + + retval = Register(bool_rprimitive) + pass_block = BasicBlock() + fail_block = BasicBlock() + exit_block = BasicBlock() + + # Chain the checks: if any succeed, jump to pass_block; else, continue + for i, desc in enumerate(descs): + is_last = i == len(descs) - 1 + next_block = fail_block if is_last else BasicBlock() + builder.add_bool_branch( + builder.primitive_op(cast(PrimitiveDescription, desc), [obj], expr.line), + pass_block, + next_block, + ) + if not is_last: + builder.activate_block(next_block) + + # If any check passed + builder.activate_block(pass_block) + builder.assign(retval, builder.true(), expr.line) + builder.goto(exit_block) + + # If all checks failed + builder.activate_block(fail_block) + builder.assign(retval, builder.false(), expr.line) + builder.goto(exit_block) + + # Return the result + builder.activate_block(exit_block) + return retval + return None diff --git a/mypyc/test-data/irbuild-isinstance.test b/mypyc/test-data/irbuild-isinstance.test index 0df9448b819f..36a9300350bd 100644 --- a/mypyc/test-data/irbuild-isinstance.test +++ b/mypyc/test-data/irbuild-isinstance.test @@ -189,3 +189,31 @@ def is_tuple(x): L0: r0 = PyTuple_Check(x) return r0 + +[case testTupleOfPrimitives] +from typing import Any + +def is_instance(x: Any) -> bool: + return isinstance(x, (str, int, bytes)) + +[out] +def is_instance(x): + x :: object + r0, r1, r2 :: bit + r3 :: bool +L0: + r0 = PyUnicode_Check(x) + if r0 goto L3 else goto L1 :: bool +L1: + r1 = PyLong_Check(x) + if r1 goto L3 else goto L2 :: bool +L2: + r2 = PyBytes_Check(x) + if r2 goto L3 else goto L4 :: bool +L3: + r3 = 1 + goto L5 +L4: + r3 = 0 +L5: + return r3 diff --git a/mypyc/test-data/run-misc.test b/mypyc/test-data/run-misc.test index 129946a4c330..1074906357ee 100644 --- a/mypyc/test-data/run-misc.test +++ b/mypyc/test-data/run-misc.test @@ -1173,3 +1173,26 @@ def test_dummy_context() -> None: with c: assert c.c == 1 assert c.c == 0 + +[case testIsInstanceTuple] +from typing import Any + +def isinstance_empty(x: Any) -> bool: + return isinstance(x, ()) +def isinstance_single(x: Any) -> bool: + return isinstance(x, (str,)) +def isinstance_multi(x: Any) -> bool: + return isinstance(x, (str, int)) + +def test_isinstance_empty() -> None: + assert isinstance_empty("a") is False + assert isinstance_empty(1) is False + assert isinstance_empty(None) is False +def test_isinstance_single() -> None: + assert isinstance_single("a") is True + assert isinstance_single(1) is False + assert isinstance_single(None) is False +def test_isinstance_multi() -> None: + assert isinstance_multi("a") is True + assert isinstance_multi(1) is True + assert isinstance_multi(None) is False From 4936099b126f4c982c68b25208c434959d1602da Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 30 Sep 2025 22:49:45 -0700 Subject: [PATCH 058/183] Sync typeshed (#19959) Source commit: https://github.com/python/typeshed/commit/91055c730ffcda6311654cf32d663858ece69bad --- mypy/typeshed/stdlib/_collections_abc.pyi | 9 +- mypy/typeshed/stdlib/_frozen_importlib.pyi | 2 +- mypy/typeshed/stdlib/_tkinter.pyi | 40 +- mypy/typeshed/stdlib/asyncio/tools.pyi | 5 + mypy/typeshed/stdlib/builtins.pyi | 2 +- .../concurrent/interpreters/_queues.pyi | 34 +- mypy/typeshed/stdlib/configparser.pyi | 6 +- mypy/typeshed/stdlib/copy.pyi | 12 +- mypy/typeshed/stdlib/curses/__init__.pyi | 5 +- mypy/typeshed/stdlib/dbm/sqlite3.pyi | 2 +- mypy/typeshed/stdlib/email/headerregistry.pyi | 2 +- mypy/typeshed/stdlib/errno.pyi | 1 + mypy/typeshed/stdlib/faulthandler.pyi | 8 +- .../stdlib/importlib/resources/_common.pyi | 2 +- .../stdlib/multiprocessing/shared_memory.pyi | 2 +- mypy/typeshed/stdlib/tkinter/__init__.pyi | 1372 ++++++++--------- mypy/typeshed/stdlib/tkinter/ttk.pyi | 326 ++-- mypy/typeshed/stdlib/types.pyi | 36 +- mypy/typeshed/stdlib/typing.pyi | 9 +- 19 files changed, 948 insertions(+), 927 deletions(-) diff --git a/mypy/typeshed/stdlib/_collections_abc.pyi b/mypy/typeshed/stdlib/_collections_abc.pyi index c63606a13ca9..319577c9284b 100644 --- a/mypy/typeshed/stdlib/_collections_abc.pyi +++ b/mypy/typeshed/stdlib/_collections_abc.pyi @@ -1,12 +1,13 @@ import sys from abc import abstractmethod from types import MappingProxyType -from typing import ( # noqa: Y022,Y038,UP035 +from typing import ( # noqa: Y022,Y038,UP035,Y057 AbstractSet as Set, AsyncGenerator as AsyncGenerator, AsyncIterable as AsyncIterable, AsyncIterator as AsyncIterator, Awaitable as Awaitable, + ByteString as ByteString, Callable as Callable, ClassVar, Collection as Collection, @@ -59,12 +60,8 @@ __all__ = [ "ValuesView", "Sequence", "MutableSequence", + "ByteString", ] -if sys.version_info < (3, 14): - from typing import ByteString as ByteString # noqa: Y057,UP035 - - __all__ += ["ByteString"] - if sys.version_info >= (3, 12): __all__ += ["Buffer"] diff --git a/mypy/typeshed/stdlib/_frozen_importlib.pyi b/mypy/typeshed/stdlib/_frozen_importlib.pyi index 93aaed82e2e1..58db64a016f3 100644 --- a/mypy/typeshed/stdlib/_frozen_importlib.pyi +++ b/mypy/typeshed/stdlib/_frozen_importlib.pyi @@ -13,7 +13,7 @@ def __import__( name: str, globals: Mapping[str, object] | None = None, locals: Mapping[str, object] | None = None, - fromlist: Sequence[str] = (), + fromlist: Sequence[str] | None = (), level: int = 0, ) -> ModuleType: ... def spec_from_loader( diff --git a/mypy/typeshed/stdlib/_tkinter.pyi b/mypy/typeshed/stdlib/_tkinter.pyi index 46366ccc1740..a3868f467c6c 100644 --- a/mypy/typeshed/stdlib/_tkinter.pyi +++ b/mypy/typeshed/stdlib/_tkinter.pyi @@ -54,34 +54,34 @@ _TkinterTraceFunc: TypeAlias = Callable[[tuple[str, ...]], object] @final class TkappType: # Please keep in sync with tkinter.Tk - def adderrorinfo(self, msg, /): ... + def adderrorinfo(self, msg: str, /): ... def call(self, command: Any, /, *args: Any) -> Any: ... - def createcommand(self, name, func, /): ... + def createcommand(self, name: str, func, /): ... if sys.platform != "win32": - def createfilehandler(self, file, mask, func, /): ... - def deletefilehandler(self, file, /): ... + def createfilehandler(self, file, mask: int, func, /): ... + def deletefilehandler(self, file, /) -> None: ... - def createtimerhandler(self, milliseconds, func, /): ... - def deletecommand(self, name, /): ... + def createtimerhandler(self, milliseconds: int, func, /): ... + def deletecommand(self, name: str, /): ... def dooneevent(self, flags: int = 0, /): ... def eval(self, script: str, /) -> str: ... - def evalfile(self, fileName, /): ... - def exprboolean(self, s, /): ... - def exprdouble(self, s, /): ... - def exprlong(self, s, /): ... - def exprstring(self, s, /): ... - def getboolean(self, arg, /): ... - def getdouble(self, arg, /): ... - def getint(self, arg, /): ... + def evalfile(self, fileName: str, /): ... + def exprboolean(self, s: str, /): ... + def exprdouble(self, s: str, /): ... + def exprlong(self, s: str, /): ... + def exprstring(self, s: str, /): ... + def getboolean(self, arg, /) -> bool: ... + def getdouble(self, arg, /) -> float: ... + def getint(self, arg, /) -> int: ... def getvar(self, *args, **kwargs): ... def globalgetvar(self, *args, **kwargs): ... def globalsetvar(self, *args, **kwargs): ... def globalunsetvar(self, *args, **kwargs): ... def interpaddr(self) -> int: ... def loadtk(self) -> None: ... - def mainloop(self, threshold: int = 0, /): ... - def quit(self): ... - def record(self, script, /): ... + def mainloop(self, threshold: int = 0, /) -> None: ... + def quit(self) -> None: ... + def record(self, script: str, /): ... def setvar(self, *ags, **kwargs): ... if sys.version_info < (3, 11): @deprecated("Deprecated since Python 3.9; removed in Python 3.11. Use `splitlist()` instead.") @@ -90,7 +90,7 @@ class TkappType: def splitlist(self, arg, /): ... def unsetvar(self, *args, **kwargs): ... def wantobjects(self, *args, **kwargs): ... - def willdispatch(self): ... + def willdispatch(self) -> None: ... if sys.version_info >= (3, 12): def gettrace(self, /) -> _TkinterTraceFunc | None: ... def settrace(self, func: _TkinterTraceFunc | None, /) -> None: ... @@ -140,5 +140,5 @@ else: /, ): ... -def getbusywaitinterval(): ... -def setbusywaitinterval(new_val, /): ... +def getbusywaitinterval() -> int: ... +def setbusywaitinterval(new_val: int, /) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/tools.pyi b/mypy/typeshed/stdlib/asyncio/tools.pyi index 65c7f27e0b85..bc8b809b9c05 100644 --- a/mypy/typeshed/stdlib/asyncio/tools.pyi +++ b/mypy/typeshed/stdlib/asyncio/tools.pyi @@ -1,3 +1,4 @@ +import sys from collections.abc import Iterable from enum import Enum from typing import NamedTuple, SupportsIndex, type_check_only @@ -37,5 +38,9 @@ class CycleFoundException(Exception): def get_all_awaited_by(pid: SupportsIndex) -> list[_AwaitedInfo]: ... def build_async_tree(result: Iterable[_AwaitedInfo], task_emoji: str = "(T)", cor_emoji: str = "") -> list[list[str]]: ... def build_task_table(result: Iterable[_AwaitedInfo]) -> list[list[int | str]]: ... + +if sys.version_info >= (3, 14): + def exit_with_permission_help_text() -> None: ... + def display_awaited_by_tasks_table(pid: SupportsIndex) -> None: ... def display_awaited_by_tasks_tree(pid: SupportsIndex) -> None: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index ef6c712e0005..e276441523c8 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -1917,7 +1917,7 @@ def __import__( name: str, globals: Mapping[str, object] | None = None, locals: Mapping[str, object] | None = None, - fromlist: Sequence[str] = (), + fromlist: Sequence[str] | None = (), level: int = 0, ) -> types.ModuleType: ... def __build_class__(func: Callable[[], CellType | Any], name: str, /, *bases: Any, metaclass: Any = ..., **kwds: Any) -> Any: ... diff --git a/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi b/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi index 7493f87809c8..bdf08d93d1e0 100644 --- a/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi +++ b/mypy/typeshed/stdlib/concurrent/interpreters/_queues.pyi @@ -45,14 +45,30 @@ if sys.version_info >= (3, 13): # needed to satisfy pyright checks for Python < def empty(self) -> bool: ... def full(self) -> bool: ... def qsize(self) -> int: ... - def put( - self, - obj: object, - timeout: SupportsIndex | None = None, - *, - unbounditems: _AnyUnbound | None = None, - _delay: float = 0.01, - ) -> None: ... + if sys.version_info >= (3, 14): + def put( + self, + obj: object, + block: bool = True, + timeout: SupportsIndex | None = None, + *, + unbounditems: _AnyUnbound | None = None, + _delay: float = 0.01, + ) -> None: ... + else: + def put( + self, + obj: object, + timeout: SupportsIndex | None = None, + *, + unbounditems: _AnyUnbound | None = None, + _delay: float = 0.01, + ) -> None: ... + def put_nowait(self, obj: object, *, unbounditems: _AnyUnbound | None = None) -> None: ... - def get(self, timeout: SupportsIndex | None = None, *, _delay: float = 0.01) -> object: ... + if sys.version_info >= (3, 14): + def get(self, block: bool = True, timeout: SupportsIndex | None = None, *, _delay: float = 0.01) -> object: ... + else: + def get(self, timeout: SupportsIndex | None = None, *, _delay: float = 0.01) -> object: ... + def get_nowait(self) -> object: ... diff --git a/mypy/typeshed/stdlib/configparser.pyi b/mypy/typeshed/stdlib/configparser.pyi index 764a8a965ea2..1909d80e3d18 100644 --- a/mypy/typeshed/stdlib/configparser.pyi +++ b/mypy/typeshed/stdlib/configparser.pyi @@ -258,9 +258,9 @@ class RawConfigParser(_Parser): ) -> None: ... def __len__(self) -> int: ... - def __getitem__(self, key: str) -> SectionProxy: ... - def __setitem__(self, key: str, value: _Section) -> None: ... - def __delitem__(self, key: str) -> None: ... + def __getitem__(self, key: _SectionName) -> SectionProxy: ... + def __setitem__(self, key: _SectionName, value: _Section) -> None: ... + def __delitem__(self, key: _SectionName) -> None: ... def __iter__(self) -> Iterator[str]: ... def __contains__(self, key: object) -> bool: ... def defaults(self) -> _Section: ... diff --git a/mypy/typeshed/stdlib/copy.pyi b/mypy/typeshed/stdlib/copy.pyi index 10d2f0ae3710..373899ea2635 100644 --- a/mypy/typeshed/stdlib/copy.pyi +++ b/mypy/typeshed/stdlib/copy.pyi @@ -1,16 +1,15 @@ import sys from typing import Any, Protocol, TypeVar, type_check_only -from typing_extensions import Self __all__ = ["Error", "copy", "deepcopy"] _T = TypeVar("_T") -_SR = TypeVar("_SR", bound=_SupportsReplace) +_RT_co = TypeVar("_RT_co", covariant=True) @type_check_only -class _SupportsReplace(Protocol): - # In reality doesn't support args, but there's no other great way to express this. - def __replace__(self, *args: Any, **kwargs: Any) -> Self: ... +class _SupportsReplace(Protocol[_RT_co]): + # In reality doesn't support args, but there's no great way to express this. + def __replace__(self, /, *_: Any, **changes: Any) -> _RT_co: ... # None in CPython but non-None in Jython PyStringMap: Any @@ -21,7 +20,8 @@ def copy(x: _T) -> _T: ... if sys.version_info >= (3, 13): __all__ += ["replace"] - def replace(obj: _SR, /, **changes: Any) -> _SR: ... + # The types accepted by `**changes` match those of `obj.__replace__`. + def replace(obj: _SupportsReplace[_RT_co], /, **changes: Any) -> _RT_co: ... class Error(Exception): ... diff --git a/mypy/typeshed/stdlib/curses/__init__.pyi b/mypy/typeshed/stdlib/curses/__init__.pyi index 2c0231c13087..3e32487ad99f 100644 --- a/mypy/typeshed/stdlib/curses/__init__.pyi +++ b/mypy/typeshed/stdlib/curses/__init__.pyi @@ -14,8 +14,9 @@ _T = TypeVar("_T") _P = ParamSpec("_P") # available after calling `curses.initscr()` -LINES: Final[int] -COLS: Final[int] +# not `Final` as it can change during the terminal resize: +LINES: int +COLS: int # available after calling `curses.start_color()` COLORS: Final[int] diff --git a/mypy/typeshed/stdlib/dbm/sqlite3.pyi b/mypy/typeshed/stdlib/dbm/sqlite3.pyi index 446a0cf155fa..e2fba93b2001 100644 --- a/mypy/typeshed/stdlib/dbm/sqlite3.pyi +++ b/mypy/typeshed/stdlib/dbm/sqlite3.pyi @@ -26,4 +26,4 @@ class _Database(MutableMapping[bytes, bytes]): def __enter__(self) -> Self: ... def __exit__(self, *args: Unused) -> None: ... -def open(filename: StrOrBytesPath, /, flag: Literal["r", "w,", "c", "n"] = "r", mode: int = 0o666) -> _Database: ... +def open(filename: StrOrBytesPath, /, flag: Literal["r", "w", "c", "n"] = "r", mode: int = 0o666) -> _Database: ... diff --git a/mypy/typeshed/stdlib/email/headerregistry.pyi b/mypy/typeshed/stdlib/email/headerregistry.pyi index dff9593b731f..bea68307e009 100644 --- a/mypy/typeshed/stdlib/email/headerregistry.pyi +++ b/mypy/typeshed/stdlib/email/headerregistry.pyi @@ -41,7 +41,7 @@ class DateHeader: max_count: ClassVar[Literal[1] | None] def init(self, name: str, *, parse_tree: TokenList, defects: Iterable[MessageDefect], datetime: _datetime) -> None: ... @property - def datetime(self) -> _datetime: ... + def datetime(self) -> _datetime | None: ... @staticmethod def value_parser(value: str) -> UnstructuredTokenList: ... @classmethod diff --git a/mypy/typeshed/stdlib/errno.pyi b/mypy/typeshed/stdlib/errno.pyi index 4f19b5aee87e..e025e1fd13b9 100644 --- a/mypy/typeshed/stdlib/errno.pyi +++ b/mypy/typeshed/stdlib/errno.pyi @@ -121,6 +121,7 @@ if sys.platform == "darwin": ESHLIBVERS: Final[int] if sys.version_info >= (3, 11): EQFULL: Final[int] + ENOTCAPABLE: Final[int] # available starting with 3.11.1 if sys.platform != "darwin": EDEADLOCK: Final[int] diff --git a/mypy/typeshed/stdlib/faulthandler.pyi b/mypy/typeshed/stdlib/faulthandler.pyi index 8f93222c9936..33d08995eb75 100644 --- a/mypy/typeshed/stdlib/faulthandler.pyi +++ b/mypy/typeshed/stdlib/faulthandler.pyi @@ -9,7 +9,13 @@ if sys.version_info >= (3, 14): def dump_c_stack(file: FileDescriptorLike = ...) -> None: ... def dump_traceback_later(timeout: float, repeat: bool = ..., file: FileDescriptorLike = ..., exit: bool = ...) -> None: ... -def enable(file: FileDescriptorLike = ..., all_threads: bool = ...) -> None: ... + +if sys.version_info >= (3, 14): + def enable(file: FileDescriptorLike = ..., all_threads: bool = ..., c_stack: bool = True) -> None: ... + +else: + def enable(file: FileDescriptorLike = ..., all_threads: bool = ...) -> None: ... + def is_enabled() -> bool: ... if sys.platform != "win32": diff --git a/mypy/typeshed/stdlib/importlib/resources/_common.pyi b/mypy/typeshed/stdlib/importlib/resources/_common.pyi index 3dd961bb657b..11a93ca82d8d 100644 --- a/mypy/typeshed/stdlib/importlib/resources/_common.pyi +++ b/mypy/typeshed/stdlib/importlib/resources/_common.pyi @@ -21,7 +21,7 @@ if sys.version_info >= (3, 11): @overload def files(anchor: Anchor | None = None) -> Traversable: ... @overload - @deprecated("First parameter to files is renamed to 'anchor'") + @deprecated("Deprecated since Python 3.12; will be removed in Python 3.15. Use `anchor` parameter instead.") def files(package: Anchor | None = None) -> Traversable: ... else: diff --git a/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi b/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi index 1a12812c27e4..f75a372a69a2 100644 --- a/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi @@ -15,7 +15,7 @@ class SharedMemory: def __init__(self, name: str | None = None, create: bool = False, size: int = 0) -> None: ... @property - def buf(self) -> memoryview: ... + def buf(self) -> memoryview | None: ... @property def name(self) -> str: ... @property diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index 54dd70baf199..b653545f1d9c 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -173,21 +173,8 @@ EXCEPTION: Final = _tkinter.EXCEPTION # # You can also read the manual pages online: https://www.tcl.tk/doc/ -# Some widgets have an option named -compound that accepts different values -# than the _Compound defined here. Many other options have similar things. -_Anchor: TypeAlias = Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] # manual page: Tk_GetAnchor -_ButtonCommand: TypeAlias = str | Callable[[], Any] # accepts string of tcl code, return value is returned from Button.invoke() -_Compound: TypeAlias = Literal["top", "left", "center", "right", "bottom", "none"] # -compound in manual page named 'options' # manual page: Tk_GetCursor _Cursor: TypeAlias = str | tuple[str] | tuple[str, str] | tuple[str, str, str] | tuple[str, str, str, str] -# example when it's sequence: entry['invalidcommand'] = [entry.register(print), '%P'] -_EntryValidateCommand: TypeAlias = str | list[str] | tuple[str, ...] | Callable[[], bool] -_ImageSpec: TypeAlias = _Image | str # str can be from e.g. tkinter.image_names() -_Relief: TypeAlias = Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] # manual page: Tk_GetRelief -_ScreenUnits: TypeAlias = str | float # Often the right type instead of int. Manual page: Tk_GetPixels -# -xscrollcommand and -yscrollcommand in 'options' manual page -_XYScrollCommand: TypeAlias = str | Callable[[float, float], object] -_TakeFocusValue: TypeAlias = bool | Literal[0, 1, ""] | Callable[[str], bool | None] # -takefocus in manual page named 'options' if sys.version_info >= (3, 11): @type_check_only @@ -333,12 +320,12 @@ class Variable: @deprecated("Deprecated since Python 3.14. Use `trace_remove()` instead.") def trace_vdelete(self, mode, cbname) -> None: ... @deprecated("Deprecated since Python 3.14. Use `trace_info()` instead.") - def trace_vinfo(self): ... + def trace_vinfo(self) -> list[Incomplete]: ... else: def trace(self, mode, callback) -> str: ... def trace_variable(self, mode, callback) -> str: ... def trace_vdelete(self, mode, cbname) -> None: ... - def trace_vinfo(self): ... + def trace_vinfo(self) -> list[Incomplete]: ... def __eq__(self, other: object) -> bool: ... def __del__(self) -> None: ... @@ -373,14 +360,14 @@ def mainloop(n: int = 0) -> None: ... getint = int getdouble = float -def getboolean(s): ... +def getboolean(s) -> bool: ... _Ts = TypeVarTuple("_Ts") @type_check_only class _GridIndexInfo(TypedDict, total=False): - minsize: _ScreenUnits - pad: _ScreenUnits + minsize: float | str + pad: float | str uniform: str | None weight: int @@ -403,9 +390,9 @@ class Misc: def wait_visibility(self, window: Misc | None = None) -> None: ... def setvar(self, name: str = "PY_VAR", value: str = "1") -> None: ... def getvar(self, name: str = "PY_VAR"): ... - def getint(self, s): ... - def getdouble(self, s): ... - def getboolean(self, s): ... + def getint(self, s) -> int: ... + def getdouble(self, s) -> float: ... + def getboolean(self, s) -> bool: ... def focus_set(self) -> None: ... focus = focus_set def focus_force(self) -> None: ... @@ -473,13 +460,13 @@ class Misc: def winfo_atom(self, name: str, displayof: Literal[0] | Misc | None = 0) -> int: ... def winfo_atomname(self, id: int, displayof: Literal[0] | Misc | None = 0) -> str: ... def winfo_cells(self) -> int: ... - def winfo_children(self) -> list[Widget]: ... # Widget because it can't be Toplevel or Tk + def winfo_children(self) -> list[Widget | Toplevel]: ... def winfo_class(self) -> str: ... def winfo_colormapfull(self) -> bool: ... def winfo_containing(self, rootX: int, rootY: int, displayof: Literal[0] | Misc | None = 0) -> Misc | None: ... def winfo_depth(self) -> int: ... def winfo_exists(self) -> bool: ... - def winfo_fpixels(self, number: _ScreenUnits) -> float: ... + def winfo_fpixels(self, number: float | str) -> float: ... def winfo_geometry(self) -> str: ... def winfo_height(self) -> int: ... def winfo_id(self) -> int: ... @@ -489,7 +476,7 @@ class Misc: def winfo_name(self) -> str: ... def winfo_parent(self) -> str: ... # return value needs nametowidget() def winfo_pathname(self, id: int, displayof: Literal[0] | Misc | None = 0): ... - def winfo_pixels(self, number: _ScreenUnits) -> int: ... + def winfo_pixels(self, number: float | str) -> int: ... def winfo_pointerx(self) -> int: ... def winfo_pointerxy(self) -> tuple[int, int]: ... def winfo_pointery(self) -> int: ... @@ -580,7 +567,7 @@ class Misc: @overload def pack_propagate(self) -> None: ... propagate = pack_propagate - def grid_anchor(self, anchor: _Anchor | None = None) -> None: ... + def grid_anchor(self, anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] | None = None) -> None: ... anchor = grid_anchor @overload def grid_bbox( @@ -596,8 +583,8 @@ class Misc: index: int | str | list[int] | tuple[int, ...], cnf: _GridIndexInfo = {}, *, - minsize: _ScreenUnits = ..., - pad: _ScreenUnits = ..., + minsize: float | str = ..., + pad: float | str = ..., uniform: str = ..., weight: int = ..., ) -> _GridIndexInfo | MaybeNone: ... # can be None but annoying to check @@ -606,14 +593,14 @@ class Misc: index: int | str | list[int] | tuple[int, ...], cnf: _GridIndexInfo = {}, *, - minsize: _ScreenUnits = ..., - pad: _ScreenUnits = ..., + minsize: float | str = ..., + pad: float | str = ..., uniform: str = ..., weight: int = ..., ) -> _GridIndexInfo | MaybeNone: ... # can be None but annoying to check columnconfigure = grid_columnconfigure rowconfigure = grid_rowconfigure - def grid_location(self, x: _ScreenUnits, y: _ScreenUnits) -> tuple[int, int]: ... + def grid_location(self, x: float | str, y: float | str) -> tuple[int, int]: ... @overload def grid_propagate(self, flag: bool) -> None: ... @overload @@ -632,32 +619,32 @@ class Misc: sequence: str, *, above: Misc | int = ..., - borderwidth: _ScreenUnits = ..., + borderwidth: float | str = ..., button: int = ..., count: int = ..., data: Any = ..., # anything with usable str() value delta: int = ..., detail: str = ..., focus: bool = ..., - height: _ScreenUnits = ..., + height: float | str = ..., keycode: int = ..., keysym: str = ..., mode: str = ..., override: bool = ..., place: Literal["PlaceOnTop", "PlaceOnBottom"] = ..., root: Misc | int = ..., - rootx: _ScreenUnits = ..., - rooty: _ScreenUnits = ..., + rootx: float | str = ..., + rooty: float | str = ..., sendevent: bool = ..., serial: int = ..., state: int | str = ..., subwindow: Misc | int = ..., time: int = ..., warp: bool = ..., - width: _ScreenUnits = ..., + width: float | str = ..., when: Literal["now", "tail", "head", "mark"] = ..., - x: _ScreenUnits = ..., - y: _ScreenUnits = ..., + x: float | str = ..., + y: float | str = ..., ) -> None: ... def event_info(self, virtual: str | None = None) -> tuple[str, ...]: ... def image_names(self) -> tuple[str, ...]: ... @@ -681,23 +668,23 @@ class XView: @overload def xview(self) -> tuple[float, float]: ... @overload - def xview(self, *args): ... + def xview(self, *args) -> None: ... def xview_moveto(self, fraction: float) -> None: ... @overload def xview_scroll(self, number: int, what: Literal["units", "pages"]) -> None: ... @overload - def xview_scroll(self, number: _ScreenUnits, what: Literal["pixels"]) -> None: ... + def xview_scroll(self, number: float | str, what: Literal["pixels"]) -> None: ... class YView: @overload def yview(self) -> tuple[float, float]: ... @overload - def yview(self, *args): ... + def yview(self, *args) -> None: ... def yview_moveto(self, fraction: float) -> None: ... @overload def yview_scroll(self, number: int, what: Literal["units", "pages"]) -> None: ... @overload - def yview_scroll(self, number: _ScreenUnits, what: Literal["pixels"]) -> None: ... + def yview_scroll(self, number: float | str, what: Literal["pixels"]) -> None: ... if sys.platform == "darwin": @type_check_only @@ -993,21 +980,21 @@ class Tk(Misc, Wm): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., menu: Menu = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., - takefocus: _TakeFocusValue = ..., - width: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1018,27 +1005,27 @@ class Tk(Misc, Wm): # Tk has __getattr__ so that tk_instance.foo falls back to tk_instance.tk.foo # Please keep in sync with _tkinter.TkappType. # Some methods are intentionally missing because they are inherited from Misc instead. - def adderrorinfo(self, msg, /): ... + def adderrorinfo(self, msg: str, /): ... def call(self, command: Any, /, *args: Any) -> Any: ... - def createcommand(self, name, func, /): ... + def createcommand(self, name: str, func, /): ... if sys.platform != "win32": - def createfilehandler(self, file, mask, func, /): ... - def deletefilehandler(self, file, /): ... + def createfilehandler(self, file, mask: int, func, /): ... + def deletefilehandler(self, file, /) -> None: ... - def createtimerhandler(self, milliseconds, func, /): ... - def dooneevent(self, flags: int = ..., /): ... + def createtimerhandler(self, milliseconds: int, func, /): ... + def dooneevent(self, flags: int = 0, /): ... def eval(self, script: str, /) -> str: ... - def evalfile(self, fileName, /): ... - def exprboolean(self, s, /): ... - def exprdouble(self, s, /): ... - def exprlong(self, s, /): ... - def exprstring(self, s, /): ... + def evalfile(self, fileName: str, /): ... + def exprboolean(self, s: str, /): ... + def exprdouble(self, s: str, /): ... + def exprlong(self, s: str, /): ... + def exprstring(self, s: str, /): ... def globalgetvar(self, *args, **kwargs): ... def globalsetvar(self, *args, **kwargs): ... def globalunsetvar(self, *args, **kwargs): ... def interpaddr(self) -> int: ... def loadtk(self) -> None: ... - def record(self, script, /): ... + def record(self, script: str, /): ... if sys.version_info < (3, 11): @deprecated("Deprecated since Python 3.9; removed in Python 3.11. Use `splitlist()` instead.") def split(self, arg, /): ... @@ -1046,7 +1033,7 @@ class Tk(Misc, Wm): def splitlist(self, arg, /): ... def unsetvar(self, *args, **kwargs): ... def wantobjects(self, *args, **kwargs): ... - def willdispatch(self): ... + def willdispatch(self) -> None: ... def Tcl(screenName: str | None = None, baseName: str | None = None, className: str = "Tk", useTk: bool = False) -> Tk: ... @@ -1056,11 +1043,11 @@ _InMiscNonTotal = TypedDict("_InMiscNonTotal", {"in": Misc}, total=False) @type_check_only class _PackInfo(_InMiscTotal): # 'before' and 'after' never appear in _PackInfo - anchor: _Anchor + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] expand: bool fill: Literal["none", "x", "y", "both"] side: Literal["left", "right", "top", "bottom"] - # Paddings come out as int or tuple of int, even though any _ScreenUnits + # Paddings come out as int or tuple of int, even though any screen units # can be specified in pack(). ipadx: int ipady: int @@ -1069,7 +1056,7 @@ class _PackInfo(_InMiscTotal): class Pack: # _PackInfo is not the valid type for cnf because pad stuff accepts any - # _ScreenUnits instead of int only. I didn't bother to create another + # screen units instead of int only. I didn't bother to create another # TypedDict for cnf because it appears to be a legacy thing that was # replaced by **kwargs. def pack_configure( @@ -1077,15 +1064,15 @@ class Pack: cnf: Mapping[str, Any] | None = {}, *, after: Misc = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., before: Misc = ..., expand: bool | Literal[0, 1] = 0, fill: Literal["none", "x", "y", "both"] = ..., side: Literal["left", "right", "top", "bottom"] = ..., - ipadx: _ScreenUnits = ..., - ipady: _ScreenUnits = ..., - padx: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., - pady: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., + ipadx: float | str = ..., + ipady: float | str = ..., + padx: float | str | tuple[float | str, float | str] = ..., + pady: float | str | tuple[float | str, float | str] = ..., in_: Misc = ..., **kw: Any, # allow keyword argument named 'in', see #4836 ) -> None: ... @@ -1097,7 +1084,7 @@ class Pack: @type_check_only class _PlaceInfo(_InMiscNonTotal): # empty dict if widget hasn't been placed - anchor: _Anchor + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] bordermode: Literal["inside", "outside", "ignore"] width: str # can be int()ed (even after e.g. widget.place(height='2.3c') or similar) height: str # can be int()ed @@ -1113,12 +1100,12 @@ class Place: self, cnf: Mapping[str, Any] | None = {}, *, - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., bordermode: Literal["inside", "outside", "ignore"] = ..., - width: _ScreenUnits = ..., - height: _ScreenUnits = ..., - x: _ScreenUnits = ..., - y: _ScreenUnits = ..., + width: float | str = ..., + height: float | str = ..., + x: float | str = ..., + y: float | str = ..., # str allowed for compatibility with place_info() relheight: str | float = ..., relwidth: str | float = ..., @@ -1153,10 +1140,10 @@ class Grid: columnspan: int = ..., row: int = ..., rowspan: int = ..., - ipadx: _ScreenUnits = ..., - ipady: _ScreenUnits = ..., - padx: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., - pady: _ScreenUnits | tuple[_ScreenUnits, _ScreenUnits] = ..., + ipadx: float | str = ..., + ipady: float | str = ..., + padx: float | str | tuple[float | str, float | str] = ..., + pady: float | str | tuple[float | str, float | str] = ..., sticky: str = ..., # consists of letters 'n', 's', 'w', 'e', may contain repeats, may be empty in_: Misc = ..., **kw: Any, # allow keyword argument named 'in', see #4836 @@ -1170,8 +1157,8 @@ class Grid: class BaseWidget(Misc): master: Misc - widgetName: Incomplete - def __init__(self, master, widgetName, cnf={}, kw={}, extra=()) -> None: ... + widgetName: str + def __init__(self, master, widgetName: str, cnf={}, kw={}, extra=()) -> None: ... def destroy(self) -> None: ... # This class represents any widget except Toplevel or Tk. @@ -1201,28 +1188,28 @@ class Toplevel(BaseWidget, Wm): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = 0, + bd: float | str = 0, bg: str = ..., - border: _ScreenUnits = 0, - borderwidth: _ScreenUnits = 0, + border: float | str = 0, + borderwidth: float | str = 0, class_: str = "Toplevel", colormap: Literal["new", ""] | Misc = "", container: bool = False, cursor: _Cursor = "", - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, + highlightthickness: float | str = 0, menu: Menu = ..., name: str = ..., - padx: _ScreenUnits = 0, - pady: _ScreenUnits = 0, - relief: _Relief = "flat", + padx: float | str = 0, + pady: float | str = 0, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", screen: str = "", # can't be changed after creating widget - takefocus: _TakeFocusValue = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, use: int = ..., visual: str | tuple[str, int] = "", - width: _ScreenUnits = 0, + width: float | str = 0, ) -> None: ... @overload def configure( @@ -1230,21 +1217,21 @@ class Toplevel(BaseWidget, Wm): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., menu: Menu = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., - takefocus: _TakeFocusValue = ..., - width: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1258,15 +1245,15 @@ class Button(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = "center", + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = "center", background: str = ..., - bd: _ScreenUnits = ..., # same as borderwidth + bd: float | str = ..., # same as borderwidth bg: str = ..., # same as background bitmap: str = "", - border: _ScreenUnits = ..., # same as borderwidth - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = "", - compound: _Compound = "none", + border: float | str = ..., # same as borderwidth + borderwidth: float | str = ..., + command: str | Callable[[], Any] = "", + compound: Literal["top", "left", "center", "right", "bottom", "none"] = "none", cursor: _Cursor = "", default: Literal["normal", "active", "disabled"] = "disabled", disabledforeground: str = ..., @@ -1274,30 +1261,30 @@ class Button(Widget): font: _FontDescription = "TkDefaultFont", foreground: str = ..., # width and height must be int for buttons containing just text, but - # ints are also valid _ScreenUnits - height: _ScreenUnits = 0, + # buttons with an image accept any screen units. + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 1, - image: _ImageSpec = "", + highlightthickness: float | str = 1, + image: _Image | str = "", justify: Literal["left", "center", "right"] = "center", name: str = ..., - overrelief: _Relief | Literal[""] = "", - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = "", + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = ..., repeatinterval: int = ..., state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", text: float | str = "", # We allow the textvariable to be any Variable, not necessarily # StringVar. This is useful for e.g. a button that displays the value # of an IntVar. textvariable: Variable = ..., underline: int = -1, - width: _ScreenUnits = 0, - wraplength: _ScreenUnits = 0, + width: float | str = 0, + wraplength: float | str = 0, ) -> None: ... @overload def configure( @@ -1306,40 +1293,40 @@ class Button(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = ..., - compound: _Compound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., cursor: _Cursor = ..., default: Literal["normal", "active", "disabled"] = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - image: _ImageSpec = ..., + highlightthickness: float | str = ..., + image: _Image | str = ..., justify: Literal["left", "center", "right"] = ..., - overrelief: _Relief | Literal[""] = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = ..., repeatinterval: int = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., underline: int = ..., - width: _ScreenUnits = ..., - wraplength: _ScreenUnits = ..., + width: float | str = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1354,41 +1341,39 @@ class Canvas(Widget, XView, YView): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = 0, + bd: float | str = 0, bg: str = ..., - border: _ScreenUnits = 0, - borderwidth: _ScreenUnits = 0, + border: float | str = 0, + borderwidth: float | str = 0, closeenough: float = 1.0, confine: bool = True, cursor: _Cursor = "", - # canvas manual page has a section named COORDINATES, and the first - # part of it describes _ScreenUnits. - height: _ScreenUnits = ..., + height: float | str = ..., # see COORDINATES in canvas manual page highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = 0, + insertborderwidth: float | str = 0, insertofftime: int = 300, insertontime: int = 600, - insertwidth: _ScreenUnits = 2, + insertwidth: float | str = 2, name: str = ..., offset=..., # undocumented - relief: _Relief = "flat", + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", # Setting scrollregion to None doesn't reset it back to empty, # but setting it to () does. - scrollregion: tuple[_ScreenUnits, _ScreenUnits, _ScreenUnits, _ScreenUnits] | tuple[()] = (), + scrollregion: tuple[float | str, float | str, float | str, float | str] | tuple[()] = (), selectbackground: str = ..., - selectborderwidth: _ScreenUnits = 1, + selectborderwidth: float | str = 1, selectforeground: str = ..., # man page says that state can be 'hidden', but it can't state: Literal["normal", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", - width: _ScreenUnits = ..., - xscrollcommand: _XYScrollCommand = "", - xscrollincrement: _ScreenUnits = 0, - yscrollcommand: _XYScrollCommand = "", - yscrollincrement: _ScreenUnits = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", + width: float | str = ..., + xscrollcommand: str | Callable[[float, float], object] = "", + xscrollincrement: float | str = 0, + yscrollcommand: str | Callable[[float, float], object] = "", + yscrollincrement: float | str = 0, ) -> None: ... @overload def configure( @@ -1396,35 +1381,35 @@ class Canvas(Widget, XView, YView): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., closeenough: float = ..., confine: bool = ..., cursor: _Cursor = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = ..., + insertborderwidth: float | str = ..., insertofftime: int = ..., insertontime: int = ..., - insertwidth: _ScreenUnits = ..., + insertwidth: float | str = ..., offset=..., # undocumented - relief: _Relief = ..., - scrollregion: tuple[_ScreenUnits, _ScreenUnits, _ScreenUnits, _ScreenUnits] | tuple[()] = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + scrollregion: tuple[float | str, float | str, float | str, float | str] | tuple[()] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., state: Literal["normal", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., - width: _ScreenUnits = ..., - xscrollcommand: _XYScrollCommand = ..., - xscrollincrement: _ScreenUnits = ..., - yscrollcommand: _XYScrollCommand = ..., - yscrollincrement: _ScreenUnits = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + width: float | str = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., + xscrollincrement: float | str = ..., + yscrollcommand: str | Callable[[float, float], object] = ..., + yscrollincrement: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1434,20 +1419,20 @@ class Canvas(Widget, XView, YView): def addtag_all(self, newtag: str) -> None: ... def addtag_below(self, newtag: str, tagOrId: str | int) -> None: ... def addtag_closest( - self, newtag: str, x: _ScreenUnits, y: _ScreenUnits, halo: _ScreenUnits | None = None, start: str | int | None = None + self, newtag: str, x: float | str, y: float | str, halo: float | str | None = None, start: str | int | None = None ) -> None: ... - def addtag_enclosed(self, newtag: str, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: _ScreenUnits) -> None: ... - def addtag_overlapping(self, newtag: str, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: _ScreenUnits) -> None: ... + def addtag_enclosed(self, newtag: str, x1: float | str, y1: float | str, x2: float | str, y2: float | str) -> None: ... + def addtag_overlapping(self, newtag: str, x1: float | str, y1: float | str, x2: float | str, y2: float | str) -> None: ... def addtag_withtag(self, newtag: str, tagOrId: str | int) -> None: ... def find(self, *args): ... # internal method def find_above(self, tagOrId: str | int) -> tuple[int, ...]: ... def find_all(self) -> tuple[int, ...]: ... def find_below(self, tagOrId: str | int) -> tuple[int, ...]: ... def find_closest( - self, x: _ScreenUnits, y: _ScreenUnits, halo: _ScreenUnits | None = None, start: str | int | None = None + self, x: float | str, y: float | str, halo: float | str | None = None, start: str | int | None = None ) -> tuple[int, ...]: ... - def find_enclosed(self, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: _ScreenUnits) -> tuple[int, ...]: ... - def find_overlapping(self, x1: _ScreenUnits, y1: _ScreenUnits, x2: _ScreenUnits, y2: float) -> tuple[int, ...]: ... + def find_enclosed(self, x1: float | str, y1: float | str, x2: float | str, y2: float | str) -> tuple[int, ...]: ... + def find_overlapping(self, x1: float | str, y1: float | str, x2: float | str, y2: float) -> tuple[int, ...]: ... def find_withtag(self, tagOrId: str | int) -> tuple[int, ...]: ... # Incompatible with Misc.bbox(), tkinter violates LSP def bbox(self, *args: str | int) -> tuple[int, int, int, int]: ... # type: ignore[override] @@ -1492,25 +1477,25 @@ class Canvas(Widget, XView, YView): activedash: str | int | list[int] | tuple[int, ...] = ..., activefill: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., arrow: Literal["first", "last", "both"] = ..., arrowshape: tuple[float, float, float] = ..., capstyle: Literal["round", "projecting", "butt"] = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_line( @@ -1522,25 +1507,25 @@ class Canvas(Widget, XView, YView): activedash: str | int | list[int] | tuple[int, ...] = ..., activefill: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., arrow: Literal["first", "last", "both"] = ..., arrowshape: tuple[float, float, float] = ..., capstyle: Literal["round", "projecting", "butt"] = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_line( @@ -1558,25 +1543,25 @@ class Canvas(Widget, XView, YView): activedash: str | int | list[int] | tuple[int, ...] = ..., activefill: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., arrow: Literal["first", "last", "both"] = ..., arrowshape: tuple[float, float, float] = ..., capstyle: Literal["round", "projecting", "butt"] = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_oval( @@ -1592,24 +1577,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_oval( @@ -1623,24 +1608,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_oval( @@ -1660,24 +1645,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_polygon( @@ -1693,27 +1678,27 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_polygon( @@ -1727,27 +1712,27 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_polygon( @@ -1767,27 +1752,27 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., joinstyle: Literal["round", "bevel", "miter"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., smooth: bool = ..., splinesteps: float = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_rectangle( @@ -1803,24 +1788,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_rectangle( @@ -1834,24 +1819,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_rectangle( @@ -1871,24 +1856,24 @@ class Canvas(Widget, XView, YView): activeoutline: str = ..., activeoutlinestipple: str = ..., activestipple: str = ..., - activewidth: _ScreenUnits = ..., + activewidth: float | str = ..., dash: str | int | list[int] | tuple[int, ...] = ..., - dashoffset: _ScreenUnits = ..., + dashoffset: float | str = ..., disableddash: str | int | list[int] | tuple[int, ...] = ..., disabledfill: str = ..., disabledoutline: str = ..., disabledoutlinestipple: str = ..., disabledstipple: str = ..., - disabledwidth: _ScreenUnits = ..., + disabledwidth: float | str = ..., fill: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., outline: str = ..., - outlineoffset: _ScreenUnits = ..., + outlineoffset: float | str = ..., outlinestipple: str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_text( @@ -1899,19 +1884,19 @@ class Canvas(Widget, XView, YView): *, activefill: str = ..., activestipple: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., angle: float | str = ..., disabledfill: str = ..., disabledstipple: str = ..., fill: str = ..., font: _FontDescription = ..., justify: Literal["left", "center", "right"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., text: float | str = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_text( @@ -1921,19 +1906,19 @@ class Canvas(Widget, XView, YView): *, activefill: str = ..., activestipple: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., angle: float | str = ..., disabledfill: str = ..., disabledstipple: str = ..., fill: str = ..., font: _FontDescription = ..., justify: Literal["left", "center", "right"] = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., state: Literal["normal", "hidden", "disabled"] = ..., stipple: str = ..., tags: str | list[str] | tuple[str, ...] = ..., text: float | str = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> int: ... @overload def create_window( @@ -1942,11 +1927,11 @@ class Canvas(Widget, XView, YView): y: float, /, *, - anchor: _Anchor = ..., - height: _ScreenUnits = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., + height: float | str = ..., state: Literal["normal", "hidden", "disabled"] = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., window: Widget = ..., ) -> int: ... @overload @@ -1955,11 +1940,11 @@ class Canvas(Widget, XView, YView): coords: tuple[float, float] | list[int] | list[float], /, *, - anchor: _Anchor = ..., - height: _ScreenUnits = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., + height: float | str = ..., state: Literal["normal", "hidden", "disabled"] = ..., tags: str | list[str] | tuple[str, ...] = ..., - width: _ScreenUnits = ..., + width: float | str = ..., window: Widget = ..., ) -> int: ... def dchars(self, *args) -> None: ... @@ -1992,9 +1977,7 @@ class Canvas(Widget, XView, YView): def tag_raise(self, first: str | int, second: str | int | None = ..., /) -> None: ... def tkraise(self, first: str | int, second: str | int | None = ..., /) -> None: ... # type: ignore[override] def lift(self, first: str | int, second: str | int | None = ..., /) -> None: ... # type: ignore[override] - def scale( - self, tagOrId: str | int, xOrigin: _ScreenUnits, yOrigin: _ScreenUnits, xScale: float, yScale: float, / - ) -> None: ... + def scale(self, tagOrId: str | int, xOrigin: float | str, yOrigin: float | str, xScale: float, yScale: float, /) -> None: ... def scan_mark(self, x, y) -> None: ... def scan_dragto(self, x, y, gain: int = 10) -> None: ... def select_adjust(self, tagOrId, index) -> None: ... @@ -2012,29 +1995,29 @@ class Checkbutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = "center", + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = "center", background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = "", - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = "", - compound: _Compound = "none", + border: float | str = ..., + borderwidth: float | str = ..., + command: str | Callable[[], Any] = "", + compound: Literal["top", "left", "center", "right", "bottom", "none"] = "none", cursor: _Cursor = "", disabledforeground: str = ..., fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 1, - image: _ImageSpec = "", + highlightthickness: float | str = 1, + image: _Image | str = "", indicatoron: bool = True, justify: Literal["left", "center", "right"] = "center", name: str = ..., - offrelief: _Relief = ..., + offrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., # The checkbutton puts a value to its variable when it's checked or # unchecked. We don't restrict the type of that value here, so # Any-typing is fine. @@ -2047,22 +2030,22 @@ class Checkbutton(Widget): # done by setting variable to empty string (the default). offvalue: Any = 0, onvalue: Any = 1, - overrelief: _Relief | Literal[""] = "", - padx: _ScreenUnits = 1, - pady: _ScreenUnits = 1, - relief: _Relief = "flat", + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = "", + padx: float | str = 1, + pady: float | str = 1, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", selectcolor: str = ..., - selectimage: _ImageSpec = "", + selectimage: _Image | str = "", state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", text: float | str = "", textvariable: Variable = ..., - tristateimage: _ImageSpec = "", + tristateimage: _Image | str = "", tristatevalue: Any = "", underline: int = -1, variable: Variable | Literal[""] = ..., - width: _ScreenUnits = 0, - wraplength: _ScreenUnits = 0, + width: float | str = 0, + wraplength: float | str = 0, ) -> None: ... @overload def configure( @@ -2071,46 +2054,46 @@ class Checkbutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = ..., - compound: _Compound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., cursor: _Cursor = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - image: _ImageSpec = ..., + highlightthickness: float | str = ..., + image: _Image | str = ..., indicatoron: bool = ..., justify: Literal["left", "center", "right"] = ..., - offrelief: _Relief = ..., + offrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., offvalue: Any = ..., onvalue: Any = ..., - overrelief: _Relief | Literal[""] = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., - tristateimage: _ImageSpec = ..., + tristateimage: _Image | str = ..., tristatevalue: Any = ..., underline: int = ..., variable: Variable | Literal[""] = ..., - width: _ScreenUnits = ..., - wraplength: _ScreenUnits = ..., + width: float | str = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2128,10 +2111,10 @@ class Entry(Widget, XView): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = "xterm", disabledbackground: str = ..., disabledforeground: str = ..., @@ -2141,30 +2124,30 @@ class Entry(Widget, XView): foreground: str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = 0, + insertborderwidth: float | str = 0, insertofftime: int = 300, insertontime: int = 600, - insertwidth: _ScreenUnits = ..., - invalidcommand: _EntryValidateCommand = "", - invcmd: _EntryValidateCommand = "", # same as invalidcommand + insertwidth: float | str = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", + invcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", # same as invalidcommand justify: Literal["left", "center", "right"] = "left", name: str = ..., readonlybackground: str = ..., - relief: _Relief = "sunken", + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "sunken", selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., show: str = "", state: Literal["normal", "disabled", "readonly"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", textvariable: Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", - validatecommand: _EntryValidateCommand = "", - vcmd: _EntryValidateCommand = "", # same as validatecommand + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", + vcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", # same as validatecommand width: int = 20, - xscrollcommand: _XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload def configure( @@ -2172,10 +2155,10 @@ class Entry(Widget, XView): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., disabledbackground: str = ..., disabledforeground: str = ..., @@ -2185,29 +2168,29 @@ class Entry(Widget, XView): foreground: str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = ..., + insertborderwidth: float | str = ..., insertofftime: int = ..., insertontime: int = ..., - insertwidth: _ScreenUnits = ..., - invalidcommand: _EntryValidateCommand = ..., - invcmd: _EntryValidateCommand = ..., + insertwidth: float | str = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., + invcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., readonlybackground: str = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., show: str = ..., state: Literal["normal", "disabled", "readonly"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: _EntryValidateCommand = ..., - vcmd: _EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., + vcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., width: int = ..., - xscrollcommand: _XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2239,25 +2222,25 @@ class Frame(Widget): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = 0, + bd: float | str = 0, bg: str = ..., - border: _ScreenUnits = 0, - borderwidth: _ScreenUnits = 0, + border: float | str = 0, + borderwidth: float | str = 0, class_: str = "Frame", # can't be changed with configure() colormap: Literal["new", ""] | Misc = "", # can't be changed with configure() container: bool = False, # can't be changed with configure() cursor: _Cursor = "", - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, + highlightthickness: float | str = 0, name: str = ..., - padx: _ScreenUnits = 0, - pady: _ScreenUnits = 0, - relief: _Relief = "flat", - takefocus: _TakeFocusValue = 0, + padx: float | str = 0, + pady: float | str = 0, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, visual: str | tuple[str, int] = "", # can't be changed with configure() - width: _ScreenUnits = 0, + width: float | str = 0, ) -> None: ... @overload def configure( @@ -2265,20 +2248,20 @@ class Frame(Widget): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., - takefocus: _TakeFocusValue = ..., - width: _ScreenUnits = ..., + highlightthickness: float | str = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2292,36 +2275,36 @@ class Label(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = "center", + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = "center", background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = "", - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - compound: _Compound = "none", + border: float | str = ..., + borderwidth: float | str = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = "none", cursor: _Cursor = "", disabledforeground: str = ..., fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, - image: _ImageSpec = "", + highlightthickness: float | str = 0, + image: _Image | str = "", justify: Literal["left", "center", "right"] = "center", name: str = ..., - padx: _ScreenUnits = 1, - pady: _ScreenUnits = 1, - relief: _Relief = "flat", + padx: float | str = 1, + pady: float | str = 1, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, text: float | str = "", textvariable: Variable = ..., underline: int = -1, - width: _ScreenUnits = 0, - wraplength: _ScreenUnits = 0, + width: float | str = 0, + wraplength: float | str = 0, ) -> None: ... @overload def configure( @@ -2330,35 +2313,35 @@ class Label(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - compound: _Compound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., cursor: _Cursor = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - image: _ImageSpec = ..., + highlightthickness: float | str = ..., + image: _Image | str = ..., justify: Literal["left", "center", "right"] = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., underline: int = ..., - width: _ScreenUnits = ..., - wraplength: _ScreenUnits = ..., + width: float | str = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2372,10 +2355,10 @@ class Listbox(Widget, XView, YView): *, activestyle: Literal["dotbox", "none", "underline"] = ..., background: str = ..., - bd: _ScreenUnits = 1, + bd: float | str = 1, bg: str = ..., - border: _ScreenUnits = 1, - borderwidth: _ScreenUnits = 1, + border: float | str = 1, + borderwidth: float | str = 1, cursor: _Cursor = "", disabledforeground: str = ..., exportselection: bool | Literal[0, 1] = 1, @@ -2385,7 +2368,7 @@ class Listbox(Widget, XView, YView): height: int = 10, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., justify: Literal["left", "center", "right"] = "left", # There's no tkinter.ListVar, but seems like bare tkinter.Variable # actually works for this: @@ -2398,9 +2381,9 @@ class Listbox(Widget, XView, YView): # ('foo', 'bar', 'baz') listvariable: Variable = ..., name: str = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = 0, + selectborderwidth: float | str = 0, selectforeground: str = ..., # from listbox man page: "The value of the [selectmode] option may be # arbitrary, but the default bindings expect it to be either single, @@ -2411,10 +2394,10 @@ class Listbox(Widget, XView, YView): selectmode: str | Literal["single", "browse", "multiple", "extended"] = "browse", # noqa: Y051 setgrid: bool = False, state: Literal["normal", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", width: int = 20, - xscrollcommand: _XYScrollCommand = "", - yscrollcommand: _XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", + yscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload def configure( @@ -2423,10 +2406,10 @@ class Listbox(Widget, XView, YView): *, activestyle: Literal["dotbox", "none", "underline"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., disabledforeground: str = ..., exportselection: bool = ..., @@ -2436,20 +2419,20 @@ class Listbox(Widget, XView, YView): height: int = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., justify: Literal["left", "center", "right"] = ..., listvariable: Variable = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., selectmode: str | Literal["single", "browse", "multiple", "extended"] = ..., # noqa: Y051 setgrid: bool = ..., state: Literal["normal", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., width: int = ..., - xscrollcommand: _XYScrollCommand = ..., - yscrollcommand: _XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., + yscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2485,13 +2468,13 @@ class Menu(Widget): cnf: dict[str, Any] | None = {}, *, activebackground: str = ..., - activeborderwidth: _ScreenUnits = ..., + activeborderwidth: float | str = ..., activeforeground: str = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = "arrow", disabledforeground: str = ..., fg: str = ..., @@ -2499,9 +2482,9 @@ class Menu(Widget): foreground: str = ..., name: str = ..., postcommand: Callable[[], object] | str = "", - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectcolor: str = ..., - takefocus: _TakeFocusValue = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, tearoff: bool | Literal[0, 1] = 1, # I guess tearoffcommand arguments are supposed to be widget objects, # but they are widget name strings. Use nametowidget() to handle the @@ -2516,22 +2499,22 @@ class Menu(Widget): cnf: dict[str, Any] | None = None, *, activebackground: str = ..., - activeborderwidth: _ScreenUnits = ..., + activeborderwidth: float | str = ..., activeforeground: str = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., postcommand: Callable[[], object] | str = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectcolor: str = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., tearoff: bool = ..., tearoffcommand: Callable[[str, str], object] | str = ..., title: str = ..., @@ -2555,11 +2538,11 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., label: str = ..., menu: Menu = ..., state: Literal["normal", "active", "disabled"] = ..., @@ -2576,17 +2559,17 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., indicatoron: bool = ..., label: str = ..., offvalue: Any = ..., onvalue: Any = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., variable: Variable = ..., @@ -2602,11 +2585,11 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., label: str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., @@ -2622,15 +2605,15 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., indicatoron: bool = ..., label: str = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., value: Any = ..., @@ -2649,11 +2632,11 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., label: str = ..., menu: Menu = ..., state: Literal["normal", "active", "disabled"] = ..., @@ -2671,17 +2654,17 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., indicatoron: bool = ..., label: str = ..., offvalue: Any = ..., onvalue: Any = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., variable: Variable = ..., @@ -2698,11 +2681,11 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., label: str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., @@ -2719,15 +2702,15 @@ class Menu(Widget): bitmap: str = ..., columnbreak: int = ..., command: Callable[[], object] | str = ..., - compound: _Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., font: _FontDescription = ..., foreground: str = ..., hidemargin: bool = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., indicatoron: bool = ..., label: str = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., underline: int = ..., value: Any = ..., @@ -2756,39 +2739,39 @@ class Menubutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = "", - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - compound: _Compound = "none", + border: float | str = ..., + borderwidth: float | str = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = "none", cursor: _Cursor = "", direction: Literal["above", "below", "left", "right", "flush"] = "below", disabledforeground: str = ..., fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, - image: _ImageSpec = "", + highlightthickness: float | str = 0, + image: _Image | str = "", indicatoron: bool = ..., justify: Literal["left", "center", "right"] = ..., menu: Menu = ..., name: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = "flat", + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, text: float | str = "", textvariable: Variable = ..., underline: int = -1, - width: _ScreenUnits = 0, - wraplength: _ScreenUnits = 0, + width: float | str = 0, + wraplength: float | str = 0, ) -> None: ... @overload def configure( @@ -2797,38 +2780,38 @@ class Menubutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - compound: _Compound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., cursor: _Cursor = ..., direction: Literal["above", "below", "left", "right", "flush"] = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - image: _ImageSpec = ..., + highlightthickness: float | str = ..., + image: _Image | str = ..., indicatoron: bool = ..., justify: Literal["left", "center", "right"] = ..., menu: Menu = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., underline: int = ..., - width: _ScreenUnits = ..., - wraplength: _ScreenUnits = ..., + width: float | str = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2840,58 +2823,58 @@ class Message(Widget): master: Misc | None = None, cnf: dict[str, Any] | None = {}, *, - anchor: _Anchor = "center", + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = "center", aspect: int = 150, background: str = ..., - bd: _ScreenUnits = 1, + bd: float | str = 1, bg: str = ..., - border: _ScreenUnits = 1, - borderwidth: _ScreenUnits = 1, + border: float | str = 1, + borderwidth: float | str = 1, cursor: _Cursor = "", fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, + highlightthickness: float | str = 0, justify: Literal["left", "center", "right"] = "left", name: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = "flat", - takefocus: _TakeFocusValue = 0, + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, text: float | str = "", textvariable: Variable = ..., # there's width but no height - width: _ScreenUnits = 0, + width: float | str = 0, ) -> None: ... @overload def configure( self, cnf: dict[str, Any] | None = None, *, - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., aspect: int = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., justify: Literal["left", "center", "right"] = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., - takefocus: _TakeFocusValue = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -2905,46 +2888,46 @@ class Radiobutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = "center", + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = "center", background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = "", - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = "", - compound: _Compound = "none", + border: float | str = ..., + borderwidth: float | str = ..., + command: str | Callable[[], Any] = "", + compound: Literal["top", "left", "center", "right", "bottom", "none"] = "none", cursor: _Cursor = "", disabledforeground: str = ..., fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 1, - image: _ImageSpec = "", + highlightthickness: float | str = 1, + image: _Image | str = "", indicatoron: bool = True, justify: Literal["left", "center", "right"] = "center", name: str = ..., - offrelief: _Relief = ..., - overrelief: _Relief | Literal[""] = "", - padx: _ScreenUnits = 1, - pady: _ScreenUnits = 1, - relief: _Relief = "flat", + offrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = "", + padx: float | str = 1, + pady: float | str = 1, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", selectcolor: str = ..., - selectimage: _ImageSpec = "", + selectimage: _Image | str = "", state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", text: float | str = "", textvariable: Variable = ..., - tristateimage: _ImageSpec = "", + tristateimage: _Image | str = "", tristatevalue: Any = "", underline: int = -1, value: Any = "", variable: Variable | Literal[""] = ..., - width: _ScreenUnits = 0, - wraplength: _ScreenUnits = 0, + width: float | str = 0, + wraplength: float | str = 0, ) -> None: ... @overload def configure( @@ -2953,45 +2936,45 @@ class Radiobutton(Widget): *, activebackground: str = ..., activeforeground: str = ..., - anchor: _Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bitmap: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., - command: _ButtonCommand = ..., - compound: _Compound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., cursor: _Cursor = ..., disabledforeground: str = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., - image: _ImageSpec = ..., + highlightthickness: float | str = ..., + image: _Image | str = ..., indicatoron: bool = ..., justify: Literal["left", "center", "right"] = ..., - offrelief: _Relief = ..., - overrelief: _Relief | Literal[""] = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + offrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + overrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove", ""] = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectcolor: str = ..., - selectimage: _ImageSpec = ..., + selectimage: _Image | str = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: Variable = ..., - tristateimage: _ImageSpec = ..., + tristateimage: _Image | str = ..., tristatevalue: Any = ..., underline: int = ..., value: Any = ..., variable: Variable | Literal[""] = ..., - width: _ScreenUnits = ..., - wraplength: _ScreenUnits = ..., + width: float | str = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -3009,11 +2992,11 @@ class Scale(Widget): *, activebackground: str = ..., background: str = ..., - bd: _ScreenUnits = 1, + bd: float | str = 1, bg: str = ..., bigincrement: float = 0.0, - border: _ScreenUnits = 1, - borderwidth: _ScreenUnits = 1, + border: float | str = 1, + borderwidth: float | str = 1, # don't know why the callback gets string instead of float command: str | Callable[[str], object] = "", cursor: _Cursor = "", @@ -3024,25 +3007,25 @@ class Scale(Widget): from_: float = 0.0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., label: str = "", - length: _ScreenUnits = 100, + length: float | str = 100, name: str = ..., orient: Literal["horizontal", "vertical"] = "vertical", - relief: _Relief = "flat", + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", repeatdelay: int = 300, repeatinterval: int = 100, resolution: float = 1.0, showvalue: bool = True, - sliderlength: _ScreenUnits = 30, - sliderrelief: _Relief = "raised", + sliderlength: float | str = 30, + sliderrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "raised", state: Literal["normal", "active", "disabled"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", tickinterval: float = 0.0, to: float = 100.0, troughcolor: str = ..., variable: IntVar | DoubleVar = ..., - width: _ScreenUnits = 15, + width: float | str = 15, ) -> None: ... @overload def configure( @@ -3051,11 +3034,11 @@ class Scale(Widget): *, activebackground: str = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., bigincrement: float = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., command: str | Callable[[str], object] = ..., cursor: _Cursor = ..., digits: int = ..., @@ -3065,24 +3048,24 @@ class Scale(Widget): from_: float = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., label: str = ..., - length: _ScreenUnits = ..., + length: float | str = ..., orient: Literal["horizontal", "vertical"] = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = ..., repeatinterval: int = ..., resolution: float = ..., showvalue: bool = ..., - sliderlength: _ScreenUnits = ..., - sliderrelief: _Relief = ..., + sliderlength: float | str = ..., + sliderrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., state: Literal["normal", "active", "disabled"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., tickinterval: float = ..., to: float = ..., troughcolor: str = ..., variable: IntVar | DoubleVar = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -3099,31 +3082,31 @@ class Scrollbar(Widget): cnf: dict[str, Any] | None = {}, *, activebackground: str = ..., - activerelief: _Relief = "raised", + activerelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "raised", background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., # There are many ways how the command may get called. Search for # 'SCROLLING COMMANDS' in scrollbar man page. There doesn't seem to # be any way to specify an overloaded callback function, so we say # that it can take any args while it can't in reality. command: Callable[..., tuple[float, float] | None] | str = "", cursor: _Cursor = "", - elementborderwidth: _ScreenUnits = -1, + elementborderwidth: float | str = -1, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, + highlightthickness: float | str = 0, jump: bool = False, name: str = ..., orient: Literal["horizontal", "vertical"] = "vertical", - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = 300, repeatinterval: int = 100, - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", troughcolor: str = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> None: ... @overload def configure( @@ -3131,26 +3114,26 @@ class Scrollbar(Widget): cnf: dict[str, Any] | None = None, *, activebackground: str = ..., - activerelief: _Relief = ..., + activerelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., command: Callable[..., tuple[float, float] | None] | str = ..., cursor: _Cursor = ..., - elementborderwidth: _ScreenUnits = ..., + elementborderwidth: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., jump: bool = ..., orient: Literal["horizontal", "vertical"] = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = ..., repeatinterval: int = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., troughcolor: str = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -3175,54 +3158,54 @@ class Text(Widget, XView, YView): *, autoseparators: bool = True, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., blockcursor: bool = False, - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = "xterm", endline: int | Literal[""] = "", exportselection: bool = True, fg: str = ..., font: _FontDescription = "TkFixedFont", foreground: str = ..., - # width is always int, but height is allowed to be ScreenUnits. + # width is always int, but height is allowed to be screen units. # This doesn't make any sense to me, and this isn't documented. # The docs seem to say that both should be integers. - height: _ScreenUnits = 24, + height: float | str = 24, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., inactiveselectbackground: str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = 0, + insertborderwidth: float | str = 0, insertofftime: int = 300, insertontime: int = 600, insertunfocussed: Literal["none", "hollow", "solid"] = "none", - insertwidth: _ScreenUnits = ..., + insertwidth: float | str = ..., maxundo: int = 0, name: str = ..., - padx: _ScreenUnits = 1, - pady: _ScreenUnits = 1, - relief: _Relief = ..., + padx: float | str = 1, + pady: float | str = 1, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., setgrid: bool = False, - spacing1: _ScreenUnits = 0, - spacing2: _ScreenUnits = 0, - spacing3: _ScreenUnits = 0, + spacing1: float | str = 0, + spacing2: float | str = 0, + spacing3: float | str = 0, startline: int | Literal[""] = "", state: Literal["normal", "disabled"] = "normal", # Literal inside Tuple doesn't actually work - tabs: _ScreenUnits | str | tuple[_ScreenUnits | str, ...] = "", + tabs: float | str | tuple[float | str, ...] = "", tabstyle: Literal["tabular", "wordprocessor"] = "tabular", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", undo: bool = False, width: int = 80, wrap: Literal["none", "char", "word"] = "char", - xscrollcommand: _XYScrollCommand = "", - yscrollcommand: _XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", + yscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload def configure( @@ -3231,49 +3214,49 @@ class Text(Widget, XView, YView): *, autoseparators: bool = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., blockcursor: bool = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., endline: int | Literal[""] = ..., exportselection: bool = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., inactiveselectbackground: str = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = ..., + insertborderwidth: float | str = ..., insertofftime: int = ..., insertontime: int = ..., insertunfocussed: Literal["none", "hollow", "solid"] = ..., - insertwidth: _ScreenUnits = ..., + insertwidth: float | str = ..., maxundo: int = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., setgrid: bool = ..., - spacing1: _ScreenUnits = ..., - spacing2: _ScreenUnits = ..., - spacing3: _ScreenUnits = ..., + spacing1: float | str = ..., + spacing2: float | str = ..., + spacing3: float | str = ..., startline: int | Literal[""] = ..., state: Literal["normal", "disabled"] = ..., - tabs: _ScreenUnits | str | tuple[_ScreenUnits | str, ...] = ..., + tabs: float | str | tuple[float | str, ...] = ..., tabstyle: Literal["tabular", "wordprocessor"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., undo: bool = ..., width: int = ..., wrap: Literal["none", "char", "word"] = ..., - xscrollcommand: _XYScrollCommand = ..., - yscrollcommand: _XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., + yscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -3482,10 +3465,10 @@ class Text(Widget, XView, YView): cnf: dict[str, Any] | None = None, *, align: Literal["baseline", "bottom", "center", "top"] = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., name: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., ) -> dict[str, tuple[str, str, str, str, str | int]] | None: ... def image_create( self, @@ -3493,10 +3476,10 @@ class Text(Widget, XView, YView): cnf: dict[str, Any] | None = {}, *, align: Literal["baseline", "bottom", "center", "top"] = ..., - image: _ImageSpec = ..., + image: _Image | str = ..., name: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., ) -> str: ... def image_names(self) -> tuple[str, ...]: ... def index(self, index: _TextIndex) -> str: ... @@ -3553,27 +3536,27 @@ class Text(Widget, XView, YView): *, background: str = ..., bgstipple: str = ..., - borderwidth: _ScreenUnits = ..., - border: _ScreenUnits = ..., # alias for borderwidth + borderwidth: float | str = ..., + border: float | str = ..., # alias for borderwidth elide: bool = ..., fgstipple: str = ..., font: _FontDescription = ..., foreground: str = ..., justify: Literal["left", "right", "center"] = ..., - lmargin1: _ScreenUnits = ..., - lmargin2: _ScreenUnits = ..., + lmargin1: float | str = ..., + lmargin2: float | str = ..., lmargincolor: str = ..., - offset: _ScreenUnits = ..., + offset: float | str = ..., overstrike: bool = ..., overstrikefg: str = ..., - relief: _Relief = ..., - rmargin: _ScreenUnits = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + rmargin: float | str = ..., rmargincolor: str = ..., selectbackground: str = ..., selectforeground: str = ..., - spacing1: _ScreenUnits = ..., - spacing2: _ScreenUnits = ..., - spacing3: _ScreenUnits = ..., + spacing1: float | str = ..., + spacing2: float | str = ..., + spacing3: float | str = ..., tabs: Any = ..., # the exact type is kind of complicated, see manual page tabstyle: Literal["tabular", "wordprocessor"] = ..., underline: bool = ..., @@ -3616,8 +3599,8 @@ class Text(Widget, XView, YView): *, align: Literal["baseline", "bottom", "center", "top"] = ..., create: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., stretch: bool | Literal[0, 1] = ..., window: Misc | str = ..., ) -> dict[str, tuple[str, str, str, str, str | int]] | None: ... @@ -3629,8 +3612,8 @@ class Text(Widget, XView, YView): *, align: Literal["baseline", "bottom", "center", "top"] = ..., create: str = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., + padx: float | str = ..., + pady: float | str = ..., stretch: bool | Literal[0, 1] = ..., window: Misc | str = ..., ) -> None: ... @@ -3643,7 +3626,6 @@ class _setit: # manual page: tk_optionMenu class OptionMenu(Menubutton): - widgetName: Incomplete menuname: Incomplete def __init__( # differs from other widgets @@ -3825,14 +3807,14 @@ class Spinbox(Widget, XView): *, activebackground: str = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., buttonbackground: str = ..., buttoncursor: _Cursor = "", - buttondownrelief: _Relief = ..., - buttonuprelief: _Relief = ..., + buttondownrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + buttonuprelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., # percent substitutions don't seem to be supported, it's similar to Entry's validation stuff command: Callable[[], object] | str | list[str] | tuple[str, ...] = "", cursor: _Cursor = "xterm", @@ -3846,35 +3828,35 @@ class Spinbox(Widget, XView): from_: float = 0.0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., increment: float = 1.0, insertbackground: str = ..., - insertborderwidth: _ScreenUnits = 0, + insertborderwidth: float | str = 0, insertofftime: int = 300, insertontime: int = 600, - insertwidth: _ScreenUnits = ..., - invalidcommand: _EntryValidateCommand = "", - invcmd: _EntryValidateCommand = "", + insertwidth: float | str = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", + invcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", justify: Literal["left", "center", "right"] = "left", name: str = ..., readonlybackground: str = ..., - relief: _Relief = "sunken", + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "sunken", repeatdelay: int = 400, repeatinterval: int = 100, selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., state: Literal["normal", "disabled", "readonly"] = "normal", - takefocus: _TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", textvariable: Variable = ..., to: float = 0.0, validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", - validatecommand: _EntryValidateCommand = "", - vcmd: _EntryValidateCommand = "", + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", + vcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", values: list[str] | tuple[str, ...] = ..., width: int = 20, wrap: bool = False, - xscrollcommand: _XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload def configure( @@ -3883,14 +3865,14 @@ class Spinbox(Widget, XView): *, activebackground: str = ..., background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., buttonbackground: str = ..., buttoncursor: _Cursor = ..., - buttondownrelief: _Relief = ..., - buttonuprelief: _Relief = ..., + buttondownrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + buttonuprelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., command: Callable[[], object] | str | list[str] | tuple[str, ...] = ..., cursor: _Cursor = ..., disabledbackground: str = ..., @@ -3903,34 +3885,34 @@ class Spinbox(Widget, XView): from_: float = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., increment: float = ..., insertbackground: str = ..., - insertborderwidth: _ScreenUnits = ..., + insertborderwidth: float | str = ..., insertofftime: int = ..., insertontime: int = ..., - insertwidth: _ScreenUnits = ..., - invalidcommand: _EntryValidateCommand = ..., - invcmd: _EntryValidateCommand = ..., + insertwidth: float | str = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., + invcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., readonlybackground: str = ..., - relief: _Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., repeatdelay: int = ..., repeatinterval: int = ..., selectbackground: str = ..., - selectborderwidth: _ScreenUnits = ..., + selectborderwidth: float | str = ..., selectforeground: str = ..., state: Literal["normal", "disabled", "readonly"] = ..., - takefocus: _TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: Variable = ..., to: float = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: _EntryValidateCommand = ..., - vcmd: _EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., + vcmd: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., values: list[str] | tuple[str, ...] = ..., width: int = ..., wrap: bool = ..., - xscrollcommand: _XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -3963,10 +3945,10 @@ class LabelFrame(Widget): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = 2, + bd: float | str = 2, bg: str = ..., - border: _ScreenUnits = 2, - borderwidth: _ScreenUnits = 2, + border: float | str = 2, + borderwidth: float | str = 2, class_: str = "Labelframe", # can't be changed with configure() colormap: Literal["new", ""] | Misc = "", # can't be changed with configure() container: bool = False, # undocumented, can't be changed with configure() @@ -3974,21 +3956,21 @@ class LabelFrame(Widget): fg: str = ..., font: _FontDescription = "TkDefaultFont", foreground: str = ..., - height: _ScreenUnits = 0, + height: float | str = 0, highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = 0, + highlightthickness: float | str = 0, # 'ne' and 'en' are valid labelanchors, but only 'ne' is a valid _Anchor. labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = "nw", labelwidget: Misc = ..., name: str = ..., - padx: _ScreenUnits = 0, - pady: _ScreenUnits = 0, - relief: _Relief = "groove", - takefocus: _TakeFocusValue = 0, + padx: float | str = 0, + pady: float | str = 0, + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "groove", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = 0, text: float | str = "", visual: str | tuple[str, int] = "", # can't be changed with configure() - width: _ScreenUnits = 0, + width: float | str = 0, ) -> None: ... @overload def configure( @@ -3996,26 +3978,26 @@ class LabelFrame(Widget): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., fg: str = ..., font: _FontDescription = ..., foreground: str = ..., - height: _ScreenUnits = ..., + height: float | str = ..., highlightbackground: str = ..., highlightcolor: str = ..., - highlightthickness: _ScreenUnits = ..., + highlightthickness: float | str = ..., labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = ..., labelwidget: Misc = ..., - padx: _ScreenUnits = ..., - pady: _ScreenUnits = ..., - relief: _Relief = ..., - takefocus: _TakeFocusValue = ..., + padx: float | str = ..., + pady: float | str = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -4028,27 +4010,27 @@ class PanedWindow(Widget): cnf: dict[str, Any] | None = {}, *, background: str = ..., - bd: _ScreenUnits = 1, + bd: float | str = 1, bg: str = ..., - border: _ScreenUnits = 1, - borderwidth: _ScreenUnits = 1, + border: float | str = 1, + borderwidth: float | str = 1, cursor: _Cursor = "", - handlepad: _ScreenUnits = 8, - handlesize: _ScreenUnits = 8, - height: _ScreenUnits = "", + handlepad: float | str = 8, + handlesize: float | str = 8, + height: float | str = "", name: str = ..., opaqueresize: bool = True, orient: Literal["horizontal", "vertical"] = "horizontal", proxybackground: str = "", - proxyborderwidth: _ScreenUnits = 2, - proxyrelief: _Relief = "flat", - relief: _Relief = "flat", + proxyborderwidth: float | str = 2, + proxyrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", sashcursor: _Cursor = "", - sashpad: _ScreenUnits = 0, - sashrelief: _Relief = "flat", - sashwidth: _ScreenUnits = 3, + sashpad: float | str = 0, + sashrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = "flat", + sashwidth: float | str = 3, showhandle: bool = False, - width: _ScreenUnits = "", + width: float | str = "", ) -> None: ... @overload def configure( @@ -4056,45 +4038,45 @@ class PanedWindow(Widget): cnf: dict[str, Any] | None = None, *, background: str = ..., - bd: _ScreenUnits = ..., + bd: float | str = ..., bg: str = ..., - border: _ScreenUnits = ..., - borderwidth: _ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: _Cursor = ..., - handlepad: _ScreenUnits = ..., - handlesize: _ScreenUnits = ..., - height: _ScreenUnits = ..., + handlepad: float | str = ..., + handlesize: float | str = ..., + height: float | str = ..., opaqueresize: bool = ..., orient: Literal["horizontal", "vertical"] = ..., proxybackground: str = ..., - proxyborderwidth: _ScreenUnits = ..., - proxyrelief: _Relief = ..., - relief: _Relief = ..., + proxyborderwidth: float | str = ..., + proxyrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., sashcursor: _Cursor = ..., - sashpad: _ScreenUnits = ..., - sashrelief: _Relief = ..., - sashwidth: _ScreenUnits = ..., + sashpad: float | str = ..., + sashrelief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., + sashwidth: float | str = ..., showhandle: bool = ..., - width: _ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... config = configure def add(self, child: Widget, **kw) -> None: ... def remove(self, child) -> None: ... - forget: Incomplete + forget = remove # type: ignore[assignment] def identify(self, x: int, y: int): ... - def proxy(self, *args): ... - def proxy_coord(self): ... - def proxy_forget(self): ... - def proxy_place(self, x, y): ... - def sash(self, *args): ... - def sash_coord(self, index): ... - def sash_mark(self, index): ... - def sash_place(self, index, x, y): ... + def proxy(self, *args) -> tuple[Incomplete, ...]: ... + def proxy_coord(self) -> tuple[Incomplete, ...]: ... + def proxy_forget(self) -> tuple[Incomplete, ...]: ... + def proxy_place(self, x, y) -> tuple[Incomplete, ...]: ... + def sash(self, *args) -> tuple[Incomplete, ...]: ... + def sash_coord(self, index) -> tuple[Incomplete, ...]: ... + def sash_mark(self, index) -> tuple[Incomplete, ...]: ... + def sash_place(self, index, x, y) -> tuple[Incomplete, ...]: ... def panecget(self, child, option): ... def paneconfigure(self, tagOrId, cnf=None, **kw): ... - paneconfig: Incomplete + paneconfig = paneconfigure def panes(self): ... def _test() -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/ttk.pyi b/mypy/typeshed/stdlib/tkinter/ttk.pyi index 86c55eba7006..1d72acd99512 100644 --- a/mypy/typeshed/stdlib/tkinter/ttk.pyi +++ b/mypy/typeshed/stdlib/tkinter/ttk.pyi @@ -39,21 +39,19 @@ def tclobjs_to_py(adict: dict[Any, Any]) -> dict[Any, Any]: ... def setup_master(master: tkinter.Misc | None = None): ... _Padding: TypeAlias = ( - tkinter._ScreenUnits - | tuple[tkinter._ScreenUnits] - | tuple[tkinter._ScreenUnits, tkinter._ScreenUnits] - | tuple[tkinter._ScreenUnits, tkinter._ScreenUnits, tkinter._ScreenUnits] - | tuple[tkinter._ScreenUnits, tkinter._ScreenUnits, tkinter._ScreenUnits, tkinter._ScreenUnits] + float + | str + | tuple[float | str] + | tuple[float | str, float | str] + | tuple[float | str, float | str, float | str] + | tuple[float | str, float | str, float | str, float | str] ) -# from ttk_widget (aka ttk::widget) manual page, differs from tkinter._Compound -_TtkCompound: TypeAlias = Literal["", "text", "image", tkinter._Compound] - # Last item (option value to apply) varies between different options so use Any. # It could also be any iterable with items matching the tuple, but that case # hasn't been added here for consistency with _Padding above. _Statespec: TypeAlias = tuple[Unpack[tuple[str, ...]], Any] -_ImageStatespec: TypeAlias = tuple[Unpack[tuple[str, ...]], tkinter._ImageSpec] +_ImageStatespec: TypeAlias = tuple[Unpack[tuple[str, ...]], tkinter._Image | str] _VsapiStatespec: TypeAlias = tuple[Unpack[tuple[str, ...]], int] class _Layout(TypedDict, total=False): @@ -69,14 +67,14 @@ _LayoutSpec: TypeAlias = list[tuple[str, _Layout | None]] # Keep these in sync with the appropriate methods in Style class _ElementCreateImageKwargs(TypedDict, total=False): border: _Padding - height: tkinter._ScreenUnits + height: float | str padding: _Padding sticky: str - width: tkinter._ScreenUnits + width: float | str _ElementCreateArgsCrossPlatform: TypeAlias = ( # Could be any sequence here but types are not homogenous so just type it as tuple - tuple[Literal["image"], tkinter._ImageSpec, Unpack[tuple[_ImageStatespec, ...]], _ElementCreateImageKwargs] + tuple[Literal["image"], tkinter._Image | str, Unpack[tuple[_ImageStatespec, ...]], _ElementCreateImageKwargs] | tuple[Literal["from"], str, str] | tuple[Literal["from"], str] # (fromelement is optional) ) @@ -88,8 +86,8 @@ if sys.platform == "win32" and sys.version_info >= (3, 13): padding: _Padding class _ElementCreateVsapiKwargsSize(TypedDict): - width: tkinter._ScreenUnits - height: tkinter._ScreenUnits + width: float | str + height: float | str _ElementCreateVsapiKwargsDict: TypeAlias = ( _ElementCreateVsapiKwargsPadding | _ElementCreateVsapiKwargsMargin | _ElementCreateVsapiKwargsSize @@ -139,14 +137,14 @@ class Style: self, elementname: str, etype: Literal["image"], - default_image: tkinter._ImageSpec, + default_image: tkinter._Image | str, /, *imagespec: _ImageStatespec, border: _Padding = ..., - height: tkinter._ScreenUnits = ..., + height: float | str = ..., padding: _Padding = ..., sticky: str = ..., - width: tkinter._ScreenUnits = ..., + width: float | str = ..., ) -> None: ... @overload def element_create(self, elementname: str, etype: Literal["from"], themename: str, fromelement: str = ..., /) -> None: ... @@ -188,8 +186,8 @@ class Style: vs_statespec: _VsapiStatespec = ..., /, *, - width: tkinter._ScreenUnits, - height: tkinter._ScreenUnits, + width: float | str, + height: float | str, ) -> None: ... def element_names(self) -> tuple[str, ...]: ... @@ -214,16 +212,16 @@ class Button(Widget): master: tkinter.Misc | None = None, *, class_: str = "", - command: tkinter._ButtonCommand = "", - compound: _TtkCompound = "", + command: str | Callable[[], Any] = "", + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = "", cursor: tkinter._Cursor = "", default: Literal["normal", "active", "disabled"] = "normal", - image: tkinter._ImageSpec = "", + image: tkinter._Image | str = "", name: str = ..., padding=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = "", textvariable: tkinter.Variable = ..., underline: int = -1, @@ -234,15 +232,15 @@ class Button(Widget): self, cnf: dict[str, Any] | None = None, *, - command: tkinter._ButtonCommand = ..., - compound: _TtkCompound = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = ..., cursor: tkinter._Cursor = ..., default: Literal["normal", "active", "disabled"] = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., padding=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: tkinter.Variable = ..., underline: int = ..., @@ -259,17 +257,17 @@ class Checkbutton(Widget): master: tkinter.Misc | None = None, *, class_: str = "", - command: tkinter._ButtonCommand = "", - compound: _TtkCompound = "", + command: str | Callable[[], Any] = "", + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = "", cursor: tkinter._Cursor = "", - image: tkinter._ImageSpec = "", + image: tkinter._Image | str = "", name: str = ..., offvalue: Any = 0, onvalue: Any = 1, padding=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = "", textvariable: tkinter.Variable = ..., underline: int = -1, @@ -284,16 +282,16 @@ class Checkbutton(Widget): self, cnf: dict[str, Any] | None = None, *, - command: tkinter._ButtonCommand = ..., - compound: _TtkCompound = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = ..., cursor: tkinter._Cursor = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., offvalue: Any = ..., onvalue: Any = ..., padding=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: tkinter.Variable = ..., underline: int = ..., @@ -317,18 +315,18 @@ class Entry(Widget, tkinter.Entry): exportselection: bool = True, font: _FontDescription = "TkTextFont", foreground: str = "", - invalidcommand: tkinter._EntryValidateCommand = "", + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", justify: Literal["left", "center", "right"] = "left", name: str = ..., show: str = "", state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", - validatecommand: tkinter._EntryValidateCommand = "", + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", width: int = 20, - xscrollcommand: tkinter._XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload # type: ignore[override] def configure( @@ -340,17 +338,17 @@ class Entry(Widget, tkinter.Entry): exportselection: bool = ..., font: _FontDescription = ..., foreground: str = ..., - invalidcommand: tkinter._EntryValidateCommand = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., show: str = ..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: tkinter._EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., width: int = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -365,17 +363,17 @@ class Entry(Widget, tkinter.Entry): exportselection: bool = ..., font: _FontDescription = ..., foreground: str = ..., - invalidcommand: tkinter._EntryValidateCommand = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., show: str = ..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: tkinter._EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., width: int = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -395,20 +393,20 @@ class Combobox(Entry): font: _FontDescription = ..., # undocumented foreground: str = ..., # undocumented height: int = 10, - invalidcommand: tkinter._EntryValidateCommand = ..., # undocumented + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., # undocumented justify: Literal["left", "center", "right"] = "left", name: str = ..., postcommand: Callable[[], object] | str = "", show=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., # undocumented - validatecommand: tkinter._EntryValidateCommand = ..., # undocumented + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., # undocumented values: list[str] | tuple[str, ...] = ..., width: int = 20, - xscrollcommand: tkinter._XYScrollCommand = ..., # undocumented + xscrollcommand: str | Callable[[float, float], object] = ..., # undocumented ) -> None: ... @overload # type: ignore[override] def configure( @@ -421,19 +419,19 @@ class Combobox(Entry): font: _FontDescription = ..., foreground: str = ..., height: int = ..., - invalidcommand: tkinter._EntryValidateCommand = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., postcommand: Callable[[], object] | str = ..., show=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: tkinter._EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., values: list[str] | tuple[str, ...] = ..., width: int = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -449,19 +447,19 @@ class Combobox(Entry): font: _FontDescription = ..., foreground: str = ..., height: int = ..., - invalidcommand: tkinter._EntryValidateCommand = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., postcommand: Callable[[], object] | str = ..., show=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: tkinter._EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., values: list[str] | tuple[str, ...] = ..., width: int = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -475,32 +473,32 @@ class Frame(Widget): self, master: tkinter.Misc | None = None, *, - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., class_: str = "", cursor: tkinter._Cursor = "", - height: tkinter._ScreenUnits = 0, + height: float | str = 0, name: str = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., style: str = "", - takefocus: tkinter._TakeFocusValue = "", - width: tkinter._ScreenUnits = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", + width: float | str = 0, ) -> None: ... @overload def configure( self, cnf: dict[str, Any] | None = None, *, - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: tkinter._Cursor = ..., - height: tkinter._ScreenUnits = ..., + height: float | str = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., - width: tkinter._ScreenUnits = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -511,54 +509,54 @@ class Label(Widget): self, master: tkinter.Misc | None = None, *, - anchor: tkinter._Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = "", - border: tkinter._ScreenUnits = ..., # alias for borderwidth - borderwidth: tkinter._ScreenUnits = ..., # undocumented + border: float | str = ..., # alias for borderwidth + borderwidth: float | str = ..., # undocumented class_: str = "", - compound: _TtkCompound = "", + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = "", cursor: tkinter._Cursor = "", font: _FontDescription = ..., foreground: str = "", - image: tkinter._ImageSpec = "", + image: tkinter._Image | str = "", justify: Literal["left", "center", "right"] = ..., name: str = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", text: float | str = "", textvariable: tkinter.Variable = ..., underline: int = -1, width: int | Literal[""] = "", - wraplength: tkinter._ScreenUnits = ..., + wraplength: float | str = ..., ) -> None: ... @overload def configure( self, cnf: dict[str, Any] | None = None, *, - anchor: tkinter._Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., background: str = ..., - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., - compound: _TtkCompound = ..., + border: float | str = ..., + borderwidth: float | str = ..., + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = ..., cursor: tkinter._Cursor = ..., font: _FontDescription = ..., foreground: str = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., justify: Literal["left", "center", "right"] = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: tkinter.Variable = ..., underline: int = ..., width: int | Literal[""] = ..., - wraplength: tkinter._ScreenUnits = ..., + wraplength: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -569,40 +567,40 @@ class Labelframe(Widget): self, master: tkinter.Misc | None = None, *, - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., # undocumented + border: float | str = ..., + borderwidth: float | str = ..., # undocumented class_: str = "", cursor: tkinter._Cursor = "", - height: tkinter._ScreenUnits = 0, + height: float | str = 0, labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = ..., labelwidget: tkinter.Misc = ..., name: str = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., # undocumented + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., # undocumented style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", text: float | str = "", underline: int = -1, - width: tkinter._ScreenUnits = 0, + width: float | str = 0, ) -> None: ... @overload def configure( self, cnf: dict[str, Any] | None = None, *, - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., cursor: tkinter._Cursor = ..., - height: tkinter._ScreenUnits = ..., + height: float | str = ..., labelanchor: Literal["nw", "n", "ne", "en", "e", "es", "se", "s", "sw", "ws", "w", "wn"] = ..., labelwidget: tkinter.Misc = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., underline: int = ..., - width: tkinter._ScreenUnits = ..., + width: float | str = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -616,16 +614,16 @@ class Menubutton(Widget): master: tkinter.Misc | None = None, *, class_: str = "", - compound: _TtkCompound = "", + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = "", cursor: tkinter._Cursor = "", direction: Literal["above", "below", "left", "right", "flush"] = "below", - image: tkinter._ImageSpec = "", + image: tkinter._Image | str = "", menu: tkinter.Menu = ..., name: str = ..., padding=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = "", textvariable: tkinter.Variable = ..., underline: int = -1, @@ -636,15 +634,15 @@ class Menubutton(Widget): self, cnf: dict[str, Any] | None = None, *, - compound: _TtkCompound = ..., + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = ..., cursor: tkinter._Cursor = ..., direction: Literal["above", "below", "left", "right", "flush"] = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., menu: tkinter.Menu = ..., padding=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: tkinter.Variable = ..., underline: int = ..., @@ -665,7 +663,7 @@ class Notebook(Widget): name: str = ..., padding: _Padding = ..., style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., width: int = 0, ) -> None: ... @overload @@ -677,7 +675,7 @@ class Notebook(Widget): height: int = ..., padding: _Padding = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., width: int = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload @@ -694,7 +692,7 @@ class Notebook(Widget): # `image` is a sequence of an image name, followed by zero or more # (sequences of one or more state names followed by an image name) image=..., - compound: tkinter._Compound = ..., + compound: Literal["top", "left", "center", "right", "bottom", "none"] = ..., underline: int = ..., ) -> None: ... def forget(self, tab_id) -> None: ... # type: ignore[override] @@ -719,7 +717,7 @@ class Panedwindow(Widget, tkinter.PanedWindow): name: str = ..., orient: Literal["vertical", "horizontal"] = "vertical", # can't be changed with configure() style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", width: int = 0, ) -> None: ... def add(self, child: tkinter.Widget, *, weight: int = ..., **kw) -> None: ... @@ -731,7 +729,7 @@ class Panedwindow(Widget, tkinter.PanedWindow): cursor: tkinter._Cursor = ..., height: int = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., width: int = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload @@ -745,7 +743,7 @@ class Panedwindow(Widget, tkinter.PanedWindow): cursor: tkinter._Cursor = ..., height: int = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., width: int = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload @@ -764,14 +762,14 @@ class Progressbar(Widget): *, class_: str = "", cursor: tkinter._Cursor = "", - length: tkinter._ScreenUnits = 100, + length: float | str = 100, maximum: float = 100, mode: Literal["determinate", "indeterminate"] = "determinate", name: str = ..., orient: Literal["horizontal", "vertical"] = "horizontal", phase: int = 0, # docs say read-only but assigning int to this works style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", value: float = 0.0, variable: tkinter.IntVar | tkinter.DoubleVar = ..., ) -> None: ... @@ -781,13 +779,13 @@ class Progressbar(Widget): cnf: dict[str, Any] | None = None, *, cursor: tkinter._Cursor = ..., - length: tkinter._ScreenUnits = ..., + length: float | str = ..., maximum: float = ..., mode: Literal["determinate", "indeterminate"] = ..., orient: Literal["horizontal", "vertical"] = ..., phase: int = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., value: float = ..., variable: tkinter.IntVar | tkinter.DoubleVar = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @@ -804,15 +802,15 @@ class Radiobutton(Widget): master: tkinter.Misc | None = None, *, class_: str = "", - command: tkinter._ButtonCommand = "", - compound: _TtkCompound = "", + command: str | Callable[[], Any] = "", + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = "", cursor: tkinter._Cursor = "", - image: tkinter._ImageSpec = "", + image: tkinter._Image | str = "", name: str = ..., padding=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = "", textvariable: tkinter.Variable = ..., underline: int = -1, @@ -825,14 +823,14 @@ class Radiobutton(Widget): self, cnf: dict[str, Any] | None = None, *, - command: tkinter._ButtonCommand = ..., - compound: _TtkCompound = ..., + command: str | Callable[[], Any] = ..., + compound: Literal["", "text", "image", "top", "left", "center", "right", "bottom", "none"] = ..., cursor: tkinter._Cursor = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., padding=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., text: float | str = ..., textvariable: tkinter.Variable = ..., underline: int = ..., @@ -855,12 +853,12 @@ class Scale(Widget, tkinter.Scale): # type: ignore[misc] command: str | Callable[[str], object] = "", cursor: tkinter._Cursor = "", from_: float = 0, - length: tkinter._ScreenUnits = 100, + length: float | str = 100, name: str = ..., orient: Literal["horizontal", "vertical"] = "horizontal", state: str = ..., # undocumented style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., to: float = 1.0, value: float = 0, variable: tkinter.IntVar | tkinter.DoubleVar = ..., @@ -873,11 +871,11 @@ class Scale(Widget, tkinter.Scale): # type: ignore[misc] command: str | Callable[[str], object] = ..., cursor: tkinter._Cursor = ..., from_: float = ..., - length: tkinter._ScreenUnits = ..., + length: float | str = ..., orient: Literal["horizontal", "vertical"] = ..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., to: float = ..., value: float = ..., variable: tkinter.IntVar | tkinter.DoubleVar = ..., @@ -893,11 +891,11 @@ class Scale(Widget, tkinter.Scale): # type: ignore[misc] command: str | Callable[[str], object] = ..., cursor: tkinter._Cursor = ..., from_: float = ..., - length: tkinter._ScreenUnits = ..., + length: float | str = ..., orient: Literal["horizontal", "vertical"] = ..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., to: float = ..., value: float = ..., variable: tkinter.IntVar | tkinter.DoubleVar = ..., @@ -918,7 +916,7 @@ class Scrollbar(Widget, tkinter.Scrollbar): # type: ignore[misc] name: str = ..., orient: Literal["horizontal", "vertical"] = "vertical", style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", ) -> None: ... @overload # type: ignore[override] def configure( @@ -929,7 +927,7 @@ class Scrollbar(Widget, tkinter.Scrollbar): # type: ignore[misc] cursor: tkinter._Cursor = ..., orient: Literal["horizontal", "vertical"] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -943,7 +941,7 @@ class Scrollbar(Widget, tkinter.Scrollbar): # type: ignore[misc] cursor: tkinter._Cursor = ..., orient: Literal["horizontal", "vertical"] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def config(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -958,7 +956,7 @@ class Separator(Widget): name: str = ..., orient: Literal["horizontal", "vertical"] = "horizontal", style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", ) -> None: ... @overload def configure( @@ -968,7 +966,7 @@ class Separator(Widget): cursor: tkinter._Cursor = ..., orient: Literal["horizontal", "vertical"] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -983,7 +981,7 @@ class Sizegrip(Widget): cursor: tkinter._Cursor = ..., name: str = ..., style: str = "", - takefocus: tkinter._TakeFocusValue = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", ) -> None: ... @overload def configure( @@ -992,7 +990,7 @@ class Sizegrip(Widget): *, cursor: tkinter._Cursor = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1013,21 +1011,21 @@ class Spinbox(Entry): format: str = "", from_: float = 0, increment: float = 1, - invalidcommand: tkinter._EntryValidateCommand = ..., # undocumented + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., # undocumented justify: Literal["left", "center", "right"] = ..., # undocumented name: str = ..., show=..., # undocumented state: str = "normal", style: str = "", - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., # undocumented to: float = 0, validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = "none", - validatecommand: tkinter._EntryValidateCommand = "", + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = "", values: list[str] | tuple[str, ...] = ..., width: int = ..., # undocumented wrap: bool = False, - xscrollcommand: tkinter._XYScrollCommand = "", + xscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload # type: ignore[override] def configure( @@ -1043,20 +1041,20 @@ class Spinbox(Entry): format: str = ..., from_: float = ..., increment: float = ..., - invalidcommand: tkinter._EntryValidateCommand = ..., + invalidcommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., justify: Literal["left", "center", "right"] = ..., show=..., state: str = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., textvariable: tkinter.Variable = ..., to: float = ..., validate: Literal["none", "focus", "focusin", "focusout", "key", "all"] = ..., - validatecommand: tkinter._EntryValidateCommand = ..., + validatecommand: str | list[str] | tuple[str, ...] | Callable[[], bool] = ..., values: list[str] | tuple[str, ...] = ..., width: int = ..., wrap: bool = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1083,7 +1081,7 @@ class _TreeviewTagDict(TypedDict): class _TreeviewHeaderDict(TypedDict): text: str image: list[str] | Literal[""] - anchor: tkinter._Anchor + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] command: str state: str # Doesn't seem to appear anywhere else than in these dicts @@ -1092,7 +1090,7 @@ class _TreeviewColumnDict(TypedDict): width: int minwidth: int stretch: bool # actually 0 or 1 - anchor: tkinter._Anchor + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] id: str class Treeview(Widget, tkinter.XView, tkinter.YView): @@ -1114,9 +1112,9 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): # surprised if someone is using it. show: Literal["tree", "headings", "tree headings", ""] | list[str] | tuple[str, ...] = ("tree", "headings"), style: str = "", - takefocus: tkinter._TakeFocusValue = ..., - xscrollcommand: tkinter._XYScrollCommand = "", - yscrollcommand: tkinter._XYScrollCommand = "", + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + xscrollcommand: str | Callable[[float, float], object] = "", + yscrollcommand: str | Callable[[float, float], object] = "", ) -> None: ... @overload def configure( @@ -1131,9 +1129,9 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): selectmode: Literal["extended", "browse", "none"] = ..., show: Literal["tree", "headings", "tree headings", ""] | list[str] | tuple[str, ...] = ..., style: str = ..., - takefocus: tkinter._TakeFocusValue = ..., - xscrollcommand: tkinter._XYScrollCommand = ..., - yscrollcommand: tkinter._XYScrollCommand = ..., + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = ..., + xscrollcommand: str | Callable[[float, float], object] = ..., + yscrollcommand: str | Callable[[float, float], object] = ..., ) -> dict[str, tuple[str, str, str, Any, Any]] | None: ... @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... @@ -1160,7 +1158,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): width: int = ..., minwidth: int = ..., stretch: bool = ..., - anchor: tkinter._Anchor = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., # id is read-only ) -> _TreeviewColumnDict | None: ... def delete(self, *items: str | int) -> None: ... @@ -1189,8 +1187,8 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): option: None = None, *, text: str = ..., - image: tkinter._ImageSpec = ..., - anchor: tkinter._Anchor = ..., + image: tkinter._Image | str = ..., + anchor: Literal["nw", "n", "ne", "w", "center", "e", "sw", "s", "se"] = ..., command: str | Callable[[], object] = ..., ) -> None: ... # Internal Method. Leave untyped: @@ -1208,7 +1206,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): *, id: str | int = ..., # same as iid text: str = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., values: list[Any] | tuple[Any, ...] = ..., open: bool = ..., tags: str | list[str] | tuple[str, ...] = ..., @@ -1234,7 +1232,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): option: None = None, *, text: str = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., values: list[Any] | tuple[Any, ...] | Literal[""] = ..., open: bool = ..., tags: str | list[str] | tuple[str, ...] = ..., @@ -1294,7 +1292,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): foreground: str = ..., background: str = ..., font: _FontDescription = ..., - image: tkinter._ImageSpec = ..., + image: tkinter._Image | str = ..., ) -> _TreeviewTagDict | MaybeNone: ... # can be None but annoying to check @overload def tag_has(self, tagname: str, item: None = None) -> tuple[str, ...]: ... @@ -1313,18 +1311,18 @@ class LabeledScale(Frame): from_: float = 0, to: float = 10, *, - border: tkinter._ScreenUnits = ..., - borderwidth: tkinter._ScreenUnits = ..., + border: float | str = ..., + borderwidth: float | str = ..., class_: str = "", compound: Literal["top", "bottom"] = "top", cursor: tkinter._Cursor = "", - height: tkinter._ScreenUnits = 0, + height: float | str = 0, name: str = ..., padding: _Padding = ..., - relief: tkinter._Relief = ..., + relief: Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] = ..., style: str = "", - takefocus: tkinter._TakeFocusValue = "", - width: tkinter._ScreenUnits = 0, + takefocus: bool | Literal[0, 1, ""] | Callable[[str], bool | None] = "", + width: float | str = 0, ) -> None: ... # destroy is overridden, signature does not change value: Any diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 591d5da2360d..ba343ce9effc 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -392,8 +392,8 @@ class CellType: cell_contents: Any _YieldT_co = TypeVar("_YieldT_co", covariant=True) -_SendT_contra = TypeVar("_SendT_contra", contravariant=True) -_ReturnT_co = TypeVar("_ReturnT_co", covariant=True) +_SendT_contra = TypeVar("_SendT_contra", contravariant=True, default=None) +_ReturnT_co = TypeVar("_ReturnT_co", covariant=True, default=None) @final class GeneratorType(Generator[_YieldT_co, _SendT_contra, _ReturnT_co]): @@ -450,16 +450,25 @@ class AsyncGeneratorType(AsyncGenerator[_YieldT_co, _SendT_contra]): def aclose(self) -> Coroutine[Any, Any, None]: ... def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... +# Non-default variations to accommodate coroutines +_SendT_nd_contra = TypeVar("_SendT_nd_contra", contravariant=True) +_ReturnT_nd_co = TypeVar("_ReturnT_nd_co", covariant=True) + @final -class CoroutineType(Coroutine[_YieldT_co, _SendT_contra, _ReturnT_co]): +class CoroutineType(Coroutine[_YieldT_co, _SendT_nd_contra, _ReturnT_nd_co]): __name__: str __qualname__: str @property def cr_await(self) -> Any | None: ... @property def cr_code(self) -> CodeType: ... - @property - def cr_frame(self) -> FrameType: ... + if sys.version_info >= (3, 12): + @property + def cr_frame(self) -> FrameType | None: ... + else: + @property + def cr_frame(self) -> FrameType: ... + @property def cr_running(self) -> bool: ... @property @@ -469,8 +478,8 @@ class CoroutineType(Coroutine[_YieldT_co, _SendT_contra, _ReturnT_co]): def cr_suspended(self) -> bool: ... def close(self) -> None: ... - def __await__(self) -> Generator[Any, None, _ReturnT_co]: ... - def send(self, arg: _SendT_contra, /) -> _YieldT_co: ... + def __await__(self) -> Generator[Any, None, _ReturnT_nd_co]: ... + def send(self, arg: _SendT_nd_contra, /) -> _YieldT_co: ... @overload def throw( self, typ: type[BaseException], val: BaseException | object = ..., tb: TracebackType | None = ..., / @@ -717,10 +726,19 @@ if sys.version_info >= (3, 10): def __args__(self) -> tuple[Any, ...]: ... @property def __parameters__(self) -> tuple[Any, ...]: ... - def __or__(self, value: Any, /) -> UnionType: ... - def __ror__(self, value: Any, /) -> UnionType: ... + # `(int | str) | Literal["foo"]` returns a generic alias to an instance of `_SpecialForm` (`Union`). + # Normally we'd express this using the return type of `_SpecialForm.__ror__`, + # but because `UnionType.__or__` accepts `Any`, type checkers will use + # the return type of `UnionType.__or__` to infer the result of this operation + # rather than `_SpecialForm.__ror__`. To mitigate this, we use `| Any` + # in the return type of `UnionType.__(r)or__`. + def __or__(self, value: Any, /) -> UnionType | Any: ... + def __ror__(self, value: Any, /) -> UnionType | Any: ... def __eq__(self, value: object, /) -> bool: ... def __hash__(self) -> int: ... + # you can only subscript a `UnionType` instance if at least one of the elements + # in the union is a generic alias instance that has a non-empty `__parameters__` + def __getitem__(self, parameters: Any) -> object: ... if sys.version_info >= (3, 13): @final diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index 15a5864613d1..ca25c92d5c34 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -41,6 +41,7 @@ __all__ = [ "AsyncIterator", "Awaitable", "BinaryIO", + "ByteString", "Callable", "ChainMap", "ClassVar", @@ -109,9 +110,6 @@ __all__ = [ "runtime_checkable", ] -if sys.version_info < (3, 14): - __all__ += ["ByteString"] - if sys.version_info >= (3, 14): __all__ += ["evaluate_forward_ref"] @@ -579,7 +577,7 @@ class Awaitable(Protocol[_T_co]): @abstractmethod def __await__(self) -> Generator[Any, Any, _T_co]: ... -# Non-default variations to accommodate couroutines, and `AwaitableGenerator` having a 4th type parameter. +# Non-default variations to accommodate coroutines, and `AwaitableGenerator` having a 4th type parameter. _SendT_nd_contra = TypeVar("_SendT_nd_contra", contravariant=True) _ReturnT_nd_co = TypeVar("_ReturnT_nd_co", covariant=True) @@ -923,8 +921,7 @@ class TextIO(IO[str]): @abstractmethod def __enter__(self) -> TextIO: ... -if sys.version_info < (3, 14): - ByteString: typing_extensions.TypeAlias = bytes | bytearray | memoryview +ByteString: typing_extensions.TypeAlias = bytes | bytearray | memoryview # Functions From 053c0545dfe48126b66af8ece091af3c0c29bddc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 1 Oct 2025 13:08:05 +0100 Subject: [PATCH 059/183] Fix crash on invalid unpack in base class (#19962) Fixes https://github.com/python/mypy/issues/19960 Fix is trivial, we can't use the "assertion" before type checking phase. I also fix missing line/column for union types. I am surprised this didn't cause problems before. --- mypy/semanal_shared.py | 4 +++- mypy/typeanal.py | 2 +- test-data/unit/check-python312.test | 10 ++++++++++ test-data/unit/check-typevar-tuple.test | 24 ++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index e94604b66381..c49b13d08f45 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -303,7 +303,9 @@ def calculate_tuple_fallback(typ: TupleType) -> None: ): items.append(unpacked_type.args[0]) else: - raise NotImplementedError + # This is called before semanal_typeargs.py fixes broken unpacks, + # where the error should also be generated. + items.append(AnyType(TypeOfAny.from_error)) else: items.append(item) fallback.args = (make_simplified_union(items),) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 81fb87fbf9ee..d7a07c9f48e3 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -634,7 +634,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ ) elif fullname == "typing.Union": items = self.anal_array(t.args) - return UnionType.make_union(items) + return UnionType.make_union(items, line=t.line, column=t.column) elif fullname == "typing.Optional": if len(t.args) != 1: self.fail( diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 382822ced861..6a72a7e4d5b5 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2121,3 +2121,13 @@ class A: ... x1: Alias1[A] # ok x2: Alias2[A] # ok + +[case testUndefinedUnpackInPEP696Base] +# Typo below is intentional. +class MyTuple[*Ts](tuple[*TS]): # E: Name "TS" is not defined + ... + +x: MyTuple[int, str] +reveal_type(x[0]) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index c668f14eaa50..927a4f037a4a 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2692,3 +2692,27 @@ tuple(a) (x,) = a (_,) = a [builtins fixtures/tuple.pyi] + +[case testNoCrashOnUndefinedUnpackInBase] +from typing import TypeVarTuple, Generic, Unpack + +Ts = TypeVarTuple("Ts") + +class MyTuple(tuple[Unpack[TsWithTypo]], Generic[Unpack[Ts]]): # E: Name "TsWithTypo" is not defined + ... + +x: MyTuple[int, str] +reveal_type(x[0]) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testNoCrashOnInvalidUnpackInBase] +from typing import TypeVarTuple, Generic, Unpack, Union + +Ts = TypeVarTuple("Ts") + +class MyTuple(tuple[Unpack[Union[int, str]]], Generic[Unpack[Ts]]): # E: "Union[int, str]" cannot be unpacked (must be tuple or TypeVarTuple) + ... + +x: MyTuple[int, str] +reveal_type(x[0]) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] From 4171da0cf394bb0b718eea3019760a8a33de0c74 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 2 Oct 2025 01:37:50 +0100 Subject: [PATCH 060/183] Delete native_internal import fallback (#19966) This should not be needed after we switched to `librt` published on PyPI. --- mypy/cache.py | 71 ++++++++++----------------------------------------- 1 file changed, 14 insertions(+), 57 deletions(-) diff --git a/mypy/cache.py b/mypy/cache.py index 08e3b05d1a75..51deac914efc 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -1,65 +1,22 @@ from __future__ import annotations from collections.abc import Sequence -from typing import TYPE_CHECKING, Final +from typing import Final from mypy_extensions import u8 - -try: - from native_internal import ( - Buffer as Buffer, - read_bool as read_bool, - read_float as read_float, - read_int as read_int, - read_str as read_str, - read_tag as read_tag, - write_bool as write_bool, - write_float as write_float, - write_int as write_int, - write_str as write_str, - write_tag as write_tag, - ) -except ImportError: - # TODO: temporary, remove this after we publish mypy-native on PyPI. - if not TYPE_CHECKING: - - class Buffer: - def __init__(self, source: bytes = b"") -> None: - raise NotImplementedError - - def getvalue(self) -> bytes: - raise NotImplementedError - - def read_int(data: Buffer) -> int: - raise NotImplementedError - - def write_int(data: Buffer, value: int) -> None: - raise NotImplementedError - - def read_tag(data: Buffer) -> u8: - raise NotImplementedError - - def write_tag(data: Buffer, value: u8) -> None: - raise NotImplementedError - - def read_str(data: Buffer) -> str: - raise NotImplementedError - - def write_str(data: Buffer, value: str) -> None: - raise NotImplementedError - - def read_bool(data: Buffer) -> bool: - raise NotImplementedError - - def write_bool(data: Buffer, value: bool) -> None: - raise NotImplementedError - - def read_float(data: Buffer) -> float: - raise NotImplementedError - - def write_float(data: Buffer, value: float) -> None: - raise NotImplementedError - +from native_internal import ( + Buffer as Buffer, + read_bool as read_bool, + read_float as read_float, + read_int as read_int, + read_str as read_str, + read_tag as read_tag, + write_bool as write_bool, + write_float as write_float, + write_int as write_int, + write_str as write_str, + write_tag as write_tag, +) # Always use this type alias to refer to type tags. Tag = u8 From baabf49b06c8b6e2c4e6d5fcd0e92cf325eef790 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 2 Oct 2025 02:06:27 -0400 Subject: [PATCH 061/183] feat: support constant folding in `ExpressionChecker.check_str_format_call` [1/1] (#19977) This PR implements constant folding in `ExpressionChecker.check_str_format_call` --- mypy/checkexpr.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 73282c94be4e..b8f9bf087467 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -18,6 +18,7 @@ from mypy.checker_shared import ExpressionCheckerSharedApi from mypy.checkmember import analyze_member_access, has_operator from mypy.checkstrformat import StringFormatterChecker +from mypy.constant_fold import constant_fold_expr from mypy.erasetype import erase_type, remove_instance_last_known_values, replace_meta_vars from mypy.errors import ErrorInfo, ErrorWatcher, report_internal_error from mypy.expandtype import ( @@ -656,11 +657,12 @@ def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) -> return ret_type def check_str_format_call(self, e: CallExpr) -> None: - """More precise type checking for str.format() calls on literals.""" + """More precise type checking for str.format() calls on literals and folded constants.""" assert isinstance(e.callee, MemberExpr) format_value = None - if isinstance(e.callee.expr, StrExpr): - format_value = e.callee.expr.value + folded_callee_expr = constant_fold_expr(e.callee.expr, "") + if isinstance(folded_callee_expr, str): + format_value = folded_callee_expr elif self.chk.has_type(e.callee.expr): typ = get_proper_type(self.chk.lookup_type(e.callee.expr)) if ( From 41efd21f143fdb250bd52d85f700b2d807aa26fd Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 2 Oct 2025 02:07:05 -0400 Subject: [PATCH 062/183] [mypyc] feat: support constant folding in `try_optimize_int_floor_divide` (#19973) This PR attempts to constant fold the divisor value in `try_optimize_int_floor_divide` I'm not sure any test changes are warranted for a small PR of this nature. --- mypyc/irbuild/expression.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 54a101bc4961..59ecc4ac2c5c 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -558,7 +558,7 @@ def transform_op_expr(builder: IRBuilder, expr: OpExpr) -> Value: # Special case some int ops to allow borrowing operands. if is_int_rprimitive(ltype) and is_int_rprimitive(rtype): if expr.op == "//": - expr = try_optimize_int_floor_divide(expr) + expr = try_optimize_int_floor_divide(builder, expr) if expr.op in int_borrow_friendly_op: borrow_left = is_borrow_friendly_expr(builder, expr.right) borrow_right = True @@ -571,11 +571,11 @@ def transform_op_expr(builder: IRBuilder, expr: OpExpr) -> Value: return builder.binary_op(left, right, expr.op, expr.line) -def try_optimize_int_floor_divide(expr: OpExpr) -> OpExpr: +def try_optimize_int_floor_divide(builder: IRBuilder, expr: OpExpr) -> OpExpr: """Replace // with a power of two with a right shift, if possible.""" - if not isinstance(expr.right, IntExpr): + divisor = constant_fold_expr(builder, expr.right) + if not isinstance(divisor, int): return expr - divisor = expr.right.value shift = divisor.bit_length() - 1 if 0 < shift < 28 and divisor == (1 << shift): return OpExpr(">>", expr.left, IntExpr(shift)) From c71fef017877d84932496f4a7d9e77891479e57f Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 2 Oct 2025 02:12:25 -0400 Subject: [PATCH 063/183] [mypyc] feat: support constant folding in `translate_str_format` (#19971) This PR adds support for constant folding inside of `translate_str_format` --- mypyc/irbuild/specialize.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 84807a7fdb53..a099e97390ea 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -78,6 +78,7 @@ uint8_rprimitive, ) from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.constant_fold import constant_fold_expr from mypyc.irbuild.for_helpers import ( comprehension_helper, sequence_from_generator_preallocate_helper, @@ -716,21 +717,18 @@ def translate_dict_setdefault(builder: IRBuilder, expr: CallExpr, callee: RefExp @specialize_function("format", str_rprimitive) def translate_str_format(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: - if ( - isinstance(callee, MemberExpr) - and isinstance(callee.expr, StrExpr) - and expr.arg_kinds.count(ARG_POS) == len(expr.arg_kinds) - ): - format_str = callee.expr.value - tokens = tokenizer_format_call(format_str) - if tokens is None: - return None - literals, format_ops = tokens - # Convert variables to strings - substitutions = convert_format_expr_to_str(builder, format_ops, expr.args, expr.line) - if substitutions is None: - return None - return join_formatted_strings(builder, literals, substitutions, expr.line) + if isinstance(callee, MemberExpr): + folded_callee = constant_fold_expr(builder, callee.expr) + if isinstance(folded_callee, str) and expr.arg_kinds.count(ARG_POS) == len(expr.arg_kinds): + tokens = tokenizer_format_call(folded_callee) + if tokens is None: + return None + literals, format_ops = tokens + # Convert variables to strings + substitutions = convert_format_expr_to_str(builder, format_ops, expr.args, expr.line) + if substitutions is None: + return None + return join_formatted_strings(builder, literals, substitutions, expr.line) return None From bf0a60c5a2a3a0ad5f685d9e3260ef9d23822a1e Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:05:45 +0200 Subject: [PATCH 064/183] Add librt as runtime dependency (#19986) `librt` is needed both at build and runtime. It's already part of mypy-requirements.txt. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 46593af0ab72..589679113c3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", + "librt>=0.1.0", ] dynamic = ["version"] From 29ee9c27f812057517e092d46750d4f7ce042d84 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 2 Oct 2025 09:43:04 -0400 Subject: [PATCH 065/183] [mypyc] feat: support constant folding in `IRBuilder.extract_int` [1/1] (#19969) This PR adds support for constant folding inside of `IRBuilder.extract_int` --- mypyc/irbuild/builder.py | 10 +++------- mypyc/irbuild/constant_fold.py | 6 ++++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 125382145991..63930123135f 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -40,7 +40,6 @@ TypeAlias, TypeInfo, TypeParam, - UnaryExpr, Var, ) from mypy.types import ( @@ -106,6 +105,7 @@ object_rprimitive, str_rprimitive, ) +from mypyc.irbuild.constant_fold import constant_fold_expr from mypyc.irbuild.context import FuncInfo, ImplicitClass from mypyc.irbuild.ll_builder import LowLevelIRBuilder from mypyc.irbuild.mapper import Mapper @@ -965,12 +965,8 @@ def maybe_spill_assignable(self, value: Value) -> Register | AssignmentTarget: return reg def extract_int(self, e: Expression) -> int | None: - if isinstance(e, IntExpr): - return e.value - elif isinstance(e, UnaryExpr) and e.op == "-" and isinstance(e.expr, IntExpr): - return -e.expr.value - else: - return None + folded = constant_fold_expr(self, e) + return folded if isinstance(folded, int) else None def get_sequence_type(self, expr: Expression) -> RType: return self.get_sequence_type_from_type(self.types[expr]) diff --git a/mypyc/irbuild/constant_fold.py b/mypyc/irbuild/constant_fold.py index 12a4b15dd40c..b1133f95b18e 100644 --- a/mypyc/irbuild/constant_fold.py +++ b/mypyc/irbuild/constant_fold.py @@ -10,7 +10,7 @@ from __future__ import annotations -from typing import Final, Union +from typing import TYPE_CHECKING, Final, Union from mypy.constant_fold import constant_fold_binary_op, constant_fold_unary_op from mypy.nodes import ( @@ -26,9 +26,11 @@ UnaryExpr, Var, ) -from mypyc.irbuild.builder import IRBuilder from mypyc.irbuild.util import bytes_from_str +if TYPE_CHECKING: + from mypyc.irbuild.builder import IRBuilder + # All possible result types of constant folding ConstantValue = Union[int, float, complex, str, bytes] CONST_TYPES: Final = (int, float, complex, str, bytes) From 536f3df8a5cddd949bd6acaa9d89c892888afbb9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 2 Oct 2025 17:20:20 +0100 Subject: [PATCH 066/183] Rename remaining refs to mypy-native to librt (#19989) We agreed that PyPI distribution name is `librt` so use it consistently. --- mypy/modulefinder.py | 8 ++++---- mypy/typeshed/stubs/librt/METADATA.toml | 1 + .../stubs/{mypy-native => librt}/native_internal.pyi | 0 mypy/typeshed/stubs/mypy-native/METADATA.toml | 1 - 4 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 mypy/typeshed/stubs/librt/METADATA.toml rename mypy/typeshed/stubs/{mypy-native => librt}/native_internal.pyi (100%) delete mode 100644 mypy/typeshed/stubs/mypy-native/METADATA.toml diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index d61c9ee3ec3f..5176b7e1df52 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -796,7 +796,7 @@ def default_lib_path( custom_typeshed_dir = os.path.abspath(custom_typeshed_dir) typeshed_dir = os.path.join(custom_typeshed_dir, "stdlib") mypy_extensions_dir = os.path.join(custom_typeshed_dir, "stubs", "mypy-extensions") - mypy_native_dir = os.path.join(custom_typeshed_dir, "stubs", "mypy-native") + librt_dir = os.path.join(custom_typeshed_dir, "stubs", "librt") versions_file = os.path.join(typeshed_dir, "VERSIONS") if not os.path.isdir(typeshed_dir) or not os.path.isfile(versions_file): print( @@ -812,13 +812,13 @@ def default_lib_path( data_dir = auto typeshed_dir = os.path.join(data_dir, "typeshed", "stdlib") mypy_extensions_dir = os.path.join(data_dir, "typeshed", "stubs", "mypy-extensions") - mypy_native_dir = os.path.join(data_dir, "typeshed", "stubs", "mypy-native") + librt_dir = os.path.join(data_dir, "typeshed", "stubs", "librt") path.append(typeshed_dir) - # Get mypy-extensions and mypy-native stubs from typeshed, since we treat them as + # Get mypy-extensions and librt stubs from typeshed, since we treat them as # "internal" libraries, similar to typing and typing-extensions. path.append(mypy_extensions_dir) - path.append(mypy_native_dir) + path.append(librt_dir) # Add fallback path that can be used if we have a broken installation. if sys.platform != "win32": diff --git a/mypy/typeshed/stubs/librt/METADATA.toml b/mypy/typeshed/stubs/librt/METADATA.toml new file mode 100644 index 000000000000..37dc09b102d4 --- /dev/null +++ b/mypy/typeshed/stubs/librt/METADATA.toml @@ -0,0 +1 @@ +version = "0.1.*" diff --git a/mypy/typeshed/stubs/mypy-native/native_internal.pyi b/mypy/typeshed/stubs/librt/native_internal.pyi similarity index 100% rename from mypy/typeshed/stubs/mypy-native/native_internal.pyi rename to mypy/typeshed/stubs/librt/native_internal.pyi diff --git a/mypy/typeshed/stubs/mypy-native/METADATA.toml b/mypy/typeshed/stubs/mypy-native/METADATA.toml deleted file mode 100644 index 76574b01cb4b..000000000000 --- a/mypy/typeshed/stubs/mypy-native/METADATA.toml +++ /dev/null @@ -1 +0,0 @@ -version = "0.0.*" From a62f273d36c281073326e94894ecb087deff760a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 2 Oct 2025 17:20:36 +0100 Subject: [PATCH 067/183] Stop calling fixed format cache experimental (#19967) It has been around for some time with no issues so far. We can stop calling it experimental so that more people try it before we make it default. --- mypy/main.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 9ebbf78ded09..19cb4f0d0a99 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1066,11 +1066,7 @@ def add_invertible_flag( incremental_group.add_argument( "--fixed-format-cache", action="store_true", - help=( - "Use experimental fast and compact fixed format cache" - if compilation_status == "yes" - else argparse.SUPPRESS - ), + help="Use new fast and compact fixed format cache", ) incremental_group.add_argument( "--skip-version-check", From 1b8841b665efdc308828ece877cb8e5141954a7b Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 2 Oct 2025 15:25:55 -0400 Subject: [PATCH 068/183] [mypyc] feat: support constant folding in `translate_ord` [1/1] (#19968) This PR adds support for constant folding inside of `translate_ord` --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/irbuild/specialize.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index a099e97390ea..e810f11bd079 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -19,7 +19,6 @@ from mypy.nodes import ( ARG_NAMED, ARG_POS, - BytesExpr, CallExpr, DictExpr, Expression, @@ -1057,9 +1056,9 @@ def translate_float(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Valu def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: return None - arg = expr.args[0] - if isinstance(arg, (StrExpr, BytesExpr)) and len(arg.value) == 1: - return Integer(ord(arg.value)) + arg = constant_fold_expr(builder, expr.args[0]) + if isinstance(arg, (str, bytes)) and len(arg) == 1: + return Integer(ord(arg)) return None From a35e84b0a6b420459a062557c155af6e22876752 Mon Sep 17 00:00:00 2001 From: James Hilton-Balfe Date: Sat, 4 Oct 2025 02:43:41 +0100 Subject: [PATCH 069/183] Allow returning Literals in __new__ (#15687) Unblocks https://github.com/python/typeshed/pull/10465 --------- Co-authored-by: Ivan Levkivskyi --- mypy/checker.py | 3 ++- mypy/checkmember.py | 2 ++ mypy/typeops.py | 2 +- mypy/types.py | 2 ++ test-data/unit/check-classes.test | 21 ++++++++++++++++++ test-data/unit/fixtures/__new__.pyi | 1 + test-data/unit/fixtures/literal__new__.pyi | 25 ++++++++++++++++++++++ 7 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 test-data/unit/fixtures/literal__new__.pyi diff --git a/mypy/checker.py b/mypy/checker.py index 96b55f321a73..3bee7b633339 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1801,7 +1801,8 @@ def check___new___signature(self, fdef: FuncDef, typ: CallableType) -> None: "but must return a subtype of", ) elif not isinstance( - get_proper_type(bound_type.ret_type), (AnyType, Instance, TupleType, UninhabitedType) + get_proper_type(bound_type.ret_type), + (AnyType, Instance, TupleType, UninhabitedType, LiteralType), ): self.fail( message_registry.NON_INSTANCE_NEW_TYPE.format( diff --git a/mypy/checkmember.py b/mypy/checkmember.py index f19a76ec6a34..719b48b14e07 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -410,6 +410,8 @@ def analyze_type_callable_member_access(name: str, typ: FunctionLike, mx: Member ret_type = tuple_fallback(ret_type) if isinstance(ret_type, TypedDictType): ret_type = ret_type.fallback + if isinstance(ret_type, LiteralType): + ret_type = ret_type.fallback if isinstance(ret_type, Instance): if not mx.is_operator: # When Python sees an operator (eg `3 == 4`), it automatically translates that diff --git a/mypy/typeops.py b/mypy/typeops.py index 298ad4d16f8c..d058bb8201d3 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -319,7 +319,7 @@ def class_callable( default_ret_type = fill_typevars(info) explicit_type = init_ret_type if is_new else orig_self_type if ( - isinstance(explicit_type, (Instance, TupleType, UninhabitedType)) + isinstance(explicit_type, (Instance, TupleType, UninhabitedType, LiteralType)) # We have to skip protocols, because it can be a subtype of a return type # by accident. Like `Hashable` is a subtype of `object`. See #11799 and isinstance(default_ret_type, Instance) diff --git a/mypy/types.py b/mypy/types.py index 38c17e240ccf..426d560c2bf7 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2278,6 +2278,8 @@ def type_object(self) -> mypy.nodes.TypeInfo: ret = ret.partial_fallback if isinstance(ret, TypedDictType): ret = ret.fallback + if isinstance(ret, LiteralType): + ret = ret.fallback assert isinstance(ret, Instance) return ret.type diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 498a2c12b6e8..c0b1114db512 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -472,6 +472,27 @@ class B(A): def __new__(cls) -> B: pass +[case testOverride__new__WithLiteralReturnPassing] +from typing import Literal + +class Falsy: + def __bool__(self) -> Literal[False]: pass + +reveal_type(bool(Falsy())) # N: Revealed type is "Literal[False]" +reveal_type(int()) # N: Revealed type is "Literal[0]" + +[builtins fixtures/literal__new__.pyi] +[typing fixtures/typing-medium.pyi] + +[case testOverride__new__WithLiteralReturnFailing] +from typing import Literal + +class Foo: + def __new__(cls) -> Literal[1]: pass # E: Incompatible return type for "__new__" (returns "Literal[1]", but must return a subtype of "Foo") + +[builtins fixtures/__new__.pyi] +[typing fixtures/typing-medium.pyi] + [case testOverride__new__AndCallObject] from typing import TypeVar, Generic diff --git a/test-data/unit/fixtures/__new__.pyi b/test-data/unit/fixtures/__new__.pyi index 401de6fb9cd1..57d3624ce92c 100644 --- a/test-data/unit/fixtures/__new__.pyi +++ b/test-data/unit/fixtures/__new__.pyi @@ -12,6 +12,7 @@ class object: class type: def __init__(self, x) -> None: pass +class float: pass class int: pass class bool: pass class str: pass diff --git a/test-data/unit/fixtures/literal__new__.pyi b/test-data/unit/fixtures/literal__new__.pyi new file mode 100644 index 000000000000..971bc39bfff4 --- /dev/null +++ b/test-data/unit/fixtures/literal__new__.pyi @@ -0,0 +1,25 @@ +from typing import Literal, Protocol, overload + +class object: + def __init__(self) -> None: pass + +class type: + def __init__(self, x) -> None: pass + +class str: pass +class dict: pass +class float: pass +class int: + def __new__(cls) -> Literal[0]: pass + +class _Truthy(Protocol): + def __bool__(self) -> Literal[True]: pass + +class _Falsy(Protocol): + def __bool__(self) -> Literal[False]: pass + +class bool(int): + @overload + def __new__(cls, __o: _Truthy) -> Literal[True]: pass + @overload + def __new__(cls, __o: _Falsy) -> Literal[False]: pass From fe313349ee9895d49e7b63ec096e3c7f76d96f2e Mon Sep 17 00:00:00 2001 From: Sigve Sebastian Farstad Date: Sat, 4 Oct 2025 04:07:21 +0200 Subject: [PATCH 070/183] Support error codes from plugins in options (#19719) Mypy has options for enabling or disabling specific error codes. These work fine, except that it is not possible to enable or disable error codes from plugins, only mypy's original error codes. The crux of the issue is that mypy validates and rejects unknown error codes passed in the options before it loads plugins and learns about the any error codes that might get registered. There are many ways to solve this. This commit tries to find a pragmatic solution where the relevant options parsing is deferred until after plugin loading. Error code validation in the config parser, where plugins are not loaded yet, is also skipped entirely, since the error code options are re-validated later anyway. This means that this commit introduces a small observable change in behavior when running with invalid error codes specified, as shown in the test test_config_file_error_codes_invalid. This fixes https://github.com/python/mypy/issues/12987. --------- Co-authored-by: John Belmonte Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ivan Levkivskyi --- mypy/build.py | 7 ++++ mypy/config_parser.py | 19 +++-------- mypy/main.py | 1 - mypy/test/teststubtest.py | 35 ++++++++++++++------ test-data/unit/check-plugin-error-codes.test | 32 ++++++++++++++++++ test-data/unit/cmdline.test | 10 ------ 6 files changed, 67 insertions(+), 37 deletions(-) create mode 100644 test-data/unit/check-plugin-error-codes.test diff --git a/mypy/build.py b/mypy/build.py index 234dd6843292..9f840499fcc2 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -143,6 +143,10 @@ def __init__(self, manager: BuildManager, graph: Graph) -> None: self.errors: list[str] = [] # Filled in by build if desired +def build_error(msg: str) -> NoReturn: + raise CompileError([f"mypy: error: {msg}"]) + + def build( sources: list[BuildSource], options: Options, @@ -241,6 +245,9 @@ def _build( errors = Errors(options, read_source=lambda path: read_py_file(path, cached_read)) plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins) + # Validate error codes after plugins are loaded. + options.process_error_codes(error_callback=build_error) + # Add catch-all .gitignore to cache dir if we created it cache_dir_existed = os.path.isdir(options.cache_dir) diff --git a/mypy/config_parser.py b/mypy/config_parser.py index 5f08f342241e..2bfd2a1e2eef 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -8,8 +8,6 @@ import sys from io import StringIO -from mypy.errorcodes import error_codes - if sys.version_info >= (3, 11): import tomllib else: @@ -87,15 +85,6 @@ def complain(x: object, additional_info: str = "") -> Never: complain(v) -def validate_codes(codes: list[str]) -> list[str]: - invalid_codes = set(codes) - set(error_codes.keys()) - if invalid_codes: - raise argparse.ArgumentTypeError( - f"Invalid error code(s): {', '.join(sorted(invalid_codes))}" - ) - return codes - - def validate_package_allow_list(allow_list: list[str]) -> list[str]: for p in allow_list: msg = f"Invalid allow list entry: {p}" @@ -209,8 +198,8 @@ def split_commas(value: str) -> list[str]: [p.strip() for p in split_commas(s)] ), "enable_incomplete_feature": lambda s: [p.strip() for p in split_commas(s)], - "disable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]), - "enable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]), + "disable_error_code": lambda s: [p.strip() for p in split_commas(s)], + "enable_error_code": lambda s: [p.strip() for p in split_commas(s)], "package_root": lambda s: [p.strip() for p in split_commas(s)], "cache_dir": expand_path, "python_executable": expand_path, @@ -234,8 +223,8 @@ def split_commas(value: str) -> list[str]: "always_false": try_split, "untyped_calls_exclude": lambda s: validate_package_allow_list(try_split(s)), "enable_incomplete_feature": try_split, - "disable_error_code": lambda s: validate_codes(try_split(s)), - "enable_error_code": lambda s: validate_codes(try_split(s)), + "disable_error_code": lambda s: try_split(s), + "enable_error_code": lambda s: try_split(s), "package_root": try_split, "exclude": str_or_array_as_list, "packages": try_split, diff --git a/mypy/main.py b/mypy/main.py index 19cb4f0d0a99..a44632ed96f9 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1462,7 +1462,6 @@ def set_strict_flags() -> None: validate_package_allow_list(options.untyped_calls_exclude) validate_package_allow_list(options.deprecated_calls_exclude) - options.process_error_codes(error_callback=parser.error) options.process_incomplete_features(error_callback=parser.error, warning_callback=print) # Compute absolute path for custom typeshed (if present). diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 28263e20099d..800f522d90a0 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -12,6 +12,8 @@ from collections.abc import Iterator from typing import Any, Callable +from pytest import raises + import mypy.stubtest from mypy import build, nodes from mypy.modulefinder import BuildSource @@ -171,7 +173,12 @@ def build_helper(source: str) -> build.BuildResult: def run_stubtest_with_stderr( - stub: str, runtime: str, options: list[str], config_file: str | None = None + stub: str, + runtime: str, + options: list[str], + config_file: str | None = None, + output: io.StringIO | None = None, + outerr: io.StringIO | None = None, ) -> tuple[str, str]: with use_tmp_dir(TEST_MODULE_NAME) as tmp_dir: with open("builtins.pyi", "w") as f: @@ -188,8 +195,8 @@ def run_stubtest_with_stderr( with open(f"{TEST_MODULE_NAME}_config.ini", "w") as f: f.write(config_file) options = options + ["--mypy-config-file", f"{TEST_MODULE_NAME}_config.ini"] - output = io.StringIO() - outerr = io.StringIO() + output = io.StringIO() if output is None else output + outerr = io.StringIO() if outerr is None else outerr with contextlib.redirect_stdout(output), contextlib.redirect_stderr(outerr): test_stubs(parse_options([TEST_MODULE_NAME] + options), use_builtins_fixtures=True) filtered_output = remove_color_code( @@ -2888,14 +2895,20 @@ def test_config_file_error_codes_invalid(self) -> None: runtime = "temp = 5\n" stub = "temp: int\n" config_file = "[mypy]\ndisable_error_code = not-a-valid-name\n" - output, outerr = run_stubtest_with_stderr( - stub=stub, runtime=runtime, options=[], config_file=config_file - ) - assert output == "Success: no issues found in 1 module\n" - assert outerr == ( - "test_module_config.ini: [mypy]: disable_error_code: " - "Invalid error code(s): not-a-valid-name\n" - ) + output = io.StringIO() + outerr = io.StringIO() + with raises(SystemExit): + run_stubtest_with_stderr( + stub=stub, + runtime=runtime, + options=[], + config_file=config_file, + output=output, + outerr=outerr, + ) + + assert output.getvalue() == "error: Invalid error code(s): not-a-valid-name\n" + assert outerr.getvalue() == "" def test_config_file_wrong_incomplete_feature(self) -> None: runtime = "x = 1\n" diff --git a/test-data/unit/check-plugin-error-codes.test b/test-data/unit/check-plugin-error-codes.test new file mode 100644 index 000000000000..95789477977e --- /dev/null +++ b/test-data/unit/check-plugin-error-codes.test @@ -0,0 +1,32 @@ +[case testCustomErrorCodeFromPluginIsTargetable] +# flags: --config-file tmp/mypy.ini --show-error-codes + +def main() -> None: + return +main() # E: Custom error [custom] + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/custom_errorcode.py + +[case testCustomErrorCodeCanBeDisabled] +# flags: --config-file tmp/mypy.ini --show-error-codes --disable-error-code=custom + +def main() -> None: + return +main() # no output expected when disabled + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/custom_errorcode.py + +[case testCustomErrorCodeCanBeReenabled] +# flags: --config-file tmp/mypy.ini --show-error-codes --disable-error-code=custom --enable-error-code=custom + +def main() -> None: + return +main() # E: Custom error [custom] + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/custom_errorcode.py diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index ff60c24b72a5..35d7b700b161 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -960,8 +960,6 @@ src/foo/bar.py: note: Common resolutions include: a) adding `__init__.py` somewh [file test.py] x = 1 [out] -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] mypy: error: Invalid error code(s): YOLO == Return code: 2 @@ -970,8 +968,6 @@ mypy: error: Invalid error code(s): YOLO [file test.py] x = 1 [out] -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] mypy: error: Invalid error code(s): YOLO == Return code: 2 @@ -980,8 +976,6 @@ mypy: error: Invalid error code(s): YOLO [file test.py] x = 1 [out] -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] mypy: error: Invalid error code(s): YOLO, YOLO2 == Return code: 2 @@ -990,8 +984,6 @@ mypy: error: Invalid error code(s): YOLO, YOLO2 [file test.py] x = 1 [out] -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] mypy: error: Invalid error code(s): YOLO == Return code: 2 @@ -1000,8 +992,6 @@ mypy: error: Invalid error code(s): YOLO [file test.py] x = 1 [out] -usage: mypy [-h] [-v] [-V] [more options; see below] - [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...] mypy: error: Invalid error code(s): YOLO == Return code: 2 From de2f375eb0da1c446e2aab1474c28b8d7707a848 Mon Sep 17 00:00:00 2001 From: "Thiago J. Barbalho" <11036045+gacheiro@users.noreply.github.com> Date: Fri, 3 Oct 2025 23:10:04 -0300 Subject: [PATCH 071/183] =?UTF-8?q?[docs]=20Replace=20`List`=20with=20buil?= =?UTF-8?q?t=E2=80=91in=20`list`=20(PEP=E2=80=AF585)=20(#20000)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updated the docs to replace typing.List with the built‑in generic list (PEP 585). The example would not work if someone copied and pasted it. --- docs/source/common_issues.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index aa325dd3b05c..e4239bd7a8ee 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -731,7 +731,7 @@ This example demonstrates both safe and unsafe overrides: class NarrowerReturn(A): # A more specific return type is fine - def test(self, t: Sequence[int]) -> List[str]: # OK + def test(self, t: Sequence[int]) -> list[str]: # OK ... class GeneralizedReturn(A): @@ -746,7 +746,7 @@ not necessary: .. code-block:: python class NarrowerArgument(A): - def test(self, t: List[int]) -> Sequence[str]: # type: ignore[override] + def test(self, t: list[int]) -> Sequence[str]: # type: ignore[override] ... .. _unreachable: From 6dc46987ddcab77ac8f164a3f35cf595180846cb Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 5 Oct 2025 21:36:30 +0300 Subject: [PATCH 072/183] [stubtest] Improve `allowlist` docs with better example (#20007) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also fix markup in several places. It used to be: Снимок экрана 2025-10-05 в 09 35 17 Notice the `--` problem. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Alex Waygood --- docs/source/stubtest.rst | 81 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/docs/source/stubtest.rst b/docs/source/stubtest.rst index 59889252f056..e7ea69290b78 100644 --- a/docs/source/stubtest.rst +++ b/docs/source/stubtest.rst @@ -99,8 +99,75 @@ to mypy build errors". In this case, you will need to mitigate those errors before stubtest will run. Despite potential overlap in errors here, stubtest is not intended as a substitute for running mypy directly. +Allowlist +********* + If you wish to ignore some of stubtest's complaints, stubtest supports a -pretty handy allowlist system. +pretty handy :option:`--allowlist` system. + +Let's say that you have this python module called ``ex``: + +.. code-block:: python + + try: + import optional_expensive_dep + except ImportError: + optional_expensive_dep = None + + first = 1 + if optional_expensive_dep: + second = 2 + +Let's say that you can't install ``optional_expensive_dep`` in CI for some reason, +but you still want to include ``second: int`` in the stub file: + +.. code-block:: python + + first: int + second: int + +In this case stubtest will correctly complain: + +.. code-block:: shell + + error: ex.second is not present at runtime + Stub: in file /.../ex.pyi:2 + builtins.int + Runtime: + MISSING + + Found 1 error (checked 1 module) + +To fix this, you can add an ``allowlist`` entry: + +.. code-block:: ini + + # Allowlist entries in `allowlist.txt` file: + + # Does not exist if `optional_expensive_dep` is not installed: + ex.second + +And now when running stubtest with ``--allowlist=allowlist.txt``, +no errors will be generated anymore. + +Allowlists also support regular expressions, +which can be useful to ignore many similar errors at once. +They can also be useful for suppressing stubtest errors that occur sometimes, +but not on every CI run. For example, if some CI workers have +``optional_expensive_dep`` installed, stubtest might complain with this message +on those workers if you had the ``ex.second`` allowlist entry: + +.. code-block:: ini + + note: unused allowlist entry ex.second + Found 1 error (checked 1 module) + +Changing ``ex.second`` to be ``(ex\.second)?`` will make this error optional, +meaning that stubtest will pass whether or not a CI runner +has``optional_expensive_dep`` installed. + +CLI +*** The rest of this section documents the command line interface of stubtest. @@ -119,15 +186,15 @@ The rest of this section documents the command line interface of stubtest. .. option:: --allowlist FILE Use file as an allowlist. Can be passed multiple times to combine multiple - allowlists. Allowlists can be created with --generate-allowlist. Allowlists - support regular expressions. + allowlists. Allowlists can be created with :option:`--generate-allowlist`. + Allowlists support regular expressions. The presence of an entry in the allowlist means stubtest will not generate any errors for the corresponding definition. .. option:: --generate-allowlist - Print an allowlist (to stdout) to be used with --allowlist + Print an allowlist (to stdout) to be used with :option:`--allowlist`. When introducing stubtest to an existing project, this is an easy way to silence all existing errors. @@ -141,17 +208,17 @@ The rest of this section documents the command line interface of stubtest. Note if an allowlist entry is a regex that matches the empty string, stubtest will never consider it unused. For example, to get - `--ignore-unused-allowlist` behaviour for a single allowlist entry like + ``--ignore-unused-allowlist`` behaviour for a single allowlist entry like ``foo.bar`` you could add an allowlist entry ``(foo\.bar)?``. This can be useful when an error only occurs on a specific platform. .. option:: --mypy-config-file FILE - Use specified mypy config file to determine mypy plugins and mypy path + Use specified mypy config *file* to determine mypy plugins and mypy path .. option:: --custom-typeshed-dir DIR - Use the custom typeshed in DIR + Use the custom typeshed in *DIR* .. option:: --check-typeshed From 3807423e9d98e678bf16b13ec8b4f909fe181908 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 5 Oct 2025 21:14:59 -0700 Subject: [PATCH 073/183] Make untyped decorator its own code (#19911) Since this apparently comes up a lot, it shouldn't just be misc (it is, however, a subcode of misc, at least at the moment, for extra backwards compatibility). Fixes https://github.com/python/mypy/issues/19148 I didn't add any tests for this and it seems like our old tests don't have codes enabled because they didn't have to be changed. I did add documentation for this, as required by the relevant test. --------- Co-authored-by: A5rocks --- docs/source/error_code_list2.rst | 23 +++++++++++++++++++++++ mypy/errorcodes.py | 4 ++++ mypy/messages.py | 6 +++++- test-data/unit/check-errorcodes.test | 15 +++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst index 125671bc2bef..bd2436061974 100644 --- a/docs/source/error_code_list2.rst +++ b/docs/source/error_code_list2.rst @@ -676,3 +676,26 @@ Example: print("red") case _: print("other") + +.. _code-untyped-decorator: + +Error if an untyped decorator makes a typed function effectively untyped [untyped-decorator] +-------------------------------------------------------------------------------------------- + +If enabled with :option:`--disallow-untyped-decorators ` +mypy generates an error if a typed function is wrapped by an untyped decorator +(as this would effectively remove the benefits of typing the function). + +Example: + +.. code-block:: python + + def printing_decorator(func): + def wrapper(*args, **kwds): + print("Calling", func) + return func(*args, **kwds) + return wrapper + # A decorated function. + @printing_decorator # E: Untyped decorator makes function "add_forty_two" untyped [untyped-decorator] + def add_forty_two(value: int) -> int: + return value + 42 diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index a96f5f723a7d..fbfa572b9439 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -302,6 +302,10 @@ def __hash__(self) -> int: sub_code_of=MISC, ) +UNTYPED_DECORATOR: Final = ErrorCode( + "untyped-decorator", "Error if an untyped decorator makes a typed function untyped", "General" +) + NARROWED_TYPE_NOT_SUBTYPE: Final = ErrorCode( "narrowed-type-not-subtype", "Warn if a TypeIs function's narrowed type is not a subtype of the original type", diff --git a/mypy/messages.py b/mypy/messages.py index 6329cad687f6..c6378c264757 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2008,7 +2008,11 @@ def untyped_decorated_function(self, typ: Type, context: Context) -> None: ) def typed_function_untyped_decorator(self, func_name: str, context: Context) -> None: - self.fail(f'Untyped decorator makes function "{func_name}" untyped', context) + self.fail( + f'Untyped decorator makes function "{func_name}" untyped', + context, + code=codes.UNTYPED_DECORATOR, + ) def bad_proto_variance( self, actual: int, tvar_name: str, expected: int, context: Context diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index bb5f658ebb50..06c5753db5a7 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -394,6 +394,21 @@ def f() -> None: def g(): pass +[case testErrorCodeUntypedDecorator] +# flags: --disallow-untyped-decorators --warn-unused-ignores +def d(f): return f + +@d # E: Untyped decorator makes function "x" untyped [untyped-decorator] +def x() -> int: return 1 +@d # type: ignore +def y() -> int: return 2 +@d # type: ignore[untyped-decorator] +def best() -> int: return 3 +@d # type: ignore[misc] # E: Unused "type: ignore" comment [unused-ignore] \ + # E: Untyped decorator makes function "z" untyped [untyped-decorator] \ + # N: Error code "untyped-decorator" not covered by "type: ignore" comment +def z() -> int: return 4 + [case testErrorCodeIndexing] from typing import Dict x: Dict[int, int] From 374fefbcfc3e75b3b5ea2550ff2826dd3474f7ef Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Mon, 6 Oct 2025 17:25:51 +0200 Subject: [PATCH 074/183] [mypyc] Support deleting attributes in __setattr__ wrapper (#19997) The `__setattr__` wrapper that mypyc generates needs to handle deleting attributes as well because `del` statements go through the same `tp_setattro` pointer but with the value argument set to `NULL`. The wrapper calls `__delattr__` in this case if it's overridden in the native class (or its parent). Handling of dynamic attributes is different without `__dict__` which makes a custom `__delattr__` required if the dynamic attributes are stored in a custom dictionary. If `__delattr__` is not overridden it calls the implementation of `object.__delattr__` which results in `AttributeError` because there's no `__dict__`. If it's defined without `__setattr__`, mypyc reports an error. It's possible to support just `__delattr__` but since it shares a slot with `__setattr__`, the wrapper generation would be more complicated. It seems like an unlikely use case to only need `__delattr__` so I think it makes sense to leave it for later. --- mypyc/irbuild/function.py | 37 +++++- mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-classes.test | 161 ++++++++++++++++++++++++++- mypyc/test-data/run-classes.test | 142 +++++++++++++++++++++++ 4 files changed, 333 insertions(+), 8 deletions(-) diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index 51bdc76495f2..c9f999597d30 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -83,7 +83,7 @@ dict_new_op, exact_dict_set_item_op, ) -from mypyc.primitives.generic_ops import generic_getattr, py_setattr_op +from mypyc.primitives.generic_ops import generic_getattr, generic_setattr, py_setattr_op from mypyc.primitives.misc_ops import register_function from mypyc.primitives.registry import builtin_names from mypyc.sametype import is_same_method_signature, is_same_type @@ -423,8 +423,10 @@ def generate_setattr_wrapper(builder: IRBuilder, cdef: ClassDef, setattr: FuncDe Returns 0 on success and -1 on failure. Restrictions are similar to the __getattr__ wrapper above. - This one is simpler because to match interpreted python semantics it's enough to always - call the user-provided function, including for names matching regular attributes. + The wrapper calls the user-defined __setattr__ when the value to set is not NULL. + When it's NULL, this means that the call to tp_setattro comes from a del statement, + so it calls __delattr__ instead. If __delattr__ is not overridden in the native class, + this will call the base implementation in object which doesn't work without __dict__. """ name = setattr.name + "__wrapper" ir = builder.mapper.type_to_ir[cdef.info] @@ -440,6 +442,27 @@ def generate_setattr_wrapper(builder: IRBuilder, cdef: ClassDef, setattr: FuncDe attr_arg = builder.add_argument("attr", object_rprimitive) value_arg = builder.add_argument("value", object_rprimitive) + call_delattr, call_setattr = BasicBlock(), BasicBlock() + null = Integer(0, object_rprimitive, line) + is_delattr = builder.add(ComparisonOp(value_arg, null, ComparisonOp.EQ, line)) + builder.add_bool_branch(is_delattr, call_delattr, call_setattr) + + builder.activate_block(call_delattr) + delattr_symbol = cdef.info.get("__delattr__") + delattr = delattr_symbol.node if delattr_symbol else None + delattr_override = delattr is not None and not delattr.fullname.startswith("builtins.") + if delattr_override: + builder.gen_method_call(builder.self(), "__delattr__", [attr_arg], None, line) + else: + # Call internal function that cpython normally calls when deleting an attribute. + # Cannot call object.__delattr__ here because it calls PyObject_SetAttr internally + # which in turn calls our wrapper and recurses infinitely. + # Note that since native classes don't have __dict__, this will raise AttributeError + # for dynamic attributes. + builder.call_c(generic_setattr, [builder.self(), attr_arg, null], line) + builder.add(Return(Integer(0, c_int_rprimitive), line)) + + builder.activate_block(call_setattr) builder.gen_method_call(builder.self(), setattr.name, [attr_arg, value_arg], None, line) builder.add(Return(Integer(0, c_int_rprimitive), line)) @@ -514,6 +537,14 @@ def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None generate_getattr_wrapper(builder, cdef, fdef) elif fdef.name == "__setattr__": generate_setattr_wrapper(builder, cdef, fdef) + elif fdef.name == "__delattr__": + setattr = cdef.info.get("__setattr__") + if not setattr or not setattr.node or setattr.node.fullname.startswith("builtins."): + builder.error( + '"__delattr__" supported only in classes that also override "__setattr__", ' + + "or inherit from a native class that overrides it.", + fdef.line, + ) def handle_non_ext_method( diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 22a6a5986cbd..4d0aaba12cab 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -46,6 +46,7 @@ def __eq__(self, x: object) -> bool: pass def __ne__(self, x: object) -> bool: pass def __str__(self) -> str: pass def __setattr__(self, k: str, v: object) -> None: pass + def __delattr__(self, k: str) -> None: pass class type: def __init__(self, o: object) -> None: ... diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a98b3a7d3dcf..a2d3b23ccfd9 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2222,18 +2222,41 @@ class AllowsInterpreted: def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "AllowsInterpreted" because it allows interpreted subclasses pass + def __delattr__(self, attr: str) -> None: + pass + class InheritsInterpreted(dict): def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "InheritsInterpreted" because it inherits from a non-native class pass + def __delattr__(self, attr: str) -> None: + pass + @mypyc_attr(native_class=False) class NonNative: - pass + def __setattr__(self, attr: str, val: object) -> None: + pass class InheritsNonNative(NonNative): def __setattr__(self, attr: str, val: object) -> None: # E: "__setattr__" not supported in class "InheritsNonNative" because it inherits from a non-native class pass + def __delattr__(self, attr: str) -> None: + pass + +[case testUnsupportedDelAttr] +class SetAttr: + def __setattr__(self, attr: str, val: object) -> None: + pass + +class NoSetAttr: + def __delattr__(self, attr: str) -> None: # E: "__delattr__" supported only in classes that also override "__setattr__", or inherit from a native class that overrides it. + pass + +class InheritedSetAttr(SetAttr): + def __delattr__(self, attr: str) -> None: + pass + [case testSetAttr] from typing import ClassVar class SetAttr: @@ -2329,11 +2352,21 @@ L6: def SetAttr.__setattr____wrapper(__mypyc_self__, attr, value): __mypyc_self__ :: __main__.SetAttr attr, value :: object - r0 :: str - r1 :: None + r0 :: bit + r1 :: i32 + r2 :: bit + r3 :: str + r4 :: None L0: - r0 = cast(str, attr) - r1 = __mypyc_self__.__setattr__(r0, value) + r0 = value == 0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = CPyObject_GenericSetAttr(__mypyc_self__, attr, 0) + r2 = r1 >= 0 :: signed + return 0 +L2: + r3 = cast(str, attr) + r4 = __mypyc_self__.__setattr__(r3, value) return 0 def test(attr, val): attr :: str @@ -2372,6 +2405,124 @@ L0: r14 = r13 >= 0 :: signed return 1 +[case testSetAttrAndDelAttr] +from typing import ClassVar +class SetAttr: + _attributes: dict[str, object] + regular_attr: int + class_var: ClassVar[str] = "x" + + def __init__(self, regular_attr: int, extra_attrs: dict[str, object], new_attr: str, new_val: object) -> None: + super().__setattr__("_attributes", extra_attrs) + object.__setattr__(self, "regular_attr", regular_attr) + + super().__setattr__(new_attr, new_val) + object.__setattr__(self, new_attr, new_val) + + def __setattr__(self, key: str, val: object) -> None: + if key == "regular_attr": + super().__setattr__("regular_attr", val) + elif key == "class_var": + raise AttributeError() + else: + self._attributes[key] = val + + def __delattr__(self, key: str) -> None: + del self._attributes[key] + +[typing fixtures/typing-full.pyi] +[out] +def SetAttr.__init__(self, regular_attr, extra_attrs, new_attr, new_val): + self :: __main__.SetAttr + regular_attr :: int + extra_attrs :: dict + new_attr :: str + new_val :: object + r0 :: i32 + r1 :: bit + r2 :: i32 + r3 :: bit +L0: + self._attributes = extra_attrs + self.regular_attr = regular_attr + r0 = CPyObject_GenericSetAttr(self, new_attr, new_val) + r1 = r0 >= 0 :: signed + r2 = CPyObject_GenericSetAttr(self, new_attr, new_val) + r3 = r2 >= 0 :: signed + return 1 +def SetAttr.__setattr__(self, key, val): + self :: __main__.SetAttr + key :: str + val :: object + r0 :: str + r1 :: bool + r2 :: int + r3 :: bool + r4 :: str + r5 :: bool + r6 :: object + r7 :: str + r8, r9 :: object + r10 :: dict + r11 :: i32 + r12 :: bit +L0: + r0 = 'regular_attr' + r1 = CPyStr_Equal(key, r0) + if r1 goto L1 else goto L2 :: bool +L1: + r2 = unbox(int, val) + self.regular_attr = r2; r3 = is_error + goto L6 +L2: + r4 = 'class_var' + r5 = CPyStr_Equal(key, r4) + if r5 goto L3 else goto L4 :: bool +L3: + r6 = builtins :: module + r7 = 'AttributeError' + r8 = CPyObject_GetAttr(r6, r7) + r9 = PyObject_Vectorcall(r8, 0, 0, 0) + CPy_Raise(r9) + unreachable +L4: + r10 = self._attributes + r11 = CPyDict_SetItem(r10, key, val) + r12 = r11 >= 0 :: signed +L5: +L6: + return 1 +def SetAttr.__setattr____wrapper(__mypyc_self__, attr, value): + __mypyc_self__ :: __main__.SetAttr + attr, value :: object + r0 :: bit + r1 :: str + r2 :: None + r3 :: str + r4 :: None +L0: + r0 = value == 0 + if r0 goto L1 else goto L2 :: bool +L1: + r1 = cast(str, attr) + r2 = __mypyc_self__.__delattr__(r1) + return 0 +L2: + r3 = cast(str, attr) + r4 = __mypyc_self__.__setattr__(r3, value) + return 0 +def SetAttr.__delattr__(self, key): + self :: __main__.SetAttr + key :: str + r0 :: dict + r1 :: i32 + r2 :: bit +L0: + r0 = self._attributes + r1 = PyObject_DelItem(r0, key) + r2 = r1 >= 0 :: signed + return 1 + [case testUntransformedSetAttr_64bit] from mypy_extensions import mypyc_attr diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index d10f7b19067c..ab1dcb926c34 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -4566,6 +4566,9 @@ class SetAttrOverridden(SetAttr): else: super().__setattr__(key, val) + def __delattr__(self, key: str) -> None: + del self._attributes[key] + @mypyc_attr(native_class=False) class SetAttrNonNative: _attributes: dict[str, object] @@ -4645,6 +4648,10 @@ def test_setattr() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_setattr_inherited() -> None: i = SetAttrInherited(99, {"one": 1}) assert i.class_var == "x" @@ -4678,6 +4685,10 @@ def test_setattr_inherited() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_setattr_overridden() -> None: i = SetAttrOverridden(99, 1, {"one": 1}) assert i.class_var == "x" @@ -4723,6 +4734,15 @@ def test_setattr_overridden() -> None: with assertRaises(AttributeError): i.const = 45 + del i.four + assert "four" not in i._attributes + + delattr(i, "three") + assert "three" not in i._attributes + + i.__delattr__("two") + assert "two" not in i._attributes + base_ref: SetAttr = i setattr(base_ref, "sub_attr", 5) assert base_ref.sub_attr == 5 @@ -4733,6 +4753,12 @@ def test_setattr_overridden() -> None: with assertRaises(AttributeError): setattr(base_ref, "subclass_var", "c") + base_ref.new_attr = "new_attr" + assert base_ref.new_attr == "new_attr" + + del base_ref.new_attr + assert "new_attr" not in base_ref._attributes + def test_setattr_nonnative() -> None: i = SetAttrNonNative(99, {"one": 1}) assert i.class_var == "x" @@ -4766,6 +4792,10 @@ def test_setattr_nonnative() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_no_setattr() -> None: i = NoSetAttr(99) i.super_setattr("attr", 100) @@ -4806,6 +4836,15 @@ def test_no_setattr_nonnative() -> None: object.__setattr__(i, "three", 102) assert i.three == 102 + del i.three + assert i.three == None + + delattr(i, "two") + assert i.two == None + + object.__delattr__(i, "one") + assert i.one == None + [typing fixtures/typing-full.pyi] [case testDunderSetAttrInterpreted] @@ -4853,6 +4892,9 @@ class SetAttrOverridden(SetAttr): else: super().__setattr__(key, val) + def __delattr__(self, key: str) -> None: + del self._attributes[key] + @mypyc_attr(native_class=False) class SetAttrNonNative: _attributes: dict[str, object] @@ -4936,6 +4978,10 @@ def test_setattr() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_setattr_inherited() -> None: i = SetAttrInherited(99, {"one": 1}) assert i.class_var == "x" @@ -4969,6 +5015,10 @@ def test_setattr_inherited() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_setattr_overridden() -> None: i = SetAttrOverridden(99, 1, {"one": 1}) assert i.class_var == "x" @@ -5014,6 +5064,15 @@ def test_setattr_overridden() -> None: with assertRaises(AttributeError): i.const = 45 + del i.four + assert "four" not in i._attributes + + delattr(i, "three") + assert "three" not in i._attributes + + i.__delattr__("two") + assert "two" not in i._attributes + base_ref: SetAttr = i setattr(base_ref, "sub_attr", 5) assert base_ref.sub_attr == 5 @@ -5024,6 +5083,12 @@ def test_setattr_overridden() -> None: with assertRaises(AttributeError): setattr(base_ref, "subclass_var", "c") + base_ref.new_attr = "new_attr" + assert base_ref.new_attr == "new_attr" + + del base_ref.new_attr + assert "new_attr" not in base_ref._attributes + def test_setattr_nonnative() -> None: i = SetAttrNonNative(99, {"one": 1}) assert i.class_var == "x" @@ -5057,6 +5122,10 @@ def test_setattr_nonnative() -> None: with assertRaises(AttributeError): i.const = 45 + # Doesn't work because there's no __delattr__. + with assertRaises(AttributeError): + del i.four + def test_no_setattr() -> None: i = NoSetAttr(99) i.super_setattr("attr", 100) @@ -5097,6 +5166,15 @@ def test_no_setattr_nonnative() -> None: object.__setattr__(i, "three", 102) assert i.three == 102 + del i.three + assert i.three == None + + delattr(i, "two") + assert i.two == None + + object.__delattr__(i, "one") + assert i.one == None + test_setattr() test_setattr_inherited() test_setattr_overridden() @@ -5105,3 +5183,67 @@ test_no_setattr() test_no_setattr_nonnative() [typing fixtures/typing-full.pyi] + +[case testDelAttrWithDeletableAttr] +from testutil import assertRaises + +class DelAttr: + __deletable__ = ["del_counter"] + + _attributes: dict[str, object] + del_counter: int = 0 + + def __init__(self) -> None: + object.__setattr__(self, "_attributes", {}) + + def __setattr__(self, key: str, val: object) -> None: + if key == "del_counter": + object.__setattr__(self, "del_counter", val) + else: + self._attributes[key] = val + + def __delattr__(self, key: str) -> None: + if key == "del_counter": + self.del_counter += 1 + else: + del self._attributes[key] + +def test_deletable_attr() -> None: + i = DelAttr() + assert i.del_counter == 0 + del i.del_counter + assert i.del_counter == 1 + +[case testDelAttrWithDeletableAttrInterpreted] +class DelAttr: + __deletable__ = ["del_counter"] + + _attributes: dict[str, object] + del_counter: int = 0 + + def __init__(self) -> None: + object.__setattr__(self, "_attributes", {}) + + def __setattr__(self, key: str, val: object) -> None: + if key == "del_counter": + object.__setattr__(self, "del_counter", val) + else: + self._attributes[key] = val + + def __delattr__(self, key: str) -> None: + if key == "del_counter": + self.del_counter += 1 + else: + del self._attributes[key] + +[file driver.py] +from native import DelAttr +from testutil import assertRaises + +def test_deletable_attr() -> None: + i = DelAttr() + assert i.del_counter == 0 + del i.del_counter + assert i.del_counter == 1 + +test_deletable_attr() From 9dc611f57639f899b5f71d286c3efe6a385dadf9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 7 Oct 2025 01:56:50 +0100 Subject: [PATCH 075/183] Pin librt version (#20010) This is to prepare for renaming `native_internal` -> `librt.internal` --- mypy-requirements.txt | 2 +- pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 6927ddd25d81..d356ca0b59d9 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.1.0 +librt==0.1.1 diff --git a/pyproject.toml b/pyproject.toml index 589679113c3b..1575a15e9909 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.1.0", + "librt==0.1.1", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.1.0", + "librt==0.1.1", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index 5402adcb682c..0983dc362c8a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.13 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.1.0 +librt==0.1.1 # via -r mypy-requirements.txt lxml==6.0.1 ; python_version < "3.15" # via -r test-requirements.in From d2a8800e314fb3aa8c88a5fe9d6000d839ca6254 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 7 Oct 2025 07:57:11 -0400 Subject: [PATCH 076/183] [mypyc] fix: reject invalid `mypyc_attr` args [1/1] (#19963) This PR emits a builder error when an invalid key is passed into `mypyc_attr`. This change could have saved me ~20 minutes or so, when the root of my problem was just a simple typo. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/irbuild/prepare.py | 2 +- mypyc/irbuild/util.py | 57 ++++++++++++++++++++++------ mypyc/test-data/irbuild-classes.test | 15 ++++++++ 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 20f2aeef8e6e..e4f43b38b0dc 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -357,7 +357,7 @@ def prepare_class_def( ir = mapper.type_to_ir[cdef.info] info = cdef.info - attrs, attrs_lines = get_mypyc_attrs(cdef) + attrs, attrs_lines = get_mypyc_attrs(cdef, path, errors) if attrs.get("allow_interpreted_subclasses") is True: ir.allow_interpreted_subclasses = True if attrs.get("serializable") is True: diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index eca2cac7e9db..3028e940f7f9 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -2,7 +2,8 @@ from __future__ import annotations -from typing import Any +from typing import Any, Final, Literal, TypedDict, cast +from typing_extensions import NotRequired from mypy.nodes import ( ARG_NAMED, @@ -31,7 +32,23 @@ from mypy.types import FINAL_DECORATOR_NAMES from mypyc.errors import Errors -DATACLASS_DECORATORS = {"dataclasses.dataclass", "attr.s", "attr.attrs"} +MYPYC_ATTRS: Final[frozenset[MypycAttr]] = frozenset( + ["native_class", "allow_interpreted_subclasses", "serializable", "free_list_len"] +) + +DATACLASS_DECORATORS: Final = frozenset(["dataclasses.dataclass", "attr.s", "attr.attrs"]) + + +MypycAttr = Literal[ + "native_class", "allow_interpreted_subclasses", "serializable", "free_list_len" +] + + +class MypycAttrs(TypedDict): + native_class: NotRequired[bool] + allow_interpreted_subclasses: NotRequired[bool] + serializable: NotRequired[bool] + free_list_len: NotRequired[int] def is_final_decorator(d: Expression) -> bool: @@ -112,21 +129,39 @@ def get_mypyc_attr_call(d: Expression) -> CallExpr | None: return None -def get_mypyc_attrs(stmt: ClassDef | Decorator) -> tuple[dict[str, Any], dict[str, int]]: +def get_mypyc_attrs( + stmt: ClassDef | Decorator, path: str, errors: Errors +) -> tuple[MypycAttrs, dict[MypycAttr, int]]: """Collect all the mypyc_attr attributes on a class definition or a function.""" - attrs: dict[str, Any] = {} - lines: dict[str, int] = {} + attrs: MypycAttrs = {} + lines: dict[MypycAttr, int] = {} + + def set_mypyc_attr(key: str, value: Any, line: int) -> None: + if key in MYPYC_ATTRS: + key = cast(MypycAttr, key) + attrs[key] = value + lines[key] = line + else: + errors.error(f'"{key}" is not a supported "mypyc_attr"', path, line) + supported_keys = '", "'.join(sorted(MYPYC_ATTRS)) + errors.note(f'supported keys: "{supported_keys}"', path, line) + for dec in stmt.decorators: - d = get_mypyc_attr_call(dec) - if d: + if d := get_mypyc_attr_call(dec): + line = d.line for name, arg in zip(d.arg_names, d.args): if name is None: if isinstance(arg, StrExpr): - attrs[arg.value] = True - lines[arg.value] = d.line + set_mypyc_attr(arg.value, True, line) + else: + errors.error( + 'All "mypyc_attr" positional arguments must be string literals.', + path, + line, + ) else: - attrs[name] = get_mypyc_attr_literal(arg) - lines[name] = d.line + arg_value = get_mypyc_attr_literal(arg) + set_mypyc_attr(name, arg_value, line) return attrs, lines diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a2d3b23ccfd9..94e89f276eeb 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2837,3 +2837,18 @@ L0: r6 = PyObject_VectorcallMethod(r3, r5, 9223372036854775812, 0) keep_alive r2, self, key, val return 1 + +[case testInvalidMypycAttr] +from mypy_extensions import mypyc_attr + +@mypyc_attr("allow_interpreted_subclasses", "invalid_arg") # E: "invalid_arg" is not a supported "mypyc_attr" \ + # N: supported keys: "allow_interpreted_subclasses", "free_list_len", "native_class", "serializable" +class InvalidArg: + pass +@mypyc_attr(invalid_kwarg=True) # E: "invalid_kwarg" is not a supported "mypyc_attr" \ + # N: supported keys: "allow_interpreted_subclasses", "free_list_len", "native_class", "serializable" +class InvalidKwarg: + pass +@mypyc_attr(str()) # E: All "mypyc_attr" positional arguments must be string literals. +class InvalidLiteral: + pass From 139071c2c3017f29cb2c9d2883a53cddd28c3493 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 7 Oct 2025 07:58:37 -0400 Subject: [PATCH 077/183] [mypyc] feat: support negative index in TupleGet op (#19990) This PR modifies `TupleGet.__init__` to automatically convert negative indexes to positive indexes instead of crashing at the assert This won't change functionality on its own, since none of the existing calling locations can pass a negative value, but will allow us to pass negative values in https://github.com/python/mypy/pull/19972 so I think we should consider this PR a prerequisite to that one --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Piotr Sawicki --- mypyc/ir/ops.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 76c1e07a79d5..ffce529f0756 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -1045,10 +1045,17 @@ class TupleGet(RegisterOp): def __init__(self, src: Value, index: int, line: int = -1, *, borrow: bool = False) -> None: super().__init__(line) + assert isinstance( + src.type, RTuple + ), f"TupleGet only operates on tuples, not {type(src.type).__name__}" + src_len = len(src.type.types) self.src = src self.index = index - assert isinstance(src.type, RTuple), "TupleGet only operates on tuples" - assert index >= 0 + if index < 0: + self.index += src_len + assert ( + self.index <= src_len - 1 + ), f"Index out of range.\nsource type: {src.type}\nindex: {index}" self.type = src.type.types[index] self.is_borrowed = borrow From f2ebd79f2484402967f5206b9fc6ce014b6760dd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 7 Oct 2025 20:46:30 +0100 Subject: [PATCH 078/183] Rename native_internal to librt.internal (#20014) Also adjust build/test logic slightly to prepare for more modules in `librt`. This PR should fix https://github.com/python/mypy/issues/20006 --- mypy-requirements.txt | 2 +- mypy/cache.py | 4 +- mypy/typeshed/stubs/librt/METADATA.toml | 2 +- mypy/typeshed/stubs/librt/librt/__init__.pyi | 0 .../internal.pyi} | 0 mypyc/build.py | 46 +++++++++++-------- mypyc/codegen/emitmodule.py | 12 ++--- mypyc/ir/rtypes.py | 2 +- .../{native_internal.c => librt_internal.c} | 28 +++++------ .../{native_internal.h => librt_internal.h} | 22 +++++---- mypyc/lib-rt/setup.py | 4 +- mypyc/options.py | 8 ++-- mypyc/primitives/misc_ops.py | 26 +++++------ mypyc/test-data/irbuild-classes.test | 6 +-- mypyc/test-data/run-classes.test | 4 +- mypyc/test/test_run.py | 11 +++-- pyproject.toml | 4 +- setup.py | 2 +- test-requirements.txt | 2 +- 19 files changed, 99 insertions(+), 86 deletions(-) create mode 100644 mypy/typeshed/stubs/librt/librt/__init__.pyi rename mypy/typeshed/stubs/librt/{native_internal.pyi => librt/internal.pyi} (100%) rename mypyc/lib-rt/{native_internal.c => librt_internal.c} (96%) rename mypyc/lib-rt/{native_internal.h => librt_internal.h} (80%) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index d356ca0b59d9..229f5624e886 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt==0.1.1 +librt>=0.2.1 diff --git a/mypy/cache.py b/mypy/cache.py index 51deac914efc..f8d3e6a05eba 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -3,8 +3,7 @@ from collections.abc import Sequence from typing import Final -from mypy_extensions import u8 -from native_internal import ( +from librt.internal import ( Buffer as Buffer, read_bool as read_bool, read_float as read_float, @@ -17,6 +16,7 @@ write_str as write_str, write_tag as write_tag, ) +from mypy_extensions import u8 # Always use this type alias to refer to type tags. Tag = u8 diff --git a/mypy/typeshed/stubs/librt/METADATA.toml b/mypy/typeshed/stubs/librt/METADATA.toml index 37dc09b102d4..a42da251bed5 100644 --- a/mypy/typeshed/stubs/librt/METADATA.toml +++ b/mypy/typeshed/stubs/librt/METADATA.toml @@ -1 +1 @@ -version = "0.1.*" +version = "0.2.*" diff --git a/mypy/typeshed/stubs/librt/librt/__init__.pyi b/mypy/typeshed/stubs/librt/librt/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/mypy/typeshed/stubs/librt/native_internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi similarity index 100% rename from mypy/typeshed/stubs/librt/native_internal.pyi rename to mypy/typeshed/stubs/librt/librt/internal.pyi diff --git a/mypyc/build.py b/mypyc/build.py index efbd0dce31db..40638b31d000 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -42,6 +42,8 @@ from mypyc.namegen import exported_name from mypyc.options import CompilerOptions +LIBRT_MODULES = [("librt.internal", "librt_internal.c")] + try: # Import setuptools so that it monkey-patch overrides distutils import setuptools @@ -492,8 +494,8 @@ def mypycify( strict_dunder_typing: bool = False, group_name: str | None = None, log_trace: bool = False, - depends_on_native_internal: bool = False, - install_native_libs: bool = False, + depends_on_librt_internal: bool = False, + install_librt: bool = False, ) -> list[Extension]: """Main entry point to building using mypyc. @@ -544,11 +546,11 @@ def mypycify( mypyc_trace.txt (derived from executed operations). This is useful for performance analysis, such as analyzing which primitive ops are used the most and on which lines. - depends_on_native_internal: This is True only for mypy itself. - install_native_libs: If True, also build the native extension modules. Normally, - those are build and published on PyPI separately, but during - tests, we want to use their development versions (i.e. from - current commit). + depends_on_librt_internal: This is True only for mypy itself. + install_librt: If True, also build the librt extension modules. Normally, + those are build and published on PyPI separately, but during + tests, we want to use their development versions (i.e. from + current commit). """ # Figure out our configuration @@ -562,7 +564,7 @@ def mypycify( strict_dunder_typing=strict_dunder_typing, group_name=group_name, log_trace=log_trace, - depends_on_native_internal=depends_on_native_internal, + depends_on_librt_internal=depends_on_librt_internal, ) # Generate all the actual important C code @@ -661,21 +663,25 @@ def mypycify( build_single_module(group_sources, cfilenames + shared_cfilenames, cflags) ) - if install_native_libs: - for name in ["native_internal.c"] + RUNTIME_C_FILES: + if install_librt: + os.makedirs("librt", exist_ok=True) + for name in RUNTIME_C_FILES: rt_file = os.path.join(build_dir, name) with open(os.path.join(include_dir(), name), encoding="utf-8") as f: write_file(rt_file, f.read()) - extensions.append( - get_extension()( - "native_internal", - sources=[ - os.path.join(build_dir, file) - for file in ["native_internal.c"] + RUNTIME_C_FILES - ], - include_dirs=[include_dir()], - extra_compile_args=cflags, + for mod, file_name in LIBRT_MODULES: + rt_file = os.path.join(build_dir, file_name) + with open(os.path.join(include_dir(), file_name), encoding="utf-8") as f: + write_file(rt_file, f.read()) + extensions.append( + get_extension()( + mod, + sources=[ + os.path.join(build_dir, file) for file in [file_name] + RUNTIME_C_FILES + ], + include_dirs=[include_dir()], + extra_compile_args=cflags, + ) ) - ) return extensions diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index ca5db52ab7da..3602b3c26e03 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -601,12 +601,12 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line(f"#define MYPYC_NATIVE{self.group_suffix}_H") ext_declarations.emit_line("#include ") ext_declarations.emit_line("#include ") - if self.compiler_options.depends_on_native_internal: - ext_declarations.emit_line("#include ") + if self.compiler_options.depends_on_librt_internal: + ext_declarations.emit_line("#include ") declarations = Emitter(self.context) - declarations.emit_line(f"#ifndef MYPYC_NATIVE_INTERNAL{self.group_suffix}_H") - declarations.emit_line(f"#define MYPYC_NATIVE_INTERNAL{self.group_suffix}_H") + declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") + declarations.emit_line(f"#define MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") declarations.emit_line("#include ") declarations.emit_line("#include ") declarations.emit_line(f'#include "__native{self.short_group_suffix}.h"') @@ -1029,8 +1029,8 @@ def emit_module_exec_func( declaration = f"int CPyExec_{exported_name(module_name)}(PyObject *module)" module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_lines(declaration, "{") - if self.compiler_options.depends_on_native_internal: - emitter.emit_line("if (import_native_internal() < 0) {") + if self.compiler_options.depends_on_librt_internal: + emitter.emit_line("if (import_librt_internal() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") emitter.emit_line("PyObject* modname = NULL;") diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 34824a59cd5c..941670ab230d 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -514,7 +514,7 @@ def __hash__(self) -> int: KNOWN_NATIVE_TYPES: Final = { name: RPrimitive(name, is_unboxed=False, is_refcounted=True) - for name in ["native_internal.Buffer"] + for name in ["librt.internal.Buffer"] } diff --git a/mypyc/lib-rt/native_internal.c b/mypyc/lib-rt/librt_internal.c similarity index 96% rename from mypyc/lib-rt/native_internal.c rename to mypyc/lib-rt/librt_internal.c index a6511a1caf25..cb9aa1025821 100644 --- a/mypyc/lib-rt/native_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -2,8 +2,8 @@ #include #include #include "CPy.h" -#define NATIVE_INTERNAL_MODULE -#include "native_internal.h" +#define LIBRT_INTERNAL_MODULE +#include "librt_internal.h" #define START_SIZE 512 #define MAX_SHORT_INT_TAGGED (255 << 1) @@ -558,7 +558,7 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames return Py_None; } -static PyMethodDef native_internal_module_methods[] = { +static PyMethodDef librt_internal_module_methods[] = { {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")}, {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, {"write_str", (PyCFunction)write_str, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a string")}, @@ -574,11 +574,11 @@ static PyMethodDef native_internal_module_methods[] = { static int NativeInternal_ABI_Version(void) { - return NATIVE_INTERNAL_ABI_VERSION; + return LIBRT_INTERNAL_ABI_VERSION; } static int -native_internal_module_exec(PyObject *m) +librt_internal_module_exec(PyObject *m) { if (PyType_Ready(&BufferType) < 0) { return -1; @@ -604,32 +604,32 @@ native_internal_module_exec(PyObject *m) (void *)read_tag_internal, (void *)NativeInternal_ABI_Version, }; - PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "native_internal._C_API", NULL); + PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { return -1; } return 0; } -static PyModuleDef_Slot native_internal_module_slots[] = { - {Py_mod_exec, native_internal_module_exec}, +static PyModuleDef_Slot librt_internal_module_slots[] = { + {Py_mod_exec, librt_internal_module_exec}, #ifdef Py_MOD_GIL_NOT_USED {Py_mod_gil, Py_MOD_GIL_NOT_USED}, #endif {0, NULL} }; -static PyModuleDef native_internal_module = { +static PyModuleDef librt_internal_module = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = "native_internal", + .m_name = "internal", .m_doc = "Mypy cache serialization utils", .m_size = 0, - .m_methods = native_internal_module_methods, - .m_slots = native_internal_module_slots, + .m_methods = librt_internal_module_methods, + .m_slots = librt_internal_module_slots, }; PyMODINIT_FUNC -PyInit_native_internal(void) +PyInit_internal(void) { - return PyModuleDef_Init(&native_internal_module); + return PyModuleDef_Init(&librt_internal_module); } diff --git a/mypyc/lib-rt/native_internal.h b/mypyc/lib-rt/librt_internal.h similarity index 80% rename from mypyc/lib-rt/native_internal.h rename to mypyc/lib-rt/librt_internal.h index 63e902a6e1bf..fd8ec2422cc5 100644 --- a/mypyc/lib-rt/native_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -1,9 +1,9 @@ -#ifndef NATIVE_INTERNAL_H -#define NATIVE_INTERNAL_H +#ifndef LIBRT_INTERNAL_H +#define LIBRT_INTERNAL_H -#define NATIVE_INTERNAL_ABI_VERSION 0 +#define LIBRT_INTERNAL_ABI_VERSION 0 -#ifdef NATIVE_INTERNAL_MODULE +#ifdef LIBRT_INTERNAL_MODULE static PyObject *Buffer_internal(PyObject *source); static PyObject *Buffer_internal_empty(void); @@ -40,17 +40,21 @@ static void **NativeInternal_API; #define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13]) static int -import_native_internal(void) +import_librt_internal(void) { - NativeInternal_API = (void **)PyCapsule_Import("native_internal._C_API", 0); + PyObject *mod = PyImport_ImportModule("librt.internal"); + if (mod == NULL) + return -1; + Py_DECREF(mod); // we import just for the side effect of making the below work. + NativeInternal_API = (void **)PyCapsule_Import("librt.internal._C_API", 0); if (NativeInternal_API == NULL) return -1; - if (NativeInternal_ABI_Version() != NATIVE_INTERNAL_ABI_VERSION) { - PyErr_SetString(PyExc_ValueError, "ABI version conflict for native_internal"); + if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) { + PyErr_SetString(PyExc_ValueError, "ABI version conflict for librt.internal"); return -1; } return 0; } #endif -#endif // NATIVE_INTERNAL_H +#endif // LIBRT_INTERNAL_H diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 36b55e44dcd1..b78ad0dbc23e 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -75,9 +75,9 @@ def run(self) -> None: setup( ext_modules=[ Extension( - "native_internal", + "librt.internal", [ - "native_internal.c", + "librt_internal.c", "init.c", "int_ops.c", "exc_ops.c", diff --git a/mypyc/options.py b/mypyc/options.py index c009d3c6a7a4..e004a0a52c95 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -17,7 +17,7 @@ def __init__( strict_dunder_typing: bool = False, group_name: str | None = None, log_trace: bool = False, - depends_on_native_internal: bool = False, + depends_on_librt_internal: bool = False, ) -> None: self.strip_asserts = strip_asserts self.multi_file = multi_file @@ -51,7 +51,7 @@ def __init__( # mypyc_trace.txt when compiled module is executed. This is useful for # performance analysis. self.log_trace = log_trace - # If enabled, add capsule imports of native_internal API. This should be used + # If enabled, add capsule imports of librt.internal API. This should be used # only for mypy itself, third-party code compiled with mypyc should not use - # native_internal. - self.depends_on_native_internal = depends_on_native_internal + # librt.internal. + self.depends_on_librt_internal = depends_on_librt_internal diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 8e6e450c64dc..18d475fe89d4 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -333,11 +333,11 @@ error_kind=ERR_NEVER, ) -buffer_rprimitive = KNOWN_NATIVE_TYPES["native_internal.Buffer"] +buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.Buffer"] # Buffer(source) function_op( - name="native_internal.Buffer", + name="librt.internal.Buffer", arg_types=[bytes_rprimitive], return_type=buffer_rprimitive, c_function_name="Buffer_internal", @@ -346,7 +346,7 @@ # Buffer() function_op( - name="native_internal.Buffer", + name="librt.internal.Buffer", arg_types=[], return_type=buffer_rprimitive, c_function_name="Buffer_internal_empty", @@ -362,7 +362,7 @@ ) function_op( - name="native_internal.write_bool", + name="librt.internal.write_bool", arg_types=[object_rprimitive, bool_rprimitive], return_type=none_rprimitive, c_function_name="write_bool_internal", @@ -370,7 +370,7 @@ ) function_op( - name="native_internal.read_bool", + name="librt.internal.read_bool", arg_types=[object_rprimitive], return_type=bool_rprimitive, c_function_name="read_bool_internal", @@ -378,7 +378,7 @@ ) function_op( - name="native_internal.write_str", + name="librt.internal.write_str", arg_types=[object_rprimitive, str_rprimitive], return_type=none_rprimitive, c_function_name="write_str_internal", @@ -386,7 +386,7 @@ ) function_op( - name="native_internal.read_str", + name="librt.internal.read_str", arg_types=[object_rprimitive], return_type=str_rprimitive, c_function_name="read_str_internal", @@ -394,7 +394,7 @@ ) function_op( - name="native_internal.write_float", + name="librt.internal.write_float", arg_types=[object_rprimitive, float_rprimitive], return_type=none_rprimitive, c_function_name="write_float_internal", @@ -402,7 +402,7 @@ ) function_op( - name="native_internal.read_float", + name="librt.internal.read_float", arg_types=[object_rprimitive], return_type=float_rprimitive, c_function_name="read_float_internal", @@ -410,7 +410,7 @@ ) function_op( - name="native_internal.write_int", + name="librt.internal.write_int", arg_types=[object_rprimitive, int_rprimitive], return_type=none_rprimitive, c_function_name="write_int_internal", @@ -418,7 +418,7 @@ ) function_op( - name="native_internal.read_int", + name="librt.internal.read_int", arg_types=[object_rprimitive], return_type=int_rprimitive, c_function_name="read_int_internal", @@ -426,7 +426,7 @@ ) function_op( - name="native_internal.write_tag", + name="librt.internal.write_tag", arg_types=[object_rprimitive, uint8_rprimitive], return_type=none_rprimitive, c_function_name="write_tag_internal", @@ -434,7 +434,7 @@ ) function_op( - name="native_internal.read_tag", + name="librt.internal.read_tag", arg_types=[object_rprimitive], return_type=uint8_rprimitive, c_function_name="read_tag_internal", diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 94e89f276eeb..c4410b3b19d2 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1452,7 +1452,7 @@ class TestOverload: [case testNativeBufferFastPath] from typing import Final from mypy_extensions import u8 -from native_internal import ( +from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag ) @@ -1476,11 +1476,11 @@ def foo() -> None: u = read_tag(b) [out] def foo(): - r0, b :: native_internal.Buffer + r0, b :: librt.internal.Buffer r1 :: str r2, r3, r4, r5, r6 :: None r7 :: bytes - r8 :: native_internal.Buffer + r8 :: librt.internal.Buffer r9, x :: str r10, y :: bool r11, z :: float diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index ab1dcb926c34..8755169fdb0b 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2710,10 +2710,10 @@ from native import Player [out] Player.MIN = -[case testBufferRoundTrip_native_libs] +[case testBufferRoundTrip_librt_internal] from typing import Final from mypy_extensions import u8 -from native_internal import ( +from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag ) diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 172a1016dd91..22ab18e97293 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -86,7 +86,7 @@ setup(name='test_run_output', ext_modules=mypycify({}, separate={}, skip_cgen_input={!r}, strip_asserts=False, - multi_file={}, opt_level='{}', install_native_libs={}), + multi_file={}, opt_level='{}', install_librt={}), ) """ @@ -239,13 +239,16 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> groups = construct_groups(sources, separate, len(module_names) > 1, None) - native_libs = "_native_libs" in testcase.name + # Use _librt_internal to test mypy-specific parts of librt (they have + # some special-casing in mypyc), for everything else use _librt suffix. + librt_internal = testcase.name.endswith("_librt_internal") + librt = librt_internal or testcase.name.endswith("_librt") try: compiler_options = CompilerOptions( multi_file=self.multi_file, separate=self.separate, strict_dunder_typing=self.strict_dunder_typing, - depends_on_native_internal=native_libs, + depends_on_librt_internal=librt_internal, ) result = emitmodule.parse_and_typecheck( sources=sources, @@ -278,7 +281,7 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> with open(setup_file, "w", encoding="utf-8") as f: f.write( setup_format.format( - module_paths, separate, cfiles, self.multi_file, opt_level, native_libs + module_paths, separate, cfiles, self.multi_file, opt_level, librt ) ) diff --git a/pyproject.toml b/pyproject.toml index 1575a15e9909..adcca65bf015 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt==0.1.1", + "librt>=0.2.1", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt==0.1.1", + "librt>=0.2.1", ] dynamic = ["version"] diff --git a/setup.py b/setup.py index 3f53d96dbd85..1d093ec3b9e2 100644 --- a/setup.py +++ b/setup.py @@ -155,7 +155,7 @@ def run(self) -> None: multi_file=sys.platform == "win32" or force_multifile, log_trace=log_trace, # Mypy itself is allowed to use native_internal extension. - depends_on_native_internal=True, + depends_on_librt_internal=True, ) else: diff --git a/test-requirements.txt b/test-requirements.txt index 0983dc362c8a..7e8e167300dc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.13 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.1.1 +librt==0.2.1 # via -r mypy-requirements.txt lxml==6.0.1 ; python_version < "3.15" # via -r test-requirements.in From 4ff18305d710b3a5491615eddf056db14d337b63 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 8 Oct 2025 03:32:12 +0200 Subject: [PATCH 079/183] Filter SyntaxWarnings during AST parsing (#20023) Especially with [PEP 765](https://peps.python.org/pep-0765/) in 3.14, Python has been getting more liberal in emitting SyntaxWarnings during AST parsing and compilation. Generally, they aren't really helpful for mypy itself. There are open discussions to add a flag which would disable these. Until that's implemented, filter the warnings manually. _This also get's rid of the warnings emitted on test code. If at some point `return in finally` will be made an error, those tests could be adjust / removed. Until then, we can continue to test these as is without issues._ ```py def func() -> None: try: x = 1/0 finally: return None # return in finally "Hello \P world" # invalid escape sequence "" is 1 # "is" with 'int' literal ``` --- mypy/fastparse.py | 5 ++++- pyproject.toml | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 6b2eb532003c..aa5c89cd0f41 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -232,9 +232,12 @@ def parse( assert options.python_version[0] >= 3 feature_version = options.python_version[1] try: - # Disable deprecation warnings about \u + # Disable + # - deprecation warnings about \u + # - syntax warnings for 'invalid escape sequence' (3.12+) and 'return in finally' (3.14+) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) + warnings.filterwarnings("ignore", category=SyntaxWarning) ast = ast3_parse(source, fnam, "exec", feature_version=feature_version) tree = ASTConverter( diff --git a/pyproject.toml b/pyproject.toml index adcca65bf015..96b05ba459b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -227,9 +227,6 @@ xfail_strict = true # Force warnings as errors filterwarnings = [ "error", - # Some testcases may contain code that emits SyntaxWarnings, and they are not yet - # handled consistently in 3.14 (PEP 765) - "default::SyntaxWarning", ] [tool.coverage.run] From 59e9e7d0bb4e40e41c002f6fb399609991beccd9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 8 Oct 2025 12:54:47 +0100 Subject: [PATCH 080/183] Fix infinite loop on empty Buffer (#20024) Currently `Buffer(b"")` (as opposite to `Buffer()`) can go into an infinite loop when resizing. Fix this by allocating one more byte, that is conveniently guaranteed by `bytes` ABI. I also move the `librt/__init__.py` hack to tests, since this is the only place where it is needed. I am still not sure why `*.so` from `site-packages` is preferred over a "local" namespace package with the same `*.so`. --- mypyc/build.py | 1 - mypyc/lib-rt/librt_internal.c | 8 +++++--- mypyc/test-data/run-classes.test | 9 +++++++++ mypyc/test/test_run.py | 6 ++++++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/mypyc/build.py b/mypyc/build.py index 40638b31d000..13648911c0b5 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -664,7 +664,6 @@ def mypycify( ) if install_librt: - os.makedirs("librt", exist_ok=True) for name in RUNTIME_C_FILES: rt_file = os.path.join(build_dir, name) with open(os.path.join(include_dir(), name), encoding="utf-8") as f: diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index cb9aa1025821..b97d6665b515 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -70,12 +70,14 @@ Buffer_init_internal(BufferObject *self, PyObject *source) { PyErr_SetString(PyExc_TypeError, "source must be a bytes object"); return -1; } - self->size = PyBytes_GET_SIZE(source); - self->end = self->size; + self->end = PyBytes_GET_SIZE(source); + // Allocate at least one byte to simplify resizing logic. + // The original bytes buffer has last null byte, so this is safe. + self->size = self->end + 1; // This returns a pointer to internal bytes data, so make our own copy. char *buf = PyBytes_AsString(source); self->buf = PyMem_Malloc(self->size); - memcpy(self->buf, buf, self->size); + memcpy(self->buf, buf, self->end); } else { self->buf = PyMem_Malloc(START_SIZE); self->size = START_SIZE; diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 8755169fdb0b..84704ce66c81 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2866,6 +2866,15 @@ test_buffer_roundtrip_interpreted() test_buffer_int_size_interpreted() test_buffer_str_size_interpreted() +[case testBufferEmpty_librt_internal] +from librt.internal import Buffer, write_int, read_int + +def test_empty() -> None: + b = Buffer(b"") + write_int(b, 42) + b1 = Buffer(b.getvalue()) + assert read_int(b1) == 42 + [case testEnumMethodCalls] from enum import Enum from typing import overload, Optional, Union diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 22ab18e97293..953f61329395 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -285,6 +285,12 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> ) ) + if librt: + # This hack forces Python to prefer the local "installation". + os.makedirs("librt", exist_ok=True) + with open(os.path.join("librt", "__init__.py"), "a"): + pass + if not run_setup(setup_file, ["build_ext", "--inplace"]): if testcase.config.getoption("--mypyc-showc"): show_c(cfiles) From eec826e940a9c5dcee54b418bc9aa6668922cf2e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 8 Oct 2025 13:24:42 +0100 Subject: [PATCH 081/183] Make lib-rt/setup.py similar to mypyc extensions (#20022) Two small things here: * Use `setuptools` extension class. * Use same optimization levels we use when compiling mypy. --- mypyc/lib-rt/setup.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index b78ad0dbc23e..afbceba060f4 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -8,10 +8,12 @@ import os import subprocess import sys -from distutils.command.build_ext import build_ext -from distutils.core import Extension, setup +from distutils import ccompiler, sysconfig from typing import Any +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext + C_APIS_TO_TEST = [ "init.c", "int_ops.c", @@ -72,6 +74,14 @@ def run(self) -> None: else: # TODO: we need a way to share our preferred C flags and get_extension() logic with # mypyc/build.py without code duplication. + compiler = ccompiler.new_compiler() + sysconfig.customize_compiler(compiler) + cflags: list[str] = [] + if compiler.compiler_type == "unix": + cflags += ["-O3"] + elif compiler.compiler_type == "msvc": + cflags += ["/O2"] + setup( ext_modules=[ Extension( @@ -85,6 +95,7 @@ def run(self) -> None: "getargsfast.c", ], include_dirs=["."], + extra_compile_args=cflags, ) ] ) From 712afc5b204e9b07e46fb8d1b12d2393b03411a1 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 8 Oct 2025 20:59:53 +0200 Subject: [PATCH 082/183] Adjust stubtest test stubs for PEP 728 (Python 3.15) (#20009) Add stubs for [PEP 728](https://peps.python.org/pep-0728/). --- mypy/test/teststubtest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 800f522d90a0..dfbde217e82f 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -88,6 +88,8 @@ class _TypedDict(Mapping[str, object]): __total__: ClassVar[bool] __readonly_keys__: ClassVar[frozenset[str]] __mutable_keys__: ClassVar[frozenset[str]] + __closed__: ClassVar[bool | None] + __extra_items__: ClassVar[Any] def overload(func: _T) -> _T: ... def type_check_only(func: _T) -> _T: ... def final(func: _T) -> _T: ... From 77b4cfb167b2ef837eddccaf338b3b52e6cf4ba5 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 9 Oct 2025 01:23:38 +0300 Subject: [PATCH 083/183] Fix `[name-defined]` false-positive in `class A[X, Y=X]:` case (#20021) This a WIP to see the test result before adding my own tests :) Closes https://github.com/python/mypy/issues/20020 --- mypy/semanal.py | 7 +++---- test-data/unit/check-python313.test | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 17dc9bfadc1f..08f9eb03c9d7 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1791,11 +1791,10 @@ def push_type_args( return None tvs.append((p.name, tv)) - for name, tv in tvs: - if self.is_defined_type_param(name): - self.fail(f'"{name}" already defined as a type parameter', context) + if self.is_defined_type_param(p.name): + self.fail(f'"{p.name}" already defined as a type parameter', context) else: - self.add_symbol(name, tv, context, no_progress=True, type_param=True) + self.add_symbol(p.name, tv, context, no_progress=True, type_param=True) return tvs diff --git a/test-data/unit/check-python313.test b/test-data/unit/check-python313.test index b46ae0fecfc4..117d20ceaf0b 100644 --- a/test-data/unit/check-python313.test +++ b/test-data/unit/check-python313.test @@ -290,3 +290,32 @@ reveal_type(A1().x) # N: Revealed type is "builtins.int" reveal_type(A2().x) # N: Revealed type is "builtins.int" reveal_type(A3().x) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultToAnotherTypeVar] +class A[X, Y = X, Z = Y]: + x: X + y: Y + z: Z + +a1: A[int] +reveal_type(a1.x) # N: Revealed type is "builtins.int" +reveal_type(a1.y) # N: Revealed type is "builtins.int" +# TODO: this must reveal `int` as well: +reveal_type(a1.z) # N: Revealed type is "X`1" + +a2: A[int, str] +reveal_type(a2.x) # N: Revealed type is "builtins.int" +reveal_type(a2.y) # N: Revealed type is "builtins.str" +reveal_type(a2.z) # N: Revealed type is "builtins.str" + +a3: A[int, str, bool] +reveal_type(a3.x) # N: Revealed type is "builtins.int" +reveal_type(a3.y) # N: Revealed type is "builtins.str" +reveal_type(a3.z) # N: Revealed type is "builtins.bool" +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultToAnotherTypeVarWrong] +class A[Y = X, X = int]: ... # E: Name "X" is not defined + +class B[Y = X]: ... # E: Name "X" is not defined +[builtins fixtures/tuple.pyi] From d51fa0098d2488ce135293f4310229c5b700f5f6 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 8 Oct 2025 17:58:26 -0700 Subject: [PATCH 084/183] Update shellcheck version in actionlint pre-commit to 0.11.0 (#20030) I was getting bizarre wasm failures from runtests running pre-commit running actionlint running shellcheck (shellcheck is written in haskell but we depend on go-shellcheck which is a packaged up version of it using wasm?). (https://github.com/python/mypy/issues/19958#issuecomment-3383449423) If I update it, I don't get bizarre wasm failures and it still doesn't complain about anything. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3b323f03b99c..a410585a52d4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: # actionlint has a shellcheck integration which extracts shell scripts in `run:` steps from GitHub Actions # and checks these with shellcheck. This is arguably its most useful feature, # but the integration only works if shellcheck is installed - - "github.com/wasilibs/go-shellcheck/cmd/shellcheck@v0.10.0" + - "github.com/wasilibs/go-shellcheck/cmd/shellcheck@v0.11.0" - repo: https://github.com/woodruffw/zizmor-pre-commit rev: v1.5.2 hooks: From c928847e8c990e0eb0306a04beda82a48eb18428 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Wed, 8 Oct 2025 17:59:11 -0700 Subject: [PATCH 085/183] [nit] clarify comment about deprecation warnings about \u (#20026) This is the same as 'invalid escape sequence', just a different type of warning on lower Python versions. Chases https://github.com/python/mypy/pull/20023. More background information is available in https://github.com/python/mypy/pull/19606, although that's probably not very helpful all told. --- mypy/fastparse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index aa5c89cd0f41..276e183a6bf0 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -233,8 +233,8 @@ def parse( feature_version = options.python_version[1] try: # Disable - # - deprecation warnings about \u - # - syntax warnings for 'invalid escape sequence' (3.12+) and 'return in finally' (3.14+) + # - deprecation warnings for 'invalid escape sequence' (Python 3.11 and below) + # - syntax warnings for 'invalid escape sequence' (3.12+) and 'return in finally' (3.14+) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore", category=SyntaxWarning) From 6e9fb5948502ccaaca4e4869121c1fe8064f8a93 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 9 Oct 2025 08:24:46 +0100 Subject: [PATCH 086/183] Update test requirements snapshot (#20031) I actually wanted this for `librt`, but it looks like there is a bunch of other deps worth updating. --- test-requirements.in | 3 ++- test-requirements.txt | 32 +++++++++++++++++--------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index df074965a1e8..556edf5077d2 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -4,7 +4,7 @@ -r mypy-requirements.txt -r build-requirements.txt attrs>=18.0 -filelock>=3.3.0 +filelock>=3.3.0,<3.20.0 # latest version is not available on 3.9 that we still support lxml>=5.3.0; python_version<'3.15' psutil>=4.0 pytest>=8.1.0 @@ -13,3 +13,4 @@ pytest-cov>=2.10.0 setuptools>=75.1.0 tomli>=1.1.0 # needed even on py311+ so the self check passes with --python-version 3.9 pre_commit>=3.5.0 +platformdirs<4.5.0 # latest version is not available on 3.9 that we still support diff --git a/test-requirements.txt b/test-requirements.txt index 7e8e167300dc..c16708dfdbaf 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,11 +4,11 @@ # # pip-compile --allow-unsafe --output-file=test-requirements.txt --strip-extras test-requirements.in # -attrs==25.3.0 +attrs==25.4.0 # via -r test-requirements.in cfgv==3.4.0 # via pre-commit -coverage==7.10.5 +coverage==7.10.7 # via pytest-cov distlib==0.4.0 # via virtualenv @@ -18,13 +18,13 @@ filelock==3.19.1 # via # -r test-requirements.in # virtualenv -identify==2.6.13 +identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.2.1 +librt==0.2.2 # via -r mypy-requirements.txt -lxml==6.0.1 ; python_version < "3.15" +lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in mypy-extensions==1.1.0 # via -r mypy-requirements.txt @@ -34,38 +34,40 @@ packaging==25.0 # via pytest pathspec==0.12.1 # via -r mypy-requirements.txt -platformdirs==4.3.8 - # via virtualenv +platformdirs==4.4.0 + # via + # -r test-requirements.in + # virtualenv pluggy==1.6.0 # via # pytest # pytest-cov pre-commit==4.3.0 # via -r test-requirements.in -psutil==7.0.0 +psutil==7.1.0 # via -r test-requirements.in pygments==2.19.2 # via pytest -pytest==8.4.1 +pytest==8.4.2 # via # -r test-requirements.in # pytest-cov # pytest-xdist -pytest-cov==6.2.1 +pytest-cov==7.0.0 # via -r test-requirements.in pytest-xdist==3.8.0 # via -r test-requirements.in -pyyaml==6.0.2 +pyyaml==6.0.3 # via pre-commit -tomli==2.2.1 +tomli==2.3.0 # via -r test-requirements.in -types-psutil==7.0.0.20250822 +types-psutil==7.0.0.20251001 # via -r build-requirements.txt types-setuptools==80.9.0.20250822 # via -r build-requirements.txt -typing-extensions==4.14.1 +typing-extensions==4.15.0 # via -r mypy-requirements.txt -virtualenv==20.34.0 +virtualenv==20.35.0 # via pre-commit # The following packages are considered to be unsafe in a requirements file: From 895d0cf9b60af0aac8c4c56709825cbca3feca1f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 9 Oct 2025 11:27:54 +0100 Subject: [PATCH 087/183] Re-run pip-compile to get rid of yanked version of virtualenv (#20037) --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c16708dfdbaf..bbaf1ce6010f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -67,7 +67,7 @@ types-setuptools==80.9.0.20250822 # via -r build-requirements.txt typing-extensions==4.15.0 # via -r mypy-requirements.txt -virtualenv==20.35.0 +virtualenv==20.34.0 # via pre-commit # The following packages are considered to be unsafe in a requirements file: From f8ecfe5658f33d5d254e7d161c25b2909691c5ce Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 9 Oct 2025 15:11:05 +0100 Subject: [PATCH 088/183] Bump librt again (#20040) Just in case to test the new MacOS wheels. Also fix `lib-rt/setup.py` docstring. --- mypyc/lib-rt/setup.py | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index afbceba060f4..299b0acd96e7 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -1,4 +1,4 @@ -"""Build script for mypyc C runtime library unit tests. +"""Build script for mypyc C runtime library and C API unit tests. The tests are written in C++ and use the Google Test framework. """ diff --git a/test-requirements.txt b/test-requirements.txt index bbaf1ce6010f..0dc2a4cf8f18 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.2.2 +librt==0.2.3 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From 826e0adcb4b6b2855788927d220b965660df9294 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Fri, 10 Oct 2025 09:05:44 -0400 Subject: [PATCH 089/183] [mypyc] feat: support constant folding in `translate_index_expr` [1/1] (#19972) This PR attempts to constant fold the index value in `translate_index_expr` I'm not sure any test changes are warranted for a small PR of this nature. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/irbuild/expression.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 59ecc4ac2c5c..f6636a0e7b62 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -590,8 +590,12 @@ def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: base = builder.accept(expr.base, can_borrow=can_borrow_base) - if isinstance(base.type, RTuple) and isinstance(index, IntExpr): - return builder.add(TupleGet(base, index.value, expr.line)) + if isinstance(base.type, RTuple): + folded_index = constant_fold_expr(builder, index) + if isinstance(folded_index, int): + length = len(base.type.types) + if -length <= folded_index <= length - 1: + return builder.add(TupleGet(base, folded_index, expr.line)) if isinstance(index, SliceExpr): value = try_gen_slice_op(builder, base, index) From 320ea65a7043e62d88ac7dfaab54bb474b6bc7b0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 10 Oct 2025 14:26:39 +0100 Subject: [PATCH 090/183] [mypyc] Refactor: adjust generator return types in a later pass (#20043) Move the inference of a more precise generator return type to a later pass. This way we have access to full class inheritance hierarchies. This is in preparation to fixing mypyc/mypyc#1141. --- mypyc/irbuild/main.py | 11 ++++--- mypyc/irbuild/mapper.py | 9 +----- mypyc/irbuild/prepare.py | 70 ++++++++++++++++++++++++---------------- 3 files changed, 51 insertions(+), 39 deletions(-) diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py index d2c8924a7298..f08911a1bc4c 100644 --- a/mypyc/irbuild/main.py +++ b/mypyc/irbuild/main.py @@ -38,8 +38,9 @@ def f(x: int) -> int: from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.prebuildvisitor import PreBuildVisitor from mypyc.irbuild.prepare import ( + adjust_generator_classes_of_methods, build_type_map, - create_generator_class_if_needed, + create_generator_class_for_func, find_singledispatch_register_impls, ) from mypyc.irbuild.visitor import IRBuilderVisitor @@ -68,6 +69,7 @@ def build_ir( """ build_type_map(mapper, modules, graph, types, options, errors) + adjust_generator_classes_of_methods(mapper) singledispatch_info = find_singledispatch_register_impls(modules, errors) result: ModuleIRs = {} @@ -87,9 +89,10 @@ def build_ir( if isinstance(fdef, FuncDef): # Make generator class name sufficiently unique. suffix = f"___{fdef.line}" - create_generator_class_if_needed( - module.fullname, None, fdef, mapper, name_suffix=suffix - ) + if fdef.is_coroutine or fdef.is_generator: + create_generator_class_for_func( + module.fullname, None, fdef, mapper, name_suffix=suffix + ) # Construct and configure builder objects (cyclic runtime dependency). visitor = IRBuilderVisitor() diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 05aa0e45c569..c986499b6f65 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -180,14 +180,7 @@ def fdef_to_sig(self, fdef: FuncDef, strict_dunders_typing: bool) -> FuncSignatu for typ, kind in zip(fdef.type.arg_types, fdef.type.arg_kinds) ] arg_pos_onlys = [name is None for name in fdef.type.arg_names] - # TODO: We could probably support decorators sometimes (static and class method?) - if (fdef.is_coroutine or fdef.is_generator) and not fdef.is_decorated: - # Give a more precise type for generators, so that we can optimize - # code that uses them. They return a generator object, which has a - # specific class. Without this, the type would have to be 'object'. - ret: RType = RInstance(self.fdef_to_generator[fdef]) - else: - ret = self.type_to_rtype(fdef.type.ret_type) + ret = self.type_to_rtype(fdef.type.ret_type) else: # Handle unannotated functions arg_types = [object_rprimitive for _ in fdef.arguments] diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index e4f43b38b0dc..2d0a1a8f03bf 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -196,8 +196,6 @@ def prepare_func_def( mapper: Mapper, options: CompilerOptions, ) -> FuncDecl: - create_generator_class_if_needed(module_name, class_name, fdef, mapper) - kind = ( FUNC_CLASSMETHOD if fdef.is_class @@ -209,38 +207,37 @@ def prepare_func_def( return decl -def create_generator_class_if_needed( +def create_generator_class_for_func( module_name: str, class_name: str | None, fdef: FuncDef, mapper: Mapper, name_suffix: str = "" -) -> None: - """If function is a generator/async function, declare a generator class. +) -> ClassIR: + """For a generator/async function, declare a generator class. Each generator and async function gets a dedicated class that implements the generator protocol with generated methods. """ - if fdef.is_coroutine or fdef.is_generator: - name = "_".join(x for x in [fdef.name, class_name] if x) + "_gen" + name_suffix - cir = ClassIR(name, module_name, is_generated=True, is_final_class=True) - cir.reuse_freed_instance = True - mapper.fdef_to_generator[fdef] = cir + assert fdef.is_coroutine or fdef.is_generator + name = "_".join(x for x in [fdef.name, class_name] if x) + "_gen" + name_suffix + cir = ClassIR(name, module_name, is_generated=True, is_final_class=True) + cir.reuse_freed_instance = True + mapper.fdef_to_generator[fdef] = cir - helper_sig = FuncSignature( - ( - RuntimeArg(SELF_NAME, object_rprimitive), - RuntimeArg("type", object_rprimitive), - RuntimeArg("value", object_rprimitive), - RuntimeArg("traceback", object_rprimitive), - RuntimeArg("arg", object_rprimitive), - # If non-NULL, used to store return value instead of raising StopIteration(retv) - RuntimeArg("stop_iter_ptr", object_pointer_rprimitive), - ), - object_rprimitive, - ) + helper_sig = FuncSignature( + ( + RuntimeArg(SELF_NAME, object_rprimitive), + RuntimeArg("type", object_rprimitive), + RuntimeArg("value", object_rprimitive), + RuntimeArg("traceback", object_rprimitive), + RuntimeArg("arg", object_rprimitive), + # If non-NULL, used to store return value instead of raising StopIteration(retv) + RuntimeArg("stop_iter_ptr", object_pointer_rprimitive), + ), + object_rprimitive, + ) - # The implementation of most generator functionality is behind this magic method. - helper_fn_decl = FuncDecl( - GENERATOR_HELPER_NAME, name, module_name, helper_sig, internal=True - ) - cir.method_decls[helper_fn_decl.name] = helper_fn_decl + # The implementation of most generator functionality is behind this magic method. + helper_fn_decl = FuncDecl(GENERATOR_HELPER_NAME, name, module_name, helper_sig, internal=True) + cir.method_decls[helper_fn_decl.name] = helper_fn_decl + return cir def prepare_method_def( @@ -811,3 +808,22 @@ def registered_impl_from_possible_register_call( if isinstance(node, Decorator): return RegisteredImpl(node.func, dispatch_type) return None + + +def adjust_generator_classes_of_methods(mapper: Mapper) -> None: + """Make optimizations and adjustments to generated generator classes of methods. + + This is a separate pass after type map has been built, since we need all classes + to be processed to analyze class hierarchies. + """ + for fdef, ir in mapper.func_to_decl.items(): + if isinstance(fdef, FuncDef) and (fdef.is_coroutine or fdef.is_generator): + gen_ir = create_generator_class_for_func(ir.module_name, ir.class_name, fdef, mapper) + # TODO: We could probably support decorators sometimes (static and class method?) + if not fdef.is_decorated: + # Give a more precise type for generators, so that we can optimize + # code that uses them. They return a generator object, which has a + # specific class. Without this, the type would have to be 'object'. + ir.sig.ret_type = RInstance(gen_ir) + if ir.bound_sig: + ir.bound_sig.ret_type = RInstance(gen_ir) From 5b7279b7dc554e8ba21a159be584da0ddf7f0010 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Fri, 10 Oct 2025 06:37:12 -0700 Subject: [PATCH 091/183] Do not sort unused error codes in unused error codes warning (#20036) I intuit the previous author of this code sorted the codes for stability, but it actually should be in default order, to match what the user typed in. This will be more intuitive for the user. In my first commit, I add the failing testUnusedIgnoreCodeOrder test. In my second commit, I fix the code. --- mypy/errors.py | 6 +++--- test-data/unit/check-ignore.test | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/mypy/errors.py b/mypy/errors.py index f1b2faf67401..1b092fb50e4a 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -806,8 +806,8 @@ def generate_unused_ignore_errors(self, file: str) -> None: continue if codes.UNUSED_IGNORE.code in ignored_codes: continue - used_ignored_codes = used_ignored_lines[line] - unused_ignored_codes = set(ignored_codes) - set(used_ignored_codes) + used_ignored_codes = set(used_ignored_lines[line]) + unused_ignored_codes = [c for c in ignored_codes if c not in used_ignored_codes] # `ignore` is used if not ignored_codes and used_ignored_codes: continue @@ -817,7 +817,7 @@ def generate_unused_ignore_errors(self, file: str) -> None: # Display detail only when `ignore[...]` specifies more than one error code unused_codes_message = "" if len(ignored_codes) > 1 and unused_ignored_codes: - unused_codes_message = f"[{', '.join(sorted(unused_ignored_codes))}]" + unused_codes_message = f"[{', '.join(unused_ignored_codes)}]" message = f'Unused "type: ignore{unused_codes_message}" comment' for unused in unused_ignored_codes: narrower = set(used_ignored_codes) & codes.sub_code_map[unused] diff --git a/test-data/unit/check-ignore.test b/test-data/unit/check-ignore.test index a4234e7a37a1..d0f6bb6aeb60 100644 --- a/test-data/unit/check-ignore.test +++ b/test-data/unit/check-ignore.test @@ -275,6 +275,16 @@ class CD(six.with_metaclass(M)): # E: Multiple metaclass definitions [builtins fixtures/tuple.pyi] +[case testUnusedIgnoreCodeOrder] +# flags: --warn-unused-ignores +5 # type: ignore[import, steven] # E: Unused "type: ignore[import, steven]" comment +-- User ordering of codes is preserved +5 # type: ignore[steven, import] # E: Unused "type: ignore[steven, import]" comment +-- Spacing is not preserved +5 # type: ignore[ steven, import ] # E: Unused "type: ignore[steven, import]" comment +-- Make sure it works as intended in more complex situations +1 + "ok" + "ok".foo # type: ignore[ operator,steven,attr-defined, import] # E: Unused "type: ignore[steven, import]" comment + [case testUnusedIgnoreTryExcept] # flags: --warn-unused-ignores try: From e03d3c1cffb2185cf3eac199db122e6364e459cc Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 11 Oct 2025 04:58:57 +0900 Subject: [PATCH 092/183] Check class references to catch non-existant classes in match cases (#20042) Fixes https://github.com/python/mypy/issues/20018. --- mypy/checkpattern.py | 36 ++++----- mypyc/test-data/irbuild-match.test | 121 ++++++++++++++-------------- test-data/unit/check-python310.test | 16 ++++ 3 files changed, 92 insertions(+), 81 deletions(-) diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index f81684d2f44a..6f00c6c43177 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -14,7 +14,7 @@ from mypy.maptype import map_instance_to_supertype from mypy.meet import narrow_declared_type from mypy.messages import MessageBuilder -from mypy.nodes import ARG_POS, Context, Expression, NameExpr, TypeAlias, TypeInfo, Var +from mypy.nodes import ARG_POS, Context, Expression, NameExpr, TypeAlias, Var from mypy.options import Options from mypy.patterns import ( AsPattern, @@ -37,6 +37,7 @@ ) from mypy.types import ( AnyType, + FunctionLike, Instance, LiteralType, NoneType, @@ -538,27 +539,20 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType: # Check class type # type_info = o.class_ref.node - if type_info is None: - typ: Type = AnyType(TypeOfAny.from_error) - elif isinstance(type_info, TypeAlias) and not type_info.no_args: + typ = self.chk.expr_checker.accept(o.class_ref) + p_typ = get_proper_type(typ) + if isinstance(type_info, TypeAlias) and not type_info.no_args: self.msg.fail(message_registry.CLASS_PATTERN_GENERIC_TYPE_ALIAS, o) return self.early_non_match() - elif isinstance(type_info, TypeInfo): - typ = fill_typevars_with_any(type_info) - elif isinstance(type_info, TypeAlias): - typ = type_info.target - elif ( - isinstance(type_info, Var) - and type_info.type is not None - and isinstance(get_proper_type(type_info.type), AnyType) - ): - typ = type_info.type - else: - if isinstance(type_info, Var) and type_info.type is not None: - name = type_info.type.str_with_options(self.options) - else: - name = type_info.name - self.msg.fail(message_registry.CLASS_PATTERN_TYPE_REQUIRED.format(name), o) + elif isinstance(p_typ, FunctionLike) and p_typ.is_type_obj(): + typ = fill_typevars_with_any(p_typ.type_object()) + elif not isinstance(p_typ, AnyType): + self.msg.fail( + message_registry.CLASS_PATTERN_TYPE_REQUIRED.format( + typ.str_with_options(self.options) + ), + o, + ) return self.early_non_match() new_type, rest_type = self.chk.conditional_types_with_intersection( @@ -697,6 +691,8 @@ def should_self_match(self, typ: Type) -> bool: typ = get_proper_type(typ) if isinstance(typ, TupleType): typ = typ.partial_fallback + if isinstance(typ, AnyType): + return False if isinstance(typ, Instance) and typ.type.get("__match_args__") is not None: # Named tuples and other subtypes of builtins that define __match_args__ # should not self match. diff --git a/mypyc/test-data/irbuild-match.test b/mypyc/test-data/irbuild-match.test index 28aff3dcfc45..1e84c385100a 100644 --- a/mypyc/test-data/irbuild-match.test +++ b/mypyc/test-data/irbuild-match.test @@ -563,10 +563,9 @@ def f(): def f(): r0, r1 :: object r2 :: bool - i :: int - r3 :: object - r4 :: str - r5, r6 :: object + r3, i, r4 :: object + r5 :: str + r6 :: object r7 :: object[1] r8 :: object_ptr r9, r10 :: object @@ -576,21 +575,22 @@ L0: r2 = CPy_TypeCheck(r1, r0) if r2 goto L1 else goto L3 :: bool L1: - i = 246 + r3 = object 123 + i = r3 L2: - r3 = builtins :: module - r4 = 'print' - r5 = CPyObject_GetAttr(r3, r4) - r6 = box(int, i) - r7 = [r6] + r4 = builtins :: module + r5 = 'print' + r6 = CPyObject_GetAttr(r4, r5) + r7 = [i] r8 = load_address r7 - r9 = PyObject_Vectorcall(r5, r8, 1, 0) - keep_alive r6 + r9 = PyObject_Vectorcall(r6, r8, 1, 0) + keep_alive i goto L4 L3: L4: r10 = box(None, 1) return r10 + [case testMatchClassPatternWithPositionalArgs_python3_10] class Position: __match_args__ = ("x", "y", "z") @@ -599,7 +599,7 @@ class Position: y: int z: int -def f(x): +def f(x) -> None: match x: case Position(1, 2, 3): print("matched") @@ -641,7 +641,7 @@ def f(x): r28 :: object r29 :: object[1] r30 :: object_ptr - r31, r32 :: object + r31 :: object L0: r0 = __main__.Position :: type r1 = PyObject_IsInstance(x, r0) @@ -687,8 +687,8 @@ L4: goto L6 L5: L6: - r32 = box(None, 1) - return r32 + return 1 + [case testMatchClassPatternWithKeywordPatterns_python3_10] class Position: x: int @@ -848,7 +848,7 @@ class C: a: int b: int -def f(x): +def f(x) -> None: match x: case C(1, 2) as y: print("matched") @@ -885,7 +885,7 @@ def f(x): r22 :: object r23 :: object[1] r24 :: object_ptr - r25, r26 :: object + r25 :: object L0: r0 = __main__.C :: type r1 = PyObject_IsInstance(x, r0) @@ -925,8 +925,8 @@ L4: goto L6 L5: L6: - r26 = box(None, 1) - return r26 + return 1 + [case testMatchClassPatternPositionalCapture_python3_10] class C: __match_args__ = ("x",) @@ -953,15 +953,14 @@ def f(x): r2 :: bit r3 :: bool r4 :: str - r5 :: object - r6, num :: int - r7 :: str - r8 :: object - r9 :: str - r10 :: object - r11 :: object[1] - r12 :: object_ptr - r13, r14 :: object + r5, num :: object + r6 :: str + r7 :: object + r8 :: str + r9 :: object + r10 :: object[1] + r11 :: object_ptr + r12, r13 :: object L0: r0 = __main__.C :: type r1 = PyObject_IsInstance(x, r0) @@ -971,22 +970,22 @@ L0: L1: r4 = 'x' r5 = CPyObject_GetAttr(x, r4) - r6 = unbox(int, r5) - num = r6 + num = r5 L2: - r7 = 'matched' - r8 = builtins :: module - r9 = 'print' - r10 = CPyObject_GetAttr(r8, r9) - r11 = [r7] - r12 = load_address r11 - r13 = PyObject_Vectorcall(r10, r12, 1, 0) - keep_alive r7 + r6 = 'matched' + r7 = builtins :: module + r8 = 'print' + r9 = CPyObject_GetAttr(r7, r8) + r10 = [r6] + r11 = load_address r10 + r12 = PyObject_Vectorcall(r9, r11, 1, 0) + keep_alive r6 goto L4 L3: L4: - r14 = box(None, 1) - return r14 + r13 = box(None, 1) + return r13 + [case testMatchMappingEmpty_python3_10] def f(x): match x: @@ -1601,35 +1600,35 @@ def f(x): def f(x): x, r0 :: object r1 :: bool - r2, y :: int - r3 :: str - r4 :: object - r5 :: str - r6 :: object - r7 :: object[1] - r8 :: object_ptr - r9, r10 :: object + y :: object + r2 :: str + r3 :: object + r4 :: str + r5 :: object + r6 :: object[1] + r7 :: object_ptr + r8, r9 :: object L0: r0 = load_address PyLong_Type r1 = CPy_TypeCheck(x, r0) if r1 goto L1 else goto L3 :: bool L1: - r2 = unbox(int, x) - y = r2 + y = x L2: - r3 = 'matched' - r4 = builtins :: module - r5 = 'print' - r6 = CPyObject_GetAttr(r4, r5) - r7 = [r3] - r8 = load_address r7 - r9 = PyObject_Vectorcall(r6, r8, 1, 0) - keep_alive r3 + r2 = 'matched' + r3 = builtins :: module + r4 = 'print' + r5 = CPyObject_GetAttr(r3, r4) + r6 = [r2] + r7 = load_address r6 + r8 = PyObject_Vectorcall(r5, r7, 1, 0) + keep_alive r2 goto L4 L3: L4: - r10 = box(None, 1) - return r10 + r9 = box(None, 1) + return r9 + [case testMatchSequenceCaptureAll_python3_10] def f(x): match x: diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 7d76c09b6151..2c4597e212ea 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -2990,3 +2990,19 @@ def foo(e: Literal[0, 1]) -> None: ... defer = unknown_module.foo + +[case testMatchErrorsIncorrectName] +class A: + pass + +match 5: + case A.blah(): # E: "type[A]" has no attribute "blah" + pass + +[case testMatchAllowsAnyClassArgsForAny] +match 5: + case BlahBlah(a, b): # E: Name "BlahBlah" is not defined + reveal_type(a) # N: Revealed type is "Any" + reveal_type(b) # N: Revealed type is "Any" + case BlahBlah(c=c): # E: Name "BlahBlah" is not defined + reveal_type(c) # N: Revealed type is "Any" From 04a586c6dbe9a29c8b2f38f037828c437682fc45 Mon Sep 17 00:00:00 2001 From: iap Date: Fri, 10 Oct 2025 21:45:17 -0400 Subject: [PATCH 093/183] stubgenc: small fix in get_default_function_sig (#19822) This small change fixes a crash in the case when a function arg has both a default value and a non-string type annotation. Here is an example: ``` def f(i: int = 0): pass ``` --- mypy/stubgenc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index e64dbcdd9d40..e0e063927aad 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -322,7 +322,7 @@ def add_args( default_value = get_default_value(i, arg) if default_value is not _Missing.VALUE: if arg in annotations: - argtype = annotations[arg] + argtype = get_annotation(arg) else: argtype = self.get_type_annotation(default_value) if argtype == "None": From 18bfc016eb2d413d1fb3389fa8f454cb40fcebd0 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Sun, 12 Oct 2025 02:15:15 +0200 Subject: [PATCH 094/183] prevent false unreachable warnings for @final instances that occur when strict optional checking is disabled (#20045) Fixes #19849 See #11717 for some background information on `--no-strict-optional`. --- mypy/typeops.py | 4 ++-- test-data/unit/check-inference.test | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index d058bb8201d3..341c96c08931 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -789,9 +789,9 @@ def false_only(t: Type) -> ProperType: if not ret_type.can_be_false: return UninhabitedType(line=t.line) elif isinstance(t, Instance): - if t.type.is_final or t.type.is_enum: + if (t.type.is_final or t.type.is_enum) and state.strict_optional: return UninhabitedType(line=t.line) - elif isinstance(t, LiteralType) and t.is_enum_literal(): + elif isinstance(t, LiteralType) and t.is_enum_literal() and state.strict_optional: return UninhabitedType(line=t.line) new_t = copy_type(t) diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 63278d6c4547..24ea61f2c715 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1964,6 +1964,31 @@ if 'x' in d: # E: "None" has no attribute "__iter__" (not iterable) reveal_type(d) # N: Revealed type is "None" [builtins fixtures/dict.pyi] +[case testNoWrongUnreachableWarningWithNoStrictOptionalAndFinalInstance] +# flags: --no-strict-optional --warn-unreachable +from typing import final, Optional + +@final +class C: ... + +x: Optional[C] +if not x: + x = C() +[builtins fixtures/dict.pyi] + +[case testNoWrongUnreachableWarningWithNoStrictOptionalAndEnumLiteral] +# flags: --no-strict-optional --warn-unreachable +from enum import Enum +from typing import Literal, Optional + +class E(Enum): + a = 1 + +x: Optional[Literal[E.a]] +if not x: + x = E.a +[builtins fixtures/dict.pyi] + [case testInferFromEmptyListWhenUsingInWithStrictEquality] # flags: --strict-equality def f() -> None: From 6aa44da630a9a277b6e7b9c77f9083bb6e00c26b Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sun, 12 Oct 2025 02:22:01 +0200 Subject: [PATCH 095/183] Prevent TypeGuardedType leak from `narrow_declared_type` as part of typevar bound (#20046) Fixes #20015, refs #18895 as a previous example of the same issue. I don't see any similar problems in other branches. --- mypy/meet.py | 4 +++- test-data/unit/check-typeguard.test | 20 ++++++++++++++++++++ test-data/unit/fixtures/typing-full.pyi | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mypy/meet.py b/mypy/meet.py index 353af59367ad..63305c2bb236 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -170,7 +170,9 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: ): # We put this branch early to get T(bound=Union[A, B]) instead of # Union[T(bound=A), T(bound=B)] that will be confusing for users. - return declared.copy_modified(upper_bound=original_narrowed) + return declared.copy_modified( + upper_bound=narrow_declared_type(declared.upper_bound, original_narrowed) + ) elif not is_overlapping_types(declared, narrowed, prohibit_none_typevar_overlap=True): if state.strict_optional: return UninhabitedType() diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 93e665e4548c..b15458d5819a 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -825,6 +825,26 @@ def handle(model: Model) -> int: return 0 [builtins fixtures/tuple.pyi] +[case testTypeGuardedTypeDoesNotLeakTypeVar] +# flags: --debug-serialize +# https://github.com/python/mypy/issues/20015 +from typing import Generic, TypeVar, TypeGuard + +class A: ... +class B: ... + +def is_a(_: object) -> TypeGuard[A]: return True +def is_b(_: object) -> TypeGuard[B]: return True + +_T = TypeVar("_T") + +class Foo(Generic[_T]): + def __init__(self, v: _T) -> None: + if is_a(v) or is_b(v): + self.v = v +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + [case testTypeGuardRestrictTypeVarUnion] from typing import Union, TypeVar from typing_extensions import TypeGuard diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index 3757e868552e..1a63deaa727d 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -34,6 +34,7 @@ no_type_check = 0 ClassVar = 0 Final = 0 TypedDict = 0 +TypeGuard = 0 NoReturn = 0 NewType = 0 Self = 0 From b3e26e7d68a792eeb207aeb8dd1e903593bfc097 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 14 Oct 2025 10:45:17 +0100 Subject: [PATCH 096/183] [mypyc] Fix inheritance of async defs (#20044) When inferring a precise generator return type for an async def (or generator), make the generate returned by an override in a subclass inherit from the base class generator. This means that the environment has to be moved to a separate class in the base class generator. Don't infer a precise generator return type when an override might have a less precise return type, since it would break LSP. Fixes mypyc/mypyc#1141. --- mypyc/codegen/emitclass.py | 8 +-- mypyc/codegen/emitmodule.py | 2 + mypyc/ir/func_ir.py | 17 ++++-- mypyc/irbuild/context.py | 6 ++- mypyc/irbuild/function.py | 12 +++-- mypyc/irbuild/generator.py | 4 +- mypyc/irbuild/prepare.py | 84 +++++++++++++++++++++++++---- mypyc/test-data/run-async.test | 74 +++++++++++++++++++++++++ mypyc/test-data/run-generators.test | 29 ++++++++++ 9 files changed, 212 insertions(+), 24 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 122f62a0d582..d64940084f12 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -410,7 +410,7 @@ def setter_name(cl: ClassIR, attribute: str, names: NameGenerator) -> str: def generate_object_struct(cl: ClassIR, emitter: Emitter) -> None: - seen_attrs: set[tuple[str, RType]] = set() + seen_attrs: set[str] = set() lines: list[str] = [] lines += ["typedef struct {", "PyObject_HEAD", "CPyVTableItem *vtable;"] if cl.has_method("__call__"): @@ -427,9 +427,11 @@ def generate_object_struct(cl: ClassIR, emitter: Emitter) -> None: lines.append(f"{BITMAP_TYPE} {attr};") bitmap_attrs.append(attr) for attr, rtype in base.attributes.items(): - if (attr, rtype) not in seen_attrs: + # Generated class may redefine certain attributes with different + # types in subclasses (this would be unsafe for user-defined classes). + if attr not in seen_attrs: lines.append(f"{emitter.ctype_spaced(rtype)}{emitter.attr(attr)};") - seen_attrs.add((attr, rtype)) + seen_attrs.add(attr) if isinstance(rtype, RTuple): emitter.declare_tuple_struct(rtype) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 3602b3c26e03..7ec315a6bd34 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -1064,6 +1064,8 @@ def emit_module_exec_func( "(PyObject *){t}_template, NULL, modname);".format(t=type_struct) ) emitter.emit_lines(f"if (unlikely(!{type_struct}))", " goto fail;") + name_prefix = cl.name_prefix(emitter.names) + emitter.emit_line(f"CPyDef_{name_prefix}_trait_vtable_setup();") emitter.emit_lines("if (CPyGlobalsInit() < 0)", " goto fail;") diff --git a/mypyc/ir/func_ir.py b/mypyc/ir/func_ir.py index 881ac5939c27..d11fef42feb5 100644 --- a/mypyc/ir/func_ir.py +++ b/mypyc/ir/func_ir.py @@ -149,8 +149,11 @@ def __init__( module_name: str, sig: FuncSignature, kind: int = FUNC_NORMAL, + *, is_prop_setter: bool = False, is_prop_getter: bool = False, + is_generator: bool = False, + is_coroutine: bool = False, implicit: bool = False, internal: bool = False, ) -> None: @@ -161,6 +164,8 @@ def __init__( self.kind = kind self.is_prop_setter = is_prop_setter self.is_prop_getter = is_prop_getter + self.is_generator = is_generator + self.is_coroutine = is_coroutine if class_name is None: self.bound_sig: FuncSignature | None = None else: @@ -219,6 +224,8 @@ def serialize(self) -> JsonDict: "kind": self.kind, "is_prop_setter": self.is_prop_setter, "is_prop_getter": self.is_prop_getter, + "is_generator": self.is_generator, + "is_coroutine": self.is_coroutine, "implicit": self.implicit, "internal": self.internal, } @@ -240,10 +247,12 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> FuncDecl: data["module_name"], FuncSignature.deserialize(data["sig"], ctx), data["kind"], - data["is_prop_setter"], - data["is_prop_getter"], - data["implicit"], - data["internal"], + is_prop_setter=data["is_prop_setter"], + is_prop_getter=data["is_prop_getter"], + is_generator=data["is_generator"], + is_coroutine=data["is_coroutine"], + implicit=data["implicit"], + internal=data["internal"], ) diff --git a/mypyc/irbuild/context.py b/mypyc/irbuild/context.py index 8d2e55ed96fb..d5a48bf838c8 100644 --- a/mypyc/irbuild/context.py +++ b/mypyc/irbuild/context.py @@ -98,7 +98,11 @@ def curr_env_reg(self) -> Value: def can_merge_generator_and_env_classes(self) -> bool: # In simple cases we can place the environment into the generator class, # instead of having two separate classes. - return self.is_generator and not self.is_nested and not self.contains_nested + if self._generator_class and not self._generator_class.ir.is_final_class: + result = False + else: + result = self.is_generator and not self.is_nested and not self.contains_nested + return result class ImplicitClass: diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index c9f999597d30..738d19ea6748 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -69,7 +69,7 @@ instantiate_callable_class, setup_callable_class, ) -from mypyc.irbuild.context import FuncInfo +from mypyc.irbuild.context import FuncInfo, GeneratorClass from mypyc.irbuild.env_class import ( add_vars_to_env, finalize_env_class, @@ -246,6 +246,12 @@ def c() -> None: is_generator = fn_info.is_generator builder.enter(fn_info, ret_type=sig.ret_type) + if is_generator: + fitem = builder.fn_info.fitem + assert isinstance(fitem, FuncDef), fitem + generator_class_ir = builder.mapper.fdef_to_generator[fitem] + builder.fn_info.generator_class = GeneratorClass(generator_class_ir) + # Functions that contain nested functions need an environment class to store variables that # are free in their nested functions. Generator functions need an environment class to # store a variable denoting the next instruction to be executed when the __next__ function @@ -357,8 +363,8 @@ def gen_func_ir( builder.module_name, sig, func_decl.kind, - func_decl.is_prop_getter, - func_decl.is_prop_setter, + is_prop_getter=func_decl.is_prop_getter, + is_prop_setter=func_decl.is_prop_setter, ) func_ir = FuncIR(func_decl, args, blocks, fitem.line, traceback_name=fitem.name) else: diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index b3a417ed6a3e..4dcd748f6eff 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -39,7 +39,7 @@ object_rprimitive, ) from mypyc.irbuild.builder import IRBuilder, calculate_arg_defaults, gen_arg_defaults -from mypyc.irbuild.context import FuncInfo, GeneratorClass +from mypyc.irbuild.context import FuncInfo from mypyc.irbuild.env_class import ( add_args_to_env, add_vars_to_env, @@ -166,10 +166,8 @@ def setup_generator_class(builder: IRBuilder) -> ClassIR: builder.fn_info.env_class = generator_class_ir else: generator_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_info.env_class) - generator_class_ir.mro = [generator_class_ir] builder.classes.append(generator_class_ir) - builder.fn_info.generator_class = GeneratorClass(generator_class_ir) return generator_class_ir diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 2d0a1a8f03bf..0f7cc7e3b3c5 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -202,7 +202,15 @@ def prepare_func_def( else (FUNC_STATICMETHOD if fdef.is_static else FUNC_NORMAL) ) sig = mapper.fdef_to_sig(fdef, options.strict_dunders_typing) - decl = FuncDecl(fdef.name, class_name, module_name, sig, kind) + decl = FuncDecl( + fdef.name, + class_name, + module_name, + sig, + kind, + is_generator=fdef.is_generator, + is_coroutine=fdef.is_coroutine, + ) mapper.func_to_decl[fdef] = decl return decl @@ -217,7 +225,7 @@ def create_generator_class_for_func( """ assert fdef.is_coroutine or fdef.is_generator name = "_".join(x for x in [fdef.name, class_name] if x) + "_gen" + name_suffix - cir = ClassIR(name, module_name, is_generated=True, is_final_class=True) + cir = ClassIR(name, module_name, is_generated=True, is_final_class=class_name is None) cir.reuse_freed_instance = True mapper.fdef_to_generator[fdef] = cir @@ -816,14 +824,70 @@ def adjust_generator_classes_of_methods(mapper: Mapper) -> None: This is a separate pass after type map has been built, since we need all classes to be processed to analyze class hierarchies. """ - for fdef, ir in mapper.func_to_decl.items(): + + generator_methods = [] + + for fdef, fn_ir in mapper.func_to_decl.items(): if isinstance(fdef, FuncDef) and (fdef.is_coroutine or fdef.is_generator): - gen_ir = create_generator_class_for_func(ir.module_name, ir.class_name, fdef, mapper) + gen_ir = create_generator_class_for_func( + fn_ir.module_name, fn_ir.class_name, fdef, mapper + ) # TODO: We could probably support decorators sometimes (static and class method?) if not fdef.is_decorated: - # Give a more precise type for generators, so that we can optimize - # code that uses them. They return a generator object, which has a - # specific class. Without this, the type would have to be 'object'. - ir.sig.ret_type = RInstance(gen_ir) - if ir.bound_sig: - ir.bound_sig.ret_type = RInstance(gen_ir) + name = fn_ir.name + precise_ret_type = True + if fn_ir.class_name is not None: + class_ir = mapper.type_to_ir[fdef.info] + subcls = class_ir.subclasses() + if subcls is None: + # Override could be of a different type, so we can't make assumptions. + precise_ret_type = False + else: + for s in subcls: + if name in s.method_decls: + m = s.method_decls[name] + if ( + m.is_generator != fn_ir.is_generator + or m.is_coroutine != fn_ir.is_coroutine + ): + # Override is of a different kind, and the optimization + # to use a precise generator return type doesn't work. + precise_ret_type = False + else: + class_ir = None + + if precise_ret_type: + # Give a more precise type for generators, so that we can optimize + # code that uses them. They return a generator object, which has a + # specific class. Without this, the type would have to be 'object'. + fn_ir.sig.ret_type = RInstance(gen_ir) + if fn_ir.bound_sig: + fn_ir.bound_sig.ret_type = RInstance(gen_ir) + if class_ir is not None: + if class_ir.is_method_final(name): + gen_ir.is_final_class = True + generator_methods.append((name, class_ir, gen_ir)) + + new_bases = {} + + for name, class_ir, gen in generator_methods: + # For generator methods, we need to have subclass generator classes inherit from + # baseclass generator classes when there are overrides to maintain LSP. + base = class_ir.real_base() + if base is not None: + if base.has_method(name): + base_sig = base.method_sig(name) + if isinstance(base_sig.ret_type, RInstance): + base_gen = base_sig.ret_type.class_ir + new_bases[gen] = base_gen + + # Add generator inheritance relationships by adjusting MROs. + for deriv, base in new_bases.items(): + if base.children is not None: + base.children.append(deriv) + while True: + deriv.mro.append(base) + deriv.base_mro.append(base) + if base not in new_bases: + break + base = new_bases[base] diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 94a1cd2e97c5..cf063310fd89 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1291,3 +1291,77 @@ class CancelledError(Exception): ... def run(x: object) -> object: ... def get_running_loop() -> Any: ... def create_task(x: object) -> Any: ... + +[case testAsyncInheritance1] +from typing import final, Coroutine, Any, TypeVar + +import asyncio + +class Base1: + async def foo(self) -> int: + return 1 + +class Derived1(Base1): + async def foo(self) -> int: + return await super().foo() + 1 + +async def base1_foo(b: Base1) -> int: + return await b.foo() + +async def derived1_foo(b: Derived1) -> int: + return await b.foo() + +def test_async_inheritance() -> None: + assert asyncio.run(base1_foo(Base1())) == 1 + assert asyncio.run(base1_foo(Derived1())) == 2 + assert asyncio.run(derived1_foo(Derived1())) == 2 + +@final +class FinalClass: + async def foo(self) -> int: + return 3 + +async def final_class_foo(b: FinalClass) -> int: + return await b.foo() + +def test_final_class() -> None: + assert asyncio.run(final_class_foo(FinalClass())) == 3 + +class Base2: + async def foo(self) -> int: + return 4 + + async def bar(self) -> int: + return 5 + +class Derived2(Base2): + # Does not override "foo" + async def bar(self) -> int: + return 6 + +async def base2_foo(b: Base2) -> int: + return await b.foo() + +def test_no_override() -> None: + assert asyncio.run(base2_foo(Base2())) == 4 + assert asyncio.run(base2_foo(Derived2())) == 4 + +class Base3: + async def foo(self) -> int: + return 7 + +class Derived3(Base3): + def foo(self) -> Coroutine[Any, Any, int]: + async def inner() -> int: + return 8 + return inner() + +async def base3_foo(b: Base3) -> int: + return await b.foo() + +def test_override_non_async() -> None: + assert asyncio.run(base3_foo(Base3())) == 7 + assert asyncio.run(base3_foo(Derived3())) == 8 + +[file asyncio/__init__.pyi] +def run(x: object) -> object: ... diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index bfbd5b83696b..c8e83173474d 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -907,3 +907,32 @@ def test_same_names() -> None: # matches the variable name in the input code, since internally it's generated # with a prefix. list(undefined()) + +[case testGeneratorInheritance] +from typing import Iterator + +class Base1: + def foo(self) -> Iterator[int]: + yield 1 + +class Derived1(Base1): + def foo(self) -> Iterator[int]: + yield 2 + yield 3 + +def base1_foo(b: Base1) -> list[int]: + a = [] + for x in b.foo(): + a.append(x) + return a + +def derived1_foo(b: Derived1) -> list[int]: + a = [] + for x in b.foo(): + a.append(x) + return a + +def test_generator_override() -> None: + assert base1_foo(Base1()) == [1] + assert base1_foo(Derived1()) == [2, 3] + assert derived1_foo(Derived1()) == [2, 3] From 8e57622c45b153c84f773bccae1db5f337f5a68c Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 14 Oct 2025 08:12:06 -0400 Subject: [PATCH 097/183] [mypyc] feat: new primitive for `int.bit_length` (#19673) This PR adds a new primitive for `int.bit_length`. --- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/int_ops.c | 64 +++++++++++++++++++++++++++++++ mypyc/primitives/int_ops.py | 18 ++++++++- mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/irbuild-int.test | 10 +++++ mypyc/test-data/run-integers.test | 14 +++++++ 6 files changed, 107 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index e9dfd8de3683..e2fe129786d3 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -148,6 +148,7 @@ CPyTagged CPyTagged_Remainder_(CPyTagged left, CPyTagged right); CPyTagged CPyTagged_BitwiseLongOp_(CPyTagged a, CPyTagged b, char op); CPyTagged CPyTagged_Rshift_(CPyTagged left, CPyTagged right); CPyTagged CPyTagged_Lshift_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_BitLength(CPyTagged self); PyObject *CPyTagged_Str(CPyTagged n); CPyTagged CPyTagged_FromFloat(double f); diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index e2c302eea576..333783ae619d 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -5,6 +5,10 @@ #include #include "CPy.h" +#ifdef _MSC_VER +#include +#endif + #ifndef _WIN32 // On 64-bit Linux and macOS, ssize_t and long are both 64 bits, and // PyLong_FromLong is faster than PyLong_FromSsize_t, so use the faster one @@ -15,6 +19,17 @@ #define CPyLong_FromSsize_t PyLong_FromSsize_t #endif +#if defined(__GNUC__) || defined(__clang__) +# if defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__) || (defined(__SIZEOF_POINTER__) && __SIZEOF_POINTER__ == 8) +# define CPY_CLZ(x) __builtin_clzll((unsigned long long)(x)) +# define CPY_BITS 64 +# else +# define CPY_CLZ(x) __builtin_clz((unsigned int)(x)) +# define CPY_BITS 32 +# endif +#endif + + CPyTagged CPyTagged_FromSsize_t(Py_ssize_t value) { // We use a Python object if the value shifted left by 1 is too // large for Py_ssize_t @@ -581,3 +596,52 @@ double CPyTagged_TrueDivide(CPyTagged x, CPyTagged y) { } return 1.0; } + +// int.bit_length() +CPyTagged CPyTagged_BitLength(CPyTagged self) { + // Handle zero + if (self == 0) { + return 0; + } + + // Fast path for small (tagged) ints + if (CPyTagged_CheckShort(self)) { + Py_ssize_t val = CPyTagged_ShortAsSsize_t(self); + Py_ssize_t absval = val < 0 ? -val : val; + int bits = 0; + if (absval) { +#if defined(_MSC_VER) + #if defined(_WIN64) + unsigned long idx; + if (_BitScanReverse64(&idx, (unsigned __int64)absval)) { + bits = (int)(idx + 1); + } + #else + unsigned long idx; + if (_BitScanReverse(&idx, (unsigned long)absval)) { + bits = (int)(idx + 1); + } + #endif +#elif defined(__GNUC__) || defined(__clang__) + bits = (int)(CPY_BITS - CPY_CLZ(absval)); +#else + // Fallback to loop if no builtin + while (absval) { + absval >>= 1; + bits++; + } +#endif + } + return bits << 1; + } + + // Slow path for big ints + PyObject *pyint = CPyTagged_AsObject(self); + int bits = _PyLong_NumBits(pyint); + Py_DECREF(pyint); + if (bits < 0) { + // _PyLong_NumBits sets an error on failure + return CPY_INT_TAG; + } + return bits << 1; +} diff --git a/mypyc/primitives/int_ops.py b/mypyc/primitives/int_ops.py index d723c9b63a86..8f43140dd255 100644 --- a/mypyc/primitives/int_ops.py +++ b/mypyc/primitives/int_ops.py @@ -31,7 +31,14 @@ str_rprimitive, void_rtype, ) -from mypyc.primitives.registry import binary_op, custom_op, function_op, load_address_op, unary_op +from mypyc.primitives.registry import ( + binary_op, + custom_op, + function_op, + load_address_op, + method_op, + unary_op, +) # Constructors for builtins.int and native int types have the same behavior. In # interpreted mode, native int types are just aliases to 'int'. @@ -305,3 +312,12 @@ def int_unary_op(name: str, c_function_name: str) -> PrimitiveDescription: c_function_name="PyLong_Check", error_kind=ERR_NEVER, ) + +# int.bit_length() +method_op( + name="bit_length", + arg_types=[int_rprimitive], + return_type=int_rprimitive, + c_function_name="CPyTagged_BitLength", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 4d0aaba12cab..5033100223a3 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -86,6 +86,7 @@ def __lt__(self, n: int) -> bool: pass def __gt__(self, n: int) -> bool: pass def __le__(self, n: int) -> bool: pass def __ge__(self, n: int) -> bool: pass + def bit_length(self) -> int: pass class str: @overload diff --git a/mypyc/test-data/irbuild-int.test b/mypyc/test-data/irbuild-int.test index bdf9127b722a..184c66fafb7c 100644 --- a/mypyc/test-data/irbuild-int.test +++ b/mypyc/test-data/irbuild-int.test @@ -210,3 +210,13 @@ L0: r0 = CPyTagged_Invert(n) x = r0 return x + +[case testIntBitLength] +def f(x: int) -> int: + return x.bit_length() +[out] +def f(x): + x, r0 :: int +L0: + r0 = CPyTagged_BitLength(x) + return r0 diff --git a/mypyc/test-data/run-integers.test b/mypyc/test-data/run-integers.test index 1163c9d942f7..c02f7d808883 100644 --- a/mypyc/test-data/run-integers.test +++ b/mypyc/test-data/run-integers.test @@ -572,3 +572,17 @@ class subc(int): [file userdefinedint.py] class int: pass + +[case testBitLength] +def bit_length(n: int) -> int: + return n.bit_length() +def bit_length_python(n: int) -> int: + return getattr(n, "bit_length")() +def test_bit_length() -> None: + for n in range(256): + i = 1 << n + assert bit_length(i) == bit_length_python(i) + assert bit_length(-(i)) == bit_length_python(-(i)) + i -= 1 + assert bit_length(i) == bit_length_python(i) + assert bit_length(-(i)) == bit_length_python(-(i)) From d69419cc452fa9a628e8ab0f6d5d7abd2b5aa015 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 14 Oct 2025 08:44:14 -0400 Subject: [PATCH 098/183] [mypyc] feat: extend `get_expr_length` to work with RTuple [2/4] (#19929) This PR extends `get_expr_length` to work with type information from RTuple types. --- mypyc/irbuild/for_helpers.py | 13 ++- mypyc/test-data/irbuild-tuple.test | 130 +++++++++++++---------------- 2 files changed, 69 insertions(+), 74 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 20440d4a26f4..715f5432cd13 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -1203,18 +1203,18 @@ def gen_cleanup(self) -> None: gen.gen_cleanup() -def get_expr_length(expr: Expression) -> int | None: +def get_expr_length(builder: IRBuilder, expr: Expression) -> int | None: if isinstance(expr, (StrExpr, BytesExpr)): return len(expr.value) elif isinstance(expr, (ListExpr, TupleExpr)): # if there are no star expressions, or we know the length of them, # we know the length of the expression - stars = [get_expr_length(i) for i in expr.items if isinstance(i, StarExpr)] + stars = [get_expr_length(builder, i) for i in expr.items if isinstance(i, StarExpr)] if None not in stars: other = sum(not isinstance(i, StarExpr) for i in expr.items) return other + sum(stars) # type: ignore [arg-type] elif isinstance(expr, StarExpr): - return get_expr_length(expr.expr) + return get_expr_length(builder, expr.expr) elif ( isinstance(expr, RefExpr) and isinstance(expr.node, Var) @@ -1227,6 +1227,11 @@ def get_expr_length(expr: Expression) -> int | None: # performance boost and can be (sometimes) figured out pretty easily. set and dict # comps *can* be done as well but will need special logic to consider the possibility # of key conflicts. Range, enumerate, zip are all simple logic. + + # we might still be able to get the length directly from the type + rtype = builder.node_type(expr) + if isinstance(rtype, RTuple): + return len(rtype.types) return None @@ -1235,7 +1240,7 @@ def get_expr_length_value( ) -> Value: rtype = builder.node_type(expr) assert is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple), rtype - length = get_expr_length(expr) + length = get_expr_length(builder, expr) if length is None: # We cannot compute the length at compile time, so we will fetch it. return builder.builder.builtin_len(expr_reg, line, use_pyssize_t=use_pyssize_t) diff --git a/mypyc/test-data/irbuild-tuple.test b/mypyc/test-data/irbuild-tuple.test index 3613c5f0101d..0fdd8e87a154 100644 --- a/mypyc/test-data/irbuild-tuple.test +++ b/mypyc/test-data/irbuild-tuple.test @@ -694,51 +694,46 @@ L0: return r1 def test(): r0, source :: tuple[int, int, int] - r1 :: object - r2 :: native_int - r3 :: bit - r4, r5, r6 :: int - r7, r8, r9 :: object - r10, r11 :: tuple - r12 :: native_int - r13 :: bit + r1, r2, r3 :: int + r4, r5, r6 :: object + r7, r8 :: tuple + r9 :: native_int + r10 :: bit + r11 :: object + r12, x :: int + r13 :: bool r14 :: object - r15, x :: int - r16 :: bool - r17 :: object - r18 :: native_int + r15 :: native_int a :: tuple L0: r0 = (2, 4, 6) source = r0 - r1 = box(tuple[int, int, int], source) - r2 = PyObject_Size(r1) - r3 = r2 >= 0 :: signed - r4 = source[0] - r5 = source[1] - r6 = source[2] - r7 = box(int, r4) - r8 = box(int, r5) - r9 = box(int, r6) - r10 = PyTuple_Pack(3, r7, r8, r9) - r11 = PyTuple_New(r2) - r12 = 0 + r1 = source[0] + r2 = source[1] + r3 = source[2] + r4 = box(int, r1) + r5 = box(int, r2) + r6 = box(int, r3) + r7 = PyTuple_Pack(3, r4, r5, r6) + r8 = PyTuple_New(3) + r9 = 0 + goto L2 L1: - r13 = r12 < r2 :: signed - if r13 goto L2 else goto L4 :: bool + r10 = r9 < 3 :: signed + if r10 goto L2 else goto L4 :: bool L2: - r14 = CPySequenceTuple_GetItemUnsafe(r10, r12) - r15 = unbox(int, r14) - x = r15 - r16 = f(x) - r17 = box(bool, r16) - CPySequenceTuple_SetItemUnsafe(r11, r12, r17) + r11 = CPySequenceTuple_GetItemUnsafe(r7, r9) + r12 = unbox(int, r11) + x = r12 + r13 = f(x) + r14 = box(bool, r13) + CPySequenceTuple_SetItemUnsafe(r8, r9, r14) L3: - r18 = r12 + 1 - r12 = r18 + r15 = r9 + 1 + r9 = r15 goto L1 L4: - a = r11 + a = r8 return 1 [case testTupleBuiltFromFinalFixedLengthTuple] @@ -762,19 +757,16 @@ L0: def test(): r0 :: tuple[int, int, int] r1 :: bool - r2 :: object - r3 :: native_int - r4 :: bit - r5, r6, r7 :: int - r8, r9, r10 :: object - r11, r12 :: tuple - r13 :: native_int - r14 :: bit + r2, r3, r4 :: int + r5, r6, r7 :: object + r8, r9 :: tuple + r10 :: native_int + r11 :: bit + r12 :: object + r13, x :: int + r14 :: bool r15 :: object - r16, x :: int - r17 :: bool - r18 :: object - r19 :: native_int + r16 :: native_int a :: tuple L0: r0 = __main__.source :: static @@ -783,34 +775,32 @@ L1: r1 = raise NameError('value for final name "source" was not set') unreachable L2: - r2 = box(tuple[int, int, int], r0) - r3 = PyObject_Size(r2) - r4 = r3 >= 0 :: signed - r5 = r0[0] - r6 = r0[1] - r7 = r0[2] - r8 = box(int, r5) - r9 = box(int, r6) - r10 = box(int, r7) - r11 = PyTuple_Pack(3, r8, r9, r10) - r12 = PyTuple_New(r3) - r13 = 0 + r2 = r0[0] + r3 = r0[1] + r4 = r0[2] + r5 = box(int, r2) + r6 = box(int, r3) + r7 = box(int, r4) + r8 = PyTuple_Pack(3, r5, r6, r7) + r9 = PyTuple_New(3) + r10 = 0 + goto L4 L3: - r14 = r13 < r3 :: signed - if r14 goto L4 else goto L6 :: bool + r11 = r10 < 3 :: signed + if r11 goto L4 else goto L6 :: bool L4: - r15 = CPySequenceTuple_GetItemUnsafe(r11, r13) - r16 = unbox(int, r15) - x = r16 - r17 = f(x) - r18 = box(bool, r17) - CPySequenceTuple_SetItemUnsafe(r12, r13, r18) + r12 = CPySequenceTuple_GetItemUnsafe(r8, r10) + r13 = unbox(int, r12) + x = r13 + r14 = f(x) + r15 = box(bool, r14) + CPySequenceTuple_SetItemUnsafe(r9, r10, r15) L5: - r19 = r13 + 1 - r13 = r19 + r16 = r10 + 1 + r10 = r16 goto L3 L6: - a = r12 + a = r9 return 1 [case testTupleBuiltFromVariableLengthTuple] From b8f57fda22b6fc5f57664e212db5578eaf4d3c84 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 14 Oct 2025 09:20:47 -0400 Subject: [PATCH 099/183] [mypyc] feat: further optimize equality check with string literals [1/1] (#19883) This PR further optimizes string equality checks against literals by getting rid of the PyUnicode_GET_LENGTH call against the literal value, which is not necessary since the value is known at compile-time I think this optimization will be helpful in cases where the non-literal string DOES match but is actually a subtype of string (actual strings instances that match would be caught by the identity check), or in cases where an exact string does NOT match. --- mypyc/irbuild/ll_builder.py | 30 +++++++++++++++++++++-- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/str_ops.c | 29 ++++++++++++++++------ mypyc/primitives/str_ops.py | 8 ++++++ mypyc/test-data/irbuild-classes.test | 8 +++--- mypyc/test-data/irbuild-dict.test | 2 +- mypyc/test-data/irbuild-str.test | 31 ++++++++++++++++++++++++ mypyc/test-data/irbuild-unreachable.test | 4 +-- 8 files changed, 96 insertions(+), 17 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 37f2add4abbd..d33497d4987b 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -8,7 +8,8 @@ import sys from collections.abc import Sequence -from typing import Callable, Final, Optional +from typing import Callable, Final, Optional, cast +from typing_extensions import TypeGuard from mypy.argmap import map_actuals_to_formals from mypy.nodes import ARG_POS, ARG_STAR, ARG_STAR2, ArgKind @@ -185,6 +186,7 @@ from mypyc.primitives.str_ops import ( str_check_if_true, str_eq, + str_eq_literal, str_ssize_t_size_op, unicode_compare, ) @@ -1551,9 +1553,33 @@ def check_tagged_short_int(self, val: Value, line: int, negated: bool = False) - def compare_strings(self, lhs: Value, rhs: Value, op: str, line: int) -> Value: """Compare two strings""" if op == "==": + # We can specialize this case if one or both values are string literals + literal_fastpath = False + + def is_string_literal(value: Value) -> TypeGuard[LoadLiteral]: + return isinstance(value, LoadLiteral) and is_str_rprimitive(value.type) + + if is_string_literal(lhs): + if is_string_literal(rhs): + # we can optimize out the check entirely in some constant-folded cases + return self.true() if lhs.value == rhs.value else self.false() + + # if lhs argument is string literal, switch sides to match specializer C api + lhs, rhs = rhs, lhs + literal_fastpath = True + elif is_string_literal(rhs): + literal_fastpath = True + + if literal_fastpath: + literal_string = cast(str, cast(LoadLiteral, rhs).value) + literal_length = Integer(len(literal_string), c_pyssize_t_rprimitive, line) + return self.primitive_op(str_eq_literal, [lhs, rhs, literal_length], line) + return self.primitive_op(str_eq, [lhs, rhs], line) + elif op == "!=": - eq = self.primitive_op(str_eq, [lhs, rhs], line) + # perform a standard equality check, then negate + eq = self.compare_strings(lhs, rhs, "==", line) return self.add(ComparisonOp(eq, self.false(), ComparisonOp.EQ, line)) # TODO: modify 'str' to use same interface as 'compare_bytes' as it would avoid diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index e2fe129786d3..c79923f69e69 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -735,6 +735,7 @@ static inline char CPyDict_CheckSize(PyObject *dict, Py_ssize_t size) { #define BOTHSTRIP 2 char CPyStr_Equal(PyObject *str1, PyObject *str2); +char CPyStr_EqualLiteral(PyObject *str, PyObject *literal_str, Py_ssize_t literal_length); PyObject *CPyStr_Build(Py_ssize_t len, ...); PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index); PyObject *CPyStr_GetItemUnsafe(PyObject *str, Py_ssize_t index); diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 337ef14fc955..f16e99bb4159 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -64,20 +64,33 @@ make_bloom_mask(int kind, const void* ptr, Py_ssize_t len) #undef BLOOM_UPDATE } -// Adapted from CPython 3.13.1 (_PyUnicode_Equal) -char CPyStr_Equal(PyObject *str1, PyObject *str2) { - if (str1 == str2) { - return 1; - } - Py_ssize_t len = PyUnicode_GET_LENGTH(str1); - if (PyUnicode_GET_LENGTH(str2) != len) +static inline char _CPyStr_Equal_NoIdentCheck(PyObject *str1, PyObject *str2, Py_ssize_t str2_length) { + // This helper function only exists to deduplicate code in CPyStr_Equal and CPyStr_EqualLiteral + Py_ssize_t str1_length = PyUnicode_GET_LENGTH(str1); + if (str1_length != str2_length) return 0; int kind = PyUnicode_KIND(str1); if (PyUnicode_KIND(str2) != kind) return 0; const void *data1 = PyUnicode_DATA(str1); const void *data2 = PyUnicode_DATA(str2); - return memcmp(data1, data2, len * kind) == 0; + return memcmp(data1, data2, str1_length * kind) == 0; +} + +// Adapted from CPython 3.13.1 (_PyUnicode_Equal) +char CPyStr_Equal(PyObject *str1, PyObject *str2) { + if (str1 == str2) { + return 1; + } + Py_ssize_t str2_length = PyUnicode_GET_LENGTH(str2); + return _CPyStr_Equal_NoIdentCheck(str1, str2, str2_length); +} + +char CPyStr_EqualLiteral(PyObject *str, PyObject *literal_str, Py_ssize_t literal_length) { + if (str == literal_str) { + return 1; + } + return _CPyStr_Equal_NoIdentCheck(str, literal_str, literal_length); } PyObject *CPyStr_GetItem(PyObject *str, CPyTagged index) { diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py index a8f4e4df74c2..d39f1f872763 100644 --- a/mypyc/primitives/str_ops.py +++ b/mypyc/primitives/str_ops.py @@ -88,6 +88,14 @@ error_kind=ERR_NEVER, ) +str_eq_literal = custom_primitive_op( + name="str_eq_literal", + c_function_name="CPyStr_EqualLiteral", + arg_types=[str_rprimitive, str_rprimitive, c_pyssize_t_rprimitive], + return_type=bool_rprimitive, + error_kind=ERR_NEVER, +) + unicode_compare = custom_op( arg_types=[str_rprimitive, str_rprimitive], return_type=c_int_rprimitive, diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index c4410b3b19d2..3280b21cf7e6 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2325,7 +2325,7 @@ def SetAttr.__setattr__(self, key, val): r12 :: bit L0: r0 = 'regular_attr' - r1 = CPyStr_Equal(key, r0) + r1 = CPyStr_EqualLiteral(key, r0, 12) if r1 goto L1 else goto L2 :: bool L1: r2 = unbox(int, val) @@ -2333,7 +2333,7 @@ L1: goto L6 L2: r4 = 'class_var' - r5 = CPyStr_Equal(key, r4) + r5 = CPyStr_EqualLiteral(key, r4, 9) if r5 goto L3 else goto L4 :: bool L3: r6 = builtins :: module @@ -2468,7 +2468,7 @@ def SetAttr.__setattr__(self, key, val): r12 :: bit L0: r0 = 'regular_attr' - r1 = CPyStr_Equal(key, r0) + r1 = CPyStr_EqualLiteral(key, r0, 12) if r1 goto L1 else goto L2 :: bool L1: r2 = unbox(int, val) @@ -2476,7 +2476,7 @@ L1: goto L6 L2: r4 = 'class_var' - r5 = CPyStr_Equal(key, r4) + r5 = CPyStr_EqualLiteral(key, r4, 9) if r5 goto L3 else goto L4 :: bool L3: r6 = builtins :: module diff --git a/mypyc/test-data/irbuild-dict.test b/mypyc/test-data/irbuild-dict.test index e0c014f07813..e7a330951ab0 100644 --- a/mypyc/test-data/irbuild-dict.test +++ b/mypyc/test-data/irbuild-dict.test @@ -410,7 +410,7 @@ L2: k = r8 v = r7 r9 = 'name' - r10 = CPyStr_Equal(k, r9) + r10 = CPyStr_EqualLiteral(k, r9, 4) if r10 goto L3 else goto L4 :: bool L3: name = v diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index 3fa39819498d..056f120c7bac 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -740,3 +740,34 @@ L2: L3: keep_alive x return r2 + +[case testStrEqLiteral] +from typing import Final +literal: Final = "literal" +def literal_rhs(x: str) -> bool: + return x == literal +def literal_lhs(x: str) -> bool: + return literal == x +def literal_both() -> bool: + return literal == "literal" +[out] +def literal_rhs(x): + x, r0 :: str + r1 :: bool +L0: + r0 = 'literal' + r1 = CPyStr_EqualLiteral(x, r0, 7) + return r1 +def literal_lhs(x): + x, r0 :: str + r1 :: bool +L0: + r0 = 'literal' + r1 = CPyStr_EqualLiteral(x, r0, 7) + return r1 +def literal_both(): + r0, r1 :: str +L0: + r0 = 'literal' + r1 = 'literal' + return 1 diff --git a/mypyc/test-data/irbuild-unreachable.test b/mypyc/test-data/irbuild-unreachable.test index a4f1ef8c7dba..8eafede66b56 100644 --- a/mypyc/test-data/irbuild-unreachable.test +++ b/mypyc/test-data/irbuild-unreachable.test @@ -20,7 +20,7 @@ L0: r2 = CPyObject_GetAttr(r0, r1) r3 = cast(str, r2) r4 = 'x' - r5 = CPyStr_Equal(r3, r4) + r5 = CPyStr_EqualLiteral(r3, r4, 1) if r5 goto L2 else goto L1 :: bool L1: r6 = r5 @@ -54,7 +54,7 @@ L0: r2 = CPyObject_GetAttr(r0, r1) r3 = cast(str, r2) r4 = 'x' - r5 = CPyStr_Equal(r3, r4) + r5 = CPyStr_EqualLiteral(r3, r4, 1) if r5 goto L2 else goto L1 :: bool L1: r6 = r5 From 843d133c12e34c781fb469fe50e02a7cacaae9ae Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Tue, 14 Oct 2025 16:50:21 +0200 Subject: [PATCH 100/183] More precise return types for `TypedDict.get` (#19897) Fixes #19896, #19902 - `TypedDict.get` now ignores the type of the default when the key is required. - `reveal_type(d.get)` now gives an appropriate list of overloads - I kept the special casing for `get(subdict, {})`, but this is not visible in the overloads. Implementing this via overloads is blocked by #19895 Some additional changes: - I added some code that ensures that the default type always appears last in the union (relevant when a union of multiple keys is given) - I ensure that the original value-type is use instead of its `proper_type`. This simplifies the return in `testRecursiveTypedDictMethods`. --------- Co-authored-by: Ivan Levkivskyi --- mypy/checkexpr.py | 2 +- mypy/plugins/default.py | 34 ++- test-data/unit/check-incremental.test | 201 ++++++++++++++ test-data/unit/check-literal.test | 19 +- test-data/unit/check-recursive-types.test | 3 +- test-data/unit/check-typeddict.test | 272 +++++++++++++++++-- test-data/unit/fixtures/typing-typeddict.pyi | 4 +- test-data/unit/pythoneval.test | 55 ++-- 8 files changed, 533 insertions(+), 57 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index b8f9bf087467..3eb54579a050 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1502,7 +1502,7 @@ def check_call_expr_with_callee_type( def check_union_call_expr(self, e: CallExpr, object_type: UnionType, member: str) -> Type: """Type check calling a member expression where the base type is a union.""" res: list[Type] = [] - for typ in object_type.relevant_items(): + for typ in flatten_nested_unions(object_type.relevant_items()): # Member access errors are already reported when visiting the member expression. with self.msg.filter_errors(): item = analyze_member_access( diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index e492b8dd7335..7a58307fc559 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -5,7 +5,7 @@ import mypy.errorcodes as codes from mypy import message_registry -from mypy.nodes import DictExpr, IntExpr, StrExpr, UnaryExpr +from mypy.nodes import DictExpr, Expression, IntExpr, StrExpr, UnaryExpr from mypy.plugin import ( AttributeContext, ClassDefContext, @@ -263,30 +263,40 @@ def typed_dict_get_callback(ctx: MethodContext) -> Type: if keys is None: return ctx.default_return_type + default_type: Type + default_arg: Expression | None + if len(ctx.arg_types) <= 1 or not ctx.arg_types[1]: + default_arg = None + default_type = NoneType() + elif len(ctx.arg_types[1]) == 1 and len(ctx.args[1]) == 1: + default_arg = ctx.args[1][0] + default_type = ctx.arg_types[1][0] + else: + return ctx.default_return_type + output_types: list[Type] = [] for key in keys: - value_type = get_proper_type(ctx.type.items.get(key)) + value_type: Type | None = ctx.type.items.get(key) if value_type is None: return ctx.default_return_type - if len(ctx.arg_types) == 1: + if key in ctx.type.required_keys: output_types.append(value_type) - elif len(ctx.arg_types) == 2 and len(ctx.arg_types[1]) == 1 and len(ctx.args[1]) == 1: - default_arg = ctx.args[1][0] + else: + # HACK to deal with get(key, {}) if ( isinstance(default_arg, DictExpr) and len(default_arg.items) == 0 - and isinstance(value_type, TypedDictType) + and isinstance(vt := get_proper_type(value_type), TypedDictType) ): - # Special case '{}' as the default for a typed dict type. - output_types.append(value_type.copy_modified(required_keys=set())) + output_types.append(vt.copy_modified(required_keys=set())) else: output_types.append(value_type) - output_types.append(ctx.arg_types[1][0]) - - if len(ctx.arg_types) == 1: - output_types.append(NoneType()) + output_types.append(default_type) + # for nicer reveal_type, put default at the end, if it is present + if default_type in output_types: + output_types = [t for t in output_types if t != default_type] + [default_type] return make_simplified_union(output_types) return ctx.default_return_type diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index e91b8778e986..94f65a950062 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -7311,3 +7311,204 @@ x = 2 [out] [rechecked bar] [stale] + + +[case testIncrementalTypedDictGetMethodTotalFalse] +import impl +[file lib.py] +from typing import TypedDict +class Unrelated: pass +D = TypedDict('D', {'x': int, 'y': str}, total=False) +[file impl.py] +pass +[file impl.py.2] +from typing import Literal +from lib import D, Unrelated +d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression +reveal_type(d.get('x')) +reveal_type(d.get('y')) +reveal_type(d.get('z')) +reveal_type(d.get('x', u)) +reveal_type(d.get('x', 1)) +reveal_type(d.get('y', None)) + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) +reveal_type(d.get(y)) +reveal_type(d.get(z)) +reveal_type(d.get(x_or_y)) +reveal_type(d.get(x_or_z)) +reveal_type(d.get(x_or_y_or_z)) + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) +reveal_type(d.get(y, u)) +reveal_type(d.get(z, u)) +reveal_type(d.get(x_or_y, u)) +reveal_type(d.get(x_or_z, u)) +reveal_type(d.get(x_or_y_or_z, u)) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +[out2] +tmp/impl.py:13: note: Revealed type is "Union[builtins.int, None]" +tmp/impl.py:14: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:15: note: Revealed type is "builtins.object" +tmp/impl.py:16: note: Revealed type is "Union[builtins.int, lib.Unrelated]" +tmp/impl.py:17: note: Revealed type is "builtins.int" +tmp/impl.py:18: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:21: note: Revealed type is "Union[builtins.int, None]" +tmp/impl.py:22: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:23: note: Revealed type is "builtins.object" +tmp/impl.py:24: note: Revealed type is "Union[builtins.int, builtins.str, None]" +tmp/impl.py:25: note: Revealed type is "builtins.object" +tmp/impl.py:26: note: Revealed type is "builtins.object" +tmp/impl.py:29: note: Revealed type is "Union[builtins.int, lib.Unrelated]" +tmp/impl.py:30: note: Revealed type is "Union[builtins.str, lib.Unrelated]" +tmp/impl.py:31: note: Revealed type is "builtins.object" +tmp/impl.py:32: note: Revealed type is "Union[builtins.int, builtins.str, lib.Unrelated]" +tmp/impl.py:33: note: Revealed type is "builtins.object" +tmp/impl.py:34: note: Revealed type is "builtins.object" + +[case testIncrementalTypedDictGetMethodTotalTrue] +import impl +[file lib.py] +from typing import TypedDict +class Unrelated: pass +D = TypedDict('D', {'x': int, 'y': str}, total=True) +[file impl.py] +pass +[file impl.py.2] +from typing import Literal +from lib import D, Unrelated +d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression +reveal_type(d.get('x')) +reveal_type(d.get('y')) +reveal_type(d.get('z')) +reveal_type(d.get('x', u)) +reveal_type(d.get('x', 1)) +reveal_type(d.get('y', None)) + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) +reveal_type(d.get(y)) +reveal_type(d.get(z)) +reveal_type(d.get(x_or_y)) +reveal_type(d.get(x_or_z)) +reveal_type(d.get(x_or_y_or_z)) + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) +reveal_type(d.get(y, u)) +reveal_type(d.get(z, u)) +reveal_type(d.get(x_or_y, u)) +reveal_type(d.get(x_or_z, u)) +reveal_type(d.get(x_or_y_or_z, u)) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +[out2] +tmp/impl.py:13: note: Revealed type is "builtins.int" +tmp/impl.py:14: note: Revealed type is "builtins.str" +tmp/impl.py:15: note: Revealed type is "builtins.object" +tmp/impl.py:16: note: Revealed type is "builtins.int" +tmp/impl.py:17: note: Revealed type is "builtins.int" +tmp/impl.py:18: note: Revealed type is "builtins.str" +tmp/impl.py:21: note: Revealed type is "builtins.int" +tmp/impl.py:22: note: Revealed type is "builtins.str" +tmp/impl.py:23: note: Revealed type is "builtins.object" +tmp/impl.py:24: note: Revealed type is "Union[builtins.int, builtins.str]" +tmp/impl.py:25: note: Revealed type is "builtins.object" +tmp/impl.py:26: note: Revealed type is "builtins.object" +tmp/impl.py:29: note: Revealed type is "builtins.int" +tmp/impl.py:30: note: Revealed type is "builtins.str" +tmp/impl.py:31: note: Revealed type is "builtins.object" +tmp/impl.py:32: note: Revealed type is "Union[builtins.int, builtins.str]" +tmp/impl.py:33: note: Revealed type is "builtins.object" +tmp/impl.py:34: note: Revealed type is "builtins.object" + + +[case testIncrementalTypedDictGetMethodTotalMixed] +import impl +[file lib.py] +from typing import TypedDict +from typing_extensions import Required, NotRequired +class Unrelated: pass +D = TypedDict('D', {'x': Required[int], 'y': NotRequired[str]}) +[file impl.py] +pass +[file impl.py.2] +from typing import Literal +from lib import D, Unrelated +d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression +reveal_type(d.get('x')) +reveal_type(d.get('y')) +reveal_type(d.get('z')) +reveal_type(d.get('x', u)) +reveal_type(d.get('x', 1)) +reveal_type(d.get('y', None)) + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) +reveal_type(d.get(y)) +reveal_type(d.get(z)) +reveal_type(d.get(x_or_y)) +reveal_type(d.get(x_or_z)) +reveal_type(d.get(x_or_y_or_z)) + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) +reveal_type(d.get(y, u)) +reveal_type(d.get(z, u)) +reveal_type(d.get(x_or_y, u)) +reveal_type(d.get(x_or_z, u)) +reveal_type(d.get(x_or_y_or_z, u)) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +[out2] +tmp/impl.py:13: note: Revealed type is "builtins.int" +tmp/impl.py:14: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:15: note: Revealed type is "builtins.object" +tmp/impl.py:16: note: Revealed type is "builtins.int" +tmp/impl.py:17: note: Revealed type is "builtins.int" +tmp/impl.py:18: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:21: note: Revealed type is "builtins.int" +tmp/impl.py:22: note: Revealed type is "Union[builtins.str, None]" +tmp/impl.py:23: note: Revealed type is "builtins.object" +tmp/impl.py:24: note: Revealed type is "Union[builtins.int, builtins.str, None]" +tmp/impl.py:25: note: Revealed type is "builtins.object" +tmp/impl.py:26: note: Revealed type is "builtins.object" +tmp/impl.py:29: note: Revealed type is "builtins.int" +tmp/impl.py:30: note: Revealed type is "Union[builtins.str, lib.Unrelated]" +tmp/impl.py:31: note: Revealed type is "builtins.object" +tmp/impl.py:32: note: Revealed type is "Union[builtins.int, builtins.str, lib.Unrelated]" +tmp/impl.py:33: note: Revealed type is "builtins.object" +tmp/impl.py:34: note: Revealed type is "builtins.object" diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index 3c9290b8dbbb..ce0ae2844bae 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -1884,7 +1884,7 @@ reveal_type(d[a_key]) # N: Revealed type is "builtins.int" reveal_type(d[b_key]) # N: Revealed type is "builtins.str" d[c_key] # E: TypedDict "Outer" has no key "c" -reveal_type(d.get(a_key, u)) # N: Revealed type is "Union[builtins.int, __main__.Unrelated]" +reveal_type(d.get(a_key, u)) # N: Revealed type is "builtins.int" reveal_type(d.get(b_key, u)) # N: Revealed type is "Union[builtins.str, __main__.Unrelated]" reveal_type(d.get(c_key, u)) # N: Revealed type is "builtins.object" @@ -1928,7 +1928,7 @@ u: Unrelated reveal_type(a[int_key_good]) # N: Revealed type is "builtins.int" reveal_type(b[int_key_good]) # N: Revealed type is "builtins.int" reveal_type(c[str_key_good]) # N: Revealed type is "builtins.int" -reveal_type(c.get(str_key_good, u)) # N: Revealed type is "Union[builtins.int, __main__.Unrelated]" +reveal_type(c.get(str_key_good, u)) # N: Revealed type is "builtins.int" reveal_type(c.get(str_key_bad, u)) # N: Revealed type is "builtins.object" a[int_key_bad] # E: Tuple index out of range @@ -1993,8 +1993,8 @@ optional_keys: Literal["d", "e"] bad_keys: Literal["a", "bad"] reveal_type(test[good_keys]) # N: Revealed type is "Union[__main__.A, __main__.B]" -reveal_type(test.get(good_keys)) # N: Revealed type is "Union[__main__.A, __main__.B, None]" -reveal_type(test.get(good_keys, 3)) # N: Revealed type is "Union[__main__.A, Literal[3]?, __main__.B]" +reveal_type(test.get(good_keys)) # N: Revealed type is "Union[__main__.A, __main__.B]" +reveal_type(test.get(good_keys, 3)) # N: Revealed type is "Union[__main__.A, __main__.B]" reveal_type(test.pop(optional_keys)) # N: Revealed type is "Union[__main__.D, __main__.E]" reveal_type(test.pop(optional_keys, 3)) # N: Revealed type is "Union[__main__.D, __main__.E, Literal[3]?]" reveal_type(test.setdefault(good_keys, AAndB())) # N: Revealed type is "Union[__main__.A, __main__.B]" @@ -2037,15 +2037,18 @@ class D2(TypedDict): d: D x: Union[D1, D2] -bad_keys: Literal['a', 'b', 'c', 'd'] good_keys: Literal['b', 'c'] +mixed_keys: Literal['a', 'b', 'c', 'd'] +bad_keys: Literal['e', 'f'] -x[bad_keys] # E: TypedDict "D1" has no key "d" \ +x[mixed_keys] # E: TypedDict "D1" has no key "d" \ # E: TypedDict "D2" has no key "a" reveal_type(x[good_keys]) # N: Revealed type is "Union[__main__.B, __main__.C]" -reveal_type(x.get(good_keys)) # N: Revealed type is "Union[__main__.B, __main__.C, None]" -reveal_type(x.get(good_keys, 3)) # N: Revealed type is "Union[__main__.B, Literal[3]?, __main__.C]" +reveal_type(x.get(good_keys)) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(x.get(good_keys, 3)) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(x.get(mixed_keys)) # N: Revealed type is "builtins.object" +reveal_type(x.get(mixed_keys, 3)) # N: Revealed type is "builtins.object" reveal_type(x.get(bad_keys)) # N: Revealed type is "builtins.object" reveal_type(x.get(bad_keys, 3)) # N: Revealed type is "builtins.object" diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 4f451aa062d6..c82111322fe1 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -690,10 +690,11 @@ class TD(TypedDict, total=False): y: TD td: TD +reveal_type(td.get("y")) # N: Revealed type is "Union[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: ...}), None]" td["y"] = {"x": 0, "y": {}} td["y"] = {"x": 0, "y": {"x": 0, "y": 42}} # E: Incompatible types (expression has type "int", TypedDict item "y" has type "TD") -reveal_type(td.get("y")) # N: Revealed type is "Union[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: ...})}), None]" +reveal_type(td.get("y")) # N: Revealed type is "Union[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: ...}), None]" s: str = td.get("y") # E: Incompatible types in assignment (expression has type "Optional[TD]", variable has type "str") td.update({"x": 0, "y": {"x": 1, "y": {}}}) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 34cae74d795b..e1a70efe9316 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -997,26 +997,149 @@ if int(): -- Other TypedDict methods -[case testTypedDictGetMethod] + +[case testTypedDictGetMethodOverloads] from typing import TypedDict -class A: pass -D = TypedDict('D', {'x': int, 'y': str}) +from typing_extensions import Required, NotRequired + +class D(TypedDict): + a: int + b: NotRequired[str] + +def test(d: D) -> None: + reveal_type(d.get) # N: Revealed type is "Overload(def (k: builtins.str) -> builtins.object, def (builtins.str, builtins.object) -> builtins.object, def [V] (builtins.str, V`4) -> builtins.object)" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictGetMethodTotalFalse] +from typing import TypedDict, Literal +class Unrelated: pass +D = TypedDict('D', {'x': int, 'y': str}, total=False) d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression reveal_type(d.get('x')) # N: Revealed type is "Union[builtins.int, None]" reveal_type(d.get('y')) # N: Revealed type is "Union[builtins.str, None]" -reveal_type(d.get('x', A())) # N: Revealed type is "Union[builtins.int, __main__.A]" +reveal_type(d.get('z')) # N: Revealed type is "builtins.object" +reveal_type(d.get('x', u)) # N: Revealed type is "Union[builtins.int, __main__.Unrelated]" reveal_type(d.get('x', 1)) # N: Revealed type is "builtins.int" reveal_type(d.get('y', None)) # N: Revealed type is "Union[builtins.str, None]" + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(d.get(y)) # N: Revealed type is "Union[builtins.str, None]" +reveal_type(d.get(z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y)) # N: Revealed type is "Union[builtins.int, builtins.str, None]" +reveal_type(d.get(x_or_z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z)) # N: Revealed type is "builtins.object" + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) # N: Revealed type is "Union[builtins.int, __main__.Unrelated]" +reveal_type(d.get(y, u)) # N: Revealed type is "Union[builtins.str, __main__.Unrelated]" +reveal_type(d.get(z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y, u)) # N: Revealed type is "Union[builtins.int, builtins.str, __main__.Unrelated]" +reveal_type(d.get(x_or_z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z, u)) # N: Revealed type is "builtins.object" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] +[case testTypedDictGetMethodTotalTrue] +from typing import TypedDict, Literal +class Unrelated: pass +D = TypedDict('D', {'x': int, 'y': str}, total=True) +d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression +reveal_type(d.get('x')) # N: Revealed type is "builtins.int" +reveal_type(d.get('y')) # N: Revealed type is "builtins.str" +reveal_type(d.get('z')) # N: Revealed type is "builtins.object" +reveal_type(d.get('x', u)) # N: Revealed type is "builtins.int" +reveal_type(d.get('x', 1)) # N: Revealed type is "builtins.int" +reveal_type(d.get('y', None)) # N: Revealed type is "builtins.str" + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) # N: Revealed type is "builtins.int" +reveal_type(d.get(y)) # N: Revealed type is "builtins.str" +reveal_type(d.get(z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y)) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(d.get(x_or_z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z)) # N: Revealed type is "builtins.object" + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) # N: Revealed type is "builtins.int" +reveal_type(d.get(y, u)) # N: Revealed type is "builtins.str" +reveal_type(d.get(z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y, u)) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(d.get(x_or_z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z, u)) # N: Revealed type is "builtins.object" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictGetMethodTotalMixed] +from typing import TypedDict, Literal +from typing_extensions import Required, NotRequired +class Unrelated: pass +D = TypedDict('D', {'x': Required[int], 'y': NotRequired[str]}) +d: D +u: Unrelated +x: Literal['x'] +y: Literal['y'] +z: Literal['z'] +x_or_y: Literal['x', 'y'] +x_or_z: Literal['x', 'z'] +x_or_y_or_z: Literal['x', 'y', 'z'] + +# test with literal expression +reveal_type(d.get('x')) # N: Revealed type is "builtins.int" +reveal_type(d.get('y')) # N: Revealed type is "Union[builtins.str, None]" +reveal_type(d.get('z')) # N: Revealed type is "builtins.object" +reveal_type(d.get('x', u)) # N: Revealed type is "builtins.int" +reveal_type(d.get('x', 1)) # N: Revealed type is "builtins.int" +reveal_type(d.get('y', None)) # N: Revealed type is "Union[builtins.str, None]" + +# test with literal type / union of literal types with implicit default +reveal_type(d.get(x)) # N: Revealed type is "builtins.int" +reveal_type(d.get(y)) # N: Revealed type is "Union[builtins.str, None]" +reveal_type(d.get(z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y)) # N: Revealed type is "Union[builtins.int, builtins.str, None]" +reveal_type(d.get(x_or_z)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z)) # N: Revealed type is "builtins.object" + +# test with literal type / union of literal types with explicit default +reveal_type(d.get(x, u)) # N: Revealed type is "builtins.int" +reveal_type(d.get(y, u)) # N: Revealed type is "Union[builtins.str, __main__.Unrelated]" +reveal_type(d.get(z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y, u)) # N: Revealed type is "Union[builtins.int, builtins.str, __main__.Unrelated]" +reveal_type(d.get(x_or_z, u)) # N: Revealed type is "builtins.object" +reveal_type(d.get(x_or_y_or_z, u)) # N: Revealed type is "builtins.object" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + [case testTypedDictGetMethodTypeContext] from typing import List, TypedDict class A: pass -D = TypedDict('D', {'x': List[int], 'y': int}) +D = TypedDict('D', {'x': List[int], 'y': int}, total=False) d: D reveal_type(d.get('x', [])) # N: Revealed type is "builtins.list[builtins.int]" -d.get('x', ['x']) # E: List item 0 has incompatible type "str"; expected "int" +reveal_type(d.get('x', ['x'])) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" a = [''] reveal_type(d.get('x', a)) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" [builtins fixtures/dict.pyi] @@ -1029,11 +1152,13 @@ d: D d.get() # E: All overload variants of "get" of "Mapping" require at least one argument \ # N: Possible overload variants: \ # N: def get(self, k: str) -> object \ - # N: def [V] get(self, k: str, default: object) -> object + # N: def get(self, str, object, /) -> object \ + # N: def [V] get(self, str, V, /) -> object d.get('x', 1, 2) # E: No overload variant of "get" of "Mapping" matches argument types "str", "int", "int" \ # N: Possible overload variants: \ # N: def get(self, k: str) -> object \ - # N: def [V] get(self, k: str, default: Union[int, V]) -> object + # N: def get(self, str, object, /) -> object \ + # N: def [V] get(self, str, Union[int, V], /) -> object x = d.get('z') reveal_type(x) # N: Revealed type is "builtins.object" s = '' @@ -1069,19 +1194,134 @@ p.get('x', 1 + 'y') # E: Unsupported operand types for + ("int" and "str") [case testTypedDictChainedGetWithEmptyDictDefault] from typing import TypedDict -C = TypedDict('C', {'a': int}) -D = TypedDict('D', {'x': C, 'y': str}) +C = TypedDict('C', {'a': int}, total=True) +D = TypedDict('D', {'x': C, 'y': str}, total=False) d: D -reveal_type(d.get('x', {})) \ - # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" -reveal_type(d.get('x', None)) \ - # N: Revealed type is "Union[TypedDict('__main__.C', {'a': builtins.int}), None]" +reveal_type(d.get('x', {})) # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" +reveal_type(d.get('x', None)) # N: Revealed type is "Union[TypedDict('__main__.C', {'a': builtins.int}), None]" reveal_type(d.get('x', {}).get('a')) # N: Revealed type is "Union[builtins.int, None]" reveal_type(d.get('x', {})['a']) # N: Revealed type is "builtins.int" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] +[case testTypedDictChainedGetWithEmptyDictDefault2] +from typing import TypedDict +C = TypedDict('C', {'a': int}, total=False) +D = TypedDict('D', {'x': C, 'y': str}, total=True) +d: D +reveal_type(d.get('x', {})) # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" +reveal_type(d.get('x', None)) # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" +reveal_type(d.get('x', {}).get('a')) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(d.get('x', {})['a']) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictChainedGetWithEmptyDictDefault3] +from typing import TypedDict +C = TypedDict('C', {'a': int}, total=True) +D = TypedDict('D', {'x': C, 'y': str}, total=True) +d: D +reveal_type(d.get('x', {})) # N: Revealed type is "TypedDict('__main__.C', {'a': builtins.int})" +reveal_type(d.get('x', None)) # N: Revealed type is "TypedDict('__main__.C', {'a': builtins.int})" +reveal_type(d.get('x', {}).get('a')) # N: Revealed type is "builtins.int" +reveal_type(d.get('x', {})['a']) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictChainedGetWithEmptyDictDefault4] +from typing import TypedDict +C = TypedDict('C', {'a': int}, total=False) +D = TypedDict('D', {'x': C, 'y': str}, total=False) +d: D +reveal_type(d.get('x', {})) # N: Revealed type is "TypedDict('__main__.C', {'a'?: builtins.int})" +reveal_type(d.get('x', None)) # N: Revealed type is "Union[TypedDict('__main__.C', {'a'?: builtins.int}), None]" +reveal_type(d.get('x', {}).get('a')) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(d.get('x', {})['a']) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictGetMethodChained] +# check that chaining with get like ``.get(key, {}).get(subkey, {})`` works. +from typing import TypedDict, Mapping +from typing_extensions import Required, NotRequired, Never + +class Total(TypedDict, total=True): # no keys optional + key_one: int + key_two: str + +class Maybe(TypedDict, total=False): # all keys are optional + key_one: int + key_two: str + +class Mixed(TypedDict): # some keys optional + key_one: Required[int] + key_two: NotRequired[str] + +class Config(TypedDict): + required_total: Required[Total] + optional_total: NotRequired[Total] + required_mixed: Required[Mixed] + optional_mixed: NotRequired[Mixed] + required_maybe: Required[Maybe] + optional_maybe: NotRequired[Maybe] + +def test_chaining(d: Config) -> None: + reveal_type( d.get("required_total", {}) ) # N: Revealed type is "TypedDict('__main__.Total', {'key_one': builtins.int, 'key_two': builtins.str})" + reveal_type( d.get("optional_total", {}) ) # N: Revealed type is "TypedDict('__main__.Total', {'key_one'?: builtins.int, 'key_two'?: builtins.str})" + reveal_type( d.get("required_maybe", {}) ) # N: Revealed type is "TypedDict('__main__.Maybe', {'key_one'?: builtins.int, 'key_two'?: builtins.str})" + reveal_type( d.get("optional_maybe", {}) ) # N: Revealed type is "TypedDict('__main__.Maybe', {'key_one'?: builtins.int, 'key_two'?: builtins.str})" + reveal_type( d.get("required_mixed", {}) ) # N: Revealed type is "TypedDict('__main__.Mixed', {'key_one': builtins.int, 'key_two'?: builtins.str})" + reveal_type( d.get("optional_mixed", {}) ) # N: Revealed type is "TypedDict('__main__.Mixed', {'key_one'?: builtins.int, 'key_two'?: builtins.str})" + + reveal_type( d.get("required_total", {}).get("key_one") ) # N: Revealed type is "builtins.int" + reveal_type( d.get("required_total", {}).get("key_two") ) # N: Revealed type is "builtins.str" + reveal_type( d.get("required_total", {}).get("bad_key") ) # N: Revealed type is "builtins.object" + reveal_type( d.get("optional_total", {}).get("key_one") ) # N: Revealed type is "Union[builtins.int, None]" + reveal_type( d.get("optional_total", {}).get("key_two") ) # N: Revealed type is "Union[builtins.str, None]" + reveal_type( d.get("optional_total", {}).get("bad_key") ) # N: Revealed type is "builtins.object" + + reveal_type( d.get("required_maybe", {}).get("key_one") ) # N: Revealed type is "Union[builtins.int, None]" + reveal_type( d.get("required_maybe", {}).get("key_two") ) # N: Revealed type is "Union[builtins.str, None]" + reveal_type( d.get("required_maybe", {}).get("bad_key") ) # N: Revealed type is "builtins.object" + reveal_type( d.get("optional_maybe", {}).get("key_one") ) # N: Revealed type is "Union[builtins.int, None]" + reveal_type( d.get("optional_maybe", {}).get("key_two") ) # N: Revealed type is "Union[builtins.str, None]" + reveal_type( d.get("optional_maybe", {}).get("bad_key") ) # N: Revealed type is "builtins.object" + + reveal_type( d.get("required_mixed", {}).get("key_one") ) # N: Revealed type is "builtins.int" + reveal_type( d.get("required_mixed", {}).get("key_two") ) # N: Revealed type is "Union[builtins.str, None]" + reveal_type( d.get("required_mixed", {}).get("bad_key") ) # N: Revealed type is "builtins.object" + reveal_type( d.get("optional_mixed", {}).get("key_one") ) # N: Revealed type is "Union[builtins.int, None]" + reveal_type( d.get("optional_mixed", {}).get("key_two") ) # N: Revealed type is "Union[builtins.str, None]" + reveal_type( d.get("optional_mixed", {}).get("bad_key") ) # N: Revealed type is "builtins.object" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictGetWithNestedUnionOfTypedDicts] +# https://github.com/python/mypy/issues/19902 +from typing import TypedDict, Union +from typing_extensions import TypeAlias, NotRequired +class A(TypedDict): + key: NotRequired[int] + +class B(TypedDict): + key: NotRequired[int] + +class C(TypedDict): + key: NotRequired[int] + +A_or_B: TypeAlias = Union[A, B] +A_or_B_or_C: TypeAlias = Union[A_or_B, C] + +def test(d: A_or_B_or_C) -> None: + reveal_type(d.get("key")) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + -- Totality (the "total" keyword argument) [case testTypedDictWithTotalTrue] @@ -1769,8 +2009,8 @@ class TDB(TypedDict): td: Union[TDA, TDB] -reveal_type(td.get('a')) # N: Revealed type is "Union[builtins.int, None]" -reveal_type(td.get('b')) # N: Revealed type is "Union[builtins.str, None, builtins.int]" +reveal_type(td.get('a')) # N: Revealed type is "builtins.int" +reveal_type(td.get('b')) # N: Revealed type is "Union[builtins.str, builtins.int]" reveal_type(td.get('c')) # N: Revealed type is "builtins.object" reveal_type(td['a']) # N: Revealed type is "builtins.int" diff --git a/test-data/unit/fixtures/typing-typeddict.pyi b/test-data/unit/fixtures/typing-typeddict.pyi index 16658c82528b..29635b651870 100644 --- a/test-data/unit/fixtures/typing-typeddict.pyi +++ b/test-data/unit/fixtures/typing-typeddict.pyi @@ -56,7 +56,9 @@ class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): @overload def get(self, k: T) -> Optional[T_co]: pass @overload - def get(self, k: T, default: Union[T_co, V]) -> Union[T_co, V]: pass + def get(self, k: T, default: T_co, /) -> Optional[T_co]: pass # type: ignore[misc] + @overload + def get(self, k: T, default: V, /) -> Union[T_co, V]: pass def values(self) -> Iterable[T_co]: pass # Approximate return type def __len__(self) -> int: ... def __contains__(self, arg: object) -> int: pass diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 93b67bfa813a..2069d082df17 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1034,24 +1034,43 @@ _program.py:17: note: Revealed type is "builtins.str" # Test that TypedDict get plugin works with typeshed stubs from typing import TypedDict class A: pass -D = TypedDict('D', {'x': int, 'y': str}) -d: D -reveal_type(d.get('x')) -reveal_type(d.get('y')) -reveal_type(d.get('z')) -d.get() -s = '' -reveal_type(d.get(s)) -[out] -_testTypedDictGet.py:6: note: Revealed type is "Union[builtins.int, None]" -_testTypedDictGet.py:7: note: Revealed type is "Union[builtins.str, None]" -_testTypedDictGet.py:8: note: Revealed type is "builtins.object" -_testTypedDictGet.py:9: error: All overload variants of "get" of "Mapping" require at least one argument -_testTypedDictGet.py:9: note: Possible overload variants: -_testTypedDictGet.py:9: note: def get(self, str, /) -> object -_testTypedDictGet.py:9: note: def get(self, str, /, default: object) -> object -_testTypedDictGet.py:9: note: def [_T] get(self, str, /, default: _T) -> object -_testTypedDictGet.py:11: note: Revealed type is "builtins.object" +D_total = TypedDict('D_total', {'x': int, 'y': str}, total=True) +D_not_total = TypedDict('D_not_total', {'x': int, 'y': str}, total=False) + +def test_total(d: D_total) -> None: + reveal_type(d.get('x')) + reveal_type(d.get('y')) + reveal_type(d.get('z')) + d.get() + s = '' + reveal_type(d.get(s)) + +def test_not_total(d: D_not_total) -> None: + reveal_type(d.get('x')) + reveal_type(d.get('y')) + reveal_type(d.get('z')) + d.get() + s = '' + reveal_type(d.get(s)) +[out] +_testTypedDictGet.py:8: note: Revealed type is "builtins.int" +_testTypedDictGet.py:9: note: Revealed type is "builtins.str" +_testTypedDictGet.py:10: note: Revealed type is "builtins.object" +_testTypedDictGet.py:11: error: All overload variants of "get" of "Mapping" require at least one argument +_testTypedDictGet.py:11: note: Possible overload variants: +_testTypedDictGet.py:11: note: def get(self, str, /) -> object +_testTypedDictGet.py:11: note: def get(self, str, /, default: object) -> object +_testTypedDictGet.py:11: note: def [_T] get(self, str, /, default: _T) -> object +_testTypedDictGet.py:13: note: Revealed type is "builtins.object" +_testTypedDictGet.py:16: note: Revealed type is "Union[builtins.int, None]" +_testTypedDictGet.py:17: note: Revealed type is "Union[builtins.str, None]" +_testTypedDictGet.py:18: note: Revealed type is "builtins.object" +_testTypedDictGet.py:19: error: All overload variants of "get" of "Mapping" require at least one argument +_testTypedDictGet.py:19: note: Possible overload variants: +_testTypedDictGet.py:19: note: def get(self, str, /) -> object +_testTypedDictGet.py:19: note: def get(self, str, /, default: object) -> object +_testTypedDictGet.py:19: note: def [_T] get(self, str, /, default: _T) -> object +_testTypedDictGet.py:21: note: Revealed type is "builtins.object" [case testTypedDictMappingMethods] from typing import TypedDict From 958e45097b5fc99a18c7f6ebfc7120f9c810ec19 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Tue, 14 Oct 2025 15:29:51 -0400 Subject: [PATCH 101/183] Fix typo in generics documentation (#20065) --- docs/source/generics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 4755c4f17ec8..bdd6e333f895 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -1165,7 +1165,7 @@ This example correctly uses a covariant type variable: See :ref:`variance-of-generics` for more about variance. -Generic protocols can also be recursive. Example (Python 3.12 synta): +Generic protocols can also be recursive. Example (Python 3.12 syntax): .. code-block:: python From 94d0d7e3e57e7ecd408c2c7324d8550e76de2f55 Mon Sep 17 00:00:00 2001 From: Joren Hammudoglu Date: Tue, 14 Oct 2025 23:55:30 +0200 Subject: [PATCH 102/183] stubtest: include function name in overload assertion messages (#20063) I just managed to cause this assertion to fail (I'll open an issue about this soon), but the traceback did not tell me *where* in the stubs this occurred. Knowing the full name of the relevant function would've made the debugging process a lot easier. --- mypy/stubtest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 4126f3959ee1..99404dbe52ab 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -968,7 +968,7 @@ def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef) -> Signature[nodes.Arg is_arg_pos_only: defaultdict[str, set[bool]] = defaultdict(set) for func in map(_resolve_funcitem_from_decorator, stub.items): - assert func is not None, "Failed to resolve decorated overload" + assert func is not None, f"Failed to resolve decorated overload of {stub.fullname!r}" args = maybe_strip_cls(stub.name, func.arguments) for index, arg in enumerate(args): if ( @@ -984,7 +984,7 @@ def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef) -> Signature[nodes.Arg all_args: dict[str, list[tuple[nodes.Argument, int]]] = {} for func in map(_resolve_funcitem_from_decorator, stub.items): - assert func is not None, "Failed to resolve decorated overload" + assert func is not None, f"Failed to resolve decorated overload of {stub.fullname!r}" args = maybe_strip_cls(stub.name, func.arguments) for index, arg in enumerate(args): # For positional-only args, we allow overloads to have different names for the same From 2c6c3959356674262d9b2c2dc43a33486e807a9c Mon Sep 17 00:00:00 2001 From: A5rocks Date: Wed, 15 Oct 2025 08:32:21 +0900 Subject: [PATCH 103/183] Make --pretty work better on multi-line issues (#20056) We can just print the first line for things where there's multiple lines being the issue. Ideally, we would print the first line, last line, and explicitly elide the lines in between. That's for a future change! (maybe) Fixes https://github.com/python/mypy/issues/18522. NOTE: this is an aesthetic thing, there's no right formatting. So I would be fine if this is closed because others think it looks worse. --- mypy/errors.py | 3 +++ test-data/unit/check-unreachable-code.test | 11 +++++++++++ test-data/unit/daemon.test | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mypy/errors.py b/mypy/errors.py index 1b092fb50e4a..69e4fb4cf065 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -1003,6 +1003,9 @@ def format_messages( marker = "^" if end_line == line and end_column > column: marker = f'^{"~" * (end_column - column - 1)}' + elif end_line != line: + # just highlight the first line instead + marker = f'^{"~" * (len(source_line_expanded) - column - 1)}' a.append(" " * (DEFAULT_SOURCE_OFFSET + column) + marker) return a diff --git a/test-data/unit/check-unreachable-code.test b/test-data/unit/check-unreachable-code.test index 645f81e89ca1..7e00671dfd11 100644 --- a/test-data/unit/check-unreachable-code.test +++ b/test-data/unit/check-unreachable-code.test @@ -1619,3 +1619,14 @@ reveal_type(bar().attr) # N: Revealed type is "Never" 1 # not unreachable reveal_type(foo().attr) # N: Revealed type is "Never" 1 # E: Statement is unreachable + +[case testUnreachableStatementPrettyHighlighting] +# flags: --warn-unreachable --pretty +def x() -> None: + assert False + if 5: + pass +[out] +main:4: error: Statement is unreachable + if 5: + ^~~~~ diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 295eb4000d81..c02f78be1834 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -122,7 +122,7 @@ Daemon stopped Daemon started foo.py:1: error: Function is missing a return type annotation def f(): - ^ + ^~~~~~~~ foo.py:1: note: Use "-> None" if function does not return a value Found 1 error in 1 file (checked 1 source file) == Return code: 1 From 2eaafe905fc00c2b6c1ea4ec74c7f84cdf09c50e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 16 Oct 2025 10:09:46 +0100 Subject: [PATCH 104/183] Add tool to convert binary cache files to JSON (#20071) Work to enable #19697 (see the issue for motivation). Copy the old `serialize` methods with some modifications and use them in the export tool. This will let us remove the old `serialize` and `deserialize` methods once we drop support for the old JSON serialization format at some point. This should be enough to support existing use cases that inspect the JSON files. Instead of directly inspecting them, users will have to run the tool first if they use the binary cache format. Example: ``` $ python -m mypy.exportjson .mypy_cache/3.13/foobar.data.ff .mypy_cache/3.13/foobar.data.ff -> .mypy_cache/3.13/foobar.data.ff.json ``` The run generates the `.mypy_cache/3.13/foobar.data.ff.json` file, which is similar to existing to cache json files. I added some tests and manually checked that the JSON file for `builtins` module is identical to the one generated by mypy. However, we won't guarantee that all new symbol table or cache features will be added to the exporter, to simplify maintenance. Also I didn't test all features of the exporter in the tests -- I just ensure that the basics work. The tool is primarily there to support existing use cases and debugging workflows, and it would be better to use `MypyFile.read(...)` in new use cases that require cache inspection. --- mypy/exportjson.py | 578 +++++++++++++++++++++++++++++++++ mypy/test/testexportjson.py | 70 ++++ setup.py | 1 + test-data/unit/exportjson.test | 280 ++++++++++++++++ 4 files changed, 929 insertions(+) create mode 100644 mypy/exportjson.py create mode 100644 mypy/test/testexportjson.py create mode 100644 test-data/unit/exportjson.test diff --git a/mypy/exportjson.py b/mypy/exportjson.py new file mode 100644 index 000000000000..09945f0ef28f --- /dev/null +++ b/mypy/exportjson.py @@ -0,0 +1,578 @@ +"""Tool to convert mypy cache file to a JSON format (print to stdout). + +Usage: + python -m mypy.exportjson .mypy_cache/.../my_module.data.ff + +The idea is to make caches introspectable once we've switched to a binary +cache format and removed support for the older JSON cache format. + +This is primarily to support existing use cases that need to inspect +cache files, and to support debugging mypy caching issues. This means that +this doesn't necessarily need to be kept 1:1 up to date with changes in the +binary cache format (to simplify maintenance -- we don't want this to slow +down mypy development). +""" + +import argparse +import json +import sys +from typing import Any, Union +from typing_extensions import TypeAlias as _TypeAlias + +from librt.internal import Buffer + +from mypy.nodes import ( + FUNCBASE_FLAGS, + FUNCDEF_FLAGS, + VAR_FLAGS, + ClassDef, + DataclassTransformSpec, + Decorator, + FuncDef, + MypyFile, + OverloadedFuncDef, + OverloadPart, + ParamSpecExpr, + SymbolNode, + SymbolTable, + SymbolTableNode, + TypeAlias, + TypeInfo, + TypeVarExpr, + TypeVarTupleExpr, + Var, + get_flags, + node_kinds, +) +from mypy.types import ( + NOT_READY, + AnyType, + CallableType, + ExtraAttrs, + Instance, + LiteralType, + NoneType, + Overloaded, + Parameters, + ParamSpecType, + TupleType, + Type, + TypeAliasType, + TypedDictType, + TypeType, + TypeVarTupleType, + TypeVarType, + UnboundType, + UninhabitedType, + UnionType, + UnpackType, + get_proper_type, +) + +Json: _TypeAlias = Union[dict[str, Any], str] + + +class Config: + def __init__(self, *, implicit_names: bool = True) -> None: + self.implicit_names = implicit_names + + +def convert_binary_cache_to_json(data: bytes, *, implicit_names: bool = True) -> Json: + tree = MypyFile.read(Buffer(data)) + return convert_mypy_file_to_json(tree, Config(implicit_names=implicit_names)) + + +def convert_mypy_file_to_json(self: MypyFile, cfg: Config) -> Json: + return { + ".class": "MypyFile", + "_fullname": self._fullname, + "names": convert_symbol_table(self.names, cfg), + "is_stub": self.is_stub, + "path": self.path, + "is_partial_stub_package": self.is_partial_stub_package, + "future_import_flags": sorted(self.future_import_flags), + } + + +def convert_symbol_table(self: SymbolTable, cfg: Config) -> Json: + data: dict[str, Any] = {".class": "SymbolTable"} + for key, value in self.items(): + # Skip __builtins__: it's a reference to the builtins + # module that gets added to every module by + # SemanticAnalyzerPass2.visit_file(), but it shouldn't be + # accessed by users of the module. + if key == "__builtins__" or value.no_serialize: + continue + if not cfg.implicit_names and key in { + "__spec__", + "__package__", + "__file__", + "__doc__", + "__annotations__", + "__name__", + }: + continue + data[key] = convert_symbol_table_node(value, cfg) + return data + + +def convert_symbol_table_node(self: SymbolTableNode, cfg: Config) -> Json: + data: dict[str, Any] = {".class": "SymbolTableNode", "kind": node_kinds[self.kind]} + if self.module_hidden: + data["module_hidden"] = True + if not self.module_public: + data["module_public"] = False + if self.implicit: + data["implicit"] = True + if self.plugin_generated: + data["plugin_generated"] = True + if self.cross_ref: + data["cross_ref"] = self.cross_ref + elif self.node is not None: + data["node"] = convert_symbol_node(self.node, cfg) + return data + + +def convert_symbol_node(self: SymbolNode, cfg: Config) -> Json: + if isinstance(self, FuncDef): + return convert_func_def(self) + elif isinstance(self, OverloadedFuncDef): + return convert_overloaded_func_def(self) + elif isinstance(self, Decorator): + return convert_decorator(self) + elif isinstance(self, Var): + return convert_var(self) + elif isinstance(self, TypeInfo): + return convert_type_info(self, cfg) + elif isinstance(self, TypeAlias): + return convert_type_alias(self) + elif isinstance(self, TypeVarExpr): + return convert_type_var_expr(self) + elif isinstance(self, ParamSpecExpr): + return convert_param_spec_expr(self) + elif isinstance(self, TypeVarTupleExpr): + return convert_type_var_tuple_expr(self) + return {"ERROR": f"{type(self)!r} unrecognized"} + + +def convert_func_def(self: FuncDef) -> Json: + return { + ".class": "FuncDef", + "name": self._name, + "fullname": self._fullname, + "arg_names": self.arg_names, + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "type": None if self.type is None else convert_type(self.type), + "flags": get_flags(self, FUNCDEF_FLAGS), + "abstract_status": self.abstract_status, + # TODO: Do we need expanded, original_def? + "dataclass_transform_spec": ( + None + if self.dataclass_transform_spec is None + else convert_dataclass_transform_spec(self.dataclass_transform_spec) + ), + "deprecated": self.deprecated, + "original_first_arg": self.original_first_arg, + } + + +def convert_dataclass_transform_spec(self: DataclassTransformSpec) -> Json: + return { + "eq_default": self.eq_default, + "order_default": self.order_default, + "kw_only_default": self.kw_only_default, + "frozen_default": self.frozen_default, + "field_specifiers": list(self.field_specifiers), + } + + +def convert_overloaded_func_def(self: OverloadedFuncDef) -> Json: + return { + ".class": "OverloadedFuncDef", + "items": [convert_overload_part(i) for i in self.items], + "type": None if self.type is None else convert_type(self.type), + "fullname": self._fullname, + "impl": None if self.impl is None else convert_overload_part(self.impl), + "flags": get_flags(self, FUNCBASE_FLAGS), + "deprecated": self.deprecated, + "setter_index": self.setter_index, + } + + +def convert_overload_part(self: OverloadPart) -> Json: + if isinstance(self, FuncDef): + return convert_func_def(self) + else: + return convert_decorator(self) + + +def convert_decorator(self: Decorator) -> Json: + return { + ".class": "Decorator", + "func": convert_func_def(self.func), + "var": convert_var(self.var), + "is_overload": self.is_overload, + } + + +def convert_var(self: Var) -> Json: + data: dict[str, Any] = { + ".class": "Var", + "name": self._name, + "fullname": self._fullname, + "type": None if self.type is None else convert_type(self.type), + "setter_type": None if self.setter_type is None else convert_type(self.setter_type), + "flags": get_flags(self, VAR_FLAGS), + } + if self.final_value is not None: + data["final_value"] = self.final_value + return data + + +def convert_type_info(self: TypeInfo, cfg: Config) -> Json: + data = { + ".class": "TypeInfo", + "module_name": self.module_name, + "fullname": self.fullname, + "names": convert_symbol_table(self.names, cfg), + "defn": convert_class_def(self.defn), + "abstract_attributes": self.abstract_attributes, + "type_vars": self.type_vars, + "has_param_spec_type": self.has_param_spec_type, + "bases": [convert_type(b) for b in self.bases], + "mro": self._mro_refs, + "_promote": [convert_type(p) for p in self._promote], + "alt_promote": None if self.alt_promote is None else convert_type(self.alt_promote), + "declared_metaclass": ( + None if self.declared_metaclass is None else convert_type(self.declared_metaclass) + ), + "metaclass_type": ( + None if self.metaclass_type is None else convert_type(self.metaclass_type) + ), + "tuple_type": None if self.tuple_type is None else convert_type(self.tuple_type), + "typeddict_type": ( + None if self.typeddict_type is None else convert_typeddict_type(self.typeddict_type) + ), + "flags": get_flags(self, TypeInfo.FLAGS), + "metadata": self.metadata, + "slots": sorted(self.slots) if self.slots is not None else None, + "deletable_attributes": self.deletable_attributes, + "self_type": convert_type(self.self_type) if self.self_type is not None else None, + "dataclass_transform_spec": ( + convert_dataclass_transform_spec(self.dataclass_transform_spec) + if self.dataclass_transform_spec is not None + else None + ), + "deprecated": self.deprecated, + } + return data + + +def convert_class_def(self: ClassDef) -> Json: + return { + ".class": "ClassDef", + "name": self.name, + "fullname": self.fullname, + "type_vars": [convert_type(v) for v in self.type_vars], + } + + +def convert_type_alias(self: TypeAlias) -> Json: + data: Json = { + ".class": "TypeAlias", + "fullname": self._fullname, + "module": self.module, + "target": convert_type(self.target), + "alias_tvars": [convert_type(v) for v in self.alias_tvars], + "no_args": self.no_args, + "normalized": self.normalized, + "python_3_12_type_alias": self.python_3_12_type_alias, + } + return data + + +def convert_type_var_expr(self: TypeVarExpr) -> Json: + return { + ".class": "TypeVarExpr", + "name": self._name, + "fullname": self._fullname, + "values": [convert_type(t) for t in self.values], + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_param_spec_expr(self: ParamSpecExpr) -> Json: + return { + ".class": "ParamSpecExpr", + "name": self._name, + "fullname": self._fullname, + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_type_var_tuple_expr(self: TypeVarTupleExpr) -> Json: + return { + ".class": "TypeVarTupleExpr", + "name": self._name, + "fullname": self._fullname, + "upper_bound": convert_type(self.upper_bound), + "tuple_fallback": convert_type(self.tuple_fallback), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_type(typ: Type) -> Json: + if type(typ) is TypeAliasType: + return convert_type_alias_type(typ) + typ = get_proper_type(typ) + if isinstance(typ, Instance): + return convert_instance(typ) + elif isinstance(typ, AnyType): + return convert_any_type(typ) + elif isinstance(typ, NoneType): + return convert_none_type(typ) + elif isinstance(typ, UnionType): + return convert_union_type(typ) + elif isinstance(typ, TupleType): + return convert_tuple_type(typ) + elif isinstance(typ, CallableType): + return convert_callable_type(typ) + elif isinstance(typ, Overloaded): + return convert_overloaded(typ) + elif isinstance(typ, LiteralType): + return convert_literal_type(typ) + elif isinstance(typ, TypeVarType): + return convert_type_var_type(typ) + elif isinstance(typ, TypeType): + return convert_type_type(typ) + elif isinstance(typ, UninhabitedType): + return convert_uninhabited_type(typ) + elif isinstance(typ, UnpackType): + return convert_unpack_type(typ) + elif isinstance(typ, ParamSpecType): + return convert_param_spec_type(typ) + elif isinstance(typ, TypeVarTupleType): + return convert_type_var_tuple_type(typ) + elif isinstance(typ, Parameters): + return convert_parameters(typ) + elif isinstance(typ, TypedDictType): + return convert_typeddict_type(typ) + elif isinstance(typ, UnboundType): + return convert_unbound_type(typ) + return {"ERROR": f"{type(typ)!r} unrecognized"} + + +def convert_instance(self: Instance) -> Json: + ready = self.type is not NOT_READY + if not self.args and not self.last_known_value and not self.extra_attrs: + if ready: + return self.type.fullname + elif self.type_ref: + return self.type_ref + + data: dict[str, Any] = { + ".class": "Instance", + "type_ref": self.type.fullname if ready else self.type_ref, + "args": [convert_type(arg) for arg in self.args], + } + if self.last_known_value is not None: + data["last_known_value"] = convert_type(self.last_known_value) + data["extra_attrs"] = convert_extra_attrs(self.extra_attrs) if self.extra_attrs else None + return data + + +def convert_extra_attrs(self: ExtraAttrs) -> Json: + return { + ".class": "ExtraAttrs", + "attrs": {k: convert_type(v) for k, v in self.attrs.items()}, + "immutable": sorted(self.immutable), + "mod_name": self.mod_name, + } + + +def convert_type_alias_type(self: TypeAliasType) -> Json: + data: Json = { + ".class": "TypeAliasType", + "type_ref": self.type_ref, + "args": [convert_type(arg) for arg in self.args], + } + return data + + +def convert_any_type(self: AnyType) -> Json: + return { + ".class": "AnyType", + "type_of_any": self.type_of_any, + "source_any": convert_type(self.source_any) if self.source_any is not None else None, + "missing_import_name": self.missing_import_name, + } + + +def convert_none_type(self: NoneType) -> Json: + return {".class": "NoneType"} + + +def convert_union_type(self: UnionType) -> Json: + return { + ".class": "UnionType", + "items": [convert_type(t) for t in self.items], + "uses_pep604_syntax": self.uses_pep604_syntax, + } + + +def convert_tuple_type(self: TupleType) -> Json: + return { + ".class": "TupleType", + "items": [convert_type(t) for t in self.items], + "partial_fallback": convert_type(self.partial_fallback), + "implicit": self.implicit, + } + + +def convert_literal_type(self: LiteralType) -> Json: + return {".class": "LiteralType", "value": self.value, "fallback": convert_type(self.fallback)} + + +def convert_type_var_type(self: TypeVarType) -> Json: + assert not self.id.is_meta_var() + return { + ".class": "TypeVarType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "values": [convert_type(v) for v in self.values], + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "variance": self.variance, + } + + +def convert_callable_type(self: CallableType) -> Json: + return { + ".class": "CallableType", + "arg_types": [convert_type(t) for t in self.arg_types], + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "arg_names": self.arg_names, + "ret_type": convert_type(self.ret_type), + "fallback": convert_type(self.fallback), + "name": self.name, + # We don't serialize the definition (only used for error messages). + "variables": [convert_type(v) for v in self.variables], + "is_ellipsis_args": self.is_ellipsis_args, + "implicit": self.implicit, + "is_bound": self.is_bound, + "type_guard": convert_type(self.type_guard) if self.type_guard is not None else None, + "type_is": convert_type(self.type_is) if self.type_is is not None else None, + "from_concatenate": self.from_concatenate, + "imprecise_arg_kinds": self.imprecise_arg_kinds, + "unpack_kwargs": self.unpack_kwargs, + } + + +def convert_overloaded(self: Overloaded) -> Json: + return {".class": "Overloaded", "items": [convert_type(t) for t in self.items]} + + +def convert_type_type(self: TypeType) -> Json: + return {".class": "TypeType", "item": convert_type(self.item)} + + +def convert_uninhabited_type(self: UninhabitedType) -> Json: + return {".class": "UninhabitedType"} + + +def convert_unpack_type(self: UnpackType) -> Json: + return {".class": "UnpackType", "type": convert_type(self.type)} + + +def convert_param_spec_type(self: ParamSpecType) -> Json: + assert not self.id.is_meta_var() + return { + ".class": "ParamSpecType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "flavor": self.flavor, + "upper_bound": convert_type(self.upper_bound), + "default": convert_type(self.default), + "prefix": convert_type(self.prefix), + } + + +def convert_type_var_tuple_type(self: TypeVarTupleType) -> Json: + assert not self.id.is_meta_var() + return { + ".class": "TypeVarTupleType", + "name": self.name, + "fullname": self.fullname, + "id": self.id.raw_id, + "namespace": self.id.namespace, + "upper_bound": convert_type(self.upper_bound), + "tuple_fallback": convert_type(self.tuple_fallback), + "default": convert_type(self.default), + "min_len": self.min_len, + } + + +def convert_parameters(self: Parameters) -> Json: + return { + ".class": "Parameters", + "arg_types": [convert_type(t) for t in self.arg_types], + "arg_kinds": [int(x.value) for x in self.arg_kinds], + "arg_names": self.arg_names, + "variables": [convert_type(tv) for tv in self.variables], + "imprecise_arg_kinds": self.imprecise_arg_kinds, + } + + +def convert_typeddict_type(self: TypedDictType) -> Json: + return { + ".class": "TypedDictType", + "items": [[n, convert_type(t)] for (n, t) in self.items.items()], + "required_keys": sorted(self.required_keys), + "readonly_keys": sorted(self.readonly_keys), + "fallback": convert_type(self.fallback), + } + + +def convert_unbound_type(self: UnboundType) -> Json: + return { + ".class": "UnboundType", + "name": self.name, + "args": [convert_type(a) for a in self.args], + "expr": self.original_str_expr, + "expr_fallback": self.original_str_fallback, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Convert binary cache files to JSON. " + "Create files in the same directory with extra .json extension." + ) + parser.add_argument( + "path", nargs="+", help="mypy cache data file to convert (.data.ff extension)" + ) + args = parser.parse_args() + fnams: list[str] = args.path + for fnam in fnams: + if not fnam.endswith(".data.ff"): + sys.exit(f"error: Expected .data.ff extension, but got {fnam}") + with open(fnam, "rb") as f: + data = f.read() + json_data = convert_binary_cache_to_json(data) + new_fnam = fnam + ".json" + with open(new_fnam, "w") as f: + json.dump(json_data, f) + print(f"{fnam} -> {new_fnam}") + + +if __name__ == "__main__": + main() diff --git a/mypy/test/testexportjson.py b/mypy/test/testexportjson.py new file mode 100644 index 000000000000..13bd96d06642 --- /dev/null +++ b/mypy/test/testexportjson.py @@ -0,0 +1,70 @@ +"""Test cases for the mypy cache JSON export tool.""" + +from __future__ import annotations + +import json +import os +import re +import sys + +from mypy import build +from mypy.errors import CompileError +from mypy.exportjson import convert_binary_cache_to_json +from mypy.modulefinder import BuildSource +from mypy.options import Options +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal + + +class TypeExportSuite(DataSuite): + required_out_section = True + files = ["exportjson.test"] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + error = False + src = "\n".join(testcase.input) + try: + options = Options() + options.use_builtins_fixtures = True + options.show_traceback = True + options.allow_empty_bodies = True + options.fixed_format_cache = True + fnam = os.path.join(self.base_path, "main.py") + with open(fnam, "w") as f: + f.write(src) + result = build.build( + sources=[BuildSource(fnam, "main")], options=options, alt_lib_path=test_temp_dir + ) + a = result.errors + error = bool(a) + + major, minor = sys.version_info[:2] + cache_dir = os.path.join(".mypy_cache", f"{major}.{minor}") + + for module in result.files: + if module in ( + "builtins", + "typing", + "_typeshed", + "__future__", + "typing_extensions", + "sys", + ): + continue + fnam = os.path.join(cache_dir, f"{module}.data.ff") + with open(fnam, "rb") as f: + json_data = convert_binary_cache_to_json(f.read(), implicit_names=False) + for line in json.dumps(json_data, indent=4).splitlines(): + if '"path": ' in line: + # We source file path is unpredictable, so filter it out + line = re.sub(r'"[^"]+\.pyi?"', "...", line) + assert "ERROR" not in line, line + a.append(line) + except CompileError as e: + a = e.messages + error = True + if error or "\n".join(testcase.output).strip() != "": + assert_string_arrays_equal( + testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})" + ) diff --git a/setup.py b/setup.py index 1d093ec3b9e2..0037624f9bbc 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ def run(self) -> None: "__main__.py", "pyinfo.py", os.path.join("dmypy", "__main__.py"), + "exportjson.py", # Uses __getattr__/__setattr__ "split_namespace.py", # Lies to mypy about code reachability diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test new file mode 100644 index 000000000000..14295281a48f --- /dev/null +++ b/test-data/unit/exportjson.test @@ -0,0 +1,280 @@ +-- Test cases for exporting mypy cache files to JSON (mypy.exportjson). +-- +-- The tool is maintained on a best effort basis so we don't attempt to have +-- full test coverage. +-- +-- Some tests only ensure that *some* JSON is generated successfully. These +-- have as [out]. + +[case testExportVar] +x = 0 +[out] +{ + ".class": "MypyFile", + "_fullname": "main", + "names": { + ".class": "SymbolTable", + "x": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "node": { + ".class": "Var", + "name": "x", + "fullname": "main.x", + "type": "builtins.int", + "setter_type": null, + "flags": [ + "is_ready", + "is_inferred", + "has_explicit_value" + ] + } + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + +[case testExportClass] +class C: + x: int +[out] +{ + ".class": "MypyFile", + "_fullname": "main", + "names": { + ".class": "SymbolTable", + "C": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "node": { + ".class": "TypeInfo", + "module_name": "main", + "fullname": "main.C", + "names": { + ".class": "SymbolTable", + "x": { + ".class": "SymbolTableNode", + "kind": "Mdef", + "node": { + ".class": "Var", + "name": "x", + "fullname": "main.C.x", + "type": "builtins.int", + "setter_type": null, + "flags": [ + "is_initialized_in_class", + "is_ready" + ] + } + } + }, + "defn": { + ".class": "ClassDef", + "name": "C", + "fullname": "main.C", + "type_vars": [] + }, + "abstract_attributes": [], + "type_vars": [], + "has_param_spec_type": false, + "bases": [ + "builtins.object" + ], + "mro": [ + "main.C", + "builtins.object" + ], + "_promote": [], + "alt_promote": null, + "declared_metaclass": null, + "metaclass_type": null, + "tuple_type": null, + "typeddict_type": null, + "flags": [], + "metadata": {}, + "slots": null, + "deletable_attributes": [], + "self_type": null, + "dataclass_transform_spec": null, + "deprecated": null + } + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + +[case testExportCrossRef] +from typing import Any +[out] +{ + ".class": "MypyFile", + "_fullname": "main", + "names": { + ".class": "SymbolTable", + "Any": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "cross_ref": "typing.Any" + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + +[case testExportFuncDef] +def foo(a: int) -> None: ... +[out] +{ + ".class": "MypyFile", + "_fullname": "main", + "names": { + ".class": "SymbolTable", + "foo": { + ".class": "SymbolTableNode", + "kind": "Gdef", + "node": { + ".class": "FuncDef", + "name": "foo", + "fullname": "main.foo", + "arg_names": [ + "a" + ], + "arg_kinds": [ + 0 + ], + "type": { + ".class": "CallableType", + "arg_types": [ + "builtins.int" + ], + "arg_kinds": [ + 0 + ], + "arg_names": [ + "a" + ], + "ret_type": { + ".class": "NoneType" + }, + "fallback": "builtins.function", + "name": "foo", + "variables": [], + "is_ellipsis_args": false, + "implicit": false, + "is_bound": false, + "type_guard": null, + "type_is": null, + "from_concatenate": false, + "imprecise_arg_kinds": false, + "unpack_kwargs": false + }, + "flags": [], + "abstract_status": 0, + "dataclass_transform_spec": null, + "deprecated": null, + "original_first_arg": "a" + } + } + }, + "is_stub": false, + "path": ..., + "is_partial_stub_package": false, + "future_import_flags": [] +} + +[case testExportDifferentTypes] +from __future__ import annotations + +from typing import Callable, Any, Literal, NoReturn, TypedDict, NamedTuple + +list_ann: list[int] +any_ann: Any +tuple_ann: tuple[int, str] +union_ann: int | None +callable_ann: Callable[[int], str] +type_type_ann: type[int] +literal_ann: Literal['x', 5, False] + +def f() -> NoReturn: + assert False + +BadType = 1 +x: BadType # type: ignore + +class TD(TypedDict): + x: int + +td = TD(x=1) + +NT = NamedTuple("NT", [("x", int)]) + +nt = NT(x=1) + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] +[out] + + +[case testExportGenericTypes] +from __future__ import annotations + +from typing import TypeVar, Callable +from typing_extensions import TypeVarTuple, ParamSpec, Unpack, Concatenate + +T = TypeVar("T") + +def ident(x: T) -> T: + return x + +Ts = TypeVarTuple("Ts") + +def ts(t: tuple[Unpack[Ts]]) -> tuple[Unpack[Ts]]: + return t + +P = ParamSpec("P") + +def pspec(f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: + f(*args, **kwargs) + +def concat(f: Callable[Concatenate[int, P], None], *args: P.args, **kwargs: P.kwargs) -> None: + f(1, *args, **kwargs) + +[builtins fixtures/tuple.pyi] +[out] + + +[case testExportDifferentNodes] +from __future__ import annotations + +import typing + +from typing import overload, TypeVar + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +def f(x: int | str) -> int | str: ... + +T = TypeVar("T") + +def deco(f: T) -> T: + return f + +@deco +def foo(x: int) -> int: ... + +X = int +x: X = 2 + +[builtins fixtures/tuple.pyi] +[out] + From fd20f348813e471d4630253d3023a5b579c47017 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 16 Oct 2025 12:54:50 +0100 Subject: [PATCH 105/183] Support writing/reading bytes in librt.internal (#20069) This is for storing actual `bytes` objects (not arbitrary sub-buffers). This will be useful for storing various hashes in cache metas, since storing them as hex strings takes twice more space and currently these hashes take a macroscopic part of the cache metas' sizes. @JukkaL for training purposes you can sync/release this one yourself (if you want to, of course). --- mypy/typeshed/stubs/librt/librt/internal.pyi | 2 + mypyc/lib-rt/librt_internal.c | 100 ++++++++++++++++++- mypyc/lib-rt/librt_internal.h | 4 + mypyc/primitives/misc_ops.py | 16 +++ mypyc/test-data/irbuild-classes.test | 61 ++++++----- mypyc/test-data/run-classes.test | 22 +++- 6 files changed, 177 insertions(+), 28 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index a47a4849fe20..8a5fc262931e 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -8,6 +8,8 @@ def write_bool(data: Buffer, value: bool) -> None: ... def read_bool(data: Buffer) -> bool: ... def write_str(data: Buffer, value: str) -> None: ... def read_str(data: Buffer) -> str: ... +def write_bytes(data: Buffer, value: bytes) -> None: ... +def read_bytes(data: Buffer) -> bytes: ... def write_float(data: Buffer, value: float) -> None: ... def read_float(data: Buffer) -> float: ... def write_int(data: Buffer, value: int) -> None: ... diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index b97d6665b515..6f6a110446ad 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -346,6 +346,100 @@ write_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames return Py_None; } +/* +bytes format: size followed by bytes + short bytes (len <= 127): single byte for size as `(uint8_t)size << 1` + long bytes: \x01 followed by size as Py_ssize_t +*/ + +static PyObject* +read_bytes_internal(PyObject *data) { + _CHECK_BUFFER(data, NULL) + + // Read length. + Py_ssize_t size; + _CHECK_READ(data, 1, NULL) + uint8_t first = _READ(data, uint8_t) + if (likely(first != LONG_STR_TAG)) { + // Common case: short bytes (len <= 127). + size = (Py_ssize_t)(first >> 1); + } else { + _CHECK_READ(data, sizeof(CPyTagged), NULL) + size = _READ(data, Py_ssize_t) + } + // Read bytes content. + char *buf = ((BufferObject *)data)->buf; + _CHECK_READ(data, size, NULL) + PyObject *res = PyBytes_FromStringAndSize( + buf + ((BufferObject *)data)->pos, (Py_ssize_t)size + ); + if (unlikely(res == NULL)) + return NULL; + ((BufferObject *)data)->pos += size; + return res; +} + +static PyObject* +read_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", 0}; + static CPyArg_Parser parser = {"O:read_bytes", kwlist, 0}; + PyObject *data; + if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { + return NULL; + } + return read_bytes_internal(data); +} + +static char +write_bytes_internal(PyObject *data, PyObject *value) { + _CHECK_BUFFER(data, CPY_NONE_ERROR) + + const char *chunk = PyBytes_AsString(value); + if (unlikely(chunk == NULL)) + return CPY_NONE_ERROR; + Py_ssize_t size = PyBytes_GET_SIZE(value); + + Py_ssize_t need; + // Write length. + if (likely(size <= MAX_SHORT_LEN)) { + // Common case: short bytes (len <= 127) store as single byte. + need = size + 1; + _CHECK_SIZE(data, need) + _WRITE(data, uint8_t, (uint8_t)size << 1) + } else { + need = size + sizeof(Py_ssize_t) + 1; + _CHECK_SIZE(data, need) + _WRITE(data, uint8_t, LONG_STR_TAG) + _WRITE(data, Py_ssize_t, size) + } + // Write bytes content. + char *buf = ((BufferObject *)data)->buf; + memcpy(buf + ((BufferObject *)data)->pos, chunk, size); + ((BufferObject *)data)->pos += size; + ((BufferObject *)data)->end += need; + return CPY_NONE; +} + +static PyObject* +write_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) { + static const char * const kwlist[] = {"data", "value", 0}; + static CPyArg_Parser parser = {"OO:write_bytes", kwlist, 0}; + PyObject *data; + PyObject *value; + if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { + return NULL; + } + if (unlikely(!PyBytes_Check(value))) { + PyErr_SetString(PyExc_TypeError, "value must be a bytes object"); + return NULL; + } + if (unlikely(write_bytes_internal(data, value) == CPY_NONE_ERROR)) { + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + /* float format: stored as a C double @@ -565,6 +659,8 @@ static PyMethodDef librt_internal_module_methods[] = { {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, {"write_str", (PyCFunction)write_str, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a string")}, {"read_str", (PyCFunction)read_str, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a string")}, + {"write_bytes", (PyCFunction)write_bytes, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write bytes")}, + {"read_bytes", (PyCFunction)read_bytes, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read bytes")}, {"write_float", (PyCFunction)write_float, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a float")}, {"read_float", (PyCFunction)read_float, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a float")}, {"write_int", (PyCFunction)write_int, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write an int")}, @@ -590,7 +686,7 @@ librt_internal_module_exec(PyObject *m) } // Export mypy internal C API, be careful with the order! - static void *NativeInternal_API[14] = { + static void *NativeInternal_API[16] = { (void *)Buffer_internal, (void *)Buffer_internal_empty, (void *)Buffer_getvalue_internal, @@ -605,6 +701,8 @@ librt_internal_module_exec(PyObject *m) (void *)write_tag_internal, (void *)read_tag_internal, (void *)NativeInternal_ABI_Version, + (void *)write_bytes_internal, + (void *)read_bytes_internal, }; PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index fd8ec2422cc5..d996b8fd95c1 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -19,6 +19,8 @@ static CPyTagged read_int_internal(PyObject *data); static char write_tag_internal(PyObject *data, uint8_t value); static uint8_t read_tag_internal(PyObject *data); static int NativeInternal_ABI_Version(void); +static char write_bytes_internal(PyObject *data, PyObject *value); +static PyObject *read_bytes_internal(PyObject *data); #else @@ -38,6 +40,8 @@ static void **NativeInternal_API; #define write_tag_internal (*(char (*)(PyObject *source, uint8_t value)) NativeInternal_API[11]) #define read_tag_internal (*(uint8_t (*)(PyObject *source)) NativeInternal_API[12]) #define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13]) +#define write_bytes_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[14]) +#define read_bytes_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[15]) static int import_librt_internal(void) diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 18d475fe89d4..c12172875e8b 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -393,6 +393,22 @@ error_kind=ERR_MAGIC, ) +function_op( + name="librt.internal.write_bytes", + arg_types=[object_rprimitive, bytes_rprimitive], + return_type=none_rprimitive, + c_function_name="write_bytes_internal", + error_kind=ERR_MAGIC, +) + +function_op( + name="librt.internal.read_bytes", + arg_types=[object_rprimitive], + return_type=bytes_rprimitive, + c_function_name="read_bytes_internal", + error_kind=ERR_MAGIC, +) + function_op( name="librt.internal.write_float", arg_types=[object_rprimitive, float_rprimitive], diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 3280b21cf7e6..27ffba45ba39 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1454,7 +1454,7 @@ from typing import Final from mypy_extensions import u8 from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, - write_int, read_int, write_tag, read_tag + write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, ) Tag = u8 @@ -1463,6 +1463,7 @@ TAG: Final[Tag] = 1 def foo() -> None: b = Buffer() write_str(b, "foo") + write_bytes(b, b"bar") write_bool(b, True) write_float(b, 0.1) write_int(b, 1) @@ -1470,6 +1471,7 @@ def foo() -> None: b = Buffer(b.getvalue()) x = read_str(b) + xb = read_bytes(b) y = read_bool(b) z = read_float(b) t = read_int(b) @@ -1478,36 +1480,43 @@ def foo() -> None: def foo(): r0, b :: librt.internal.Buffer r1 :: str - r2, r3, r4, r5, r6 :: None - r7 :: bytes - r8 :: librt.internal.Buffer - r9, x :: str - r10, y :: bool - r11, z :: float - r12, t :: int - r13, u :: u8 + r2 :: None + r3 :: bytes + r4, r5, r6, r7, r8 :: None + r9 :: bytes + r10 :: librt.internal.Buffer + r11, x :: str + r12, xb :: bytes + r13, y :: bool + r14, z :: float + r15, t :: int + r16, u :: u8 L0: r0 = Buffer_internal_empty() b = r0 r1 = 'foo' r2 = write_str_internal(b, r1) - r3 = write_bool_internal(b, 1) - r4 = write_float_internal(b, 0.1) - r5 = write_int_internal(b, 2) - r6 = write_tag_internal(b, 1) - r7 = Buffer_getvalue_internal(b) - r8 = Buffer_internal(r7) - b = r8 - r9 = read_str_internal(b) - x = r9 - r10 = read_bool_internal(b) - y = r10 - r11 = read_float_internal(b) - z = r11 - r12 = read_int_internal(b) - t = r12 - r13 = read_tag_internal(b) - u = r13 + r3 = b'bar' + r4 = write_bytes_internal(b, r3) + r5 = write_bool_internal(b, 1) + r6 = write_float_internal(b, 0.1) + r7 = write_int_internal(b, 2) + r8 = write_tag_internal(b, 1) + r9 = Buffer_getvalue_internal(b) + r10 = Buffer_internal(r9) + b = r10 + r11 = read_str_internal(b) + x = r11 + r12 = read_bytes_internal(b) + xb = r12 + r13 = read_bool_internal(b) + y = r13 + r14 = read_float_internal(b) + z = r14 + r15 = read_int_internal(b) + t = r15 + r16 = read_tag_internal(b) + u = r16 return 1 [case testEnumFastPath] diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 84704ce66c81..efa6c225ecab 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2715,7 +2715,7 @@ from typing import Final from mypy_extensions import u8 from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, - write_int, read_int, write_tag, read_tag + write_int, read_int, write_tag, read_tag, write_bytes, read_bytes ) Tag = u8 @@ -2733,6 +2733,11 @@ def test_buffer_roundtrip() -> None: write_bool(b, True) write_str(b, "bar" * 1000) write_bool(b, False) + write_bytes(b, b"bar") + write_bytes(b, b"bar" * 100) + write_bytes(b, b"") + write_bytes(b, b"a" * 127) + write_bytes(b, b"a" * 128) write_float(b, 0.1) write_int(b, 0) write_int(b, 1) @@ -2752,6 +2757,11 @@ def test_buffer_roundtrip() -> None: assert read_bool(b) is True assert read_str(b) == "bar" * 1000 assert read_bool(b) is False + assert read_bytes(b) == b"bar" + assert read_bytes(b) == b"bar" * 100 + assert read_bytes(b) == b"" + assert read_bytes(b) == b"a" * 127 + assert read_bytes(b) == b"a" * 128 assert read_float(b) == 0.1 assert read_int(b) == 0 assert read_int(b) == 1 @@ -2806,6 +2816,11 @@ def test_buffer_roundtrip_interpreted() -> None: write_bool(b, True) write_str(b, "bar" * 1000) write_bool(b, False) + write_bytes(b, b"bar") + write_bytes(b, b"bar" * 100) + write_bytes(b, b"") + write_bytes(b, b"a" * 127) + write_bytes(b, b"a" * 128) write_float(b, 0.1) write_int(b, 0) write_int(b, 1) @@ -2825,6 +2840,11 @@ def test_buffer_roundtrip_interpreted() -> None: assert read_bool(b) is True assert read_str(b) == "bar" * 1000 assert read_bool(b) is False + assert read_bytes(b) == b"bar" + assert read_bytes(b) == b"bar" * 100 + assert read_bytes(b) == b"" + assert read_bytes(b) == b"a" * 127 + assert read_bytes(b) == b"a" * 128 assert read_float(b) == 0.1 assert read_int(b) == 0 assert read_int(b) == 1 From 72b0fcacb720121ebe241c12f25ef6c66435e5c9 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Thu, 16 Oct 2025 05:36:16 -0700 Subject: [PATCH 106/183] Bump version of wasilibs/go-shellcheck to fix lint on Windows (#20050) Currently, if you try to run our lint step on Windows, shellcheck crashes with a complaint about an empty env var key. However, that has just been fixed in https://github.com/wasilibs/go-shellcheck/issues/10. Closes https://github.com/python/mypy/issues/19958, which I have already preemptively closed. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a410585a52d4..1f4282e4f65b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: # actionlint has a shellcheck integration which extracts shell scripts in `run:` steps from GitHub Actions # and checks these with shellcheck. This is arguably its most useful feature, # but the integration only works if shellcheck is installed - - "github.com/wasilibs/go-shellcheck/cmd/shellcheck@v0.11.0" + - "github.com/wasilibs/go-shellcheck/cmd/shellcheck@v0.11.1" - repo: https://github.com/woodruffw/zizmor-pre-commit rev: v1.5.2 hooks: From 9c26271945ee7c9095d769f2138f4f2e969445ce Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Thu, 16 Oct 2025 16:39:52 +0200 Subject: [PATCH 107/183] [match-case] fix matching against `typing.Callable` and `Protocol` types. (#19471) - Fixes #14014 - Partially addresses #19470 Added extra logic in `checker.py:conditional_types` function to deal with structural types such as `typing.Callable` or protocols. ## new tests - `testMatchClassPatternCallable`: tests `case Callable() as fn` usage - `testMatchClassPatternProtocol`: tests `case Proto()` usage, where `Proto` is a Protocol - `testMatchClassPatternCallbackProtocol`: tests `case Proto()` usage, where `Proto` is a Callback-Protocol - `testGenericAliasIsinstanceUnreachable`: derived from a mypy-primer failure in mesonbuild. Tests that `isinstance(x, Proto)` can produce unreachable error. - `testGenericAliasRedundantExprCompoundIfExpr`: derived from a CI failure of `python runtest.py self` of an earlier version of this PR. ## modified tests - `testOverloadOnProtocol` added annotations to overload implementation, which wasn't getting checked. Added missing return. Fixed return type in second branch. --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ivan Levkivskyi --- mypy/checker.py | 33 +++-- mypy/checkpattern.py | 10 ++ test-data/unit/check-generic-alias.test | 32 +++++ test-data/unit/check-protocols.test | 5 +- test-data/unit/check-python310.test | 156 ++++++++++++++++++++++++ 5 files changed, 225 insertions(+), 11 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 3bee7b633339..754bd59a4962 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -8174,11 +8174,15 @@ def conditional_types( ) -> tuple[Type | None, Type | None]: """Takes in the current type and a proposed type of an expression. - Returns a 2-tuple: The first element is the proposed type, if the expression - can be the proposed type. The second element is the type it would hold - if it was not the proposed type, if any. UninhabitedType means unreachable. - None means no new information can be inferred. If default is set it is returned - instead.""" + Returns a 2-tuple: + The first element is the proposed type, if the expression can be the proposed type. + (or default, if default is set and the expression is a subtype of the proposed type). + The second element is the type it would hold if it was not the proposed type, if any. + (or default, if default is set and the expression is not a subtype of the proposed type). + + UninhabitedType means unreachable. + None means no new information can be inferred. + """ if proposed_type_ranges: if len(proposed_type_ranges) == 1: target = proposed_type_ranges[0].item @@ -8190,14 +8194,25 @@ def conditional_types( current_type = try_expanding_sum_type_to_union(current_type, enum_name) proposed_items = [type_range.item for type_range in proposed_type_ranges] proposed_type = make_simplified_union(proposed_items) - if isinstance(proposed_type, AnyType): + if isinstance(get_proper_type(current_type), AnyType): + return proposed_type, current_type + elif isinstance(proposed_type, AnyType): # We don't really know much about the proposed type, so we shouldn't # attempt to narrow anything. Instead, we broaden the expr to Any to # avoid false positives return proposed_type, default - elif not any( - type_range.is_upper_bound for type_range in proposed_type_ranges - ) and is_proper_subtype(current_type, proposed_type, ignore_promotions=True): + elif not any(type_range.is_upper_bound for type_range in proposed_type_ranges) and ( + # concrete subtypes + is_proper_subtype(current_type, proposed_type, ignore_promotions=True) + # structural subtypes + or ( + ( + isinstance(proposed_type, CallableType) + or (isinstance(proposed_type, Instance) and proposed_type.type.is_protocol) + ) + and is_subtype(current_type, proposed_type, ignore_promotions=True) + ) + ): # Expression is always of one of the types in proposed_type_ranges return default, UninhabitedType() elif not is_overlapping_types(current_type, proposed_type, ignore_promotions=True): diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index 6f00c6c43177..3c51c4106909 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -51,6 +51,7 @@ UninhabitedType, UnionType, UnpackType, + callable_with_ellipsis, find_unpack_in_list, get_proper_type, split_with_prefix_and_suffix, @@ -546,6 +547,15 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType: return self.early_non_match() elif isinstance(p_typ, FunctionLike) and p_typ.is_type_obj(): typ = fill_typevars_with_any(p_typ.type_object()) + elif ( + isinstance(type_info, Var) + and type_info.type is not None + and type_info.fullname == "typing.Callable" + ): + # Create a `Callable[..., Any]` + fallback = self.chk.named_type("builtins.function") + any_type = AnyType(TypeOfAny.unannotated) + typ = callable_with_ellipsis(any_type, ret_type=any_type, fallback=fallback) elif not isinstance(p_typ, AnyType): self.msg.fail( message_registry.CLASS_PATTERN_TYPE_REQUIRED.format( diff --git a/test-data/unit/check-generic-alias.test b/test-data/unit/check-generic-alias.test index 678950a1e18b..3f088308da64 100644 --- a/test-data/unit/check-generic-alias.test +++ b/test-data/unit/check-generic-alias.test @@ -149,6 +149,38 @@ t23: collections.abc.ValuesView[str] # reveal_type(t23) # Nx Revealed type is "collections.abc.ValuesView[builtins.str]" [builtins fixtures/tuple.pyi] +[case testGenericAliasIsinstanceUnreachable] +# flags: --warn-unreachable --python-version 3.10 +from collections.abc import Iterable + +class A: ... + +def test(dependencies: list[A] | None) -> None: + if dependencies is None: + dependencies = [] + elif not isinstance(dependencies, Iterable): + dependencies = [dependencies] # E: Statement is unreachable + +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-full.pyi] + +[case testGenericAliasRedundantExprCompoundIfExpr] +# flags: --warn-unreachable --enable-error-code=redundant-expr --python-version 3.10 + +from typing import Any, reveal_type +from collections.abc import Iterable + +def test_example(x: Iterable[Any]) -> None: + if isinstance(x, Iterable) and not isinstance(x, str): # E: Left operand of "and" is always true + reveal_type(x) # N: Revealed type is "typing.Iterable[Any]" + +def test_counterexample(x: Any) -> None: + if isinstance(x, Iterable) and not isinstance(x, str): + reveal_type(x) # N: Revealed type is "typing.Iterable[Any]" + +[builtins fixtures/isinstancelist.pyi] +[typing fixtures/typing-full.pyi] + [case testGenericBuiltinTupleTyping] from typing import Tuple diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 0f19b404082e..ae6f60355512 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1506,11 +1506,12 @@ class C: pass def f(x: P1) -> int: ... @overload def f(x: P2) -> str: ... -def f(x): +def f(x: object) -> object: if isinstance(x, P1): return P1.attr1 if isinstance(x, P2): # E: Only @runtime_checkable protocols can be used with instance and class checks - return P1.attr2 + return P2.attr2 + return None reveal_type(f(C1())) # N: Revealed type is "builtins.int" reveal_type(f(C2())) # N: Revealed type is "builtins.str" diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 2c4597e212ea..3a9f12e8a550 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -11,11 +11,30 @@ match m: -- Literal Pattern -- [case testMatchLiteralPatternNarrows] +# flags: --warn-unreachable m: object match m: case 1: reveal_type(m) # N: Revealed type is "Literal[1]" + case 2: + reveal_type(m) # N: Revealed type is "Literal[2]" + case other: + reveal_type(other) # N: Revealed type is "builtins.object" + +[case testMatchLiteralPatternNarrows2] +# flags: --warn-unreachable +from typing import Any + +m: Any + +match m: + case 1: + reveal_type(m) # N: Revealed type is "Literal[1]" + case 2: + reveal_type(m) # N: Revealed type is "Literal[2]" + case other: + reveal_type(other) # N: Revealed type is "Any" [case testMatchLiteralPatternAlreadyNarrower-skip] m: bool @@ -1079,6 +1098,143 @@ match m: case Foo(): pass +[case testMatchClassPatternCallable] +# flags: --warn-unreachable +from typing import Callable, Any + +class FnImpl: + def __call__(self, x: object, /) -> int: ... + +def test_any(x: Any) -> None: + match x: + case Callable() as fn: + reveal_type(fn) # N: Revealed type is "def (*Any, **Any) -> Any" + case other: + reveal_type(other) # N: Revealed type is "Any" + +def test_object(x: object) -> None: + match x: + case Callable() as fn: + reveal_type(fn) # N: Revealed type is "def (*Any, **Any) -> Any" + case other: + reveal_type(other) # N: Revealed type is "builtins.object" + +def test_impl(x: FnImpl) -> None: + match x: + case Callable() as fn: + reveal_type(fn) # N: Revealed type is "__main__.FnImpl" + case other: + reveal_type(other) # E: Statement is unreachable + +def test_callable(x: Callable[[object], int]) -> None: + match x: + case Callable() as fn: + reveal_type(fn) # N: Revealed type is "def (builtins.object) -> builtins.int" + case other: + reveal_type(other) # E: Statement is unreachable + +[case testMatchClassPatternCallbackProtocol] +# flags: --warn-unreachable +from typing import Any, Callable +from typing_extensions import Protocol, runtime_checkable + +@runtime_checkable +class FnProto(Protocol): + def __call__(self, x: int, /) -> object: ... + +class FnImpl: + def __call__(self, x: object, /) -> int: ... + +def test_any(x: Any) -> None: + match x: + case FnProto() as fn: + reveal_type(fn) # N: Revealed type is "__main__.FnProto" + case other: + reveal_type(other) # N: Revealed type is "Any" + +def test_object(x: object) -> None: + match x: + case FnProto() as fn: + reveal_type(fn) # N: Revealed type is "__main__.FnProto" + case other: + reveal_type(other) # N: Revealed type is "builtins.object" + +def test_impl(x: FnImpl) -> None: + match x: + case FnProto() as fn: + reveal_type(fn) # N: Revealed type is "__main__.FnImpl" + case other: + reveal_type(other) # E: Statement is unreachable + +def test_callable(x: Callable[[object], int]) -> None: + match x: + case FnProto() as fn: + reveal_type(fn) # N: Revealed type is "def (builtins.object) -> builtins.int" + case other: + reveal_type(other) # E: Statement is unreachable + +[builtins fixtures/dict.pyi] + +[case testMatchClassPatternAnyCallableProtocol] +# flags: --warn-unreachable +from typing import Any, Callable +from typing_extensions import Protocol, runtime_checkable + +@runtime_checkable +class AnyCallable(Protocol): + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + +class FnImpl: + def __call__(self, x: object, /) -> int: ... + +def test_object(x: object) -> None: + match x: + case AnyCallable() as fn: + reveal_type(fn) # N: Revealed type is "__main__.AnyCallable" + case other: + reveal_type(other) # N: Revealed type is "builtins.object" + +def test_impl(x: FnImpl) -> None: + match x: + case AnyCallable() as fn: + reveal_type(fn) # N: Revealed type is "__main__.FnImpl" + case other: + reveal_type(other) # E: Statement is unreachable + +def test_callable(x: Callable[[object], int]) -> None: + match x: + case AnyCallable() as fn: + reveal_type(fn) # N: Revealed type is "def (builtins.object) -> builtins.int" + case other: + reveal_type(other) # E: Statement is unreachable + +[builtins fixtures/dict.pyi] + + +[case testMatchClassPatternProtocol] +from typing import Any +from typing_extensions import Protocol, runtime_checkable + +@runtime_checkable +class Proto(Protocol): + def foo(self, x: int, /) -> object: ... + +class Impl: + def foo(self, x: object, /) -> int: ... + +def test_object(x: object) -> None: + match x: + case Proto() as y: + reveal_type(y) # N: Revealed type is "__main__.Proto" + +def test_impl(x: Impl) -> None: + match x: + case Proto() as y: + reveal_type(y) # N: Revealed type is "__main__.Impl" + +[builtins fixtures/dict.pyi] + + [case testMatchClassPatternNestedGenerics] # From cpython test_patma.py x = [[{0: 0}]] From e1643aec5229e37cf62700dabae294b9a5344c97 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 16 Oct 2025 16:31:35 +0100 Subject: [PATCH 108/183] Switch to a more dynamic SCC processing logic (#20053) Ref https://github.com/python/mypy/issues/933 Instead of processing SCCs layer by layer, we will now process an SCC as soon as it is ready. This logic is easier to adapt for parallel processing, and should get us more benefit from parallelization (as more SCCs can be processed in parallel). I tried to make order with single worker stable and very similar (or maybe even identical) to the current order. Note I already add some methods to the build manager to emulate parallel processing, but they are not parallel _yet_. --- mypy/build.py | 378 ++++++++++++++++++++++++------------ mypy/test/testcheck.py | 10 +- mypy/test/testgraph.py | 6 +- mypyc/codegen/emitmodule.py | 4 +- 4 files changed, 265 insertions(+), 133 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 9f840499fcc2..f9137d8b1a32 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -122,6 +122,28 @@ Graph: _TypeAlias = dict[str, "State"] +class SCC: + """A simple class that represents a strongly connected component (import cycle).""" + + id_counter: ClassVar[int] = 0 + + def __init__(self, ids: set[str]) -> None: + self.id = SCC.id_counter + SCC.id_counter += 1 + # Ids of modules in this cycle. + self.mod_ids = ids + # Direct dependencies, should be populated by the caller. + self.deps: set[int] = set() + # Direct dependencies that have not been processed yet. + # Should be populated by the caller. This set may change during graph + # processing, while the above stays constant. + self.not_ready_deps: set[int] = set() + # SCCs that (directly) depend on this SCC. Note this is a list to + # make processing order more predictable. Dependents will be notified + # that they may be ready in the order in this list. + self.direct_dependents: list[int] = [] + + # TODO: Get rid of BuildResult. We might as well return a BuildManager. class BuildResult: """The result of a successful build. @@ -725,6 +747,18 @@ def __init__( self.ast_cache: dict[str, tuple[MypyFile, list[ErrorInfo]]] = {} # Number of times we used GC optimization hack for fresh SCCs. self.gc_freeze_cycles = 0 + # Mapping from SCC id to corresponding SCC instance. This is populated + # in process_graph(). + self.scc_by_id: dict[int, SCC] = {} + # Global topological order for SCCs. This exists to make order of processing + # SCCs more predictable. + self.top_order: list[int] = [] + # Stale SCCs that are queued for processing. Note that as of now we have just + # one worker, that is the same process. In the future, we will support multiple + # parallel worker processes. + self.scc_queue: list[SCC] = [] + # SCCs that have been fully processed. + self.done_sccs: set[int] = set() def dump_stats(self) -> None: if self.options.dump_build_stats: @@ -925,6 +959,23 @@ def add_stats(self, **kwds: Any) -> None: def stats_summary(self) -> Mapping[str, object]: return self.stats + def submit(self, sccs: list[SCC]) -> None: + """Submit a stale SCC for processing in current process.""" + self.scc_queue.extend(sccs) + + def wait_for_done(self, graph: Graph) -> tuple[list[SCC], bool]: + """Wait for a stale SCC processing (in process) to finish. + + Return next processed SCC and whether we have more in the queue. + This emulates the API we will have for parallel processing + in multiple worker processes. + """ + if not self.scc_queue: + return [], False + next_scc = self.scc_queue.pop(0) + process_stale_scc(graph, next_scc, self) + return [next_scc], bool(self.scc_queue) + def deps_to_json(x: dict[str, set[str]]) -> bytes: return json_dumps({k: list(v) for k, v in x.items()}) @@ -3012,7 +3063,7 @@ def dump_graph(graph: Graph, stdout: TextIO | None = None) -> None: nodes = [] sccs = sorted_components(graph) for i, ascc in enumerate(sccs): - scc = order_ascc(graph, ascc) + scc = order_ascc(graph, ascc.mod_ids) node = NodeInfo(i, scc) nodes.append(node) inv_nodes = {} # module -> node_id @@ -3203,58 +3254,51 @@ def load_graph( return graph -def process_graph(graph: Graph, manager: BuildManager) -> None: - """Process everything in dependency order.""" - sccs = sorted_components(graph) - manager.log("Found %d SCCs; largest has %d nodes" % (len(sccs), max(len(scc) for scc in sccs))) - - fresh_scc_queue: list[list[str]] = [] +def order_ascc_ex(graph: Graph, ascc: SCC) -> list[str]: + """Apply extra heuristics on top of order_ascc(). - # We're processing SCCs from leaves (those without further - # dependencies) to roots (those from which everything else can be - # reached). + This should be used only for actual SCCs, not for "inner" SCCs + we create recursively during ordering of the SCC. Currently, this + has only some special handling for builtin SCC. + """ + scc = order_ascc(graph, ascc.mod_ids) + # Make the order of the SCC that includes 'builtins' and 'typing', + # among other things, predictable. Various things may break if + # the order changes. + if "builtins" in ascc.mod_ids: + scc = sorted(scc, reverse=True) + # If builtins is in the list, move it last. (This is a bit of + # a hack, but it's necessary because the builtins module is + # part of a small cycle involving at least {builtins, abc, + # typing}. Of these, builtins must be processed last or else + # some builtin objects will be incompletely processed.) + scc.remove("builtins") + scc.append("builtins") + return scc + + +def find_stale_sccs( + sccs: list[SCC], graph: Graph, manager: BuildManager +) -> tuple[list[SCC], list[SCC]]: + """Split a list of ready SCCs into stale and fresh. + + Fresh SCCs are those where: + * We have valid cache files for all modules in the SCC. + * The interface hashes of direct dependents matches those recorded in the cache. + * There are no new (un)suppressed dependencies (files removed/added to the build). + """ + stale_sccs = [] + fresh_sccs = [] for ascc in sccs: - # Order the SCC's nodes using a heuristic. - # Note that ascc is a set, and scc is a list. - scc = order_ascc(graph, ascc) - # Make the order of the SCC that includes 'builtins' and 'typing', - # among other things, predictable. Various things may break if - # the order changes. - if "builtins" in ascc: - scc = sorted(scc, reverse=True) - # If builtins is in the list, move it last. (This is a bit of - # a hack, but it's necessary because the builtins module is - # part of a small cycle involving at least {builtins, abc, - # typing}. Of these, builtins must be processed last or else - # some builtin objects will be incompletely processed.) - scc.remove("builtins") - scc.append("builtins") - if manager.options.verbosity >= 2: - for id in scc: - manager.trace( - f"Priorities for {id}:", - " ".join( - "%s:%d" % (x, graph[id].priorities[x]) - for x in graph[id].dependencies - if x in ascc and x in graph[id].priorities - ), - ) - # Because the SCCs are presented in topological sort order, we - # don't need to look at dependencies recursively for staleness - # -- the immediate dependencies are sufficient. - stale_scc = {id for id in scc if not graph[id].is_fresh()} + stale_scc = {id for id in ascc.mod_ids if not graph[id].is_fresh()} fresh = not stale_scc - deps = set() - for id in scc: - deps.update(graph[id].dependencies) - deps -= ascc # Verify that interfaces of dependencies still present in graph are up-to-date (fresh). # Note: if a dependency is not in graph anymore, it should be considered interface-stale. # This is important to trigger any relevant updates from indirect dependencies that were # removed in load_graph(). stale_deps = set() - for id in ascc: + for id in ascc.mod_ids: for dep in graph[id].dep_hashes: if dep not in graph: stale_deps.add(dep) @@ -3262,98 +3306,101 @@ def process_graph(graph: Graph, manager: BuildManager) -> None: if graph[dep].interface_hash != graph[id].dep_hashes[dep]: stale_deps.add(dep) fresh = fresh and not stale_deps + undeps = set() if fresh: # Check if any dependencies that were suppressed according # to the cache have been added back in this run. # NOTE: Newly suppressed dependencies are handled by is_fresh(). - for id in scc: + for id in ascc.mod_ids: undeps.update(graph[id].suppressed) undeps &= graph.keys() if undeps: fresh = False + if fresh: fresh_msg = "fresh" elif undeps: fresh_msg = f"stale due to changed suppression ({' '.join(sorted(undeps))})" elif stale_scc: fresh_msg = "inherently stale" - if stale_scc != ascc: + if stale_scc != ascc.mod_ids: fresh_msg += f" ({' '.join(sorted(stale_scc))})" if stale_deps: fresh_msg += f" with stale deps ({' '.join(sorted(stale_deps))})" else: fresh_msg = f"stale due to deps ({' '.join(sorted(stale_deps))})" - scc_str = " ".join(scc) + scc_str = " ".join(ascc.mod_ids) if fresh: - manager.trace(f"Queuing {fresh_msg} SCC ({scc_str})") + manager.trace(f"Found {fresh_msg} SCC ({scc_str})") + # If there is at most one file with errors we can skip the ordering to save time. + mods_with_errors = [id for id in ascc.mod_ids if graph[id].error_lines] + if len(mods_with_errors) <= 1: + scc = mods_with_errors + else: + # Use exactly the same order as for stale SCCs for stability. + scc = order_ascc_ex(graph, ascc) for id in scc: if graph[id].error_lines: manager.flush_errors( manager.errors.simplify_path(graph[id].xpath), graph[id].error_lines, False ) - fresh_scc_queue.append(scc) + fresh_sccs.append(ascc) else: - if fresh_scc_queue: - manager.log(f"Processing {len(fresh_scc_queue)} queued fresh SCCs") - # Defer processing fresh SCCs until we actually run into a stale SCC - # and need the earlier modules to be loaded. - # - # Note that `process_graph` may end with us not having processed every - # single fresh SCC. This is intentional -- we don't need those modules - # loaded if there are no more stale SCCs to be rechecked. - # - # TODO: see if it's possible to determine if we need to process only a - # _subset_ of the past SCCs instead of having to process them all. - if ( - not manager.options.test_env - and platform.python_implementation() == "CPython" - and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES - ): - # When deserializing cache we create huge amount of new objects, so even - # with our generous GC thresholds, GC is still doing a lot of pointless - # work searching for garbage. So, we temporarily disable it when - # processing fresh SCCs, and then move all the new objects to the oldest - # generation with the freeze()/unfreeze() trick below. This is arguably - # a hack, but it gives huge performance wins for large third-party - # libraries, like torch. - gc.collect() - gc.disable() - for prev_scc in fresh_scc_queue: - process_fresh_modules(graph, prev_scc, manager) - if ( - not manager.options.test_env - and platform.python_implementation() == "CPython" - and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES - ): - manager.gc_freeze_cycles += 1 - gc.freeze() - gc.unfreeze() - gc.enable() - fresh_scc_queue = [] - size = len(scc) + size = len(ascc.mod_ids) if size == 1: - manager.log(f"Processing SCC singleton ({scc_str}) as {fresh_msg}") + manager.log(f"Scheduling SCC singleton ({scc_str}) as {fresh_msg}") else: - manager.log("Processing SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg)) - process_stale_scc(graph, scc, manager) + manager.log("Scheduling SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg)) + stale_sccs.append(ascc) + return stale_sccs, fresh_sccs - sccs_left = len(fresh_scc_queue) - nodes_left = sum(len(scc) for scc in fresh_scc_queue) - manager.add_stats(sccs_left=sccs_left, nodes_left=nodes_left) - if sccs_left: - manager.log( - "{} fresh SCCs ({} nodes) left in queue (and will remain unprocessed)".format( - sccs_left, nodes_left - ) - ) - manager.trace(str(fresh_scc_queue)) - else: - manager.log("No fresh SCCs left in queue") +def process_graph(graph: Graph, manager: BuildManager) -> None: + """Process everything in dependency order.""" + sccs = sorted_components(graph) + manager.log( + "Found %d SCCs; largest has %d nodes" % (len(sccs), max(len(scc.mod_ids) for scc in sccs)) + ) + + scc_by_id = {scc.id: scc for scc in sccs} + manager.scc_by_id = scc_by_id + manager.top_order = [scc.id for scc in sccs] -def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> list[str]: + # Prime the ready list with leaf SCCs (that have no dependencies). + ready = [] + not_ready = [] + for scc in sccs: + if not scc.deps: + ready.append(scc) + else: + not_ready.append(scc) + + still_working = False + while ready or not_ready or still_working: + stale, fresh = find_stale_sccs(ready, graph, manager) + if stale: + manager.submit(stale) + still_working = True + # We eagerly walk over fresh SCCs to reach as many stale SCCs as soon + # as possible. Only when there are no fresh SCCs, we wait on scheduled stale ones. + # This strategy, similar to a naive strategy in minesweeper game, will allow us + # to leverage parallelism as much as possible. + if fresh: + done = fresh + else: + done, still_working = manager.wait_for_done(graph) + ready = [] + for done_scc in done: + for dependent in done_scc.direct_dependents: + scc_by_id[dependent].not_ready_deps.discard(done_scc.id) + if not scc_by_id[dependent].not_ready_deps: + not_ready.remove(scc_by_id[dependent]) + ready.append(scc_by_id[dependent]) + + +def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_INDIRECT) -> list[str]: """Come up with the ideal processing order within an SCC. Using the priorities assigned by all_imported_modules_in_file(), @@ -3377,7 +3424,7 @@ def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> In practice there are only a few priority levels (less than a dozen) and in the worst case we just carry out the same algorithm - for finding SCCs N times. Thus the complexity is no worse than + for finding SCCs N times. Thus, the complexity is no worse than the complexity of the original SCC-finding algorithm -- see strongly_connected_components() below for a reference. """ @@ -3395,7 +3442,7 @@ def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> # Filtered dependencies are uniform -- order by global order. return sorted(ascc, key=lambda id: -graph[id].order) pri_max = max(pri_spread) - sccs = sorted_components(graph, ascc, pri_max) + sccs = sorted_components_inner(graph, ascc, pri_max) # The recursion is bounded by the len(pri_spread) check above. return [s for ss in sccs for s in order_ascc(graph, ss, pri_max)] @@ -3403,8 +3450,8 @@ def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> def process_fresh_modules(graph: Graph, modules: list[str], manager: BuildManager) -> None: """Process the modules in one group of modules from their cached data. - This can be used to process an SCC of modules - This involves loading the tree from JSON and then doing various cleanups. + This can be used to process an SCC of modules. This involves loading the tree (i.e. + module symbol tables) from cache file and then fixing cross-references in the symbols. """ t0 = time.time() for id in modules: @@ -3416,11 +3463,54 @@ def process_fresh_modules(graph: Graph, modules: list[str], manager: BuildManage manager.add_stats(process_fresh_time=t2 - t0, load_tree_time=t1 - t0) -def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> None: - """Process the modules in one SCC from source code. +def process_stale_scc(graph: Graph, ascc: SCC, manager: BuildManager) -> None: + """Process the modules in one SCC from source code.""" + # First verify if all transitive dependencies are loaded in the current process. + missing_sccs = set() + sccs_to_find = ascc.deps.copy() + while sccs_to_find: + dep_scc = sccs_to_find.pop() + if dep_scc in manager.done_sccs or dep_scc in missing_sccs: + continue + missing_sccs.add(dep_scc) + sccs_to_find.update(manager.scc_by_id[dep_scc].deps) + + if missing_sccs: + # Load missing SCCs from cache. + # TODO: speed-up ordering if this causes problems for large builds. + fresh_sccs_to_load = [ + manager.scc_by_id[sid] for sid in manager.top_order if sid in missing_sccs + ] + manager.log(f"Processing {len(fresh_sccs_to_load)} fresh SCCs") + if ( + not manager.options.test_env + and platform.python_implementation() == "CPython" + and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES + ): + # When deserializing cache we create huge amount of new objects, so even + # with our generous GC thresholds, GC is still doing a lot of pointless + # work searching for garbage. So, we temporarily disable it when + # processing fresh SCCs, and then move all the new objects to the oldest + # generation with the freeze()/unfreeze() trick below. This is arguably + # a hack, but it gives huge performance wins for large third-party + # libraries, like torch. + gc.collect() + gc.disable() + for prev_scc in fresh_sccs_to_load: + manager.done_sccs.add(prev_scc.id) + process_fresh_modules(graph, sorted(prev_scc.mod_ids), manager) + if ( + not manager.options.test_env + and platform.python_implementation() == "CPython" + and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES + ): + manager.gc_freeze_cycles += 1 + gc.freeze() + gc.unfreeze() + gc.enable() - Exception: If quick_and_dirty is set, use the cache for fresh modules. - """ + # Process the SCC in stable order. + scc = order_ascc_ex(graph, ascc) stale = scc for id in stale: # We may already have parsed the module, or not. @@ -3434,7 +3524,7 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No assert typing_mod, "The typing module was not parsed" mypy.semanal_main.semantic_analysis_for_scc(graph, scc, manager.errors) - # Track what modules aren't yet done so we can finish them as soon + # Track what modules aren't yet done, so we can finish them as soon # as possible, saving memory. unfinished_modules = set(stale) for id in stale: @@ -3478,27 +3568,44 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No } meta["error_lines"] = errors_by_id.get(id, []) write_cache_meta(meta, manager, meta_json) - - -def sorted_components( - graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_INDIRECT -) -> list[AbstractSet[str]]: + manager.done_sccs.add(ascc.id) + + +def prepare_sccs_full( + raw_sccs: Iterator[set[str]], edges: dict[str, list[str]] +) -> dict[SCC, set[SCC]]: + """Turn raw SCC sets into SCC objects and build dependency graph for SCCs.""" + sccs = [SCC(raw_scc) for raw_scc in raw_sccs] + scc_map = {} + for scc in sccs: + for id in scc.mod_ids: + scc_map[id] = scc + scc_deps_map: dict[SCC, set[SCC]] = {} + for scc in sccs: + for id in scc.mod_ids: + scc_deps_map.setdefault(scc, set()).update(scc_map[dep] for dep in edges[id]) + for scc in sccs: + # Remove trivial dependency on itself. + scc_deps_map[scc].discard(scc) + for dep_scc in scc_deps_map[scc]: + scc.deps.add(dep_scc.id) + scc.not_ready_deps.add(dep_scc.id) + return scc_deps_map + + +def sorted_components(graph: Graph) -> list[SCC]: """Return the graph's SCCs, topologically sorted by dependencies. The sort order is from leaves (nodes without dependencies) to roots (nodes on which no other nodes depend). - - This works for a subset of the full dependency graph too; - dependencies that aren't present in graph.keys() are ignored. """ # Compute SCCs. - if vertices is None: - vertices = set(graph) - edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} - sccs = list(strongly_connected_components(vertices, edges)) + vertices = set(graph) + edges = {id: deps_filtered(graph, vertices, id, PRI_INDIRECT) for id in vertices} + scc_dep_map = prepare_sccs_full(strongly_connected_components(vertices, edges), edges) # Topsort. res = [] - for ready in topsort(prepare_sccs(sccs, edges)): + for ready in topsort(scc_dep_map): # Sort the sets in ready by reversed smallest State.order. Examples: # # - If ready is [{x}, {y}], x.order == 1, y.order == 2, we get @@ -3507,6 +3614,27 @@ def sorted_components( # - If ready is [{a, b}, {c, d}], a.order == 1, b.order == 3, # c.order == 2, d.order == 4, the sort keys become [1, 2] # and the result is [{c, d}, {a, b}]. + sorted_ready = sorted(ready, key=lambda scc: -min(graph[id].order for id in scc.mod_ids)) + for scc in sorted_ready: + for dep in scc_dep_map[scc]: + dep.direct_dependents.append(scc.id) + res.extend(sorted_ready) + return res + + +def sorted_components_inner( + graph: Graph, vertices: AbstractSet[str], pri_max: int +) -> list[AbstractSet[str]]: + """Simplified version of sorted_components() to work with sub-graphs. + + This doesn't create SCC objects, and operates with raw sets. This function + also allows filtering dependencies to take into account when building SCCs. + This is used for heuristic ordering of modules within actual SCCs. + """ + edges = {id: deps_filtered(graph, vertices, id, pri_max) for id in vertices} + sccs = list(strongly_connected_components(vertices, edges)) + res = [] + for ready in topsort(prepare_sccs(sccs, edges)): res.extend(sorted(ready, key=lambda scc: -min(graph[id].order for id in scc))) return res diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 73f33c0323af..f59cce701ea6 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -200,7 +200,7 @@ def run_case_once( if res: if options.cache_dir != os.devnull: - self.verify_cache(module_data, res.manager, blocker) + self.verify_cache(module_data, res.manager, blocker, incremental_step) name = "targets" if incremental_step: @@ -230,7 +230,11 @@ def run_case_once( check_test_output_files(testcase, incremental_step, strip_prefix="tmp/") def verify_cache( - self, module_data: list[tuple[str, str, str]], manager: build.BuildManager, blocker: bool + self, + module_data: list[tuple[str, str, str]], + manager: build.BuildManager, + blocker: bool, + step: int, ) -> None: if not blocker: # There should be valid cache metadata for each module except @@ -240,7 +244,7 @@ def verify_cache( modules.update({module_name: path for module_name, path, text in module_data}) missing_paths = self.find_missing_cache_files(modules, manager) if missing_paths: - raise AssertionError(f"cache data missing for {missing_paths}") + raise AssertionError(f"cache data missing for {missing_paths} on run {step}") assert os.path.isfile(os.path.join(manager.options.cache_dir, ".gitignore")) cachedir_tag = os.path.join(manager.options.cache_dir, "CACHEDIR.TAG") assert os.path.isfile(cachedir_tag) diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index 238869f36fdf..c87eb66c1304 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -65,8 +65,8 @@ def test_sorted_components(self) -> None: "b": State("b", None, "import c", manager), "c": State("c", None, "import b, d", manager), } - res = sorted_components(graph) - assert_equal(res, [frozenset({"d"}), frozenset({"c", "b"}), frozenset({"a"})]) + res = [scc.mod_ids for scc in sorted_components(graph)] + assert_equal(res, [{"d"}, {"c", "b"}, {"a"}]) def test_order_ascc(self) -> None: manager = self._make_manager() @@ -76,7 +76,7 @@ def test_order_ascc(self) -> None: "b": State("b", None, "import c", manager), "c": State("c", None, "import b, d", manager), } - res = sorted_components(graph) + res = [scc.mod_ids for scc in sorted_components(graph)] assert_equal(res, [frozenset({"a", "d", "c", "b"})]) ascc = res[0] scc = order_ascc(graph, ascc) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 7ec315a6bd34..996ec4c52b08 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -280,13 +280,13 @@ def compile_modules_to_ir( # Process the graph by SCC in topological order, like we do in mypy.build for scc in sorted_components(result.graph): - scc_states = [result.graph[id] for id in scc] + scc_states = [result.graph[id] for id in scc.mod_ids] trees = [st.tree for st in scc_states if st.id in mapper.group_map and st.tree] if not trees: continue - fresh = all(id not in result.manager.rechecked_modules for id in scc) + fresh = all(id not in result.manager.rechecked_modules for id in scc.mod_ids) if fresh: load_scc_from_cache(trees, result, mapper, deser_ctx) else: From b38413d8959758167e96c9f5e140cfb4be1a94dd Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Fri, 17 Oct 2025 02:12:20 +0200 Subject: [PATCH 109/183] Sync typeshed (#20080) Source commit: https://github.com/python/typeshed/commit/11c7821a79a8ab7e1982f3ab506db16f1c4a22a9 --------- Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: AlexWaygood --- ...e-of-LiteralString-in-builtins-13743.patch | 30 +-- ...redundant-inheritances-from-Iterator.patch | 18 +- mypy/typeshed/stdlib/_csv.pyi | 7 +- .../stdlib/_frozen_importlib_external.pyi | 10 +- mypy/typeshed/stdlib/_typeshed/__init__.pyi | 3 + mypy/typeshed/stdlib/ast.pyi | 39 ++- mypy/typeshed/stdlib/builtins.pyi | 131 +++++----- mypy/typeshed/stdlib/ctypes/__init__.pyi | 4 +- mypy/typeshed/stdlib/html/parser.pyi | 13 +- mypy/typeshed/stdlib/importlib/abc.pyi | 6 +- .../stdlib/importlib/resources/__init__.pyi | 8 +- .../importlib/resources/_functional.pyi | 3 +- mypy/typeshed/stdlib/operator.pyi | 6 +- mypy/typeshed/stdlib/pdb.pyi | 5 + mypy/typeshed/stdlib/tkinter/__init__.pyi | 227 ++++++++++++------ mypy/typeshed/stdlib/turtle.pyi | 24 +- mypy/typeshed/stdlib/types.pyi | 28 +-- mypy/typeshed/stdlib/typing.pyi | 13 +- mypy/typeshed/stdlib/typing_extensions.pyi | 2 +- .../typeshed/stdlib/xml/etree/ElementTree.pyi | 30 +-- 20 files changed, 375 insertions(+), 232 deletions(-) diff --git a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch index a47d5db3cd22..f9334251c2bd 100644 --- a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch +++ b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch @@ -1,4 +1,4 @@ -From 805d7fc06a8bee350959512e0908a18a87b7f8c2 Mon Sep 17 00:00:00 2001 +From 3229a6066cff3d80d6cb923322c2d42a300d0be3 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 26 Sep 2022 12:55:07 -0700 Subject: [PATCH] Remove use of LiteralString in builtins (#13743) @@ -8,7 +8,7 @@ Subject: [PATCH] Remove use of LiteralString in builtins (#13743) 1 file changed, 1 insertion(+), 99 deletions(-) diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi -index c7ab95482..3e93da36e 100644 +index 969d16876..044e264d2 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -63,7 +63,6 @@ from typing import ( # noqa: Y022,UP035 @@ -19,10 +19,10 @@ index c7ab95482..3e93da36e 100644 ParamSpec, Self, TypeAlias, -@@ -468,31 +467,16 @@ class str(Sequence[str]): - def __new__(cls, object: object = ...) -> Self: ... +@@ -480,31 +479,16 @@ class str(Sequence[str]): + def __new__(cls, object: object = "") -> Self: ... @overload - def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... + def __new__(cls, object: ReadableBuffer, encoding: str = "utf-8", errors: str = "strict") -> Self: ... - @overload - def capitalize(self: LiteralString) -> LiteralString: ... - @overload @@ -35,23 +35,23 @@ index c7ab95482..3e93da36e 100644 - def center(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ... - @overload def center(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] - def count(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def count(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... def encode(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: ... def endswith( - self, suffix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, suffix: str | tuple[str, ...], start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> bool: ... - @overload - def expandtabs(self: LiteralString, tabsize: SupportsIndex = 8) -> LiteralString: ... - @overload def expandtabs(self, tabsize: SupportsIndex = 8) -> str: ... # type: ignore[misc] - def find(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def find(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... - @overload - def format(self: LiteralString, *args: LiteralString, **kwargs: LiteralString) -> LiteralString: ... - @overload def format(self, *args: object, **kwargs: object) -> str: ... def format_map(self, mapping: _FormatMapMapping, /) -> str: ... - def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... -@@ -508,98 +492,34 @@ class str(Sequence[str]): + def index(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... +@@ -520,98 +504,34 @@ class str(Sequence[str]): def isspace(self) -> bool: ... def istitle(self) -> bool: ... def isupper(self) -> bool: ... @@ -98,8 +98,8 @@ index c7ab95482..3e93da36e 100644 - def removesuffix(self: LiteralString, suffix: LiteralString, /) -> LiteralString: ... - @overload def removesuffix(self, suffix: str, /) -> str: ... # type: ignore[misc] - def rfind(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... - def rindex(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def rfind(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... + def rindex(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... - @overload - def rjust(self: LiteralString, width: SupportsIndex, fillchar: LiteralString = " ", /) -> LiteralString: ... - @overload @@ -125,7 +125,7 @@ index c7ab95482..3e93da36e 100644 - @overload def splitlines(self, keepends: bool = False) -> list[str]: ... # type: ignore[misc] def startswith( - self, prefix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, prefix: str | tuple[str, ...], start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> bool: ... - @overload - def strip(self: LiteralString, chars: LiteralString | None = None, /) -> LiteralString: ... @@ -150,7 +150,7 @@ index c7ab95482..3e93da36e 100644 def zfill(self, width: SupportsIndex, /) -> str: ... # type: ignore[misc] @staticmethod @overload -@@ -610,39 +530,21 @@ class str(Sequence[str]): +@@ -622,39 +542,21 @@ class str(Sequence[str]): @staticmethod @overload def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ... @@ -192,5 +192,5 @@ index c7ab95482..3e93da36e 100644 def __getnewargs__(self) -> tuple[str]: ... def __format__(self, format_spec: str, /) -> str: ... -- -2.50.1 +2.51.1 diff --git a/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch b/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch index fdcc14cec3c6..7110eff5f148 100644 --- a/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch +++ b/misc/typeshed_patches/0001-Revert-Remove-redundant-inheritances-from-Iterator.patch @@ -1,4 +1,4 @@ -From 438dbb1300b77331940d7db8f010e97305745116 Mon Sep 17 00:00:00 2001 +From 7678bc3f80e4d3f04a0ff0ee3a7d51f49ae4c465 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 21 Dec 2024 22:36:38 +0100 Subject: [PATCH] Revert Remove redundant inheritances from Iterator in @@ -36,10 +36,10 @@ index d663f5d93..f43178e4d 100644 @property def _exception(self) -> BaseException | None: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi -index f2dd00079..784ee7eac 100644 +index 044e264d2..6d813f172 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi -@@ -1209,7 +1209,7 @@ class frozenset(AbstractSet[_T_co]): +@@ -1210,7 +1210,7 @@ class frozenset(AbstractSet[_T_co]): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... @disjoint_base @@ -48,7 +48,7 @@ index f2dd00079..784ee7eac 100644 def __new__(cls, iterable: Iterable[_T], start: int = 0) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> tuple[int, _T]: ... -@@ -1405,7 +1405,7 @@ else: +@@ -1404,7 +1404,7 @@ else: exit: _sitebuiltins.Quitter @disjoint_base @@ -57,7 +57,7 @@ index f2dd00079..784ee7eac 100644 @overload def __new__(cls, function: None, iterable: Iterable[_T | None], /) -> Self: ... @overload -@@ -1469,7 +1469,7 @@ license: _sitebuiltins._Printer +@@ -1468,7 +1468,7 @@ license: _sitebuiltins._Printer def locals() -> dict[str, Any]: ... @disjoint_base @@ -66,7 +66,7 @@ index f2dd00079..784ee7eac 100644 # 3.14 adds `strict` argument. if sys.version_info >= (3, 14): @overload -@@ -1776,7 +1776,7 @@ def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = None) -> complex +@@ -1775,7 +1775,7 @@ def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = None) -> complex quit: _sitebuiltins.Quitter @disjoint_base @@ -75,7 +75,7 @@ index f2dd00079..784ee7eac 100644 @overload def __new__(cls, sequence: Reversible[_T], /) -> Iterator[_T]: ... # type: ignore[misc] @overload -@@ -1840,7 +1840,7 @@ def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... +@@ -1839,7 +1839,7 @@ def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... @overload def vars(object: Any = ..., /) -> dict[str, Any]: ... @disjoint_base @@ -83,7 +83,7 @@ index f2dd00079..784ee7eac 100644 +class zip(Iterator[_T_co]): if sys.version_info >= (3, 10): @overload - def __new__(cls, *, strict: bool = ...) -> zip[Any]: ... + def __new__(cls, *, strict: bool = False) -> zip[Any]: ... diff --git a/mypy/typeshed/stdlib/csv.pyi b/mypy/typeshed/stdlib/csv.pyi index 2c8e7109c..4ed0ab1d8 100644 --- a/mypy/typeshed/stdlib/csv.pyi @@ -326,5 +326,5 @@ index 6b0f1ba94..882cd143c 100644 @property def connection(self) -> Connection: ... -- -2.51.0 +2.51.1 diff --git a/mypy/typeshed/stdlib/_csv.pyi b/mypy/typeshed/stdlib/_csv.pyi index 4128178c18b3..ea90766afee6 100644 --- a/mypy/typeshed/stdlib/_csv.pyi +++ b/mypy/typeshed/stdlib/_csv.pyi @@ -92,7 +92,7 @@ else: def writerows(self, rows: Iterable[Iterable[Any]]) -> None: ... def writer( - csvfile: SupportsWrite[str], + fileobj: SupportsWrite[str], /, dialect: _DialectLike = "excel", *, @@ -106,7 +106,7 @@ def writer( strict: bool = False, ) -> _writer: ... def reader( - csvfile: Iterable[str], + iterable: Iterable[str], /, dialect: _DialectLike = "excel", *, @@ -121,7 +121,8 @@ def reader( ) -> _reader: ... def register_dialect( name: str, - dialect: type[Dialect | csv.Dialect] = ..., + /, + dialect: type[Dialect | csv.Dialect] | str = "excel", *, delimiter: str = ",", quotechar: str | None = '"', diff --git a/mypy/typeshed/stdlib/_frozen_importlib_external.pyi b/mypy/typeshed/stdlib/_frozen_importlib_external.pyi index 71642c65dc07..4778be3af1f3 100644 --- a/mypy/typeshed/stdlib/_frozen_importlib_external.pyi +++ b/mypy/typeshed/stdlib/_frozen_importlib_external.pyi @@ -100,7 +100,7 @@ class SourceLoader(_LoaderBasics): def get_source(self, fullname: str) -> str | None: ... def path_stats(self, path: str) -> Mapping[str, Any]: ... def source_to_code( - self, data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, path: ReadableBuffer | StrPath + self, data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, path: bytes | StrPath ) -> types.CodeType: ... def get_code(self, fullname: str) -> types.CodeType | None: ... @@ -109,8 +109,8 @@ class FileLoader: path: str def __init__(self, fullname: str, path: str) -> None: ... def get_data(self, path: str) -> bytes: ... - def get_filename(self, name: str | None = None) -> str: ... - def load_module(self, name: str | None = None) -> types.ModuleType: ... + def get_filename(self, fullname: str | None = None) -> str: ... + def load_module(self, fullname: str | None = None) -> types.ModuleType: ... if sys.version_info >= (3, 10): def get_resource_reader(self, name: str | None = None) -> importlib.readers.FileReader: ... else: @@ -126,7 +126,7 @@ class SourceFileLoader(importlib.abc.FileLoader, FileLoader, importlib.abc.Sourc def source_to_code( # type: ignore[override] # incompatible with InspectLoader.source_to_code self, data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, - path: ReadableBuffer | StrPath, + path: bytes | StrPath, *, _optimize: int = -1, ) -> types.CodeType: ... @@ -137,7 +137,7 @@ class SourcelessFileLoader(importlib.abc.FileLoader, FileLoader, _LoaderBasics): class ExtensionFileLoader(FileLoader, _LoaderBasics, importlib.abc.ExecutionLoader): def __init__(self, name: str, path: str) -> None: ... - def get_filename(self, name: str | None = None) -> str: ... + def get_filename(self, fullname: str | None = None) -> str: ... def get_source(self, fullname: str) -> None: ... def create_module(self, spec: ModuleSpec) -> types.ModuleType: ... def exec_module(self, module: types.ModuleType) -> None: ... diff --git a/mypy/typeshed/stdlib/_typeshed/__init__.pyi b/mypy/typeshed/stdlib/_typeshed/__init__.pyi index 25054b601a4f..b786923880e1 100644 --- a/mypy/typeshed/stdlib/_typeshed/__init__.pyi +++ b/mypy/typeshed/stdlib/_typeshed/__init__.pyi @@ -142,6 +142,9 @@ class SupportsIter(Protocol[_T_co]): class SupportsAiter(Protocol[_T_co]): def __aiter__(self) -> _T_co: ... +class SupportsLen(Protocol): + def __len__(self) -> int: ... + class SupportsLenAndGetItem(Protocol[_T_co]): def __len__(self) -> int: ... def __getitem__(self, k: int, /) -> _T_co: ... diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index d360c2ed60e5..e66e609ee664 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -1744,10 +1744,20 @@ if sys.version_info < (3, 14): _T = _TypeVar("_T", bound=AST) if sys.version_info >= (3, 13): + @overload + def parse( + source: _T, + filename: str | bytes | os.PathLike[Any] = "", + mode: Literal["exec", "eval", "func_type", "single"] = "exec", + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + optimize: Literal[-1, 0, 1, 2] = -1, + ) -> _T: ... @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any] = "", + filename: str | bytes | os.PathLike[Any] = "", mode: Literal["exec"] = "exec", *, type_comments: bool = False, @@ -1757,7 +1767,7 @@ if sys.version_info >= (3, 13): @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["eval"], *, type_comments: bool = False, @@ -1767,7 +1777,7 @@ if sys.version_info >= (3, 13): @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["func_type"], *, type_comments: bool = False, @@ -1777,7 +1787,7 @@ if sys.version_info >= (3, 13): @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["single"], *, type_comments: bool = False, @@ -1814,7 +1824,7 @@ if sys.version_info >= (3, 13): @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any] = "", + filename: str | bytes | os.PathLike[Any] = "", mode: str = "exec", *, type_comments: bool = False, @@ -1823,10 +1833,19 @@ if sys.version_info >= (3, 13): ) -> mod: ... else: + @overload + def parse( + source: _T, + filename: str | bytes | os.PathLike[Any] = "", + mode: Literal["exec", "eval", "func_type", "single"] = "exec", + *, + type_comments: bool = False, + feature_version: None | int | tuple[int, int] = None, + ) -> _T: ... @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any] = "", + filename: str | bytes | os.PathLike[Any] = "", mode: Literal["exec"] = "exec", *, type_comments: bool = False, @@ -1835,7 +1854,7 @@ else: @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["eval"], *, type_comments: bool = False, @@ -1844,7 +1863,7 @@ else: @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["func_type"], *, type_comments: bool = False, @@ -1853,7 +1872,7 @@ else: @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any], + filename: str | bytes | os.PathLike[Any], mode: Literal["single"], *, type_comments: bool = False, @@ -1886,7 +1905,7 @@ else: @overload def parse( source: str | ReadableBuffer, - filename: str | ReadableBuffer | os.PathLike[Any] = "", + filename: str | bytes | os.PathLike[Any] = "", mode: str = "exec", *, type_comments: bool = False, diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index e276441523c8..ddf81db181bf 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -225,8 +225,10 @@ class type: @classmethod def __prepare__(metacls, name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object]: ... if sys.version_info >= (3, 10): - def __or__(self, value: Any, /) -> types.UnionType: ... - def __ror__(self, value: Any, /) -> types.UnionType: ... + # `int | str` produces an instance of `UnionType`, but `int | int` produces an instance of `type`, + # and `abc.ABC | abc.ABC` produces an instance of `abc.ABCMeta`. + def __or__(self: _typeshed.Self, value: Any, /) -> types.UnionType | _typeshed.Self: ... + def __ror__(self: _typeshed.Self, value: Any, /) -> types.UnionType | _typeshed.Self: ... if sys.version_info >= (3, 12): __type_params__: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] __annotations__: dict[str, AnnotationForm] @@ -249,7 +251,7 @@ _LiteralInteger = _PositiveInteger | _NegativeInteger | Literal[0] # noqa: Y026 @disjoint_base class int: @overload - def __new__(cls, x: ConvertibleToInt = ..., /) -> Self: ... + def __new__(cls, x: ConvertibleToInt = 0, /) -> Self: ... @overload def __new__(cls, x: str | bytes | bytearray, /, base: SupportsIndex) -> Self: ... def as_integer_ratio(self) -> tuple[int, Literal[1]]: ... @@ -359,7 +361,7 @@ class int: @disjoint_base class float: - def __new__(cls, x: ConvertibleToFloat = ..., /) -> Self: ... + def __new__(cls, x: ConvertibleToFloat = 0, /) -> Self: ... def as_integer_ratio(self) -> tuple[int, int]: ... def hex(self) -> str: ... def is_integer(self) -> bool: ... @@ -429,8 +431,8 @@ class complex: @overload def __new__( cls, - real: complex | SupportsComplex | SupportsFloat | SupportsIndex = ..., - imag: complex | SupportsFloat | SupportsIndex = ..., + real: complex | SupportsComplex | SupportsFloat | SupportsIndex = 0, + imag: complex | SupportsFloat | SupportsIndex = 0, ) -> Self: ... @overload def __new__(cls, real: str | SupportsComplex | SupportsFloat | SupportsIndex | complex) -> Self: ... @@ -474,22 +476,22 @@ class _TranslateTable(Protocol): @disjoint_base class str(Sequence[str]): @overload - def __new__(cls, object: object = ...) -> Self: ... + def __new__(cls, object: object = "") -> Self: ... @overload - def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... + def __new__(cls, object: ReadableBuffer, encoding: str = "utf-8", errors: str = "strict") -> Self: ... def capitalize(self) -> str: ... # type: ignore[misc] def casefold(self) -> str: ... # type: ignore[misc] def center(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] - def count(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def count(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... def encode(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: ... def endswith( - self, suffix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, suffix: str | tuple[str, ...], start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> bool: ... def expandtabs(self, tabsize: SupportsIndex = 8) -> str: ... # type: ignore[misc] - def find(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def find(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... def format(self, *args: object, **kwargs: object) -> str: ... def format_map(self, mapping: _FormatMapMapping, /) -> str: ... - def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def index(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... def isalnum(self) -> bool: ... def isalpha(self) -> bool: ... def isascii(self) -> bool: ... @@ -514,8 +516,8 @@ class str(Sequence[str]): def removeprefix(self, prefix: str, /) -> str: ... # type: ignore[misc] def removesuffix(self, suffix: str, /) -> str: ... # type: ignore[misc] - def rfind(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... - def rindex(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... + def rfind(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... + def rindex(self, sub: str, start: SupportsIndex | None = None, end: SupportsIndex | None = None, /) -> int: ... def rjust(self, width: SupportsIndex, fillchar: str = " ", /) -> str: ... # type: ignore[misc] def rpartition(self, sep: str, /) -> tuple[str, str, str]: ... # type: ignore[misc] def rsplit(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ... # type: ignore[misc] @@ -523,7 +525,7 @@ class str(Sequence[str]): def split(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[str]: ... # type: ignore[misc] def splitlines(self, keepends: bool = False) -> list[str]: ... # type: ignore[misc] def startswith( - self, prefix: str | tuple[str, ...], start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, prefix: str | tuple[str, ...], start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> bool: ... def strip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] def swapcase(self) -> str: ... # type: ignore[misc] @@ -564,29 +566,29 @@ class bytes(Sequence[int]): @overload def __new__(cls, o: Iterable[SupportsIndex] | SupportsIndex | SupportsBytes | ReadableBuffer, /) -> Self: ... @overload - def __new__(cls, string: str, /, encoding: str, errors: str = ...) -> Self: ... + def __new__(cls, string: str, /, encoding: str, errors: str = "strict") -> Self: ... @overload def __new__(cls) -> Self: ... def capitalize(self) -> bytes: ... def center(self, width: SupportsIndex, fillchar: bytes = b" ", /) -> bytes: ... def count( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: ... def endswith( self, suffix: ReadableBuffer | tuple[ReadableBuffer, ...], - start: SupportsIndex | None = ..., - end: SupportsIndex | None = ..., + start: SupportsIndex | None = None, + end: SupportsIndex | None = None, /, ) -> bool: ... def expandtabs(self, tabsize: SupportsIndex = 8) -> bytes: ... def find( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... - def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ... + def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = 1) -> str: ... def index( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def isalnum(self) -> bool: ... def isalpha(self) -> bool: ... @@ -605,10 +607,10 @@ class bytes(Sequence[int]): def removeprefix(self, prefix: ReadableBuffer, /) -> bytes: ... def removesuffix(self, suffix: ReadableBuffer, /) -> bytes: ... def rfind( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def rindex( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def rjust(self, width: SupportsIndex, fillchar: bytes | bytearray = b" ", /) -> bytes: ... def rpartition(self, sep: ReadableBuffer, /) -> tuple[bytes, bytes, bytes]: ... @@ -619,8 +621,8 @@ class bytes(Sequence[int]): def startswith( self, prefix: ReadableBuffer | tuple[ReadableBuffer, ...], - start: SupportsIndex | None = ..., - end: SupportsIndex | None = ..., + start: SupportsIndex | None = None, + end: SupportsIndex | None = None, /, ) -> bool: ... def strip(self, bytes: ReadableBuffer | None = None, /) -> bytes: ... @@ -665,30 +667,30 @@ class bytearray(MutableSequence[int]): @overload def __init__(self, ints: Iterable[SupportsIndex] | SupportsIndex | ReadableBuffer, /) -> None: ... @overload - def __init__(self, string: str, /, encoding: str, errors: str = ...) -> None: ... + def __init__(self, string: str, /, encoding: str, errors: str = "strict") -> None: ... def append(self, item: SupportsIndex, /) -> None: ... def capitalize(self) -> bytearray: ... def center(self, width: SupportsIndex, fillchar: bytes = b" ", /) -> bytearray: ... def count( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def copy(self) -> bytearray: ... def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: ... def endswith( self, suffix: ReadableBuffer | tuple[ReadableBuffer, ...], - start: SupportsIndex | None = ..., - end: SupportsIndex | None = ..., + start: SupportsIndex | None = None, + end: SupportsIndex | None = None, /, ) -> bool: ... def expandtabs(self, tabsize: SupportsIndex = 8) -> bytearray: ... def extend(self, iterable_of_ints: Iterable[SupportsIndex], /) -> None: ... def find( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... - def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ... + def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = 1) -> str: ... def index( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def insert(self, index: SupportsIndex, item: SupportsIndex, /) -> None: ... def isalnum(self) -> bool: ... @@ -710,10 +712,10 @@ class bytearray(MutableSequence[int]): def removesuffix(self, suffix: ReadableBuffer, /) -> bytearray: ... def replace(self, old: ReadableBuffer, new: ReadableBuffer, count: SupportsIndex = -1, /) -> bytearray: ... def rfind( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def rindex( - self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., / + self, sub: ReadableBuffer | SupportsIndex, start: SupportsIndex | None = None, end: SupportsIndex | None = None, / ) -> int: ... def rjust(self, width: SupportsIndex, fillchar: bytes | bytearray = b" ", /) -> bytearray: ... def rpartition(self, sep: ReadableBuffer, /) -> tuple[bytearray, bytearray, bytearray]: ... @@ -724,8 +726,8 @@ class bytearray(MutableSequence[int]): def startswith( self, prefix: ReadableBuffer | tuple[ReadableBuffer, ...], - start: SupportsIndex | None = ..., - end: SupportsIndex | None = ..., + start: SupportsIndex | None = None, + end: SupportsIndex | None = None, /, ) -> bool: ... def strip(self, bytes: ReadableBuffer | None = None, /) -> bytearray: ... @@ -839,7 +841,7 @@ class memoryview(Sequence[_I]): def tolist(self) -> list[int]: ... def toreadonly(self) -> memoryview: ... def release(self) -> None: ... - def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = ...) -> str: ... + def hex(self, sep: str | bytes = ..., bytes_per_sep: SupportsIndex = 1) -> str: ... def __buffer__(self, flags: int, /) -> memoryview: ... def __release_buffer__(self, buffer: memoryview, /) -> None: ... @@ -852,7 +854,7 @@ class memoryview(Sequence[_I]): @final class bool(int): - def __new__(cls, o: object = ..., /) -> Self: ... + def __new__(cls, o: object = False, /) -> Self: ... # The following overloads could be represented more elegantly with a TypeVar("_B", bool, int), # however mypy has a bug regarding TypeVar constraints (https://github.com/python/mypy/issues/11880). @overload @@ -925,7 +927,7 @@ class slice(Generic[_StartT_co, _StopT_co, _StepT_co]): @disjoint_base class tuple(Sequence[_T_co]): - def __new__(cls, iterable: Iterable[_T_co] = ..., /) -> Self: ... + def __new__(cls, iterable: Iterable[_T_co] = (), /) -> Self: ... def __len__(self) -> int: ... def __contains__(self, key: object, /) -> bool: ... @overload @@ -1225,7 +1227,7 @@ class range(Sequence[int]): @overload def __new__(cls, stop: SupportsIndex, /) -> Self: ... @overload - def __new__(cls, start: SupportsIndex, stop: SupportsIndex, step: SupportsIndex = ..., /) -> Self: ... + def __new__(cls, start: SupportsIndex, stop: SupportsIndex, step: SupportsIndex = 1, /) -> Self: ... def count(self, value: int, /) -> int: ... def index(self, value: int, /) -> int: ... # type: ignore[override] def __len__(self) -> int: ... @@ -1250,10 +1252,10 @@ class property: def __init__( self, - fget: Callable[[Any], Any] | None = ..., - fset: Callable[[Any, Any], None] | None = ..., - fdel: Callable[[Any], None] | None = ..., - doc: str | None = ..., + fget: Callable[[Any], Any] | None = None, + fset: Callable[[Any, Any], None] | None = None, + fdel: Callable[[Any], None] | None = None, + doc: str | None = None, ) -> None: ... def getter(self, fget: Callable[[Any], Any], /) -> property: ... def setter(self, fset: Callable[[Any, Any], None], /) -> property: ... @@ -1301,7 +1303,7 @@ if sys.version_info >= (3, 10): @overload def compile( source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, - filename: str | ReadableBuffer | PathLike[Any], + filename: str | bytes | PathLike[Any], mode: str, flags: Literal[0], dont_inherit: bool = False, @@ -1312,7 +1314,7 @@ def compile( @overload def compile( source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, - filename: str | ReadableBuffer | PathLike[Any], + filename: str | bytes | PathLike[Any], mode: str, *, dont_inherit: bool = False, @@ -1322,7 +1324,7 @@ def compile( @overload def compile( source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, - filename: str | ReadableBuffer | PathLike[Any], + filename: str | bytes | PathLike[Any], mode: str, flags: Literal[1024], dont_inherit: bool = False, @@ -1333,7 +1335,7 @@ def compile( @overload def compile( source: str | ReadableBuffer | _ast.Module | _ast.Expression | _ast.Interactive, - filename: str | ReadableBuffer | PathLike[Any], + filename: str | bytes | PathLike[Any], mode: str, flags: int, dont_inherit: bool = False, @@ -1840,18 +1842,25 @@ def vars(object: Any = ..., /) -> dict[str, Any]: ... class zip(Iterator[_T_co]): if sys.version_info >= (3, 10): @overload - def __new__(cls, *, strict: bool = ...) -> zip[Any]: ... + def __new__(cls, *, strict: bool = False) -> zip[Any]: ... @overload - def __new__(cls, iter1: Iterable[_T1], /, *, strict: bool = ...) -> zip[tuple[_T1]]: ... + def __new__(cls, iter1: Iterable[_T1], /, *, strict: bool = False) -> zip[tuple[_T1]]: ... @overload - def __new__(cls, iter1: Iterable[_T1], iter2: Iterable[_T2], /, *, strict: bool = ...) -> zip[tuple[_T1, _T2]]: ... + def __new__(cls, iter1: Iterable[_T1], iter2: Iterable[_T2], /, *, strict: bool = False) -> zip[tuple[_T1, _T2]]: ... @overload def __new__( - cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], /, *, strict: bool = ... + cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], /, *, strict: bool = False ) -> zip[tuple[_T1, _T2, _T3]]: ... @overload def __new__( - cls, iter1: Iterable[_T1], iter2: Iterable[_T2], iter3: Iterable[_T3], iter4: Iterable[_T4], /, *, strict: bool = ... + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + /, + *, + strict: bool = False, ) -> zip[tuple[_T1, _T2, _T3, _T4]]: ... @overload def __new__( @@ -1863,7 +1872,7 @@ class zip(Iterator[_T_co]): iter5: Iterable[_T5], /, *, - strict: bool = ..., + strict: bool = False, ) -> zip[tuple[_T1, _T2, _T3, _T4, _T5]]: ... @overload def __new__( @@ -1876,7 +1885,7 @@ class zip(Iterator[_T_co]): iter6: Iterable[Any], /, *iterables: Iterable[Any], - strict: bool = ..., + strict: bool = False, ) -> zip[tuple[Any, ...]]: ... else: @overload @@ -1990,8 +1999,8 @@ class AssertionError(Exception): ... if sys.version_info >= (3, 10): @disjoint_base class AttributeError(Exception): - def __init__(self, *args: object, name: str | None = ..., obj: object = ...) -> None: ... - name: str + def __init__(self, *args: object, name: str | None = None, obj: object = None) -> None: ... + name: str | None obj: object else: @@ -2002,7 +2011,7 @@ class EOFError(Exception): ... @disjoint_base class ImportError(Exception): - def __init__(self, *args: object, name: str | None = ..., path: str | None = ...) -> None: ... + def __init__(self, *args: object, name: str | None = None, path: str | None = None) -> None: ... name: str | None path: str | None msg: str # undocumented @@ -2015,8 +2024,8 @@ class MemoryError(Exception): ... if sys.version_info >= (3, 10): @disjoint_base class NameError(Exception): - def __init__(self, *args: object, name: str | None = ...) -> None: ... - name: str + def __init__(self, *args: object, name: str | None = None) -> None: ... + name: str | None else: class NameError(Exception): ... diff --git a/mypy/typeshed/stdlib/ctypes/__init__.pyi b/mypy/typeshed/stdlib/ctypes/__init__.pyi index 9da972240abb..19bd261c67e0 100644 --- a/mypy/typeshed/stdlib/ctypes/__init__.pyi +++ b/mypy/typeshed/stdlib/ctypes/__init__.pyi @@ -23,7 +23,7 @@ from _ctypes import ( set_errno as set_errno, sizeof as sizeof, ) -from _typeshed import StrPath +from _typeshed import StrPath, SupportsBool, SupportsLen from ctypes._endian import BigEndianStructure as BigEndianStructure, LittleEndianStructure as LittleEndianStructure from types import GenericAlias from typing import Any, ClassVar, Final, Generic, Literal, TypeVar, overload, type_check_only @@ -217,7 +217,7 @@ class py_object(_CanCastTo, _SimpleCData[_T]): class c_bool(_SimpleCData[bool]): _type_: ClassVar[Literal["?"]] - def __init__(self, value: bool = ...) -> None: ... + def __init__(self, value: SupportsBool | SupportsLen | None = ...) -> None: ... class c_byte(_SimpleCData[int]): _type_: ClassVar[Literal["b"]] diff --git a/mypy/typeshed/stdlib/html/parser.pyi b/mypy/typeshed/stdlib/html/parser.pyi index 8b3fce0010b7..7edd39e8c703 100644 --- a/mypy/typeshed/stdlib/html/parser.pyi +++ b/mypy/typeshed/stdlib/html/parser.pyi @@ -1,4 +1,3 @@ -import sys from _markupbase import ParserBase from re import Pattern from typing import Final @@ -7,9 +6,8 @@ __all__ = ["HTMLParser"] class HTMLParser(ParserBase): CDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] - if sys.version_info >= (3, 13): - # Added in 3.13.6 - RCDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] + # Added in Python 3.9.23, 3.10.18, 3.11.13, 3.12.11, 3.13.6 + RCDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] def __init__(self, *, convert_charrefs: bool = True) -> None: ... def feed(self, data: str) -> None: ... @@ -32,11 +30,8 @@ class HTMLParser(ParserBase): def parse_html_declaration(self, i: int) -> int: ... # undocumented def parse_pi(self, i: int) -> int: ... # undocumented def parse_starttag(self, i: int) -> int: ... # undocumented - if sys.version_info >= (3, 13): - # `escapable` parameter added in 3.13.6 - def set_cdata_mode(self, elem: str, *, escapable: bool = False) -> None: ... # undocumented - else: - def set_cdata_mode(self, elem: str) -> None: ... # undocumented + # `escapable` parameter added in Python 3.9.23, 3.10.18, 3.11.13, 3.12.11, 3.13.6 + def set_cdata_mode(self, elem: str, *, escapable: bool = False) -> None: ... # undocumented rawdata: str # undocumented cdata_elem: str | None # undocumented convert_charrefs: bool # undocumented diff --git a/mypy/typeshed/stdlib/importlib/abc.pyi b/mypy/typeshed/stdlib/importlib/abc.pyi index 72031e0e3bd2..ef7761f7119b 100644 --- a/mypy/typeshed/stdlib/importlib/abc.pyi +++ b/mypy/typeshed/stdlib/importlib/abc.pyi @@ -53,7 +53,7 @@ class InspectLoader(Loader): def exec_module(self, module: types.ModuleType) -> None: ... @staticmethod def source_to_code( - data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, path: ReadableBuffer | StrPath = "" + data: ReadableBuffer | str | _ast.Module | _ast.Expression | _ast.Interactive, path: bytes | StrPath = "" ) -> types.CodeType: ... class ExecutionLoader(InspectLoader): @@ -114,8 +114,8 @@ class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader path: str def __init__(self, fullname: str, path: str) -> None: ... def get_data(self, path: str) -> bytes: ... - def get_filename(self, name: str | None = None) -> str: ... - def load_module(self, name: str | None = None) -> types.ModuleType: ... + def get_filename(self, fullname: str | None = None) -> str: ... + def load_module(self, fullname: str | None = None) -> types.ModuleType: ... if sys.version_info < (3, 11): class ResourceReader(metaclass=ABCMeta): diff --git a/mypy/typeshed/stdlib/importlib/resources/__init__.pyi b/mypy/typeshed/stdlib/importlib/resources/__init__.pyi index e672a619bd17..28adc37da4a4 100644 --- a/mypy/typeshed/stdlib/importlib/resources/__init__.pyi +++ b/mypy/typeshed/stdlib/importlib/resources/__init__.pyi @@ -5,7 +5,7 @@ from contextlib import AbstractContextManager from pathlib import Path from types import ModuleType from typing import Any, BinaryIO, Literal, TextIO -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated if sys.version_info >= (3, 11): from importlib.resources.abc import Traversable @@ -64,7 +64,11 @@ else: def read_text(package: Package, resource: Resource, encoding: str = "utf-8", errors: str = "strict") -> str: ... def path(package: Package, resource: Resource) -> AbstractContextManager[Path, Literal[False]]: ... def is_resource(package: Package, name: str) -> bool: ... - def contents(package: Package) -> Iterator[str]: ... + if sys.version_info >= (3, 11): + @deprecated("Deprecated since Python 3.11. Use `files(anchor).iterdir()`.") + def contents(package: Package) -> Iterator[str]: ... + else: + def contents(package: Package) -> Iterator[str]: ... if sys.version_info >= (3, 11): from importlib.resources._common import as_file as as_file diff --git a/mypy/typeshed/stdlib/importlib/resources/_functional.pyi b/mypy/typeshed/stdlib/importlib/resources/_functional.pyi index 50f3405f9a00..71e01bcd3d5e 100644 --- a/mypy/typeshed/stdlib/importlib/resources/_functional.pyi +++ b/mypy/typeshed/stdlib/importlib/resources/_functional.pyi @@ -9,7 +9,7 @@ if sys.version_info >= (3, 13): from io import TextIOWrapper from pathlib import Path from typing import BinaryIO, Literal, overload - from typing_extensions import Unpack + from typing_extensions import Unpack, deprecated def open_binary(anchor: Anchor, *path_names: StrPath) -> BinaryIO: ... @overload @@ -27,4 +27,5 @@ if sys.version_info >= (3, 13): def read_text(anchor: Anchor, *path_names: StrPath, encoding: str | None, errors: str | None = "strict") -> str: ... def path(anchor: Anchor, *path_names: StrPath) -> AbstractContextManager[Path, Literal[False]]: ... def is_resource(anchor: Anchor, *path_names: StrPath) -> bool: ... + @deprecated("Deprecated since Python 3.11. Use `files(anchor).iterdir()`.") def contents(anchor: Anchor, *path_names: StrPath) -> Iterator[str]: ... diff --git a/mypy/typeshed/stdlib/operator.pyi b/mypy/typeshed/stdlib/operator.pyi index bc2b5e026617..2f919514b0b8 100644 --- a/mypy/typeshed/stdlib/operator.pyi +++ b/mypy/typeshed/stdlib/operator.pyi @@ -205,8 +205,10 @@ class itemgetter(Generic[_T_co]): # "tuple[int, int]" is incompatible with protocol "SupportsIndex" # preventing [_T_co, ...] instead of [Any, ...] # - # A suspected mypy issue prevents using [..., _T] instead of [..., Any] here. - # https://github.com/python/mypy/issues/14032 + # If we can't infer a literal key from __new__ (ie: `itemgetter[Literal[0]]` for `itemgetter(0)`), + # then we can't annotate __call__'s return type or it'll break on tuples + # + # These issues are best demonstrated by the `itertools.check_itertools_recipes.unique_justseen` test. def __call__(self, obj: SupportsGetItem[Any, Any]) -> Any: ... @final diff --git a/mypy/typeshed/stdlib/pdb.pyi b/mypy/typeshed/stdlib/pdb.pyi index 0c16f48e2e22..2f114b20572d 100644 --- a/mypy/typeshed/stdlib/pdb.pyi +++ b/mypy/typeshed/stdlib/pdb.pyi @@ -5,6 +5,7 @@ from cmd import Cmd from collections.abc import Callable, Iterable, Mapping, Sequence from inspect import _SourceObjectType from linecache import _ModuleGlobals +from rlcompleter import Completer from types import CodeType, FrameType, TracebackType from typing import IO, Any, ClassVar, Final, Literal, TypeVar from typing_extensions import ParamSpec, Self, TypeAlias @@ -200,6 +201,10 @@ class Pdb(Bdb, Cmd): def completenames(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... # type: ignore[override] if sys.version_info >= (3, 12): def set_convenience_variable(self, frame: FrameType, name: str, value: Any) -> None: ... + if sys.version_info >= (3, 13) and sys.version_info < (3, 14): + # Added in 3.13.8. + @property + def rlcompleter(self) -> type[Completer]: ... def _select_frame(self, number: int) -> None: ... def _getval_except(self, arg: str, frame: FrameType | None = None) -> object: ... diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index b653545f1d9c..ef57faa2b009 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -3145,7 +3145,6 @@ class Scrollbar(Widget): def get(self) -> tuple[float, float, float, float] | tuple[float, float]: ... def set(self, first: float | str, last: float | str) -> None: ... -_TextIndex: TypeAlias = _tkinter.Tcl_Obj | str | float | Misc _WhatToCount: TypeAlias = Literal[ "chars", "displaychars", "displayindices", "displaylines", "indices", "lines", "xpixels", "ypixels" ] @@ -3261,20 +3260,37 @@ class Text(Widget, XView, YView): @overload def configure(self, cnf: str) -> tuple[str, str, str, Any, Any]: ... config = configure - def bbox(self, index: _TextIndex) -> tuple[int, int, int, int] | None: ... # type: ignore[override] - def compare(self, index1: _TextIndex, op: Literal["<", "<=", "==", ">=", ">", "!="], index2: _TextIndex) -> bool: ... + def bbox(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> tuple[int, int, int, int] | None: ... # type: ignore[override] + def compare( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + op: Literal["<", "<=", "==", ">=", ">", "!="], + index2: str | float | _tkinter.Tcl_Obj | Widget, + ) -> bool: ... if sys.version_info >= (3, 13): @overload - def count(self, index1: _TextIndex, index2: _TextIndex, *, return_ints: Literal[True]) -> int: ... + def count( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + *, + return_ints: Literal[True], + ) -> int: ... @overload def count( - self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], /, *, return_ints: Literal[True] + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg: _WhatToCount | Literal["update"], + /, + *, + return_ints: Literal[True], ) -> int: ... @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: Literal["update"], arg2: _WhatToCount, /, @@ -3284,8 +3300,8 @@ class Text(Widget, XView, YView): @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount, arg2: Literal["update"], /, @@ -3294,13 +3310,20 @@ class Text(Widget, XView, YView): ) -> int: ... @overload def count( - self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /, *, return_ints: Literal[True] + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg1: _WhatToCount, + arg2: _WhatToCount, + /, + *, + return_ints: Literal[True], ) -> tuple[int, int]: ... @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount | Literal["update"], arg2: _WhatToCount | Literal["update"], arg3: _WhatToCount | Literal["update"], @@ -3309,12 +3332,18 @@ class Text(Widget, XView, YView): return_ints: Literal[True], ) -> tuple[int, ...]: ... @overload - def count(self, index1: _TextIndex, index2: _TextIndex, *, return_ints: Literal[False] = False) -> tuple[int] | None: ... + def count( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + *, + return_ints: Literal[False] = False, + ) -> tuple[int] | None: ... @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg: _WhatToCount | Literal["update"], /, *, @@ -3323,8 +3352,8 @@ class Text(Widget, XView, YView): @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: Literal["update"], arg2: _WhatToCount, /, @@ -3334,8 +3363,8 @@ class Text(Widget, XView, YView): @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount, arg2: Literal["update"], /, @@ -3345,8 +3374,8 @@ class Text(Widget, XView, YView): @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount, arg2: _WhatToCount, /, @@ -3356,8 +3385,8 @@ class Text(Widget, XView, YView): @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount | Literal["update"], arg2: _WhatToCount | Literal["update"], arg3: _WhatToCount | Literal["update"], @@ -3367,22 +3396,49 @@ class Text(Widget, XView, YView): ) -> tuple[int, ...]: ... else: @overload - def count(self, index1: _TextIndex, index2: _TextIndex) -> tuple[int] | None: ... + def count( + self, index1: str | float | _tkinter.Tcl_Obj | Widget, index2: str | float | _tkinter.Tcl_Obj | Widget + ) -> tuple[int] | None: ... @overload def count( - self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], / + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg: _WhatToCount | Literal["update"], + /, ) -> tuple[int] | None: ... @overload - def count(self, index1: _TextIndex, index2: _TextIndex, arg1: Literal["update"], arg2: _WhatToCount, /) -> int | None: ... + def count( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg1: Literal["update"], + arg2: _WhatToCount, + /, + ) -> int | None: ... @overload - def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: Literal["update"], /) -> int | None: ... + def count( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg1: _WhatToCount, + arg2: Literal["update"], + /, + ) -> int | None: ... @overload - def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /) -> tuple[int, int]: ... + def count( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + arg1: _WhatToCount, + arg2: _WhatToCount, + /, + ) -> tuple[int, int]: ... @overload def count( self, - index1: _TextIndex, - index2: _TextIndex, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, arg1: _WhatToCount | Literal["update"], arg2: _WhatToCount | Literal["update"], arg3: _WhatToCount | Literal["update"], @@ -3394,13 +3450,15 @@ class Text(Widget, XView, YView): def debug(self, boolean: None = None) -> bool: ... @overload def debug(self, boolean: bool) -> None: ... - def delete(self, index1: _TextIndex, index2: _TextIndex | None = None) -> None: ... - def dlineinfo(self, index: _TextIndex) -> tuple[int, int, int, int, int] | None: ... + def delete( + self, index1: str | float | _tkinter.Tcl_Obj | Widget, index2: str | float | _tkinter.Tcl_Obj | Widget | None = None + ) -> None: ... + def dlineinfo(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> tuple[int, int, int, int, int] | None: ... @overload def dump( self, - index1: _TextIndex, - index2: _TextIndex | None = None, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None = None, command: None = None, *, all: bool = ..., @@ -3413,8 +3471,8 @@ class Text(Widget, XView, YView): @overload def dump( self, - index1: _TextIndex, - index2: _TextIndex | None, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None, command: Callable[[str, str, str], object] | str, *, all: bool = ..., @@ -3427,8 +3485,8 @@ class Text(Widget, XView, YView): @overload def dump( self, - index1: _TextIndex, - index2: _TextIndex | None = None, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None = None, *, command: Callable[[str, str, str], object] | str, all: bool = ..., @@ -3447,21 +3505,27 @@ class Text(Widget, XView, YView): def edit_reset(self) -> None: ... # actually returns empty string def edit_separator(self) -> None: ... # actually returns empty string def edit_undo(self) -> None: ... # actually returns empty string - def get(self, index1: _TextIndex, index2: _TextIndex | None = None) -> str: ... + def get( + self, index1: str | float | _tkinter.Tcl_Obj | Widget, index2: str | float | _tkinter.Tcl_Obj | Widget | None = None + ) -> str: ... @overload - def image_cget(self, index: _TextIndex, option: Literal["image", "name"]) -> str: ... + def image_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["image", "name"]) -> str: ... @overload - def image_cget(self, index: _TextIndex, option: Literal["padx", "pady"]) -> int: ... + def image_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["padx", "pady"]) -> int: ... @overload - def image_cget(self, index: _TextIndex, option: Literal["align"]) -> Literal["baseline", "bottom", "center", "top"]: ... + def image_cget( + self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["align"] + ) -> Literal["baseline", "bottom", "center", "top"]: ... @overload - def image_cget(self, index: _TextIndex, option: str) -> Any: ... + def image_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: str) -> Any: ... @overload - def image_configure(self, index: _TextIndex, cnf: str) -> tuple[str, str, str, str, str | int]: ... + def image_configure( + self, index: str | float | _tkinter.Tcl_Obj | Widget, cnf: str + ) -> tuple[str, str, str, str, str | int]: ... @overload def image_configure( self, - index: _TextIndex, + index: str | float | _tkinter.Tcl_Obj | Widget, cnf: dict[str, Any] | None = None, *, align: Literal["baseline", "bottom", "center", "top"] = ..., @@ -3472,7 +3536,7 @@ class Text(Widget, XView, YView): ) -> dict[str, tuple[str, str, str, str, str | int]] | None: ... def image_create( self, - index: _TextIndex, + index: str | float | _tkinter.Tcl_Obj | Widget, cnf: dict[str, Any] | None = {}, *, align: Literal["baseline", "bottom", "center", "top"] = ..., @@ -3482,28 +3546,36 @@ class Text(Widget, XView, YView): pady: float | str = ..., ) -> str: ... def image_names(self) -> tuple[str, ...]: ... - def index(self, index: _TextIndex) -> str: ... - def insert(self, index: _TextIndex, chars: str, *args: str | list[str] | tuple[str, ...]) -> None: ... + def index(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> str: ... + def insert( + self, index: str | float | _tkinter.Tcl_Obj | Widget, chars: str, *args: str | list[str] | tuple[str, ...] + ) -> None: ... @overload def mark_gravity(self, markName: str, direction: None = None) -> Literal["left", "right"]: ... @overload def mark_gravity(self, markName: str, direction: Literal["left", "right"]) -> None: ... # actually returns empty string def mark_names(self) -> tuple[str, ...]: ... - def mark_set(self, markName: str, index: _TextIndex) -> None: ... + def mark_set(self, markName: str, index: str | float | _tkinter.Tcl_Obj | Widget) -> None: ... def mark_unset(self, *markNames: str) -> None: ... - def mark_next(self, index: _TextIndex) -> str | None: ... - def mark_previous(self, index: _TextIndex) -> str | None: ... + def mark_next(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> str | None: ... + def mark_previous(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> str | None: ... # **kw of peer_create is same as the kwargs of Text.__init__ def peer_create(self, newPathName: str | Text, cnf: dict[str, Any] = {}, **kw) -> None: ... def peer_names(self) -> tuple[_tkinter.Tcl_Obj, ...]: ... - def replace(self, index1: _TextIndex, index2: _TextIndex, chars: str, *args: str | list[str] | tuple[str, ...]) -> None: ... + def replace( + self, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget, + chars: str, + *args: str | list[str] | tuple[str, ...], + ) -> None: ... def scan_mark(self, x: int, y: int) -> None: ... def scan_dragto(self, x: int, y: int) -> None: ... def search( self, pattern: str, - index: _TextIndex, - stopindex: _TextIndex | None = None, + index: str | float | _tkinter.Tcl_Obj | Widget, + stopindex: str | float | _tkinter.Tcl_Obj | Widget | None = None, forwards: bool | None = None, backwards: bool | None = None, exact: bool | None = None, @@ -3512,8 +3584,10 @@ class Text(Widget, XView, YView): count: Variable | None = None, elide: bool | None = None, ) -> str: ... # returns empty string for not found - def see(self, index: _TextIndex) -> None: ... - def tag_add(self, tagName: str, index1: _TextIndex, *args: _TextIndex) -> None: ... + def see(self, index: str | float | _tkinter.Tcl_Obj | Widget) -> None: ... + def tag_add( + self, tagName: str, index1: str | float | _tkinter.Tcl_Obj | Widget, *args: str | float | _tkinter.Tcl_Obj | Widget + ) -> None: ... # tag_bind stuff is very similar to Canvas @overload def tag_bind( @@ -3568,33 +3642,50 @@ class Text(Widget, XView, YView): tag_config = tag_configure def tag_delete(self, first_tag_name: str, /, *tagNames: str) -> None: ... # error if no tag names given def tag_lower(self, tagName: str, belowThis: str | None = None) -> None: ... - def tag_names(self, index: _TextIndex | None = None) -> tuple[str, ...]: ... + def tag_names(self, index: str | float | _tkinter.Tcl_Obj | Widget | None = None) -> tuple[str, ...]: ... def tag_nextrange( - self, tagName: str, index1: _TextIndex, index2: _TextIndex | None = None + self, + tagName: str, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None = None, ) -> tuple[str, str] | tuple[()]: ... def tag_prevrange( - self, tagName: str, index1: _TextIndex, index2: _TextIndex | None = None + self, + tagName: str, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None = None, ) -> tuple[str, str] | tuple[()]: ... def tag_raise(self, tagName: str, aboveThis: str | None = None) -> None: ... def tag_ranges(self, tagName: str) -> tuple[_tkinter.Tcl_Obj, ...]: ... # tag_remove and tag_delete are different - def tag_remove(self, tagName: str, index1: _TextIndex, index2: _TextIndex | None = None) -> None: ... + def tag_remove( + self, + tagName: str, + index1: str | float | _tkinter.Tcl_Obj | Widget, + index2: str | float | _tkinter.Tcl_Obj | Widget | None = None, + ) -> None: ... @overload - def window_cget(self, index: _TextIndex, option: Literal["padx", "pady"]) -> int: ... + def window_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["padx", "pady"]) -> int: ... @overload - def window_cget(self, index: _TextIndex, option: Literal["stretch"]) -> bool: ... # actually returns Literal[0, 1] + def window_cget( + self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["stretch"] + ) -> bool: ... # actually returns Literal[0, 1] @overload - def window_cget(self, index: _TextIndex, option: Literal["align"]) -> Literal["baseline", "bottom", "center", "top"]: ... + def window_cget( + self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["align"] + ) -> Literal["baseline", "bottom", "center", "top"]: ... @overload # window is set to a widget, but read as the string name. - def window_cget(self, index: _TextIndex, option: Literal["create", "window"]) -> str: ... + def window_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: Literal["create", "window"]) -> str: ... @overload - def window_cget(self, index: _TextIndex, option: str) -> Any: ... + def window_cget(self, index: str | float | _tkinter.Tcl_Obj | Widget, option: str) -> Any: ... @overload - def window_configure(self, index: _TextIndex, cnf: str) -> tuple[str, str, str, str, str | int]: ... + def window_configure( + self, index: str | float | _tkinter.Tcl_Obj | Widget, cnf: str + ) -> tuple[str, str, str, str, str | int]: ... @overload def window_configure( self, - index: _TextIndex, + index: str | float | _tkinter.Tcl_Obj | Widget, cnf: dict[str, Any] | None = None, *, align: Literal["baseline", "bottom", "center", "top"] = ..., @@ -3607,7 +3698,7 @@ class Text(Widget, XView, YView): window_config = window_configure def window_create( self, - index: _TextIndex, + index: str | float | _tkinter.Tcl_Obj | Widget, cnf: dict[str, Any] | None = {}, *, align: Literal["baseline", "bottom", "center", "top"] = ..., diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index 39a995de2612..9b9b329bd74b 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -221,16 +221,20 @@ class Terminator(Exception): ... class TurtleGraphicsError(Exception): ... class Shape: - def __init__(self, type_: str, data: _PolygonCoords | PhotoImage | None = None) -> None: ... + def __init__( + self, type_: Literal["polygon", "image", "compound"], data: _PolygonCoords | PhotoImage | None = None + ) -> None: ... def addcomponent(self, poly: _PolygonCoords, fill: _Color, outline: _Color | None = None) -> None: ... class TurtleScreen(TurtleScreenBase): - def __init__(self, cv: Canvas, mode: str = "standard", colormode: float = 1.0, delay: int = 10) -> None: ... + def __init__( + self, cv: Canvas, mode: Literal["standard", "logo", "world"] = "standard", colormode: float = 1.0, delay: int = 10 + ) -> None: ... def clear(self) -> None: ... @overload def mode(self, mode: None = None) -> str: ... @overload - def mode(self, mode: str) -> None: ... + def mode(self, mode: Literal["standard", "logo", "world"]) -> None: ... def setworldcoordinates(self, llx: float, lly: float, urx: float, ury: float) -> None: ... def register_shape(self, name: str, shape: _PolygonCoords | Shape | None = None) -> None: ... @overload @@ -289,7 +293,7 @@ class TNavigator: DEFAULT_MODE: str DEFAULT_ANGLEOFFSET: int DEFAULT_ANGLEORIENT: int - def __init__(self, mode: str = "standard") -> None: ... + def __init__(self, mode: Literal["standard", "logo", "world"] = "standard") -> None: ... def reset(self) -> None: ... def degrees(self, fullcircle: float = 360.0) -> None: ... def radians(self) -> None: ... @@ -333,11 +337,11 @@ class TNavigator: seth = setheading class TPen: - def __init__(self, resizemode: str = "noresize") -> None: ... + def __init__(self, resizemode: Literal["auto", "user", "noresize"] = "noresize") -> None: ... @overload def resizemode(self, rmode: None = None) -> str: ... @overload - def resizemode(self, rmode: str) -> None: ... + def resizemode(self, rmode: Literal["auto", "user", "noresize"]) -> None: ... @overload def pensize(self, width: None = None) -> int: ... @overload @@ -389,7 +393,7 @@ class TPen: fillcolor: _Color = ..., pensize: int = ..., speed: int = ..., - resizemode: str = ..., + resizemode: Literal["auto", "user", "noresize"] = ..., stretchfactor: tuple[float, float] = ..., outline: int = ..., tilt: float = ..., @@ -524,7 +528,7 @@ def clear() -> None: ... @overload def mode(mode: None = None) -> str: ... @overload -def mode(mode: str) -> None: ... +def mode(mode: Literal["standard", "logo", "world"]) -> None: ... def setworldcoordinates(llx: float, lly: float, urx: float, ury: float) -> None: ... def register_shape(name: str, shape: _PolygonCoords | Shape | None = None) -> None: ... @overload @@ -634,7 +638,7 @@ seth = setheading @overload def resizemode(rmode: None = None) -> str: ... @overload -def resizemode(rmode: str) -> None: ... +def resizemode(rmode: Literal["auto", "user", "noresize"]) -> None: ... @overload def pensize(width: None = None) -> int: ... @overload @@ -683,7 +687,7 @@ def pen( fillcolor: _Color = ..., pensize: int = ..., speed: int = ..., - resizemode: str = ..., + resizemode: Literal["auto", "user", "noresize"] = ..., stretchfactor: tuple[float, float] = ..., outline: int = ..., tilt: float = ..., diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index ba343ce9effc..649e463ff71f 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -65,7 +65,7 @@ if sys.version_info >= (3, 13): _T1 = TypeVar("_T1") _T2 = TypeVar("_T2") -_KT = TypeVar("_KT") +_KT_co = TypeVar("_KT_co", covariant=True) _VT_co = TypeVar("_VT_co", covariant=True) # Make sure this class definition stays roughly in line with `builtins.function` @@ -309,27 +309,27 @@ class CodeType: __replace__ = replace @final -class MappingProxyType(Mapping[_KT, _VT_co]): +class MappingProxyType(Mapping[_KT_co, _VT_co]): # type: ignore[type-var] # pyright: ignore[reportInvalidTypeArguments] __hash__: ClassVar[None] # type: ignore[assignment] - def __new__(cls, mapping: SupportsKeysAndGetItem[_KT, _VT_co]) -> Self: ... - def __getitem__(self, key: _KT, /) -> _VT_co: ... - def __iter__(self) -> Iterator[_KT]: ... + def __new__(cls, mapping: SupportsKeysAndGetItem[_KT_co, _VT_co]) -> Self: ... + def __getitem__(self, key: _KT_co, /) -> _VT_co: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __iter__(self) -> Iterator[_KT_co]: ... def __len__(self) -> int: ... def __eq__(self, value: object, /) -> bool: ... - def copy(self) -> dict[_KT, _VT_co]: ... - def keys(self) -> KeysView[_KT]: ... + def copy(self) -> dict[_KT_co, _VT_co]: ... + def keys(self) -> KeysView[_KT_co]: ... def values(self) -> ValuesView[_VT_co]: ... - def items(self) -> ItemsView[_KT, _VT_co]: ... + def items(self) -> ItemsView[_KT_co, _VT_co]: ... @overload - def get(self, key: _KT, /) -> _VT_co | None: ... + def get(self, key: _KT_co, /) -> _VT_co | None: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] # Covariant type as parameter @overload - def get(self, key: _KT, default: _VT_co, /) -> _VT_co: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] # Covariant type as parameter + def get(self, key: _KT_co, default: _VT_co, /) -> _VT_co: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] # Covariant type as parameter @overload - def get(self, key: _KT, default: _T2, /) -> _VT_co | _T2: ... + def get(self, key: _KT_co, default: _T2, /) -> _VT_co | _T2: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] # Covariant type as parameter def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... - def __reversed__(self) -> Iterator[_KT]: ... - def __or__(self, value: Mapping[_T1, _T2], /) -> dict[_KT | _T1, _VT_co | _T2]: ... - def __ror__(self, value: Mapping[_T1, _T2], /) -> dict[_KT | _T1, _VT_co | _T2]: ... + def __reversed__(self) -> Iterator[_KT_co]: ... + def __or__(self, value: Mapping[_T1, _T2], /) -> dict[_KT_co | _T1, _VT_co | _T2]: ... + def __ror__(self, value: Mapping[_T1, _T2], /) -> dict[_KT_co | _T1, _VT_co | _T2]: ... if sys.version_info >= (3, 12): @disjoint_base diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index ca25c92d5c34..2ca65dad4562 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -1133,14 +1133,23 @@ if sys.version_info >= (3, 10): def _type_repr(obj: object) -> str: ... if sys.version_info >= (3, 12): + _TypeParameter: typing_extensions.TypeAlias = ( + TypeVar + | typing_extensions.TypeVar + | ParamSpec + | typing_extensions.ParamSpec + | TypeVarTuple + | typing_extensions.TypeVarTuple + ) + def override(method: _F, /) -> _F: ... @final class TypeAliasType: - def __new__(cls, name: str, value: Any, *, type_params: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] = ()) -> Self: ... + def __new__(cls, name: str, value: Any, *, type_params: tuple[_TypeParameter, ...] = ()) -> Self: ... @property def __value__(self) -> Any: ... # AnnotationForm @property - def __type_params__(self) -> tuple[TypeVar | ParamSpec | TypeVarTuple, ...]: ... + def __type_params__(self) -> tuple[_TypeParameter, ...]: ... @property def __parameters__(self) -> tuple[Any, ...]: ... # AnnotationForm @property diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index f5ea13f67733..5fd3f4578a8b 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -239,7 +239,7 @@ class _TypedDict(Mapping[str, object], metaclass=abc.ABCMeta): __readonly_keys__: ClassVar[frozenset[str]] __mutable_keys__: ClassVar[frozenset[str]] # PEP 728 - __closed__: ClassVar[bool] + __closed__: ClassVar[bool | None] __extra_items__: ClassVar[AnnotationForm] def copy(self) -> Self: ... # Using Never so that only calls using mypy plugin hook that specialize the signature diff --git a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi index e8f737778040..d42db1bc0c57 100644 --- a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi +++ b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi @@ -93,7 +93,7 @@ class Element(Generic[_Tag]): def __init__(self, tag: _Tag, attrib: dict[str, str] = {}, **extra: str) -> None: ... def append(self, subelement: Element[Any], /) -> None: ... def clear(self) -> None: ... - def extend(self, elements: Iterable[Element], /) -> None: ... + def extend(self, elements: Iterable[Element[Any]], /) -> None: ... def find(self, path: str, namespaces: dict[str, str] | None = None) -> Element | None: ... def findall(self, path: str, namespaces: dict[str, str] | None = None) -> list[Element]: ... @overload @@ -104,7 +104,7 @@ class Element(Generic[_Tag]): def get(self, key: str, default: None = None) -> str | None: ... @overload def get(self, key: str, default: _T) -> str | _T: ... - def insert(self, index: int, subelement: Element, /) -> None: ... + def insert(self, index: int, subelement: Element[Any], /) -> None: ... def items(self) -> ItemsView[str, str]: ... def iter(self, tag: str | None = None) -> Generator[Element, None, None]: ... @overload @@ -115,7 +115,7 @@ class Element(Generic[_Tag]): def keys(self) -> dict_keys[str, str]: ... # makeelement returns the type of self in Python impl, but not in C impl def makeelement(self, tag: _OtherTag, attrib: dict[str, str], /) -> Element[_OtherTag]: ... - def remove(self, subelement: Element, /) -> None: ... + def remove(self, subelement: Element[Any], /) -> None: ... def set(self, key: str, value: str, /) -> None: ... def __copy__(self) -> Element[_Tag]: ... # returns the type of self in Python impl, but not in C impl def __deepcopy__(self, memo: Any, /) -> Element: ... # Only exists in C impl @@ -128,15 +128,15 @@ class Element(Generic[_Tag]): # Doesn't actually exist at runtime, but instance of the class are indeed iterable due to __getitem__. def __iter__(self) -> Iterator[Element]: ... @overload - def __setitem__(self, key: SupportsIndex, value: Element, /) -> None: ... + def __setitem__(self, key: SupportsIndex, value: Element[Any], /) -> None: ... @overload - def __setitem__(self, key: slice, value: Iterable[Element], /) -> None: ... + def __setitem__(self, key: slice, value: Iterable[Element[Any]], /) -> None: ... # Doesn't really exist in earlier versions, where __len__ is called implicitly instead @deprecated("Testing an element's truth value is deprecated.") def __bool__(self) -> bool: ... -def SubElement(parent: Element, tag: str, attrib: dict[str, str] = ..., **extra: str) -> Element: ... +def SubElement(parent: Element[Any], tag: str, attrib: dict[str, str] = ..., **extra: str) -> Element: ... def Comment(text: str | None = None) -> Element[_ElementCallable]: ... def ProcessingInstruction(target: str, text: str | None = None) -> Element[_ElementCallable]: ... @@ -155,7 +155,7 @@ class QName: _Root = TypeVar("_Root", Element, Element | None, default=Element | None) class ElementTree(Generic[_Root]): - def __init__(self, element: Element | None = None, file: _FileRead | None = None) -> None: ... + def __init__(self, element: Element[Any] | None = None, file: _FileRead | None = None) -> None: ... def getroot(self) -> _Root: ... def parse(self, source: _FileRead, parser: XMLParser | None = None) -> Element: ... def iter(self, tag: str | None = None) -> Generator[Element, None, None]: ... @@ -186,7 +186,7 @@ HTML_EMPTY: Final[set[str]] def register_namespace(prefix: str, uri: str) -> None: ... @overload def tostring( - element: Element, + element: Element[Any], encoding: None = None, method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -196,7 +196,7 @@ def tostring( ) -> bytes: ... @overload def tostring( - element: Element, + element: Element[Any], encoding: Literal["unicode"], method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -206,7 +206,7 @@ def tostring( ) -> str: ... @overload def tostring( - element: Element, + element: Element[Any], encoding: str, method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -216,7 +216,7 @@ def tostring( ) -> Any: ... @overload def tostringlist( - element: Element, + element: Element[Any], encoding: None = None, method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -226,7 +226,7 @@ def tostringlist( ) -> list[bytes]: ... @overload def tostringlist( - element: Element, + element: Element[Any], encoding: Literal["unicode"], method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -236,7 +236,7 @@ def tostringlist( ) -> list[str]: ... @overload def tostringlist( - element: Element, + element: Element[Any], encoding: str, method: Literal["xml", "html", "text", "c14n"] | None = None, *, @@ -244,8 +244,8 @@ def tostringlist( default_namespace: str | None = None, short_empty_elements: bool = True, ) -> list[Any]: ... -def dump(elem: Element | ElementTree[Any]) -> None: ... -def indent(tree: Element | ElementTree[Any], space: str = " ", level: int = 0) -> None: ... +def dump(elem: Element[Any] | ElementTree[Any]) -> None: ... +def indent(tree: Element[Any] | ElementTree[Any], space: str = " ", level: int = 0) -> None: ... def parse(source: _FileRead, parser: XMLParser[Any] | None = None) -> ElementTree[Element]: ... # This class is defined inside the body of iterparse From 37333c7741fecc2980e7ec6c983d3f7ce9186cb2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 17 Oct 2025 19:16:05 +0100 Subject: [PATCH 110/183] Make metas more compact; fix indirect suppression (#20075) This makes cache metas ~twice smaller with two things: * Don't store individual options that don't require any special handling, just take a hash of them all. This behavior is disabled if `--debug-cache` is set. * Store `dep_hashes` as a list instead of a dictionary. Note that while implementing the second part, the assert I added failed, so I started digging and fond that suppression was handled inconsistency for indirect dependencies. I am fixing this here, now indirect dependencies are (un)suppressed just like any other dependency. Note this allowed to simplify/delete some parts of the code. I the next PR I am going to use fixed format serialization for `CacheMeta` (when enabled). --------- Co-authored-by: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> --- mypy/build.py | 112 ++++++++++++++++++++++-------------------------- mypy/options.py | 3 +- 2 files changed, 52 insertions(+), 63 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index f9137d8b1a32..489fcf69c22c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -363,7 +363,7 @@ class CacheMeta(NamedTuple): # dep_prios and dep_lines are in parallel with dependencies + suppressed dep_prios: list[int] dep_lines: list[int] - dep_hashes: dict[str, str] + dep_hashes: list[str] interface_hash: str # hash representing the public interface error_lines: list[str] version_id: str # mypy version for cache invalidation @@ -403,7 +403,7 @@ def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta: meta.get("options"), meta.get("dep_prios", []), meta.get("dep_lines", []), - meta.get("dep_hashes", {}), + meta.get("dep_hashes", []), meta.get("interface_hash", ""), meta.get("error_lines", []), meta.get("version_id", sentinel), @@ -1310,8 +1310,7 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str Args: id: module ID path: module path - cache_dir: cache directory - pyversion: Python version (major, minor) + options: build options Returns: A tuple with the file names to be used for the meta JSON, the @@ -1328,7 +1327,7 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str # Solve this by rewriting the paths as relative to the root dir. # This only makes sense when using the filesystem backed cache. root = _cache_dir_prefix(options) - return (os.path.relpath(pair[0], root), os.path.relpath(pair[1], root), None) + return os.path.relpath(pair[0], root), os.path.relpath(pair[1], root), None prefix = os.path.join(*id.split(".")) is_package = os.path.basename(path).startswith("__init__.py") if is_package: @@ -1341,7 +1340,20 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str data_suffix = ".data.ff" else: data_suffix = ".data.json" - return (prefix + ".meta.json", prefix + data_suffix, deps_json) + return prefix + ".meta.json", prefix + data_suffix, deps_json + + +def options_snapshot(id: str, manager: BuildManager) -> dict[str, object]: + """Make compact snapshot of options for a module. + + Separately store only the options we may compare individually, and take a hash + of everything else. If --debug-cache is specified, fall back to full snapshot. + """ + snapshot = manager.options.clone_for_module(id).select_options_affecting_cache() + if manager.options.debug_cache: + return snapshot + platform_opt = snapshot.pop("platform") + return {"platform": platform_opt, "other_options": hash_digest(json_dumps(snapshot))} def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | None: @@ -1403,7 +1415,7 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No # Ignore cache if (relevant) options aren't the same. # Note that it's fine to mutilate cached_options since it's only used here. cached_options = m.options - current_options = manager.options.clone_for_module(id).select_options_affecting_cache() + current_options = options_snapshot(id, manager) if manager.options.skip_version_check: # When we're lax about version we're also lax about platform. cached_options["platform"] = current_options["platform"] @@ -1556,7 +1568,7 @@ def validate_meta( "data_mtime": meta.data_mtime, "dependencies": meta.dependencies, "suppressed": meta.suppressed, - "options": (manager.options.clone_for_module(id).select_options_affecting_cache()), + "options": options_snapshot(id, manager), "dep_prios": meta.dep_prios, "dep_lines": meta.dep_lines, "dep_hashes": meta.dep_hashes, @@ -1701,7 +1713,6 @@ def write_cache( # updates made by inline config directives in the file. This is # important, or otherwise the options would never match when # verifying the cache. - options = manager.options.clone_for_module(id) assert source_hash is not None meta = { "id": id, @@ -1712,7 +1723,7 @@ def write_cache( "data_mtime": data_mtime, "dependencies": dependencies, "suppressed": suppressed, - "options": options.select_options_affecting_cache(), + "options": options_snapshot(id, manager), "dep_prios": dep_prios, "dep_lines": dep_lines, "interface_hash": interface_hash, @@ -2029,7 +2040,10 @@ def __init__( self.priorities = {id: pri for id, pri in zip(all_deps, self.meta.dep_prios)} assert len(all_deps) == len(self.meta.dep_lines) self.dep_line_map = {id: line for id, line in zip(all_deps, self.meta.dep_lines)} - self.dep_hashes = self.meta.dep_hashes + assert len(self.meta.dep_hashes) == len(self.meta.dependencies) + self.dep_hashes = { + k: v for (k, v) in zip(self.meta.dependencies, self.meta.dep_hashes) + } self.error_lines = self.meta.error_lines if temporary: self.load_tree(temporary=True) @@ -2346,6 +2360,7 @@ def compute_dependencies(self) -> None: self.suppressed_set = set() self.priorities = {} # id -> priority self.dep_line_map = {} # id -> line + self.dep_hashes = {} dep_entries = manager.all_imported_modules_in_file( self.tree ) + self.manager.plugin.get_additional_deps(self.tree) @@ -2433,7 +2448,7 @@ def finish_passes(self) -> None: # We should always patch indirect dependencies, even in full (non-incremental) builds, # because the cache still may be written, and it must be correct. - self._patch_indirect_dependencies( + self.patch_indirect_dependencies( # Two possible sources of indirect dependencies: # * Symbols not directly imported in this module but accessed via an attribute # or via a re-export (vast majority of these recorded in semantic analysis). @@ -2470,21 +2485,17 @@ def free_state(self) -> None: self._type_checker.reset() self._type_checker = None - def _patch_indirect_dependencies(self, module_refs: set[str], types: set[Type]) -> None: - assert None not in types - valid = self.valid_references() + def patch_indirect_dependencies(self, module_refs: set[str], types: set[Type]) -> None: + assert self.ancestors is not None + existing_deps = set(self.dependencies + self.suppressed + self.ancestors) + existing_deps.add(self.id) encountered = self.manager.indirection_detector.find_modules(types) | module_refs - extra = encountered - valid - - for dep in sorted(extra): + for dep in sorted(encountered - existing_deps): if dep not in self.manager.modules: continue - if dep not in self.suppressed_set and dep not in self.manager.missing_modules: - self.add_dependency(dep) - self.priorities[dep] = PRI_INDIRECT - elif dep not in self.suppressed_set and dep in self.manager.missing_modules: - self.suppress_dependency(dep) + self.add_dependency(dep) + self.priorities[dep] = PRI_INDIRECT def compute_fine_grained_deps(self) -> dict[str, set[str]]: assert self.tree is not None @@ -2514,16 +2525,6 @@ def update_fine_grained_deps(self, deps: dict[str, set[str]]) -> None: merge_dependencies(self.compute_fine_grained_deps(), deps) type_state.update_protocol_deps(deps) - def valid_references(self) -> set[str]: - assert self.ancestors is not None - valid_refs = set(self.dependencies + self.suppressed + self.ancestors) - valid_refs.add(self.id) - - if "os" in valid_refs: - valid_refs.add("os.path") - - return valid_refs - def write_cache(self) -> tuple[dict[str, Any], str] | None: assert self.tree is not None, "Internal error: method must be called on parsed file only" # We don't support writing cache files in fine-grained incremental mode. @@ -2577,14 +2578,16 @@ def verify_dependencies(self, suppressed_only: bool = False) -> None: """ manager = self.manager assert self.ancestors is not None + # Strip out indirect dependencies. See comment in build.load_graph(). if suppressed_only: - all_deps = self.suppressed + all_deps = [dep for dep in self.suppressed if self.priorities.get(dep) != PRI_INDIRECT] else: - # Strip out indirect dependencies. See comment in build.load_graph(). dependencies = [ - dep for dep in self.dependencies if self.priorities.get(dep) != PRI_INDIRECT + dep + for dep in self.dependencies + self.suppressed + if self.priorities.get(dep) != PRI_INDIRECT ] - all_deps = dependencies + self.suppressed + self.ancestors + all_deps = dependencies + self.ancestors for dep in all_deps: if dep in manager.modules: continue @@ -3250,6 +3253,13 @@ def load_graph( if dep in graph and dep in st.suppressed_set: # Previously suppressed file is now visible st.add_dependency(dep) + # In the loop above we skip indirect dependencies, so to make indirect dependencies behave + # more consistently with regular ones, we suppress them manually here (when needed). + for st in graph.values(): + indirect = [dep for dep in st.dependencies if st.priorities.get(dep) == PRI_INDIRECT] + for dep in indirect: + if dep not in graph: + st.suppress_dependency(dep) manager.plugin.set_modules(manager.modules) return graph @@ -3284,8 +3294,9 @@ def find_stale_sccs( Fresh SCCs are those where: * We have valid cache files for all modules in the SCC. + * There are no changes in dependencies (files removed from/added to the build). * The interface hashes of direct dependents matches those recorded in the cache. - * There are no new (un)suppressed dependencies (files removed/added to the build). + The first and second conditions are verified by is_fresh(). """ stale_sccs = [] fresh_sccs = [] @@ -3294,34 +3305,15 @@ def find_stale_sccs( fresh = not stale_scc # Verify that interfaces of dependencies still present in graph are up-to-date (fresh). - # Note: if a dependency is not in graph anymore, it should be considered interface-stale. - # This is important to trigger any relevant updates from indirect dependencies that were - # removed in load_graph(). stale_deps = set() for id in ascc.mod_ids: for dep in graph[id].dep_hashes: - if dep not in graph: - stale_deps.add(dep) - continue - if graph[dep].interface_hash != graph[id].dep_hashes[dep]: + if dep in graph and graph[dep].interface_hash != graph[id].dep_hashes[dep]: stale_deps.add(dep) fresh = fresh and not stale_deps - undeps = set() - if fresh: - # Check if any dependencies that were suppressed according - # to the cache have been added back in this run. - # NOTE: Newly suppressed dependencies are handled by is_fresh(). - for id in ascc.mod_ids: - undeps.update(graph[id].suppressed) - undeps &= graph.keys() - if undeps: - fresh = False - if fresh: fresh_msg = "fresh" - elif undeps: - fresh_msg = f"stale due to changed suppression ({' '.join(sorted(undeps))})" elif stale_scc: fresh_msg = "inherently stale" if stale_scc != ascc.mod_ids: @@ -3563,9 +3555,7 @@ def process_stale_scc(graph: Graph, ascc: SCC, manager: BuildManager) -> None: if meta_tuple is None: continue meta, meta_json = meta_tuple - meta["dep_hashes"] = { - dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph - } + meta["dep_hashes"] = [graph[dep].interface_hash for dep in graph[id].dependencies] meta["error_lines"] = errors_by_id.get(id, []) write_cache_meta(meta, manager, meta_json) manager.done_sccs.add(ascc.id) diff --git a/mypy/options.py b/mypy/options.py index b1456934c6c9..209759763a5a 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -5,7 +5,6 @@ import sys import sysconfig import warnings -from collections.abc import Mapping from re import Pattern from typing import Any, Callable, Final @@ -621,7 +620,7 @@ def compile_glob(self, s: str) -> Pattern[str]: expr += re.escape("." + part) if part != "*" else r"(\..*)?" return re.compile(expr + "\\Z") - def select_options_affecting_cache(self) -> Mapping[str, object]: + def select_options_affecting_cache(self) -> dict[str, object]: result: dict[str, object] = {} for opt in OPTIONS_AFFECTING_CACHE: val = getattr(self, opt) From 80d00663650cec5e957f0923093473ed72f83536 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 18 Oct 2025 15:35:33 +0200 Subject: [PATCH 111/183] Run CI with Python 3.14.0 final (#20086) --- .github/workflows/test.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 47f725170bd8..de3f8877ee67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,19 +59,25 @@ jobs: toxenv: py tox_extra_args: "-n 4" test_mypyc: true - - name: Test suite with py313-windows-64 - python: '3.13' - os: windows-latest - toxenv: py - tox_extra_args: "-n 4" - - - name: Test suite with py314-dev-ubuntu - python: '3.14-dev' + - name: Test suite with py314-ubuntu, mypyc-compiled + python: '3.14' os: ubuntu-24.04-arm toxenv: py tox_extra_args: "-n 4" - # allow_failure: true test_mypyc: true + - name: Test suite with py314-windows-64 + python: '3.14' + os: windows-latest + toxenv: py + tox_extra_args: "-n 4" + + # - name: Test suite with py315-dev-ubuntu + # python: '3.15-dev' + # os: ubuntu-24.04-arm + # toxenv: py + # tox_extra_args: "-n 4" + # # allow_failure: true + # test_mypyc: true - name: mypyc runtime tests with py39-macos python: '3.9.21' From c2a82b95bd68f7f41ebbb823123a2120dd0c770c Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Sun, 19 Oct 2025 23:29:43 +0200 Subject: [PATCH 112/183] Do not emit unreachable warnings for lines that return `NotImplemented`. (#20083) I think no one has complained so far. I just encountered this (in my understanding) lack in `TypeChecker.is_noop_for_reachability` working on #20068. --- mypy/checker.py | 4 +++- test-data/unit/check-unreachable-code.test | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 754bd59a4962..b6a9bb3b22cd 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3147,6 +3147,8 @@ def is_noop_for_reachability(self, s: Statement) -> bool: """ if isinstance(s, AssertStmt) and is_false_literal(s.expr): return True + elif isinstance(s, ReturnStmt) and is_literal_not_implemented(s.expr): + return True elif isinstance(s, (RaiseStmt, PassStmt)): return True elif isinstance(s, ExpressionStmt): @@ -8281,7 +8283,7 @@ def is_literal_none(n: Expression) -> bool: return isinstance(n, NameExpr) and n.fullname == "builtins.None" -def is_literal_not_implemented(n: Expression) -> bool: +def is_literal_not_implemented(n: Expression | None) -> bool: return isinstance(n, NameExpr) and n.fullname == "builtins.NotImplemented" diff --git a/test-data/unit/check-unreachable-code.test b/test-data/unit/check-unreachable-code.test index 7e00671dfd11..34d800404903 100644 --- a/test-data/unit/check-unreachable-code.test +++ b/test-data/unit/check-unreachable-code.test @@ -1620,6 +1620,26 @@ reveal_type(bar().attr) # N: Revealed type is "Never" reveal_type(foo().attr) # N: Revealed type is "Never" 1 # E: Statement is unreachable +[case testIgnoreReturningNotImplemented] +# flags: --warn-unreachable + +class C: + def __add__(self, o: C) -> C: + if not isinstance(o, C): + return NotImplemented + return C() + def __sub__(self, o: C) -> C: + if isinstance(o, C): + return C() + return NotImplemented + def __mul__(self, o: C) -> C: + if isinstance(o, C): + return C() + else: + return NotImplemented + +[builtins fixtures/isinstance.pyi] + [case testUnreachableStatementPrettyHighlighting] # flags: --warn-unreachable --pretty def x() -> None: From 583c5f7efe5b851c140dbd2458619182137261bc Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 20 Oct 2025 20:16:52 +0200 Subject: [PATCH 113/183] Allow type parameters reusing the name missing from current module (#20081) Fixes #18507. Fixes #19526. Fixes #19946 (already closed without reproducer, but star imports may cause such issues, `testPEP695TypeVarNameClashStarImport` fails on current master). Even if a type parameter is reusing the name from outer scope, it still cannot be a redefinition, so we can safely store it even in presence of unresolved star imports or variables that do not refer to a type. --- mypy/semanal.py | 9 +++++-- test-data/unit/check-python312.test | 37 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 08f9eb03c9d7..d7b50bd09496 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1794,7 +1794,9 @@ def push_type_args( if self.is_defined_type_param(p.name): self.fail(f'"{p.name}" already defined as a type parameter', context) else: - self.add_symbol(p.name, tv, context, no_progress=True, type_param=True) + assert self.add_symbol( + p.name, tv, context, no_progress=True, type_param=True + ), "Type parameter should not be discarded" return tvs @@ -6830,6 +6832,7 @@ def add_symbol_table_node( else: # see note in docstring describing None contexts self.defer() + if ( existing is not None and context is not None @@ -6849,7 +6852,9 @@ def add_symbol_table_node( self.add_redefinition(names, name, symbol) if not (isinstance(new, (FuncDef, Decorator)) and self.set_original_def(old, new)): self.name_already_defined(name, context, existing) - elif name not in self.missing_names[-1] and "*" not in self.missing_names[-1]: + elif type_param or ( + name not in self.missing_names[-1] and "*" not in self.missing_names[-1] + ): names[name] = symbol if not no_progress: self.progress = True diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 6a72a7e4d5b5..be46ff6ee5c0 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1228,6 +1228,43 @@ class C[T]: def f[S, S](x: S) -> S: # E: "S" already defined as a type parameter return x +[case testPEP695TypeVarNameClashNoCrashForwardReference] +# https://github.com/python/mypy/issues/18507 +from typing import TypeVar +T = TypeVar("T", bound=Foo) # E: Name "Foo" is used before definition + +class Foo: ... +class Bar[T]: ... + +[case testPEP695TypeVarNameClashNoCrashDeferredSymbol] +# https://github.com/python/mypy/issues/19526 +T = Unknown # E: Name "Unknown" is not defined + +class Foo[T]: ... +class Bar[*T]: ... +class Baz[**T]: ... +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeVarNameClashTypeAlias] +type Tb = object +type Ta[Tb] = 'B[Tb]' +class A[Ta]: ... +class B[Tb](A[Ta]): ... + +[case testPEP695TypeVarNameClashStarImport] +# Similar to +# https://github.com/python/mypy/issues/19946 +import a + +[file a.py] +from b import * +class Foo[T]: ... + +[file b.py] +from a import * +class Bar[T]: ... +[builtins fixtures/tuple.pyi] + [case testPEP695ClassDecorator] from typing import Any From b266dd1a238e867e6b135c54664a76dae5a00fcb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 20 Oct 2025 23:27:56 +0100 Subject: [PATCH 114/183] Use fixed format for cache metas (#20088) This makes cache meta files ~1.5x smaller. (I hoped together with previous diff it will give us 4x, but it is more like 3x). Implementation is mostly straightforward, here are some comments: * I make `CacheMeta` a regular class, it doesn't need to be immutable IMO (since we actually mutate it in few places). * I remove all uses of untyped dicts in favour of `CacheMeta` (note this might make JSON format slightly slower actually, but difference is below noise level) * Instead of manually checking some individual keys, I use blanket `try/except (KeyError, ValueError)` when deserializing metas. * In one place (where we update meta file after _read_), I update the loaded view to match the updated file 1:1. This should be more robust. * I still use JSON dumps for options and plugins snapshots. Serializing these using FF is tricky (and will be simpler with type tags). * I rename `data_json`/`meta_json` paths to `data_file`/`meta_file` everywhere. --- mypy-requirements.txt | 2 +- mypy/build.py | 277 ++++++++++++++---------------------- mypy/cache.py | 157 +++++++++++++++++++- mypy/util.py | 8 ++ mypyc/codegen/emitmodule.py | 3 +- pyproject.toml | 4 +- test-requirements.txt | 2 +- 7 files changed, 277 insertions(+), 176 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 229f5624e886..622a8c3f3613 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.2.1 +librt>=0.3.0 diff --git a/mypy/build.py b/mypy/build.py index 489fcf69c22c..0058fb7eaaa0 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -25,21 +25,11 @@ import time import types from collections.abc import Iterator, Mapping, Sequence, Set as AbstractSet -from typing import ( - TYPE_CHECKING, - Any, - Callable, - ClassVar, - Final, - NamedTuple, - NoReturn, - TextIO, - TypedDict, -) +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Final, NoReturn, TextIO, TypedDict from typing_extensions import TypeAlias as _TypeAlias import mypy.semanal_main -from mypy.cache import Buffer +from mypy.cache import Buffer, CacheMeta from mypy.checker import TypeChecker from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error @@ -55,6 +45,7 @@ decode_python_encoding, get_mypy_comments, hash_digest, + hash_digest_bytes, is_stub_package_file, is_sub_path_normabs, is_typeshed_file, @@ -349,28 +340,6 @@ def normpath(path: str, options: Options) -> str: return os.path.abspath(path) -class CacheMeta(NamedTuple): - id: str - path: str - mtime: int - size: int - hash: str - dependencies: list[str] # names of imported modules - data_mtime: int # mtime of data_json - data_json: str # path of .data.json - suppressed: list[str] # dependencies that weren't imported - options: dict[str, object] | None # build options - # dep_prios and dep_lines are in parallel with dependencies + suppressed - dep_prios: list[int] - dep_lines: list[int] - dep_hashes: list[str] - interface_hash: str # hash representing the public interface - error_lines: list[str] - version_id: str # mypy version for cache invalidation - ignore_all: bool # if errors were ignored - plugin_data: Any # config data from plugins - - # NOTE: dependencies + suppressed == all reachable imports; # suppressed contains those reachable imports that were prevented by # silent mode or simply not found. @@ -382,36 +351,6 @@ class FgDepMeta(TypedDict): mtime: int -def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta: - """Build a CacheMeta object from a json metadata dictionary - - Args: - meta: JSON metadata read from the metadata cache file - data_json: Path to the .data.json file containing the AST trees - """ - sentinel: Any = None # Values to be validated by the caller - return CacheMeta( - meta.get("id", sentinel), - meta.get("path", sentinel), - int(meta["mtime"]) if "mtime" in meta else sentinel, - meta.get("size", sentinel), - meta.get("hash", sentinel), - meta.get("dependencies", []), - int(meta["data_mtime"]) if "data_mtime" in meta else sentinel, - data_json, - meta.get("suppressed", []), - meta.get("options"), - meta.get("dep_prios", []), - meta.get("dep_lines", []), - meta.get("dep_hashes", []), - meta.get("interface_hash", ""), - meta.get("error_lines", []), - meta.get("version_id", sentinel), - meta.get("ignore_all", True), - meta.get("plugin_data", None), - ) - - # Priorities used for imports. (Here, top-level includes inside a class.) # These are used to determine a more predictable order in which the # nodes in an import cycle are processed. @@ -1234,7 +1173,7 @@ def _load_json_file( try: t1 = time.time() result = json_loads(data) - manager.add_stats(data_json_load_time=time.time() - t1) + manager.add_stats(data_file_load_time=time.time() - t1) except json.JSONDecodeError: manager.errors.set_file(file, None, manager.options) manager.errors.report( @@ -1313,8 +1252,8 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str options: build options Returns: - A tuple with the file names to be used for the meta JSON, the - data JSON, and the fine-grained deps JSON, respectively. + A tuple with the file names to be used for the meta file, the + data file, and the fine-grained deps JSON, respectively. """ if options.cache_map: pair = options.cache_map.get(normpath(path, options)) @@ -1338,9 +1277,11 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str deps_json = prefix + ".deps.json" if options.fixed_format_cache: data_suffix = ".data.ff" + meta_suffix = ".meta.ff" else: data_suffix = ".data.json" - return prefix + ".meta.json", prefix + data_suffix, deps_json + meta_suffix = ".meta.json" + return prefix + meta_suffix, prefix + data_suffix, deps_json def options_snapshot(id: str, manager: BuildManager) -> dict[str, object]: @@ -1369,47 +1310,50 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No valid; otherwise None. """ # TODO: May need to take more build options into account - meta_json, data_json, _ = get_cache_names(id, path, manager.options) - manager.trace(f"Looking for {id} at {meta_json}") + meta_file, data_file, _ = get_cache_names(id, path, manager.options) + manager.trace(f"Looking for {id} at {meta_file}") + meta: bytes | dict[str, Any] | None t0 = time.time() - meta = _load_json_file( - meta_json, manager, log_success=f"Meta {id} ", log_error=f"Could not load cache for {id}: " - ) + if manager.options.fixed_format_cache: + meta = _load_ff_file(meta_file, manager, log_error=f"Could not load cache for {id}: ") + if meta is None: + return None + else: + meta = _load_json_file( + meta_file, + manager, + log_success=f"Meta {id} ", + log_error=f"Could not load cache for {id}: ", + ) + if meta is None: + return None + if not isinstance(meta, dict): + manager.log( # type: ignore[unreachable] + f"Could not load cache for {id}: meta cache is not a dict: {repr(meta)}" + ) + return None t1 = time.time() - if meta is None: - return None - if not isinstance(meta, dict): - manager.log(f"Could not load cache for {id}: meta cache is not a dict: {repr(meta)}") # type: ignore[unreachable] + if isinstance(meta, bytes): + data_io = Buffer(meta) + m = CacheMeta.read(data_io, data_file) + else: + m = CacheMeta.deserialize(meta, data_file) + if m is None: + manager.log(f"Metadata abandoned for {id}: attributes are missing") return None - m = cache_meta_from_dict(meta, data_json) t2 = time.time() manager.add_stats( load_meta_time=t2 - t0, load_meta_load_time=t1 - t0, load_meta_from_dict_time=t2 - t1 ) - # Don't check for path match, that is dealt with in validate_meta(). - # - # TODO: these `type: ignore`s wouldn't be necessary - # if the type annotations for CacheMeta were more accurate - # (all of these attributes can be `None`) - if ( - m.id != id - or m.mtime is None # type: ignore[redundant-expr] - or m.size is None # type: ignore[redundant-expr] - or m.dependencies is None # type: ignore[redundant-expr] - or m.data_mtime is None - ): - manager.log(f"Metadata abandoned for {id}: attributes are missing") + # Ignore cache if generated by an older mypy version. + if m.version_id != manager.version_id and not manager.options.skip_version_check: + manager.log(f"Metadata abandoned for {id}: different mypy version") return None - # Ignore cache if generated by an older mypy version. - if ( - (m.version_id != manager.version_id and not manager.options.skip_version_check) - or m.options is None - or len(m.dependencies) + len(m.suppressed) != len(m.dep_prios) - or len(m.dependencies) + len(m.suppressed) != len(m.dep_lines) - ): - manager.log(f"Metadata abandoned for {id}: new attributes are missing") + total_deps = len(m.dependencies) + len(m.suppressed) + if len(m.dep_prios) != total_deps or len(m.dep_lines) != total_deps: + manager.log(f"Metadata abandoned for {id}: broken dependencies") return None # Ignore cache if (relevant) options aren't the same. @@ -1479,11 +1423,11 @@ def validate_meta( bazel = manager.options.bazel assert path is not None, "Internal error: meta was provided without a path" if not manager.options.skip_cache_mtime_checks: - # Check data_json; assume if its mtime matches it's good. + # Check data_file; assume if its mtime matches it's good. try: - data_mtime = manager.getmtime(meta.data_json) + data_mtime = manager.getmtime(meta.data_file) except OSError: - manager.log(f"Metadata abandoned for {id}: failed to stat data_json") + manager.log(f"Metadata abandoned for {id}: failed to stat data_file") return None if data_mtime != meta.data_mtime: manager.log(f"Metadata abandoned for {id}: data cache is modified") @@ -1534,7 +1478,8 @@ def validate_meta( qmtime, qsize, qhash = manager.quickstart_state[path] if int(qmtime) == mtime and qsize == size and qhash == meta.hash: manager.log(f"Metadata fresh (by quickstart) for {id}: file {path}") - meta = meta._replace(mtime=mtime, path=path) + meta.mtime = mtime + meta.path = path return meta t0 = time.time() @@ -1557,36 +1502,18 @@ def validate_meta( else: t0 = time.time() # Optimization: update mtime and path (otherwise, this mismatch will reappear). - meta = meta._replace(mtime=mtime, path=path) - # Construct a dict we can pass to json.dumps() (compare to write_cache()). - meta_dict = { - "id": id, - "path": path, - "mtime": mtime, - "size": size, - "hash": source_hash, - "data_mtime": meta.data_mtime, - "dependencies": meta.dependencies, - "suppressed": meta.suppressed, - "options": options_snapshot(id, manager), - "dep_prios": meta.dep_prios, - "dep_lines": meta.dep_lines, - "dep_hashes": meta.dep_hashes, - "interface_hash": meta.interface_hash, - "error_lines": meta.error_lines, - "version_id": manager.version_id, - "ignore_all": meta.ignore_all, - "plugin_data": meta.plugin_data, - } - meta_bytes = json_dumps(meta_dict, manager.options.debug_cache) - meta_json, _, _ = get_cache_names(id, path, manager.options) + meta.mtime = mtime + meta.path = path + meta.size = size + meta.options = options_snapshot(id, manager) + meta_file, _, _ = get_cache_names(id, path, manager.options) manager.log( "Updating mtime for {}: file {}, meta {}, mtime {}".format( - id, path, meta_json, meta.mtime + id, path, meta_file, meta.mtime ) ) + write_cache_meta(meta, manager, meta_file) t1 = time.time() - manager.metastore.write(meta_json, meta_bytes) # Ignore errors, just an optimization. manager.add_stats(validate_update_time=time.time() - t1, validate_munging_time=t1 - t0) return meta @@ -1612,11 +1539,11 @@ def write_cache( suppressed: list[str], dep_prios: list[int], dep_lines: list[int], - old_interface_hash: str, + old_interface_hash: bytes, source_hash: str, ignore_all: bool, manager: BuildManager, -) -> tuple[str, tuple[dict[str, Any], str] | None]: +) -> tuple[bytes, tuple[CacheMeta, str] | None]: """Write cache files for a module. Note that this mypy's behavior is still correct when any given @@ -1637,7 +1564,7 @@ def write_cache( manager: the build manager (for pyversion, log/trace) Returns: - A tuple containing the interface hash and inner tuple with cache meta JSON + A tuple containing the interface hash and inner tuple with CacheMeta that should be written and path to cache file (inner tuple may be None, if the cache data could not be written). """ @@ -1646,8 +1573,8 @@ def write_cache( bazel = manager.options.bazel # Obtain file paths. - meta_json, data_json, _ = get_cache_names(id, path, manager.options) - manager.log(f"Writing {id} {path} {meta_json} {data_json}") + meta_file, data_file, _ = get_cache_names(id, path, manager.options) + manager.log(f"Writing {id} {path} {meta_file} {data_file}") # Update tree.path so that in bazel mode it's made relative (since # sometimes paths leak out). @@ -1664,7 +1591,7 @@ def write_cache( else: data = tree.serialize() data_bytes = json_dumps(data, manager.options.debug_cache) - interface_hash = hash_digest(data_bytes + json_dumps(plugin_data)) + interface_hash = hash_digest_bytes(data_bytes + json_dumps(plugin_data)) # Obtain and set up metadata st = manager.get_stat(path) @@ -1672,7 +1599,7 @@ def write_cache( manager.log(f"Cannot get stat for {path}") # Remove apparently-invalid cache files. # (This is purely an optimization.) - for filename in [data_json, meta_json]: + for filename in [data_file, meta_file]: try: os.remove(filename) except OSError: @@ -1686,10 +1613,10 @@ def write_cache( manager.trace(f"Interface for {id} is unchanged") else: manager.trace(f"Interface for {id} has changed") - if not metastore.write(data_json, data_bytes): + if not metastore.write(data_file, data_bytes): # Most likely the error is the replace() call # (see https://github.com/python/mypy/issues/3215). - manager.log(f"Error writing data JSON file {data_json}") + manager.log(f"Error writing cache data file {data_file}") # Let's continue without writing the meta file. Analysis: # If the replace failed, we've changed nothing except left # behind an extraneous temporary file; if the replace @@ -1701,9 +1628,9 @@ def write_cache( return interface_hash, None try: - data_mtime = manager.getmtime(data_json) + data_mtime = manager.getmtime(data_file) except OSError: - manager.log(f"Error in os.stat({data_json!r}), skipping cache write") + manager.log(f"Error in os.stat({data_file!r}), skipping cache write") return interface_hash, None mtime = 0 if bazel else int(st.st_mtime) @@ -1714,35 +1641,45 @@ def write_cache( # important, or otherwise the options would never match when # verifying the cache. assert source_hash is not None - meta = { - "id": id, - "path": path, - "mtime": mtime, - "size": size, - "hash": source_hash, - "data_mtime": data_mtime, - "dependencies": dependencies, - "suppressed": suppressed, - "options": options_snapshot(id, manager), - "dep_prios": dep_prios, - "dep_lines": dep_lines, - "interface_hash": interface_hash, - "version_id": manager.version_id, - "ignore_all": ignore_all, - "plugin_data": plugin_data, - } - return interface_hash, (meta, meta_json) + meta = CacheMeta( + id=id, + path=path, + mtime=mtime, + size=size, + hash=source_hash, + dependencies=dependencies, + data_mtime=data_mtime, + data_file=data_file, + suppressed=suppressed, + options=options_snapshot(id, manager), + dep_prios=dep_prios, + dep_lines=dep_lines, + interface_hash=interface_hash, + version_id=manager.version_id, + ignore_all=ignore_all, + plugin_data=plugin_data, + # These two will be filled by the caller. + dep_hashes=[], + error_lines=[], + ) + return interface_hash, (meta, meta_file) -def write_cache_meta(meta: dict[str, Any], manager: BuildManager, meta_json: str) -> None: +def write_cache_meta(meta: CacheMeta, manager: BuildManager, meta_file: str) -> None: # Write meta cache file metastore = manager.metastore - meta_str = json_dumps(meta, manager.options.debug_cache) - if not metastore.write(meta_json, meta_str): + if manager.options.fixed_format_cache: + data_io = Buffer() + meta.write(data_io) + meta_bytes = data_io.getvalue() + else: + meta_dict = meta.serialize() + meta_bytes = json_dumps(meta_dict, manager.options.debug_cache) + if not metastore.write(meta_file, meta_bytes): # Most likely the error is the replace() call # (see https://github.com/python/mypy/issues/3215). # The next run will simply find the cache entry out of date. - manager.log(f"Error writing meta JSON file {meta_json}") + manager.log(f"Error writing cache meta file {meta_file}") """Dependency manager. @@ -1918,7 +1855,7 @@ class State: dep_line_map: dict[str, int] # Map from dependency id to its last observed interface hash - dep_hashes: dict[str, str] = {} + dep_hashes: dict[str, bytes] = {} # List of errors reported for this file last time. error_lines: list[str] = [] @@ -1933,7 +1870,7 @@ class State: caller_line = 0 # Contains a hash of the public interface in incremental mode - interface_hash: str = "" + interface_hash: bytes = b"" # Options, specialized for this file options: Options @@ -2152,10 +2089,10 @@ def load_tree(self, temporary: bool = False) -> None: data: bytes | dict[str, Any] | None if self.options.fixed_format_cache: - data = _load_ff_file(self.meta.data_json, self.manager, "Could not load tree: ") + data = _load_ff_file(self.meta.data_file, self.manager, "Could not load tree: ") else: data = _load_json_file( - self.meta.data_json, self.manager, "Load tree ", "Could not load tree: " + self.meta.data_file, self.manager, "Load tree ", "Could not load tree: " ) if data is None: return @@ -2525,7 +2462,7 @@ def update_fine_grained_deps(self, deps: dict[str, set[str]]) -> None: merge_dependencies(self.compute_fine_grained_deps(), deps) type_state.update_protocol_deps(deps) - def write_cache(self) -> tuple[dict[str, Any], str] | None: + def write_cache(self) -> tuple[CacheMeta, str] | None: assert self.tree is not None, "Internal error: method must be called on parsed file only" # We don't support writing cache files in fine-grained incremental mode. if ( @@ -3554,10 +3491,10 @@ def process_stale_scc(graph: Graph, ascc: SCC, manager: BuildManager) -> None: meta_tuple = meta_tuples[id] if meta_tuple is None: continue - meta, meta_json = meta_tuple - meta["dep_hashes"] = [graph[dep].interface_hash for dep in graph[id].dependencies] - meta["error_lines"] = errors_by_id.get(id, []) - write_cache_meta(meta, manager, meta_json) + meta, meta_file = meta_tuple + meta.dep_hashes = [graph[dep].interface_hash for dep in graph[id].dependencies] + meta.error_lines = errors_by_id.get(id, []) + write_cache_meta(meta, manager, meta_file) manager.done_sccs.add(ascc.id) diff --git a/mypy/cache.py b/mypy/cache.py index f8d3e6a05eba..aeb0e8810fd6 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -1,16 +1,18 @@ from __future__ import annotations from collections.abc import Sequence -from typing import Final +from typing import Any, Final from librt.internal import ( Buffer as Buffer, read_bool as read_bool, + read_bytes as read_bytes, read_float as read_float, read_int as read_int, read_str as read_str, read_tag as read_tag, write_bool as write_bool, + write_bytes as write_bytes, write_float as write_float, write_int as write_int, write_str as write_str, @@ -18,6 +20,148 @@ ) from mypy_extensions import u8 +from mypy.util import json_dumps, json_loads + + +class CacheMeta: + """Class representing cache metadata for a module.""" + + def __init__( + self, + *, + id: str, + path: str, + mtime: int, + size: int, + hash: str, + dependencies: list[str], + data_mtime: int, + data_file: str, + suppressed: list[str], + options: dict[str, object], + dep_prios: list[int], + dep_lines: list[int], + dep_hashes: list[bytes], + interface_hash: bytes, + error_lines: list[str], + version_id: str, + ignore_all: bool, + plugin_data: Any, + ) -> None: + self.id = id + self.path = path + self.mtime = mtime # source file mtime + self.size = size # source file size + self.hash = hash # source file hash (as a hex string for historical reasons) + self.dependencies = dependencies # names of imported modules + self.data_mtime = data_mtime # mtime of data_file + self.data_file = data_file # path of .data.json or .data.ff + self.suppressed = suppressed # dependencies that weren't imported + self.options = options # build options snapshot + # dep_prios and dep_lines are both aligned with dependencies + suppressed + self.dep_prios = dep_prios + self.dep_lines = dep_lines + # dep_hashes list is aligned with dependencies only + self.dep_hashes = dep_hashes # list of interface_hash for dependencies + self.interface_hash = interface_hash # hash representing the public interface + self.error_lines = error_lines + self.version_id = version_id # mypy version for cache invalidation + self.ignore_all = ignore_all # if errors were ignored + self.plugin_data = plugin_data # config data from plugins + + def serialize(self) -> dict[str, Any]: + return { + "id": self.id, + "path": self.path, + "mtime": self.mtime, + "size": self.size, + "hash": self.hash, + "data_mtime": self.data_mtime, + "dependencies": self.dependencies, + "suppressed": self.suppressed, + "options": self.options, + "dep_prios": self.dep_prios, + "dep_lines": self.dep_lines, + "dep_hashes": [dep.hex() for dep in self.dep_hashes], + "interface_hash": self.interface_hash.hex(), + "error_lines": self.error_lines, + "version_id": self.version_id, + "ignore_all": self.ignore_all, + "plugin_data": self.plugin_data, + } + + @classmethod + def deserialize(cls, meta: dict[str, Any], data_file: str) -> CacheMeta | None: + try: + return CacheMeta( + id=meta["id"], + path=meta["path"], + mtime=meta["mtime"], + size=meta["size"], + hash=meta["hash"], + dependencies=meta["dependencies"], + data_mtime=meta["data_mtime"], + data_file=data_file, + suppressed=meta["suppressed"], + options=meta["options"], + dep_prios=meta["dep_prios"], + dep_lines=meta["dep_lines"], + dep_hashes=[bytes.fromhex(dep) for dep in meta["dep_hashes"]], + interface_hash=bytes.fromhex(meta["interface_hash"]), + error_lines=meta["error_lines"], + version_id=meta["version_id"], + ignore_all=meta["ignore_all"], + plugin_data=meta["plugin_data"], + ) + except (KeyError, ValueError): + return None + + def write(self, data: Buffer) -> None: + write_str(data, self.id) + write_str(data, self.path) + write_int(data, self.mtime) + write_int(data, self.size) + write_str(data, self.hash) + write_str_list(data, self.dependencies) + write_int(data, self.data_mtime) + write_str_list(data, self.suppressed) + write_bytes(data, json_dumps(self.options)) + write_int_list(data, self.dep_prios) + write_int_list(data, self.dep_lines) + write_bytes_list(data, self.dep_hashes) + write_bytes(data, self.interface_hash) + write_str_list(data, self.error_lines) + write_str(data, self.version_id) + write_bool(data, self.ignore_all) + write_bytes(data, json_dumps(self.plugin_data)) + + @classmethod + def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: + try: + return CacheMeta( + id=read_str(data), + path=read_str(data), + mtime=read_int(data), + size=read_int(data), + hash=read_str(data), + dependencies=read_str_list(data), + data_mtime=read_int(data), + data_file=data_file, + suppressed=read_str_list(data), + options=json_loads(read_bytes(data)), + dep_prios=read_int_list(data), + dep_lines=read_int_list(data), + dep_hashes=read_bytes_list(data), + interface_hash=read_bytes(data), + error_lines=read_str_list(data), + version_id=read_str(data), + ignore_all=read_bool(data), + plugin_data=json_loads(read_bytes(data)), + ) + except ValueError: + return None + + # Always use this type alias to refer to type tags. Tag = u8 @@ -112,6 +256,17 @@ def write_str_list(data: Buffer, value: Sequence[str]) -> None: write_str(data, item) +def read_bytes_list(data: Buffer) -> list[bytes]: + size = read_int(data) + return [read_bytes(data) for _ in range(size)] + + +def write_bytes_list(data: Buffer, value: Sequence[bytes]) -> None: + write_int(data, len(value)) + for item in value: + write_bytes(data, item) + + def read_str_opt_list(data: Buffer) -> list[str | None]: size = read_int(data) return [read_str_opt(data) for _ in range(size)] diff --git a/mypy/util.py b/mypy/util.py index d7ff2a367fa2..c919ff87f5b0 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -569,6 +569,14 @@ def hash_digest(data: bytes) -> str: return hashlib.sha1(data).hexdigest() +def hash_digest_bytes(data: bytes) -> bytes: + """Compute a hash digest of some data. + + Similar to above but returns a bytes object. + """ + return hashlib.sha1(data).digest() + + def parse_gray_color(cup: bytes) -> str: """Reproduce a gray color in ANSI escape sequence""" assert sys.platform != "win32", "curses is not available on Windows" diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 996ec4c52b08..da60e145a790 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -341,7 +341,8 @@ def compile_ir_to_c( def get_ir_cache_name(id: str, path: str, options: Options) -> str: meta_path, _, _ = get_cache_names(id, path, options) - return meta_path.replace(".meta.json", ".ir.json") + # Mypy uses JSON cache even with --fixed-format-cache (for now). + return meta_path.replace(".meta.json", ".ir.json").replace(".meta.ff", ".ir.json") def get_state_ir_cache_name(state: State) -> str: diff --git a/pyproject.toml b/pyproject.toml index 96b05ba459b1..f9f6c01b5c1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.2.1", + "librt>=0.3.0", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.2.1", + "librt>=0.3.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index 0dc2a4cf8f18..b9ff4ffe085b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.2.3 +librt==0.3.0 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From cb0da626b15e30c0bfb0ca9f7b3867fb10fdaedb Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Thu, 23 Oct 2025 15:09:57 -0700 Subject: [PATCH 115/183] Use Pybind11 3.0.0+ (#20095) Pybind11 had a new major version bump to [3.0.0](https://github.com/pybind/pybind11/releases/tag/v3.0.0). Also fix bug in workflow to run with the new folder name. Signed-off-by: Michael Carlstrom --- .github/workflows/test_stubgenc.yml | 2 +- .../pybind11_fixtures/__init__.pyi | 11 ++-- .../pybind11_fixtures/demo.pyi | 15 ++--- .../pybind11_fixtures/__init__.pyi | 29 ++++----- .../pybind11_fixtures/demo.pyi | 63 ++++++++++--------- test-data/pybind11_fixtures/pyproject.toml | 2 +- 6 files changed, 63 insertions(+), 59 deletions(-) diff --git a/.github/workflows/test_stubgenc.yml b/.github/workflows/test_stubgenc.yml index 4676acf8695b..6cf3cb71c3ff 100644 --- a/.github/workflows/test_stubgenc.yml +++ b/.github/workflows/test_stubgenc.yml @@ -11,7 +11,7 @@ on: - 'mypy/stubgenc.py' - 'mypy/stubdoc.py' - 'mypy/stubutil.py' - - 'test-data/stubgen/**' + - 'test-data/pybind11_fixtures/**' permissions: contents: read diff --git a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi index 90afb46d6d94..c841b207c130 100644 --- a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi @@ -1,4 +1,5 @@ -import os +import pathlib +import typing from . import demo as demo from typing import overload @@ -6,12 +7,12 @@ class StaticMethods: def __init__(self, *args, **kwargs) -> None: ... @overload @staticmethod - def overloaded_static_method(value: int) -> int: ... + def overloaded_static_method(value: typing.SupportsInt) -> int: ... @overload @staticmethod - def overloaded_static_method(value: float) -> float: ... + def overloaded_static_method(value: typing.SupportsFloat) -> float: ... @staticmethod - def some_static_method(a: int, b: int) -> int: ... + def some_static_method(a: typing.SupportsInt, b: typing.SupportsInt) -> int: ... class TestStruct: field_readwrite: int @@ -23,5 +24,5 @@ class TestStruct: def func_incomplete_signature(*args, **kwargs): ... def func_returning_optional() -> int | None: ... def func_returning_pair() -> tuple[int, float]: ... -def func_returning_path() -> os.PathLike: ... +def func_returning_path() -> pathlib.Path: ... def func_returning_vector() -> list[float]: ... diff --git a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi index 87b8ec0e4ad6..09e75e1ad4aa 100644 --- a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi @@ -1,3 +1,4 @@ +import typing from typing import ClassVar, overload PI: float @@ -9,7 +10,7 @@ class Point: __entries: ClassVar[dict] = ... degree: ClassVar[Point.AngleUnit] = ... radian: ClassVar[Point.AngleUnit] = ... - def __init__(self, value: int) -> None: ... + def __init__(self, value: typing.SupportsInt) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... @@ -26,7 +27,7 @@ class Point: inch: ClassVar[Point.LengthUnit] = ... mm: ClassVar[Point.LengthUnit] = ... pixel: ClassVar[Point.LengthUnit] = ... - def __init__(self, value: int) -> None: ... + def __init__(self, value: typing.SupportsInt) -> None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... def __index__(self) -> int: ... @@ -46,16 +47,16 @@ class Point: @overload def __init__(self) -> None: ... @overload - def __init__(self, x: float, y: float) -> None: ... + def __init__(self, x: typing.SupportsFloat, y: typing.SupportsFloat) -> None: ... def as_list(self) -> list[float]: ... @overload - def distance_to(self, x: float, y: float) -> float: ... + def distance_to(self, x: typing.SupportsFloat, y: typing.SupportsFloat) -> float: ... @overload def distance_to(self, other: Point) -> float: ... @property def length(self) -> float: ... def answer() -> int: ... -def midpoint(left: float, right: float) -> float: ... -def sum(arg0: int, arg1: int) -> int: ... -def weighted_midpoint(left: float, right: float, alpha: float = ...) -> float: ... +def midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat) -> float: ... +def sum(arg0: typing.SupportsInt, arg1: typing.SupportsInt) -> int: ... +def weighted_midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat, alpha: typing.SupportsFloat = ...) -> float: ... diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi index 0eeb788d4278..701a837580b4 100644 --- a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi @@ -1,4 +1,5 @@ -import os +import pathlib +import typing from . import demo as demo from typing import overload @@ -7,27 +8,27 @@ class StaticMethods: """Initialize self. See help(type(self)) for accurate signature.""" @overload @staticmethod - def overloaded_static_method(value: int) -> int: + def overloaded_static_method(value: typing.SupportsInt) -> int: """overloaded_static_method(*args, **kwargs) Overloaded function. - 1. overloaded_static_method(value: int) -> int + 1. overloaded_static_method(value: typing.SupportsInt) -> int - 2. overloaded_static_method(value: float) -> float + 2. overloaded_static_method(value: typing.SupportsFloat) -> float """ @overload @staticmethod - def overloaded_static_method(value: float) -> float: + def overloaded_static_method(value: typing.SupportsFloat) -> float: """overloaded_static_method(*args, **kwargs) Overloaded function. - 1. overloaded_static_method(value: int) -> int + 1. overloaded_static_method(value: typing.SupportsInt) -> int - 2. overloaded_static_method(value: float) -> float + 2. overloaded_static_method(value: typing.SupportsFloat) -> float """ @staticmethod - def some_static_method(a: int, b: int) -> int: - """some_static_method(a: int, b: int) -> int + def some_static_method(a: typing.SupportsInt, b: typing.SupportsInt) -> int: + """some_static_method(a: typing.SupportsInt, b: typing.SupportsInt) -> int None """ @@ -46,10 +47,10 @@ class TestStruct: def func_incomplete_signature(*args, **kwargs): """func_incomplete_signature() -> dummy_sub_namespace::HasNoBinding""" def func_returning_optional() -> int | None: - """func_returning_optional() -> Optional[int]""" + """func_returning_optional() -> int | None""" def func_returning_pair() -> tuple[int, float]: - """func_returning_pair() -> Tuple[int, float]""" -def func_returning_path() -> os.PathLike: - """func_returning_path() -> os.PathLike""" + """func_returning_pair() -> tuple[int, float]""" +def func_returning_path() -> pathlib.Path: + """func_returning_path() -> pathlib.Path""" def func_returning_vector() -> list[float]: - """func_returning_vector() -> List[float]""" + """func_returning_vector() -> list[float]""" diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi index 6e285f202f1a..580aa2700178 100644 --- a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi @@ -1,3 +1,4 @@ +import typing from typing import ClassVar, overload PI: float @@ -14,23 +15,23 @@ class Point: __entries: ClassVar[dict] = ... degree: ClassVar[Point.AngleUnit] = ... radian: ClassVar[Point.AngleUnit] = ... - def __init__(self, value: int) -> None: - """__init__(self: pybind11_fixtures.demo.Point.AngleUnit, value: int) -> None""" + def __init__(self, value: typing.SupportsInt) -> None: + """__init__(self: pybind11_fixtures.demo.Point.AngleUnit, value: typing.SupportsInt) -> None""" def __eq__(self, other: object) -> bool: - """__eq__(self: object, other: object) -> bool""" + """__eq__(self: object, other: object, /) -> bool""" def __hash__(self) -> int: - """__hash__(self: object) -> int""" + """__hash__(self: object, /) -> int""" def __index__(self) -> int: - """__index__(self: pybind11_fixtures.demo.Point.AngleUnit) -> int""" + """__index__(self: pybind11_fixtures.demo.Point.AngleUnit, /) -> int""" def __int__(self) -> int: - """__int__(self: pybind11_fixtures.demo.Point.AngleUnit) -> int""" + """__int__(self: pybind11_fixtures.demo.Point.AngleUnit, /) -> int""" def __ne__(self, other: object) -> bool: - """__ne__(self: object, other: object) -> bool""" + """__ne__(self: object, other: object, /) -> bool""" @property def name(self) -> str: - """name(self: handle) -> str + """name(self: object, /) -> str - name(self: handle) -> str + name(self: object, /) -> str """ @property def value(self) -> int: @@ -49,23 +50,23 @@ class Point: inch: ClassVar[Point.LengthUnit] = ... mm: ClassVar[Point.LengthUnit] = ... pixel: ClassVar[Point.LengthUnit] = ... - def __init__(self, value: int) -> None: - """__init__(self: pybind11_fixtures.demo.Point.LengthUnit, value: int) -> None""" + def __init__(self, value: typing.SupportsInt) -> None: + """__init__(self: pybind11_fixtures.demo.Point.LengthUnit, value: typing.SupportsInt) -> None""" def __eq__(self, other: object) -> bool: - """__eq__(self: object, other: object) -> bool""" + """__eq__(self: object, other: object, /) -> bool""" def __hash__(self) -> int: - """__hash__(self: object) -> int""" + """__hash__(self: object, /) -> int""" def __index__(self) -> int: - """__index__(self: pybind11_fixtures.demo.Point.LengthUnit) -> int""" + """__index__(self: pybind11_fixtures.demo.Point.LengthUnit, /) -> int""" def __int__(self) -> int: - """__int__(self: pybind11_fixtures.demo.Point.LengthUnit) -> int""" + """__int__(self: pybind11_fixtures.demo.Point.LengthUnit, /) -> int""" def __ne__(self, other: object) -> bool: - """__ne__(self: object, other: object) -> bool""" + """__ne__(self: object, other: object, /) -> bool""" @property def name(self) -> str: - """name(self: handle) -> str + """name(self: object, /) -> str - name(self: handle) -> str + name(self: object, /) -> str """ @property def value(self) -> int: @@ -84,25 +85,25 @@ class Point: 1. __init__(self: pybind11_fixtures.demo.Point) -> None - 2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None + 2. __init__(self: pybind11_fixtures.demo.Point, x: typing.SupportsFloat, y: typing.SupportsFloat) -> None """ @overload - def __init__(self, x: float, y: float) -> None: + def __init__(self, x: typing.SupportsFloat, y: typing.SupportsFloat) -> None: """__init__(*args, **kwargs) Overloaded function. 1. __init__(self: pybind11_fixtures.demo.Point) -> None - 2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None + 2. __init__(self: pybind11_fixtures.demo.Point, x: typing.SupportsFloat, y: typing.SupportsFloat) -> None """ def as_list(self) -> list[float]: - """as_list(self: pybind11_fixtures.demo.Point) -> List[float]""" + """as_list(self: pybind11_fixtures.demo.Point) -> list[float]""" @overload - def distance_to(self, x: float, y: float) -> float: + def distance_to(self, x: typing.SupportsFloat, y: typing.SupportsFloat) -> float: """distance_to(*args, **kwargs) Overloaded function. - 1. distance_to(self: pybind11_fixtures.demo.Point, x: float, y: float) -> float + 1. distance_to(self: pybind11_fixtures.demo.Point, x: typing.SupportsFloat, y: typing.SupportsFloat) -> float 2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float """ @@ -111,7 +112,7 @@ class Point: """distance_to(*args, **kwargs) Overloaded function. - 1. distance_to(self: pybind11_fixtures.demo.Point, x: float, y: float) -> float + 1. distance_to(self: pybind11_fixtures.demo.Point, x: typing.SupportsFloat, y: typing.SupportsFloat) -> float 2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float """ @@ -124,12 +125,12 @@ def answer() -> int: answer docstring, with end quote" ''' -def midpoint(left: float, right: float) -> float: - """midpoint(left: float, right: float) -> float""" -def sum(arg0: int, arg1: int) -> int: - '''sum(arg0: int, arg1: int) -> int +def midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat) -> float: + """midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat) -> float""" +def sum(arg0: typing.SupportsInt, arg1: typing.SupportsInt) -> int: + '''sum(arg0: typing.SupportsInt, arg1: typing.SupportsInt) -> int multiline docstring test, edge case quotes """\'\'\' ''' -def weighted_midpoint(left: float, right: float, alpha: float = ...) -> float: - """weighted_midpoint(left: float, right: float, alpha: float = 0.5) -> float""" +def weighted_midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat, alpha: typing.SupportsFloat = ...) -> float: + """weighted_midpoint(left: typing.SupportsFloat, right: typing.SupportsFloat, alpha: typing.SupportsFloat = 0.5) -> float""" diff --git a/test-data/pybind11_fixtures/pyproject.toml b/test-data/pybind11_fixtures/pyproject.toml index 773d036e62f5..56eaa5072065 100644 --- a/test-data/pybind11_fixtures/pyproject.toml +++ b/test-data/pybind11_fixtures/pyproject.toml @@ -4,7 +4,7 @@ requires = [ "wheel", # Officially supported pybind11 version. This is pinned to guarantee 100% reproducible CI. # As a result, the version needs to be bumped manually at will. - "pybind11==2.9.2", + "pybind11==3.0.1", ] build-backend = "setuptools.build_meta" From fb16e938d12f6b24b5edf8023d4adf8e0e36abb4 Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Fri, 24 Oct 2025 00:21:48 +0200 Subject: [PATCH 116/183] [match-case] Fix narrowing of class pattern with union-argument. (#19517) Fixes #19468 Based on earlier PR #19473 - refactored the `conditional_types` function. - return `UninhabitedType(), default` when no ranges are given. This corresponds to `isinstance(x, ())`, with an empty tuple, which always returns `False` at runtime. - modified #19473 to change the proposed type, rather than directly returning. This is essential to maintain the behavior of the unit test `testIsinstanceWithOverlappingPromotionTypes` - Added special casing in `restrict_subtype_away`: if the second argument is a `TypeVar`, replace it with its upper bound (crucial to get correct result in `testNarrowSelfType`) - Allow `TypeChecker.get_isinstance_type` to return empty list (fixes `isinstance(x, ())` behavior). ## Modified tests - `testIsInstanceWithEmtpy2ndArg` now correctly infers unreachable for `isinstance(x, ())`. - `testNarrowingUnionMixins` now predicts the same results as pyright playground ## New Tests - `testMatchNarrowDownUnionUsingClassPattern` (https://mypy-play.net/?mypy=1.17.0&python=3.12&gist=e9ec514f49903022bd32a82ae1774abd) --- mypy/checker.py | 130 ++++++++++++++++----------- test-data/unit/check-isinstance.test | 3 +- test-data/unit/check-narrowing.test | 2 +- test-data/unit/check-python310.test | 16 ++++ 4 files changed, 95 insertions(+), 56 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index b6a9bb3b22cd..2dd7f10e6f35 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -7905,6 +7905,10 @@ def is_writable_attribute(self, node: Node) -> bool: return False def get_isinstance_type(self, expr: Expression) -> list[TypeRange] | None: + """Get the type(s) resulting from an isinstance check. + + Returns an empty list for isinstance(x, ()). + """ if isinstance(expr, OpExpr) and expr.op == "|": left = self.get_isinstance_type(expr.left) if left is None and is_literal_none(expr.left): @@ -7944,11 +7948,6 @@ def get_isinstance_type(self, expr: Expression) -> list[TypeRange] | None: types.append(TypeRange(typ, is_upper_bound=False)) else: # we didn't see an actual type, but rather a variable with unknown value return None - if not types: - # this can happen if someone has empty tuple as 2nd argument to isinstance - # strictly speaking, we should return UninhabitedType but for simplicity we will simply - # refuse to do any type inference for now - return None return types def is_literal_enum(self, n: Expression) -> bool: @@ -8185,59 +8184,82 @@ def conditional_types( UninhabitedType means unreachable. None means no new information can be inferred. """ - if proposed_type_ranges: - if len(proposed_type_ranges) == 1: - target = proposed_type_ranges[0].item - target = get_proper_type(target) - if isinstance(target, LiteralType) and ( - target.is_enum_literal() or isinstance(target.value, bool) - ): - enum_name = target.fallback.type.fullname - current_type = try_expanding_sum_type_to_union(current_type, enum_name) - proposed_items = [type_range.item for type_range in proposed_type_ranges] - proposed_type = make_simplified_union(proposed_items) - if isinstance(get_proper_type(current_type), AnyType): - return proposed_type, current_type - elif isinstance(proposed_type, AnyType): - # We don't really know much about the proposed type, so we shouldn't - # attempt to narrow anything. Instead, we broaden the expr to Any to - # avoid false positives - return proposed_type, default - elif not any(type_range.is_upper_bound for type_range in proposed_type_ranges) and ( - # concrete subtypes - is_proper_subtype(current_type, proposed_type, ignore_promotions=True) - # structural subtypes - or ( - ( - isinstance(proposed_type, CallableType) - or (isinstance(proposed_type, Instance) and proposed_type.type.is_protocol) - ) - and is_subtype(current_type, proposed_type, ignore_promotions=True) - ) + if proposed_type_ranges is None: + # An isinstance check, but we don't understand the type + return current_type, default + + if not proposed_type_ranges: + # This is the case for `if isinstance(x, ())` which always returns False. + return UninhabitedType(), default + + if len(proposed_type_ranges) == 1: + # expand e.g. bool -> Literal[True] | Literal[False] + target = proposed_type_ranges[0].item + target = get_proper_type(target) + if isinstance(target, LiteralType) and ( + target.is_enum_literal() or isinstance(target.value, bool) ): - # Expression is always of one of the types in proposed_type_ranges - return default, UninhabitedType() - elif not is_overlapping_types(current_type, proposed_type, ignore_promotions=True): - # Expression is never of any type in proposed_type_ranges - return UninhabitedType(), default - else: - # we can only restrict when the type is precise, not bounded - proposed_precise_type = UnionType.make_union( - [ - type_range.item - for type_range in proposed_type_ranges - if not type_range.is_upper_bound - ] - ) - remaining_type = restrict_subtype_away( - current_type, - proposed_precise_type, + enum_name = target.fallback.type.fullname + current_type = try_expanding_sum_type_to_union(current_type, enum_name) + + proper_type = get_proper_type(current_type) + # factorize over union types: isinstance(A|B, C) -> yes = A_yes | B_yes + if isinstance(proper_type, UnionType): + result: list[tuple[Type | None, Type | None]] = [ + conditional_types( + union_item, + proposed_type_ranges, + default=union_item, consider_runtime_isinstance=consider_runtime_isinstance, ) - return proposed_type, remaining_type + for union_item in get_proper_types(proper_type.items) + ] + # separate list of tuples into two lists + yes_types, no_types = zip(*result) + proposed_type = make_simplified_union([t for t in yes_types if t is not None]) else: - # An isinstance check, but we don't understand the type - return current_type, default + proposed_items = [type_range.item for type_range in proposed_type_ranges] + proposed_type = make_simplified_union(proposed_items) + + if isinstance(proper_type, AnyType): + return proposed_type, current_type + elif isinstance(proposed_type, AnyType): + # We don't really know much about the proposed type, so we shouldn't + # attempt to narrow anything. Instead, we broaden the expr to Any to + # avoid false positives + return proposed_type, default + elif not any(type_range.is_upper_bound for type_range in proposed_type_ranges) and ( + # concrete subtypes + is_proper_subtype(current_type, proposed_type, ignore_promotions=True) + # structural subtypes + or ( + ( + isinstance(proposed_type, CallableType) + or (isinstance(proposed_type, Instance) and proposed_type.type.is_protocol) + ) + and is_subtype(current_type, proposed_type, ignore_promotions=True) + ) + ): + # Expression is always of one of the types in proposed_type_ranges + return default, UninhabitedType() + elif not is_overlapping_types(current_type, proposed_type, ignore_promotions=True): + # Expression is never of any type in proposed_type_ranges + return UninhabitedType(), default + else: + # we can only restrict when the type is precise, not bounded + proposed_precise_type = UnionType.make_union( + [ + type_range.item + for type_range in proposed_type_ranges + if not type_range.is_upper_bound + ] + ) + remaining_type = restrict_subtype_away( + current_type, + proposed_precise_type, + consider_runtime_isinstance=consider_runtime_isinstance, + ) + return proposed_type, remaining_type def conditional_types_to_typemaps( diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 5043d5422108..acd4b588f98c 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -1483,11 +1483,12 @@ def f(x: Union[int, A], a: Type[A]) -> None: [builtins fixtures/isinstancelist.pyi] [case testIsInstanceWithEmtpy2ndArg] +# flags: --warn-unreachable from typing import Union def f(x: Union[int, str]) -> None: if isinstance(x, ()): - reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(x) # E: Statement is unreachable else: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [builtins fixtures/isinstancelist.pyi] diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 7fffd3ce94e5..00d33c86414f 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2639,7 +2639,7 @@ def baz(item: Base) -> None: reveal_type(item) # N: Revealed type is "Union[__main__., __main__.]" if isinstance(item, FooMixin): - reveal_type(item) # N: Revealed type is "__main__.FooMixin" + reveal_type(item) # N: Revealed type is "__main__." item.foo() else: reveal_type(item) # N: Revealed type is "__main__." diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 3a9f12e8a550..1e27e30d4b04 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -1936,6 +1936,22 @@ def union(x: str | bool) -> None: reveal_type(x) # N: Revealed type is "Union[builtins.str, Literal[False]]" [builtins fixtures/tuple.pyi] +[case testMatchNarrowDownUnionUsingClassPattern] + +class Foo: ... +class Bar(Foo): ... + +def test_1(bar: Bar) -> None: + match bar: + case Foo() as foo: + reveal_type(foo) # N: Revealed type is "__main__.Bar" + +def test_2(bar: Bar | str) -> None: + match bar: + case Foo() as foo: + reveal_type(foo) # N: Revealed type is "__main__.Bar" + + [case testMatchAssertFalseToSilenceFalsePositives] class C: a: int | str From 0ece662da24111c25345fdba5e69346bbd5c287f Mon Sep 17 00:00:00 2001 From: Randolf Scholz Date: Mon, 27 Oct 2025 17:41:46 +0100 Subject: [PATCH 117/183] [PEP 696] Fix swapping TypeVars with defaults. (#19449) - Fixes #19444. (added `testTypeVarDefaultsSwap`) - Fixes #19362 (added `testTypeVarDefaultsSwap2`) Changed the logic for recursion guards of `TypeVarType`: Instead of always substituting `repl = repl.accept(self)`, and situationally updating `repl.default = repl.default.accept(self)` if the result is a `TypeVarType`, we now always update `repl.default = repl.default.accept(self)` a priori and then only choose the expanded `repl.accept(self)` if the result is a concrete type. ## New Tests - `testTypeVarDefaultsSwap` (https://mypy-play.net/?mypy=1.17.0&python=3.12&gist=d5a025a31ae3c8b9e2a36f4738aa1991) - `testTypeVarDefaultsSwap2` (https://mypy-play.net/?mypy=1.17.0&python=3.12&gist=d3ed42c82f7144967c97d846c4c041ef) PS: closed earlier PRs #19447, since it contained debugging changes, and #19448 because it didn't solve #19362. --- mypy/expandtype.py | 6 ++-- test-data/unit/check-typevar-defaults.test | 32 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index e2a42317141f..be1c2dd91e77 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -251,9 +251,9 @@ def visit_type_var(self, t: TypeVarType) -> Type: if (tvar_id := repl.id) in self.recursive_tvar_guard: return self.recursive_tvar_guard[tvar_id] or repl self.recursive_tvar_guard[tvar_id] = None - repl = repl.accept(self) - if isinstance(repl, TypeVarType): - repl.default = repl.default.accept(self) + repl.default = repl.default.accept(self) + expanded = repl.accept(self) # Note: `expanded is repl` may be true. + repl = repl if isinstance(expanded, TypeVarType) else expanded self.recursive_tvar_guard[tvar_id] = repl return repl diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test index 22270e17787e..103c0e782797 100644 --- a/test-data/unit/check-typevar-defaults.test +++ b/test-data/unit/check-typevar-defaults.test @@ -416,6 +416,38 @@ def func_c4( reveal_type(m) # N: Revealed type is "__main__.ClassC4[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] +[case testTypeVarDefaultsSwap] +from typing import TypeVar, Generic + +T = TypeVar("T") +X = TypeVar("X", default=object) +Y = TypeVar("Y", default=object) + + +class Foo(Generic[T, Y]): + def test(self) -> None: + reveal_type( Foo[Y, T]() ) # N: Revealed type is "__main__.Foo[Y`2 = builtins.object, T`1]" + + +class Bar(Generic[X, Y]): + def test(self) -> None: + reveal_type( Bar[Y, X]() ) # N: Revealed type is "__main__.Bar[Y`2 = builtins.object, X`1 = builtins.object]" + + +[case testTypeVarDefaultsSwap2] +from typing import TypeVar, Generic + +X = TypeVar("X", default=object) +Y = TypeVar("Y", default=object) +U = TypeVar("U", default=object) +V = TypeVar("V", default=object) + +class Transform(Generic[X, Y]): + def invert(self) -> "Transform[Y, X]": ... + +class Foo(Transform[U, V], Generic[U, V]): + def invert(self) -> "Foo[V, U]": ... + [case testTypeVarDefaultsClassRecursive1] # flags: --disallow-any-generics from typing import Generic, TypeVar, List From 00df9a65e196f775d98947a2ca0c377afa4cb236 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 27 Oct 2025 11:11:41 -0600 Subject: [PATCH 118/183] Fix an INTERNAL ERROR when creating cobertura output for namespace package (#20112) Fixes https://github.com/python/mypy/issues/19843 This fixes an Internal Error where namespace packages were not supported properly. We have to use `os.path.isdir(path)` instead of trying to catch an `IsADirectoryError` exception because of a bug on Windows which causes it to throw a `PermissionError` instead in [the relevant situation](https://discuss.python.org/t/permissionerror-errno-13-permission-denied-python-2023/22360/8), which makes `except IsADirectoryError` unreliable. (We also can't just `except (IsADirectoryError, PermissionError)` because what if there is an actual permission error?) --- mypy/report.py | 5 +---- test-data/unit/reports.test | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/mypy/report.py b/mypy/report.py index 39cd80ed38bf..90a62c8a4c7c 100644 --- a/mypy/report.py +++ b/mypy/report.py @@ -141,12 +141,9 @@ def should_skip_path(path: str) -> bool: def iterate_python_lines(path: str) -> Iterator[tuple[int, str]]: """Return an iterator over (line number, line text) from a Python file.""" - try: + if not os.path.isdir(path): # can happen with namespace packages with tokenize.open(path) as input_file: yield from enumerate(input_file, 1) - except IsADirectoryError: - # can happen with namespace packages - pass class FuncCounterVisitor(TraverserVisitor): diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 82c3869bb855..714610314c88 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -547,3 +547,37 @@ namespace_packages = True + +[case testReportCoberturaCrashOnNamespacePackages] +# cmd: mypy --cobertura-xml-report report -p folder +-- Regression test for https://github.com/python/mypy/issues/19843 +[file folder/subfolder/something.py] +-- This output is not important, but due to the way tests are run we need to check it. +[outfile report/cobertura.xml] + + + $PWD + + + + + + + + + + + + + + + + + + + + + + + + From 28536b5338df20465a8bc98fd34859e0714cba46 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 27 Oct 2025 11:46:20 -0600 Subject: [PATCH 119/183] Fix IsADirectoryError for namespace packages when using --linecoverage-report (#20109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #18128. This fixes an Internal Error where namespace packages were not supported properly. This fix was inspired by @sterliakov noticing that this bug was very similar to #19843, which has a similar fix. Note that we use `os.path.isdir(tree.path)` instead of trying to catch an `IsADirectoryError` exception because of a bug on Windows which causes it to throw a `PermissionError` instead in [the relevant situation](https://discuss.python.org/t/permissionerror-errno-13-permission-denied-python-2023/22360/8), which makes `except IsADirectoryError` unreliable. (We also can't just `except (IsADirectoryError, PermissionError)` because what if there is an actual permission error?) Anyway, we just early-return — which, based on my manual testing and reading the code, seems to be the right thing to do here. --------- Co-authored-by: Ivan Levkivskyi --- mypy/report.py | 3 +++ test-data/unit/reports.test | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/mypy/report.py b/mypy/report.py index 90a62c8a4c7c..398127a33026 100644 --- a/mypy/report.py +++ b/mypy/report.py @@ -421,6 +421,9 @@ def on_file( type_map: dict[Expression, Type], options: Options, ) -> None: + if os.path.isdir(tree.path): # can happen with namespace packages + return + with open(tree.path) as f: tree_source = f.readlines() diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 714610314c88..396414575948 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -548,6 +548,14 @@ namespace_packages = True +[case testReportIsADirectoryErrorCrashOnNamespacePackages] +# cmd: mypy --linecoverage-report report -p folder +-- Regression test for https://github.com/python/mypy/issues/18128 +-- "IsADirectoryError for namespace packages when using --linecoverage-report" +[file folder/subfolder/something.py] +class Something: + pass + [case testReportCoberturaCrashOnNamespacePackages] # cmd: mypy --cobertura-xml-report report -p folder -- Regression test for https://github.com/python/mypy/issues/19843 From 31a915afe5292e25fb627a73ab94b45f23eceb02 Mon Sep 17 00:00:00 2001 From: bzoracler <50305397+bzoracler@users.noreply.github.com> Date: Tue, 28 Oct 2025 06:57:44 +1300 Subject: [PATCH 120/183] Use dummy concrete type instead of `Any` when checking protocol variance (#20110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #20108. Variance checks for protocols follow a procedure roughly equivalent to that described in [typing.python.org - Variance Inference](https://typing.python.org/en/latest/spec/generics.html#variance-inference). A major difference in mypy's current implementation is in Step 3: > Create two specialized versions of the class. We’ll refer to these as `upper` and `lower` specializations. In both of these specializations, replace all type parameters other than the one being inferred by a dummy type instance (a concrete anonymous class that is assumed to meet the bounds or constraints of the type parameter). Mypy currently uses `Any` rather than a concrete dummy type. This causes issues during overload subtype checks in the example reported in the original issue, as the specialisations when checking variance suitability of `_T2_contra` look like: ```python from typing import TypeVar, Protocol, overload _T1_contra = TypeVar("_T1_contra", contravariant=True) _T2_contra = TypeVar("_T2_contra", contravariant=True) class A(Protocol[<_T1_contra=Any>, _T2_contra]): @overload def method(self, a: <_T1_contra=Any>) -> None: ... @overload def method(self, a: _T2_contra) -> None: ... ``` This PR replaces the use of `Any` with a dummy concrete type in the entire protocol variance check to more closely follow the variance inference algorithm in the spec and fixes this overload issue. --- mypy/checker.py | 13 +++++++++---- test-data/unit/check-protocols.test | 13 +++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 2dd7f10e6f35..c1cb29652e88 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -395,6 +395,8 @@ class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi): # A helper state to produce unique temporary names on demand. _unique_id: int + # Fake concrete type used when checking variance + _variance_dummy_type: Instance | None def __init__( self, @@ -469,6 +471,7 @@ def __init__( self.pattern_checker = PatternChecker(self, self.msg, self.plugin, options) self._unique_id = 0 + self._variance_dummy_type = None @property def expr_checker(self) -> mypy.checkexpr.ExpressionChecker: @@ -2918,17 +2921,19 @@ def check_protocol_variance(self, defn: ClassDef) -> None: info = defn.info object_type = Instance(info.mro[-1], []) tvars = info.defn.type_vars + if self._variance_dummy_type is None: + _, dummy_info = self.make_fake_typeinfo("", "Dummy", "Dummy", []) + self._variance_dummy_type = Instance(dummy_info, []) + dummy = self._variance_dummy_type for i, tvar in enumerate(tvars): if not isinstance(tvar, TypeVarType): # Variance of TypeVarTuple and ParamSpec is underspecified by PEPs. continue up_args: list[Type] = [ - object_type if i == j else AnyType(TypeOfAny.special_form) - for j, _ in enumerate(tvars) + object_type if i == j else dummy.copy_modified() for j, _ in enumerate(tvars) ] down_args: list[Type] = [ - UninhabitedType() if i == j else AnyType(TypeOfAny.special_form) - for j, _ in enumerate(tvars) + UninhabitedType() if i == j else dummy.copy_modified() for j, _ in enumerate(tvars) ] up, down = Instance(info, up_args), Instance(info, down_args) # TODO: add advanced variance checks for recursive protocols diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index ae6f60355512..e7971cd5b5d8 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1373,6 +1373,19 @@ main:16: note: def meth(self, x: int) -> int main:16: note: @overload main:16: note: def meth(self, x: bytes) -> str +[case testProtocolWithMultiContravariantTypeVarOverloads] +from typing import overload, Protocol, TypeVar + +T1 = TypeVar("T1", contravariant=True) +T2 = TypeVar("T2", contravariant=True) + +class A(Protocol[T1, T2]): + @overload + def method(self, a: T1) -> None: ... + @overload + def method(self, a: T2) -> None: ... + + -- Join and meet with protocol types -- --------------------------------- From 842a8fdcaa711c92c1067d373098844eb7d1ab57 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 27 Oct 2025 12:16:15 -0600 Subject: [PATCH 121/183] Add a regression test for #15979, and fix linecount-report for Windows (#20111) https://github.com/python/mypy/pull/16019, which fixed this issue (for linux) was not accompanied by a regression test. Thus, nobody noticed that it doesn't work on Windows! This fixes an Internal Error where namespace packages were not supported properly. This fix was inspired by @sterliakov noticing that https://github.com/python/mypy/issues/18128 was very similar to #19843, which has a similar fix. Note that we use `os.path.isdir(tree.path)` instead of trying to catch an `IsADirectoryError` exception because of a bug on Windows which causes it to throw a `PermissionError` instead in [the relevant situation](https://discuss.python.org/t/permissionerror-errno-13-permission-denied-python-2023/22360/8), which makes `except IsADirectoryError` unreliable. (We also can't just `except (IsADirectoryError, PermissionError)` because what if there is an actual permission error?) --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ivan Levkivskyi --- mypy/report.py | 5 ++--- test-data/unit/reports.test | 7 +++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/mypy/report.py b/mypy/report.py index 398127a33026..4a0b965077f6 100644 --- a/mypy/report.py +++ b/mypy/report.py @@ -169,11 +169,10 @@ def on_file( ) -> None: # Count physical lines. This assumes the file's encoding is a # superset of ASCII (or at least uses \n in its line endings). - try: + if not os.path.isdir(tree.path): # can happen with namespace packages with open(tree.path, "rb") as f: physical_lines = len(f.readlines()) - except IsADirectoryError: - # can happen with namespace packages + else: physical_lines = 0 func_counter = FuncCounterVisitor() diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 396414575948..cce2f7295e3b 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -548,6 +548,13 @@ namespace_packages = True +[case testLinecountReportCrashOnNamespacePackages] +# cmd: mypy --linecount-report report -p folder +-- Regression test for https://github.com/python/mypy/issues/15979 +[file folder/subfolder/something.py] +class Something: + pass + [case testReportIsADirectoryErrorCrashOnNamespacePackages] # cmd: mypy --linecoverage-report report -p folder -- Regression test for https://github.com/python/mypy/issues/18128 From 841db1f7e537b1ac15c8866bec574c04ce125554 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 28 Oct 2025 12:57:53 +0100 Subject: [PATCH 122/183] Remember the pair in `is_overlapping_types` if at least one of them is an alias (#20127) Fixes #20107. One recursive alias is enough to trigger infinite recursion --- mypy/meet.py | 2 +- test-data/unit/check-recursive-types.test | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mypy/meet.py b/mypy/meet.py index 63305c2bb236..1cb291ff90d5 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -346,7 +346,7 @@ def is_overlapping_types( seen_types = set() elif (left, right) in seen_types: return True - if isinstance(left, TypeAliasType) and isinstance(right, TypeAliasType): + if is_recursive_pair(left, right): seen_types.add((left, right)) left, right = get_proper_types((left, right)) diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index c82111322fe1..c09f1e6b90c0 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -1024,3 +1024,19 @@ L = list[T] A = L[A] a: A = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "A") + +[case testRecursiveAliasInstanceOverlapCheck] +# flags: --warn-unreachable +from typing_extensions import TypeAlias + +OneClass: TypeAlias = 'list[OneClass]' + +class TwoClass(list['TwoClass']): + pass + +def f(obj: OneClass) -> None: + if isinstance(obj, TwoClass): + reveal_type(obj) # N: Revealed type is "__main__.TwoClass" + else: + reveal_type(obj) # N: Revealed type is "builtins.list[...]" +[builtins fixtures/isinstancelist.pyi] From 054f7212d8c79d7b8e672f1cd5f207d6f9ec7fba Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 29 Oct 2025 00:32:30 +0100 Subject: [PATCH 123/183] Discard partials remaining after inference failure (#20126) Fixes #16573. Fixes #3031. When we infer something with Never as a replacement for a partial type, we cannot ignore that result entirely: if we do that, any use of that variable down the road will still refer to `partial`, causing reachability issues and unexpected partials leaks. --- mypy/checker.py | 16 ++++-- test-data/unit/check-inference.test | 77 ++++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 9 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index c1cb29652e88..63e128f78310 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3299,9 +3299,19 @@ def check_assignment( del partial_types[var] lvalue_type = var.type else: - # Try to infer a partial type. No need to check the return value, as - # an error will be reported elsewhere. - self.infer_partial_type(lvalue_type.var, lvalue, rvalue_type) + # Try to infer a partial type. + if not self.infer_partial_type(var, lvalue, rvalue_type): + # If that also failed, give up and let the caller know that we + # cannot read their mind. The definition site will be reported later. + # Calling .put() directly because the newly inferred type is + # not a subtype of None - we are not looking for narrowing + fallback = self.inference_error_fallback_type(rvalue_type) + self.binder.put(lvalue, fallback) + # Same as self.set_inference_error_fallback_type but inlined + # to avoid computing fallback twice. + # We are replacing partial now, so the variable type + # should remain optional. + self.set_inferred_type(var, lvalue, make_optional_type(fallback)) elif ( is_literal_none(rvalue) and isinstance(lvalue, NameExpr) diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 24ea61f2c715..9ed9c5e9ec78 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -2522,7 +2522,6 @@ class C: def __init__(self) -> None: self.x = [] # E: Need type annotation for "x" (hint: "x: list[] = ...") [builtins fixtures/list.pyi] -[out] [case testNoCrashOnPartialVariable] from typing import Tuple, TypeVar @@ -2534,7 +2533,6 @@ x = None (x,) = f('') reveal_type(x) # N: Revealed type is "builtins.str" [builtins fixtures/tuple.pyi] -[out] [case testNoCrashOnPartialVariable2] # flags: --no-local-partial-types @@ -2543,11 +2541,10 @@ T = TypeVar('T', bound=str) def f() -> Tuple[T]: ... -x = None +x = None # E: Need type annotation for "x" if int(): (x,) = f() [builtins fixtures/tuple.pyi] -[out] [case testNoCrashOnPartialVariable3] from typing import Tuple, TypeVar @@ -2559,7 +2556,76 @@ x = None (x, x) = f('') reveal_type(x) # N: Revealed type is "builtins.str" [builtins fixtures/tuple.pyi] -[out] + +[case testRejectsPartialWithUninhabited] +from typing import Generic, TypeVar +T = TypeVar('T') + +class Foo(Generic[T]): ... + +def check() -> None: + x = None # E: Need type annotation for "x" + if int(): + x = Foo() + reveal_type(x) # N: Revealed type is "__main__.Foo[Any]" + reveal_type(x) # N: Revealed type is "Union[__main__.Foo[Any], None]" + +[case testRejectsPartialWithUninhabited2] +from typing import Generic, TypeVar +T = TypeVar('T') + +class Foo(Generic[T]): ... + +x = None # E: Need type annotation for "x" + +def check() -> None: + global x + x = Foo() + reveal_type(x) # N: Revealed type is "__main__.Foo[Any]" + +reveal_type(x) # N: Revealed type is "Union[__main__.Foo[Any], None]" + +[case testRejectsPartialWithUninhabited3] +# Without force-rejecting Partial, this crashes: +# https://github.com/python/mypy/issues/16573 +from typing import Generic, TypeVar +T = TypeVar('T') + +class Foo(Generic[T]): ... + +def check() -> None: + client = None # E: Need type annotation for "client" + + if client := Foo(): + reveal_type(client) # N: Revealed type is "__main__.Foo[Any]" + + reveal_type(client) # N: Revealed type is "Union[__main__.Foo[Any], None]" + + client = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "Optional[Foo[Any]]") + reveal_type(client) # N: Revealed type is "Union[__main__.Foo[Any], None]" + +[case testRejectsPartialWithUninhabitedIndependently] +from typing import Generic, TypeVar +T = TypeVar('T') + +class Foo(Generic[T]): ... + +client = None # E: Need type annotation for "client" + +def bad() -> None: + global client + client = Foo() + reveal_type(client) # N: Revealed type is "__main__.Foo[Any]" + +def good() -> None: + global client + client = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "Optional[Foo[Any]]") + reveal_type(client) # N: Revealed type is "Union[__main__.Foo[Any], None]" + +def bad2() -> None: + global client + client = Foo() + reveal_type(client) # N: Revealed type is "__main__.Foo[Any]" [case testInferenceNestedTuplesFromGenericIterable] from typing import Tuple, TypeVar @@ -2574,7 +2640,6 @@ def main() -> None: reveal_type(a) # N: Revealed type is "builtins.int" reveal_type(b) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] -[out] [case testDontMarkUnreachableAfterInferenceUninhabited] from typing import TypeVar From 16984327a45300a0563498f2823aac5c3d2b11af Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 30 Oct 2025 06:56:51 -0700 Subject: [PATCH 124/183] Sync pythoncapi_compat.h (#20143) --- mypyc/lib-rt/exc_ops.c | 4 +- mypyc/lib-rt/librt_internal.c | 4 +- mypyc/lib-rt/misc_ops.c | 4 +- mypyc/lib-rt/pythoncapi_compat.h | 407 ++++++++++++++++++++++++++++++- mypyc/lib-rt/str_ops.c | 4 +- 5 files changed, 410 insertions(+), 13 deletions(-) diff --git a/mypyc/lib-rt/exc_ops.c b/mypyc/lib-rt/exc_ops.c index d8307ecf21f8..85498420d2af 100644 --- a/mypyc/lib-rt/exc_ops.c +++ b/mypyc/lib-rt/exc_ops.c @@ -1,3 +1,5 @@ +#include "pythoncapi_compat.h" + // Exception related primitive operations // // These are registered in mypyc.primitives.exc_ops. @@ -24,7 +26,7 @@ void CPy_Reraise(void) { } void CPyErr_SetObjectAndTraceback(PyObject *type, PyObject *value, PyObject *traceback) { - if (!PyType_Check(type) && value == Py_None) { + if (!PyType_Check(type) && Py_IsNone(value)) { // The first argument must be an exception instance value = type; type = (PyObject *)Py_TYPE(value); diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 6f6a110446ad..af8dcbccaa43 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -1,3 +1,5 @@ +#include "pythoncapi_compat.h" + #define PY_SSIZE_T_CLEAN #include #include @@ -245,7 +247,7 @@ write_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname PyErr_SetString(PyExc_TypeError, "value must be a bool"); return NULL; } - if (unlikely(write_bool_internal(data, value == Py_True) == CPY_NONE_ERROR)) { + if (unlikely(write_bool_internal(data, Py_IsTrue(value)) == CPY_NONE_ERROR)) { return NULL; } Py_INCREF(Py_None); diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index ca09c347b4ff..8e5bfffba759 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1,3 +1,5 @@ +#include "pythoncapi_compat.h" + // Misc primitive operations + C helpers // // These are registered in mypyc.primitives.misc_ops. @@ -750,7 +752,7 @@ CPy_Super(PyObject *builtins, PyObject *self) { static bool import_single(PyObject *mod_id, PyObject **mod_static, PyObject *globals_id, PyObject *globals_name, PyObject *globals) { - if (*mod_static == Py_None) { + if (Py_IsNone(*mod_static)) { CPyModule *mod = PyImport_Import(mod_id); if (mod == NULL) { return false; diff --git a/mypyc/lib-rt/pythoncapi_compat.h b/mypyc/lib-rt/pythoncapi_compat.h index f94e50a3479f..b16075fcc9b8 100644 --- a/mypyc/lib-rt/pythoncapi_compat.h +++ b/mypyc/lib-rt/pythoncapi_compat.h @@ -34,16 +34,16 @@ extern "C" { # define _Py_CAST(type, expr) ((type)(expr)) #endif -#ifndef _Py_NULL // Static inline functions should use _Py_NULL rather than using directly NULL // to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, // _Py_NULL is defined as nullptr. -#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ - || (defined(__cplusplus) && __cplusplus >= 201103) -# define _Py_NULL nullptr -#else -# define _Py_NULL NULL -#endif +#ifndef _Py_NULL +# if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + || (defined(__cplusplus) && __cplusplus >= 201103) +# define _Py_NULL nullptr +# else +# define _Py_NULL NULL +# endif #endif // Cast argument to PyObject* type. @@ -1456,6 +1456,18 @@ PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, return res; } +static inline int +PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, + const char *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)strlen(str); + } + + return _PyUnicodeWriter_WriteASCIIString((_PyUnicodeWriter*)writer, + str, size); +} + static inline int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) @@ -1479,7 +1491,8 @@ PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) { if (!PyUnicode_Check(str)) { - PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + PyErr_Format(PyExc_TypeError, "expect str, not %s", + Py_TYPE(str)->tp_name); return -1; } if (start < 0 || start > end) { @@ -1978,7 +1991,7 @@ static inline int Py_fclose(FILE *file) #endif -#if 0x03090000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030E0000 && !defined(PYPY_VERSION) +#if 0x03080000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030E0000 && !defined(PYPY_VERSION) static inline PyObject* PyConfig_Get(const char *name) { @@ -2019,7 +2032,9 @@ PyConfig_Get(const char *name) PYTHONCAPI_COMPAT_SPEC(module_search_paths, WSTR_LIST, "path"), PYTHONCAPI_COMPAT_SPEC(optimization_level, UINT, _Py_NULL), PYTHONCAPI_COMPAT_SPEC(parser_debug, BOOL, _Py_NULL), +#if 0x03090000 <= PY_VERSION_HEX PYTHONCAPI_COMPAT_SPEC(platlibdir, WSTR, "platlibdir"), +#endif PYTHONCAPI_COMPAT_SPEC(prefix, WSTR_OPT, "prefix"), PYTHONCAPI_COMPAT_SPEC(pycache_prefix, WSTR_OPT, "pycache_prefix"), PYTHONCAPI_COMPAT_SPEC(quiet, BOOL, _Py_NULL), @@ -2198,6 +2213,380 @@ PyConfig_GetInt(const char *name, int *value) } #endif // PY_VERSION_HEX > 0x03090000 && !defined(PYPY_VERSION) +// gh-133144 added PyUnstable_Object_IsUniquelyReferenced() to Python 3.14.0b1. +// Adapted from _PyObject_IsUniquelyReferenced() implementation. +#if PY_VERSION_HEX < 0x030E00B0 +static inline int PyUnstable_Object_IsUniquelyReferenced(PyObject *obj) +{ +#if !defined(Py_GIL_DISABLED) + return Py_REFCNT(obj) == 1; +#else + // NOTE: the entire ob_ref_shared field must be zero, including flags, to + // ensure that other threads cannot concurrently create new references to + // this object. + return (_Py_IsOwnedByCurrentThread(obj) && + _Py_atomic_load_uint32_relaxed(&obj->ob_ref_local) == 1 && + _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared) == 0); +#endif +} +#endif + + +#if PY_VERSION_HEX < 0x030F0000 +static inline PyObject* +PySys_GetAttrString(const char *name) +{ +#if PY_VERSION_HEX >= 0x03000000 + PyObject *value = Py_XNewRef(PySys_GetObject(name)); +#else + PyObject *value = Py_XNewRef(PySys_GetObject((char*)name)); +#endif + if (value != NULL) { + return value; + } + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_RuntimeError, "lost sys.%s", name); + } + return NULL; +} + +static inline PyObject* +PySys_GetAttr(PyObject *name) +{ +#if PY_VERSION_HEX >= 0x03000000 + const char *name_str = PyUnicode_AsUTF8(name); +#else + const char *name_str = PyString_AsString(name); +#endif + if (name_str == NULL) { + return NULL; + } + + return PySys_GetAttrString(name_str); +} + +static inline int +PySys_GetOptionalAttrString(const char *name, PyObject **value) +{ +#if PY_VERSION_HEX >= 0x03000000 + *value = Py_XNewRef(PySys_GetObject(name)); +#else + *value = Py_XNewRef(PySys_GetObject((char*)name)); +#endif + if (*value != NULL) { + return 1; + } + return 0; +} + +static inline int +PySys_GetOptionalAttr(PyObject *name, PyObject **value) +{ +#if PY_VERSION_HEX >= 0x03000000 + const char *name_str = PyUnicode_AsUTF8(name); +#else + const char *name_str = PyString_AsString(name); +#endif + if (name_str == NULL) { + *value = NULL; + return -1; + } + + return PySys_GetOptionalAttrString(name_str, value); +} +#endif // PY_VERSION_HEX < 0x030F00A1 + + +#if PY_VERSION_HEX < 0x030F00A1 +typedef struct PyBytesWriter { + char small_buffer[256]; + PyObject *obj; + Py_ssize_t size; +} PyBytesWriter; + +static inline Py_ssize_t +_PyBytesWriter_GetAllocated(PyBytesWriter *writer) +{ + if (writer->obj == NULL) { + return sizeof(writer->small_buffer); + } + else { + return PyBytes_GET_SIZE(writer->obj); + } +} + + +static inline int +_PyBytesWriter_Resize_impl(PyBytesWriter *writer, Py_ssize_t size, + int resize) +{ + int overallocate = resize; + assert(size >= 0); + + if (size <= _PyBytesWriter_GetAllocated(writer)) { + return 0; + } + + if (overallocate) { +#ifdef MS_WINDOWS + /* On Windows, overallocate by 50% is the best factor */ + if (size <= (PY_SSIZE_T_MAX - size / 2)) { + size += size / 2; + } +#else + /* On Linux, overallocate by 25% is the best factor */ + if (size <= (PY_SSIZE_T_MAX - size / 4)) { + size += size / 4; + } +#endif + } + + if (writer->obj != NULL) { + if (_PyBytes_Resize(&writer->obj, size)) { + return -1; + } + assert(writer->obj != NULL); + } + else { + writer->obj = PyBytes_FromStringAndSize(NULL, size); + if (writer->obj == NULL) { + return -1; + } + + if (resize) { + assert((size_t)size > sizeof(writer->small_buffer)); + memcpy(PyBytes_AS_STRING(writer->obj), + writer->small_buffer, + sizeof(writer->small_buffer)); + } + } + return 0; +} + +static inline void* +PyBytesWriter_GetData(PyBytesWriter *writer) +{ + if (writer->obj == NULL) { + return writer->small_buffer; + } + else { + return PyBytes_AS_STRING(writer->obj); + } +} + +static inline Py_ssize_t +PyBytesWriter_GetSize(PyBytesWriter *writer) +{ + return writer->size; +} + +static inline void +PyBytesWriter_Discard(PyBytesWriter *writer) +{ + if (writer == NULL) { + return; + } + + Py_XDECREF(writer->obj); + PyMem_Free(writer); +} + +static inline PyBytesWriter* +PyBytesWriter_Create(Py_ssize_t size) +{ + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must be >= 0"); + return NULL; + } + + PyBytesWriter *writer = (PyBytesWriter*)PyMem_Malloc(sizeof(PyBytesWriter)); + if (writer == NULL) { + PyErr_NoMemory(); + return NULL; + } + + writer->obj = NULL; + writer->size = 0; + + if (size >= 1) { + if (_PyBytesWriter_Resize_impl(writer, size, 0) < 0) { + PyBytesWriter_Discard(writer); + return NULL; + } + writer->size = size; + } + return writer; +} + +static inline PyObject* +PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) +{ + PyObject *result; + if (size == 0) { + result = PyBytes_FromStringAndSize("", 0); + } + else if (writer->obj != NULL) { + if (size != PyBytes_GET_SIZE(writer->obj)) { + if (_PyBytes_Resize(&writer->obj, size)) { + goto error; + } + } + result = writer->obj; + writer->obj = NULL; + } + else { + result = PyBytes_FromStringAndSize(writer->small_buffer, size); + } + PyBytesWriter_Discard(writer); + return result; + +error: + PyBytesWriter_Discard(writer); + return NULL; +} + +static inline PyObject* +PyBytesWriter_Finish(PyBytesWriter *writer) +{ + return PyBytesWriter_FinishWithSize(writer, writer->size); +} + +static inline PyObject* +PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) +{ + Py_ssize_t size = (char*)buf - (char*)PyBytesWriter_GetData(writer); + if (size < 0 || size > _PyBytesWriter_GetAllocated(writer)) { + PyBytesWriter_Discard(writer); + PyErr_SetString(PyExc_ValueError, "invalid end pointer"); + return NULL; + } + + return PyBytesWriter_FinishWithSize(writer, size); +} + +static inline int +PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) +{ + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must be >= 0"); + return -1; + } + if (_PyBytesWriter_Resize_impl(writer, size, 1) < 0) { + return -1; + } + writer->size = size; + return 0; +} + +static inline int +PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t size) +{ + if (size < 0 && writer->size + size < 0) { + PyErr_SetString(PyExc_ValueError, "invalid size"); + return -1; + } + if (size > PY_SSIZE_T_MAX - writer->size) { + PyErr_NoMemory(); + return -1; + } + size = writer->size + size; + + if (_PyBytesWriter_Resize_impl(writer, size, 1) < 0) { + return -1; + } + writer->size = size; + return 0; +} + +static inline void* +PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, + Py_ssize_t size, void *buf) +{ + Py_ssize_t pos = (char*)buf - (char*)PyBytesWriter_GetData(writer); + if (PyBytesWriter_Grow(writer, size) < 0) { + return NULL; + } + return (char*)PyBytesWriter_GetData(writer) + pos; +} + +static inline int +PyBytesWriter_WriteBytes(PyBytesWriter *writer, + const void *bytes, Py_ssize_t size) +{ + if (size < 0) { + size_t len = strlen((const char*)bytes); + if (len > (size_t)PY_SSIZE_T_MAX) { + PyErr_NoMemory(); + return -1; + } + size = (Py_ssize_t)len; + } + + Py_ssize_t pos = writer->size; + if (PyBytesWriter_Grow(writer, size) < 0) { + return -1; + } + char *buf = (char*)PyBytesWriter_GetData(writer); + memcpy(buf + pos, bytes, (size_t)size); + return 0; +} + +static inline int +PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) + Py_GCC_ATTRIBUTE((format(printf, 2, 3))); + +static inline int +PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + PyObject *str = PyBytes_FromFormatV(format, vargs); + va_end(vargs); + + if (str == NULL) { + return -1; + } + int res = PyBytesWriter_WriteBytes(writer, + PyBytes_AS_STRING(str), + PyBytes_GET_SIZE(str)); + Py_DECREF(str); + return res; +} +#endif // PY_VERSION_HEX < 0x030F00A1 + + +#if PY_VERSION_HEX < 0x030F00A1 +static inline PyObject* +PyTuple_FromArray(PyObject *const *array, Py_ssize_t size) +{ + PyObject *tuple = PyTuple_New(size); + if (tuple == NULL) { + return NULL; + } + for (Py_ssize_t i=0; i < size; i++) { + PyObject *item = array[i]; + PyTuple_SET_ITEM(tuple, i, Py_NewRef(item)); + } + return tuple; +} +#endif + + +#if PY_VERSION_HEX < 0x030F00A1 +static inline Py_hash_t +PyUnstable_Unicode_GET_CACHED_HASH(PyObject *op) +{ +#ifdef PYPY_VERSION + (void)op; // unused argument + return -1; +#elif PY_VERSION_HEX >= 0x03000000 + return ((PyASCIIObject*)op)->hash; +#else + return ((PyUnicodeObject*)op)->hash; +#endif +} +#endif + #ifdef __cplusplus } diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index f16e99bb4159..721a2bbb10b9 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -1,3 +1,5 @@ +#include "pythoncapi_compat.h" + // String primitive operations // // These are registered in mypyc.primitives.str_ops. @@ -321,7 +323,7 @@ static PyObject *_PyStr_XStrip(PyObject *self, int striptype, PyObject *sepobj) // Copied from do_strip function in cpython.git/Objects/unicodeobject.c@0ef4ffeefd1737c18dc9326133c7894d58108c2e. PyObject *_CPyStr_Strip(PyObject *self, int strip_type, PyObject *sep) { - if (sep == NULL || sep == Py_None) { + if (sep == NULL || Py_IsNone(sep)) { Py_ssize_t len, i, j; // This check is needed from Python 3.9 and earlier. From 49e9a9b369a2d63c946dbc0a97ddc9e0b7cc2546 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Oct 2025 23:07:58 +0000 Subject: [PATCH 125/183] Expose buffer cache layout version (#20145) This idea is quite simple: each time we change how an object is cached in the buffer we bump this version (even if we don't change the ABI version). Notes: * This is *not* the same as ABI version, if the ABI version is different, the capsule import will fail and you can't even use this version of `librt`. We can however use buffer cache layout version to abandon cache files that would result in reading garbage data. * I can't actually start using this in cache metas in the same PR, to avoid breaking self-check in CI. If there are no comments objections, I will merge this soon. Then release a new `librt` version, and make second PR that actually uses this feature. While I am at it I fix a bug in the float deserialization (magic value was not handled). --- mypy/typeshed/stubs/librt/librt/internal.pyi | 1 + mypyc/lib-rt/librt_internal.c | 14 +++++++++++++- mypyc/lib-rt/librt_internal.h | 2 ++ mypyc/primitives/misc_ops.py | 10 +++++++++- mypyc/test-data/irbuild-classes.test | 6 +++++- mypyc/test-data/run-classes.test | 6 +++++- 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index 8a5fc262931e..8654e31c100e 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -16,3 +16,4 @@ def write_int(data: Buffer, value: int) -> None: ... def read_int(data: Buffer) -> int: ... def write_tag(data: Buffer, value: u8) -> None: ... def read_tag(data: Buffer) -> u8: ... +def cache_version() -> u8: ... diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index af8dcbccaa43..b37136be0784 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -656,6 +656,16 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames return Py_None; } +static uint8_t +cache_version_internal(void) { + return 0; +} + +static PyObject* +cache_version(PyObject *self, PyObject *Py_UNUSED(ignored)) { + return PyLong_FromLong(cache_version_internal()); +} + static PyMethodDef librt_internal_module_methods[] = { {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")}, {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, @@ -669,6 +679,7 @@ static PyMethodDef librt_internal_module_methods[] = { {"read_int", (PyCFunction)read_int, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read an int")}, {"write_tag", (PyCFunction)write_tag, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a short int")}, {"read_tag", (PyCFunction)read_tag, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a short int")}, + {"cache_version", (PyCFunction)cache_version, METH_NOARGS, PyDoc_STR("cache format version")}, {NULL, NULL, 0, NULL} }; @@ -688,7 +699,7 @@ librt_internal_module_exec(PyObject *m) } // Export mypy internal C API, be careful with the order! - static void *NativeInternal_API[16] = { + static void *NativeInternal_API[17] = { (void *)Buffer_internal, (void *)Buffer_internal_empty, (void *)Buffer_getvalue_internal, @@ -705,6 +716,7 @@ librt_internal_module_exec(PyObject *m) (void *)NativeInternal_ABI_Version, (void *)write_bytes_internal, (void *)read_bytes_internal, + (void *)cache_version_internal, }; PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index d996b8fd95c1..1d16e1cb127f 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -21,6 +21,7 @@ static uint8_t read_tag_internal(PyObject *data); static int NativeInternal_ABI_Version(void); static char write_bytes_internal(PyObject *data, PyObject *value); static PyObject *read_bytes_internal(PyObject *data); +static uint8_t cache_version_internal(void); #else @@ -42,6 +43,7 @@ static void **NativeInternal_API; #define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13]) #define write_bytes_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[14]) #define read_bytes_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[15]) +#define cache_version_internal (*(uint8_t (*)(void)) NativeInternal_API[16]) static int import_librt_internal(void) diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index c12172875e8b..10f4bc001e29 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -422,7 +422,7 @@ arg_types=[object_rprimitive], return_type=float_rprimitive, c_function_name="read_float_internal", - error_kind=ERR_MAGIC, + error_kind=ERR_MAGIC_OVERLAPPING, ) function_op( @@ -456,3 +456,11 @@ c_function_name="read_tag_internal", error_kind=ERR_MAGIC_OVERLAPPING, ) + +function_op( + name="librt.internal.cache_version", + arg_types=[], + return_type=uint8_rprimitive, + c_function_name="cache_version_internal", + error_kind=ERR_NEVER, +) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 27ffba45ba39..a8ee7213ef96 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1455,6 +1455,7 @@ from mypy_extensions import u8 from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, + cache_version, ) Tag = u8 @@ -1476,6 +1477,7 @@ def foo() -> None: z = read_float(b) t = read_int(b) u = read_tag(b) + v = cache_version() [out] def foo(): r0, b :: librt.internal.Buffer @@ -1490,7 +1492,7 @@ def foo(): r13, y :: bool r14, z :: float r15, t :: int - r16, u :: u8 + r16, u, r17, v :: u8 L0: r0 = Buffer_internal_empty() b = r0 @@ -1517,6 +1519,8 @@ L0: t = r15 r16 = read_tag_internal(b) u = r16 + r17 = cache_version_internal() + v = r17 return 1 [case testEnumFastPath] diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index efa6c225ecab..e08f2fd7007d 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2715,7 +2715,8 @@ from typing import Final from mypy_extensions import u8 from librt.internal import ( Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, - write_int, read_int, write_tag, read_tag, write_bytes, read_bytes + write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, + cache_version, ) Tag = u8 @@ -2724,6 +2725,7 @@ TAG_B: Final[Tag] = 255 TAG_SPECIAL: Final[Tag] = 239 def test_buffer_basic() -> None: + assert cache_version() == 0 b = Buffer(b"foo") assert b.getvalue() == b"foo" @@ -2739,6 +2741,7 @@ def test_buffer_roundtrip() -> None: write_bytes(b, b"a" * 127) write_bytes(b, b"a" * 128) write_float(b, 0.1) + write_float(b, -113.0) write_int(b, 0) write_int(b, 1) write_tag(b, TAG_A) @@ -2763,6 +2766,7 @@ def test_buffer_roundtrip() -> None: assert read_bytes(b) == b"a" * 127 assert read_bytes(b) == b"a" * 128 assert read_float(b) == 0.1 + assert read_float(b) == -113.0 assert read_int(b) == 0 assert read_int(b) == 1 assert read_tag(b) == TAG_A From 99bc45f68839c798273e7799c9487f3f3e2bc2b6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 30 Oct 2025 23:08:18 +0000 Subject: [PATCH 126/183] Use self-descriptive cache with type tags (#20137) This should make our cache format more future proof w.r.t lazy deserialization and other features: * This makes cache ~5% larger (data + meta) * This makes interpreted performance ~5% worse (just the serialization steps, *not* overall time) * No visible performance impact when compiled (probably because all extra indirections like `read_str()` -> `read_str_bare()` etc are C calls). Note that I now serialize various little arbitrary JSON blobs (like plugin data etc) using fixed format, since with the tags we can do this. --- mypy/cache.py | 290 +++++++++++++++++++++++++++------- mypy/nodes.py | 200 ++++++++++++++--------- mypy/types.py | 245 ++++++++++++++++++---------- mypyc/lib-rt/librt_internal.c | 4 + 4 files changed, 529 insertions(+), 210 deletions(-) diff --git a/mypy/cache.py b/mypy/cache.py index aeb0e8810fd6..0d2db67fac94 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -1,27 +1,70 @@ +""" +This module contains high-level logic for fixed format serialization. + +Lower-level parts are implemented in C in mypyc/lib-rt/librt_internal.c +Short summary of low-level functionality: +* integers are automatically serialized as 1, 2, or 4 bytes, or arbitrary length. +* str/bytes are serialized as size (1, 2, or 4 bytes) followed by bytes buffer. +* floats are serialized as C doubles. + +At high-level we add type tags as needed so that our format is self-descriptive. +More precisely: +* False, True, and None are stored as just a tag: 0, 1, 2 correspondingly. +* builtin primitives like int/str/bytes/float are stored as their type tag followed + by bare (low-level) representation of the value. Reserved tag range for primitives is + 3 ... 19. +* generic (heterogeneous) list are stored as tag, followed by bare size, followed by + sequence of tagged values. +* homogeneous lists of primitives are stored as tag, followed by bare size, followed + by sequence of bare values. +* reserved tag range for sequence-like builtins is 20 ... 29 +* currently we have only one mapping-like format: string-keyed dictionary with heterogeneous + values. It is stored as tag, followed by bare size, followed by sequence of pairs: bare + string key followed by tagged value. +* reserved tag range for mapping-like builtins is 30 ... 39 +* there is an additional reserved tag range 40 ... 49 for any other builtin collections. +* custom classes (like types, symbols etc.) are stored as tag, followed by a sequence of + tagged field values, followed by a special end tag 255. Names of class fields are + *not* stored, the caller should know the field names and order for the given class tag. +* reserved tag range for symbols (TypeInfo, Var, etc) is 50 ... 79. +* class Instance is the only exception from the above format (since it is the most common one). + It has two extra formats: few most common instances like "builtins.object" are stored as + instance tag followed by a secondary tag, other plain non-generic instances are stored as + instance tag followed by secondary tag followed by fullname as bare string. All generic + readers must handle these. +* reserved tag range for Instance type formats is 80 ... 99, for other types it is 100 ... 149. +* tag 254 is reserved for if we would ever need to extend the tag range to indicated second tag + page. Tags 150 ... 253 are free for everything else (e.g. AST nodes etc). + +General convention is that custom classes implement write() and read() methods for FF +serialization. The write method should write both class tag and end tag. The read method +conventionally *does not* read the start tag (to simplify logic for unions). Known exceptions +are MypyFile.read() and SymbolTableNode.read(), since those two never appear in a union. +""" + from __future__ import annotations from collections.abc import Sequence -from typing import Any, Final +from typing import Any, Final, Union +from typing_extensions import TypeAlias as _TypeAlias from librt.internal import ( Buffer as Buffer, read_bool as read_bool, - read_bytes as read_bytes, - read_float as read_float, - read_int as read_int, - read_str as read_str, + read_bytes as read_bytes_bare, + read_float as read_float_bare, + read_int as read_int_bare, + read_str as read_str_bare, read_tag as read_tag, write_bool as write_bool, - write_bytes as write_bytes, - write_float as write_float, - write_int as write_int, - write_str as write_str, + write_bytes as write_bytes_bare, + write_float as write_float_bare, + write_int as write_int_bare, + write_str as write_str_bare, write_tag as write_tag, ) from mypy_extensions import u8 -from mypy.util import json_dumps, json_loads - class CacheMeta: """Class representing cache metadata for a module.""" @@ -125,7 +168,7 @@ def write(self, data: Buffer) -> None: write_str_list(data, self.dependencies) write_int(data, self.data_mtime) write_str_list(data, self.suppressed) - write_bytes(data, json_dumps(self.options)) + write_json(data, self.options) write_int_list(data, self.dep_prios) write_int_list(data, self.dep_lines) write_bytes_list(data, self.dep_hashes) @@ -133,7 +176,9 @@ def write(self, data: Buffer) -> None: write_str_list(data, self.error_lines) write_str(data, self.version_id) write_bool(data, self.ignore_all) - write_bytes(data, json_dumps(self.plugin_data)) + # Plugin data may be not a dictionary, so we use + # a more generic write_json_value() here. + write_json_value(data, self.plugin_data) @classmethod def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: @@ -148,7 +193,7 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: data_mtime=read_int(data), data_file=data_file, suppressed=read_str_list(data), - options=json_loads(read_bytes(data)), + options=read_json(data), dep_prios=read_int_list(data), dep_lines=read_int_list(data), dep_hashes=read_bytes_list(data), @@ -156,7 +201,7 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: error_lines=read_str_list(data), version_id=read_str(data), ignore_all=read_bool(data), - plugin_data=json_loads(read_bytes(data)), + plugin_data=read_json_value(data), ) except ValueError: return None @@ -165,114 +210,243 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: # Always use this type alias to refer to type tags. Tag = u8 -LITERAL_INT: Final[Tag] = 1 -LITERAL_STR: Final[Tag] = 2 -LITERAL_BOOL: Final[Tag] = 3 -LITERAL_FLOAT: Final[Tag] = 4 -LITERAL_COMPLEX: Final[Tag] = 5 -LITERAL_NONE: Final[Tag] = 6 +# Primitives. +LITERAL_FALSE: Final[Tag] = 0 +LITERAL_TRUE: Final[Tag] = 1 +LITERAL_NONE: Final[Tag] = 2 +LITERAL_INT: Final[Tag] = 3 +LITERAL_STR: Final[Tag] = 4 +LITERAL_BYTES: Final[Tag] = 5 +LITERAL_FLOAT: Final[Tag] = 6 +LITERAL_COMPLEX: Final[Tag] = 7 + +# Collections. +LIST_GEN: Final[Tag] = 20 +LIST_INT: Final[Tag] = 21 +LIST_STR: Final[Tag] = 22 +LIST_BYTES: Final[Tag] = 23 +DICT_STR_GEN: Final[Tag] = 30 + +# Misc classes. +EXTRA_ATTRS: Final[Tag] = 150 +DT_SPEC: Final[Tag] = 151 + +END_TAG: Final[Tag] = 255 def read_literal(data: Buffer, tag: Tag) -> int | str | bool | float: if tag == LITERAL_INT: - return read_int(data) + return read_int_bare(data) elif tag == LITERAL_STR: - return read_str(data) - elif tag == LITERAL_BOOL: - return read_bool(data) + return read_str_bare(data) + elif tag == LITERAL_FALSE: + return False + elif tag == LITERAL_TRUE: + return True elif tag == LITERAL_FLOAT: - return read_float(data) + return read_float_bare(data) assert False, f"Unknown literal tag {tag}" +# There is an intentional asymmetry between read and write for literals because +# None and/or complex values are only allowed in some contexts but not in others. def write_literal(data: Buffer, value: int | str | bool | float | complex | None) -> None: if isinstance(value, bool): - write_tag(data, LITERAL_BOOL) write_bool(data, value) elif isinstance(value, int): write_tag(data, LITERAL_INT) - write_int(data, value) + write_int_bare(data, value) elif isinstance(value, str): write_tag(data, LITERAL_STR) - write_str(data, value) + write_str_bare(data, value) elif isinstance(value, float): write_tag(data, LITERAL_FLOAT) - write_float(data, value) + write_float_bare(data, value) elif isinstance(value, complex): write_tag(data, LITERAL_COMPLEX) - write_float(data, value.real) - write_float(data, value.imag) + write_float_bare(data, value.real) + write_float_bare(data, value.imag) else: write_tag(data, LITERAL_NONE) +def read_int(data: Buffer) -> int: + assert read_tag(data) == LITERAL_INT + return read_int_bare(data) + + +def write_int(data: Buffer, value: int) -> None: + write_tag(data, LITERAL_INT) + write_int_bare(data, value) + + +def read_str(data: Buffer) -> str: + assert read_tag(data) == LITERAL_STR + return read_str_bare(data) + + +def write_str(data: Buffer, value: str) -> None: + write_tag(data, LITERAL_STR) + write_str_bare(data, value) + + +def read_bytes(data: Buffer) -> bytes: + assert read_tag(data) == LITERAL_BYTES + return read_bytes_bare(data) + + +def write_bytes(data: Buffer, value: bytes) -> None: + write_tag(data, LITERAL_BYTES) + write_bytes_bare(data, value) + + def read_int_opt(data: Buffer) -> int | None: - if read_bool(data): - return read_int(data) - return None + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + assert tag == LITERAL_INT + return read_int_bare(data) def write_int_opt(data: Buffer, value: int | None) -> None: if value is not None: - write_bool(data, True) - write_int(data, value) + write_tag(data, LITERAL_INT) + write_int_bare(data, value) else: - write_bool(data, False) + write_tag(data, LITERAL_NONE) def read_str_opt(data: Buffer) -> str | None: - if read_bool(data): - return read_str(data) - return None + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + assert tag == LITERAL_STR + return read_str_bare(data) def write_str_opt(data: Buffer, value: str | None) -> None: if value is not None: - write_bool(data, True) - write_str(data, value) + write_tag(data, LITERAL_STR) + write_str_bare(data, value) else: - write_bool(data, False) + write_tag(data, LITERAL_NONE) def read_int_list(data: Buffer) -> list[int]: - size = read_int(data) - return [read_int(data) for _ in range(size)] + assert read_tag(data) == LIST_INT + size = read_int_bare(data) + return [read_int_bare(data) for _ in range(size)] def write_int_list(data: Buffer, value: list[int]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_INT) + write_int_bare(data, len(value)) for item in value: - write_int(data, item) + write_int_bare(data, item) def read_str_list(data: Buffer) -> list[str]: - size = read_int(data) - return [read_str(data) for _ in range(size)] + assert read_tag(data) == LIST_STR + size = read_int_bare(data) + return [read_str_bare(data) for _ in range(size)] def write_str_list(data: Buffer, value: Sequence[str]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_STR) + write_int_bare(data, len(value)) for item in value: - write_str(data, item) + write_str_bare(data, item) def read_bytes_list(data: Buffer) -> list[bytes]: - size = read_int(data) - return [read_bytes(data) for _ in range(size)] + assert read_tag(data) == LIST_BYTES + size = read_int_bare(data) + return [read_bytes_bare(data) for _ in range(size)] def write_bytes_list(data: Buffer, value: Sequence[bytes]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_BYTES) + write_int_bare(data, len(value)) for item in value: - write_bytes(data, item) + write_bytes_bare(data, item) def read_str_opt_list(data: Buffer) -> list[str | None]: - size = read_int(data) + assert read_tag(data) == LIST_GEN + size = read_int_bare(data) return [read_str_opt(data) for _ in range(size)] def write_str_opt_list(data: Buffer, value: list[str | None]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_GEN) + write_int_bare(data, len(value)) for item in value: write_str_opt(data, item) + + +JsonValue: _TypeAlias = Union[None, int, str, bool, list["JsonValue"], dict[str, "JsonValue"]] + + +def read_json_value(data: Buffer) -> JsonValue: + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + if tag == LITERAL_FALSE: + return False + if tag == LITERAL_TRUE: + return True + if tag == LITERAL_INT: + return read_int_bare(data) + if tag == LITERAL_STR: + return read_str_bare(data) + if tag == LIST_GEN: + size = read_int_bare(data) + return [read_json_value(data) for _ in range(size)] + if tag == DICT_STR_GEN: + size = read_int_bare(data) + return {read_str_bare(data): read_json_value(data) for _ in range(size)} + assert False, f"Invalid JSON tag: {tag}" + + +# Currently tuples are used by mypyc plugin. They will be normalized to +# JSON lists after a roundtrip. +def write_json_value(data: Buffer, value: JsonValue | tuple[JsonValue, ...]) -> None: + if value is None: + write_tag(data, LITERAL_NONE) + elif isinstance(value, bool): + write_bool(data, value) + elif isinstance(value, int): + write_tag(data, LITERAL_INT) + write_int_bare(data, value) + elif isinstance(value, str): + write_tag(data, LITERAL_STR) + write_str_bare(data, value) + elif isinstance(value, (list, tuple)): + write_tag(data, LIST_GEN) + write_int_bare(data, len(value)) + for val in value: + write_json_value(data, val) + elif isinstance(value, dict): + write_tag(data, DICT_STR_GEN) + write_int_bare(data, len(value)) + for key in sorted(value): + write_str_bare(data, key) + write_json_value(data, value[key]) + else: + assert False, f"Invalid JSON value: {value}" + + +# These are functions for JSON *dictionaries* specifically. Unfortunately, we +# must use imprecise types here, because the callers use imprecise types. +def read_json(data: Buffer) -> dict[str, Any]: + assert read_tag(data) == DICT_STR_GEN + size = read_int_bare(data) + return {read_str_bare(data): read_json_value(data) for _ in range(size)} + + +def write_json(data: Buffer, value: dict[str, Any]) -> None: + write_tag(data, DICT_STR_GEN) + write_int_bare(data, len(value)) + for key in sorted(value): + write_str_bare(data, key) + write_json_value(data, value[key]) diff --git a/mypy/nodes.py b/mypy/nodes.py index 040f3fc28dce..6cf984e5a218 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import os from abc import abstractmethod from collections import defaultdict @@ -11,19 +10,31 @@ from typing import TYPE_CHECKING, Any, Callable, Final, Optional, TypeVar, Union, cast from typing_extensions import TypeAlias as _TypeAlias, TypeGuard +from librt.internal import ( + read_float as read_float_bare, + read_int as read_int_bare, + read_str as read_str_bare, + write_int as write_int_bare, + write_str as write_str_bare, +) from mypy_extensions import trait import mypy.strconv from mypy.cache import ( + DICT_STR_GEN, + DT_SPEC, + END_TAG, + LIST_GEN, + LIST_STR, LITERAL_COMPLEX, LITERAL_NONE, Buffer, Tag, read_bool, - read_float, read_int, read_int_list, read_int_opt, + read_json, read_literal, read_str, read_str_list, @@ -34,6 +45,7 @@ write_int, write_int_list, write_int_opt, + write_json, write_literal, write_str, write_str_list, @@ -432,6 +444,7 @@ def write(self, data: Buffer) -> None: write_str(data, self.path) write_bool(data, self.is_partial_stub_package) write_str_list(data, sorted(self.future_import_flags)) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> MypyFile: @@ -444,6 +457,7 @@ def read(cls, data: Buffer) -> MypyFile: tree.is_partial_stub_package = read_bool(data) tree.future_import_flags = set(read_str_list(data)) tree.is_cache_skeleton = True + assert read_tag(data) == END_TAG return tree @@ -720,30 +734,33 @@ def deserialize(cls, data: JsonDict) -> OverloadedFuncDef: def write(self, data: Buffer) -> None: write_tag(data, OVERLOADED_FUNC_DEF) - write_int(data, len(self.items)) + write_tag(data, LIST_GEN) + write_int_bare(data, len(self.items)) for item in self.items: item.write(data) mypy.types.write_type_opt(data, self.type) write_str(data, self._fullname) if self.impl is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.impl.write(data) write_flags(data, self, FUNCBASE_FLAGS) write_str_opt(data, self.deprecated) write_int_opt(data, self.setter_index) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> OverloadedFuncDef: - res = OverloadedFuncDef([read_overload_part(data) for _ in range(read_int(data))]) + assert read_tag(data) == LIST_GEN + res = OverloadedFuncDef([read_overload_part(data) for _ in range(read_int_bare(data))]) typ = mypy.types.read_type_opt(data) if typ is not None: assert isinstance(typ, mypy.types.ProperType) res.type = typ res._fullname = read_str(data) - if read_bool(data): - res.impl = read_overload_part(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + res.impl = read_overload_part(data, tag) # set line for empty overload items, as not set in __init__ if len(res.items) > 0: res.set_line(res.impl.line) @@ -751,6 +768,7 @@ def read(cls, data: Buffer) -> OverloadedFuncDef: res.deprecated = read_str_opt(data) res.setter_index = read_int_opt(data) # NOTE: res.info will be set in the fixup phase. + assert read_tag(data) == END_TAG return res def is_dynamic(self) -> bool: @@ -1039,19 +1057,20 @@ def write(self, data: Buffer) -> None: write_int_list(data, [int(ak.value) for ak in self.arg_kinds]) write_int(data, self.abstract_status) if self.dataclass_transform_spec is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.dataclass_transform_spec.write(data) write_str_opt(data, self.deprecated) write_str_opt(data, self.original_first_arg) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> FuncDef: name = read_str(data) typ: mypy.types.FunctionLike | None = None - if read_bool(data): - typ = mypy.types.read_function_like(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + typ = mypy.types.read_function_like(data, tag) ret = FuncDef(name, [], Block([]), typ) ret._fullname = read_str(data) read_flags(data, ret, FUNCDEF_FLAGS) @@ -1059,7 +1078,9 @@ def read(cls, data: Buffer) -> FuncDef: ret.arg_names = read_str_opt_list(data) ret.arg_kinds = [ARG_KINDS[ak] for ak in read_int_list(data)] ret.abstract_status = read_int(data) - if read_bool(data): + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == DT_SPEC ret.dataclass_transform_spec = DataclassTransformSpec.read(data) ret.deprecated = read_str_opt(data) ret.original_first_arg = read_str_opt(data) @@ -1067,6 +1088,7 @@ def read(cls, data: Buffer) -> FuncDef: del ret.arguments del ret.max_pos del ret.min_args + assert read_tag(data) == END_TAG return ret @@ -1146,6 +1168,7 @@ def write(self, data: Buffer) -> None: self.func.write(data) self.var.write(data) write_bool(data, self.is_overload) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Decorator: @@ -1155,6 +1178,7 @@ def read(cls, data: Buffer) -> Decorator: var = Var.read(data) dec = Decorator(func, [], var) dec.is_overload = read_bool(data) + assert read_tag(data) == END_TAG return dec def is_dynamic(self) -> bool: @@ -1341,6 +1365,7 @@ def write(self, data: Buffer) -> None: write_str(data, self._fullname) write_flags(data, self, VAR_FLAGS) write_literal(data, self.final_value) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Var: @@ -1348,18 +1373,20 @@ def read(cls, data: Buffer) -> Var: typ = mypy.types.read_type_opt(data) v = Var(name, typ) setter_type: mypy.types.CallableType | None = None - if read_bool(data): - assert read_tag(data) == mypy.types.CALLABLE_TYPE + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == mypy.types.CALLABLE_TYPE setter_type = mypy.types.CallableType.read(data) v.setter_type = setter_type v.is_ready = False # Override True default set in __init__ v._fullname = read_str(data) read_flags(data, v, VAR_FLAGS) - marker = read_tag(data) - if marker == LITERAL_COMPLEX: - v.final_value = complex(read_float(data), read_float(data)) - elif marker != LITERAL_NONE: - v.final_value = read_literal(data, marker) + tag = read_tag(data) + if tag == LITERAL_COMPLEX: + v.final_value = complex(read_float_bare(data), read_float_bare(data)) + elif tag != LITERAL_NONE: + v.final_value = read_literal(data, tag) + assert read_tag(data) == END_TAG return v @@ -1477,15 +1504,13 @@ def write(self, data: Buffer) -> None: write_str(data, self.name) mypy.types.write_type_list(data, self.type_vars) write_str(data, self.fullname) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ClassDef: - res = ClassDef( - read_str(data), - Block([]), - [mypy.types.read_type_var_like(data) for _ in range(read_int(data))], - ) + res = ClassDef(read_str(data), Block([]), mypy.types.read_type_var_likes(data)) res.fullname = read_str(data) + assert read_tag(data) == END_TAG return res @@ -2913,10 +2938,11 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarExpr: - return TypeVarExpr( + ret = TypeVarExpr( read_str(data), read_str(data), mypy.types.read_type_list(data), @@ -2924,6 +2950,8 @@ def read(cls, data: Buffer) -> TypeVarExpr: mypy.types.read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class ParamSpecExpr(TypeVarLikeExpr): @@ -2962,16 +2990,19 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ParamSpecExpr: - return ParamSpecExpr( + ret = ParamSpecExpr( read_str(data), read_str(data), mypy.types.read_type(data), mypy.types.read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class TypeVarTupleExpr(TypeVarLikeExpr): @@ -3031,12 +3062,13 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarTupleExpr: assert read_tag(data) == mypy.types.INSTANCE fallback = mypy.types.Instance.read(data) - return TypeVarTupleExpr( + ret = TypeVarTupleExpr( read_str(data), read_str(data), mypy.types.read_type(data), @@ -3044,6 +3076,8 @@ def read(cls, data: Buffer) -> TypeVarTupleExpr: mypy.types.read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class TypeAliasExpr(Expression): @@ -3934,20 +3968,19 @@ def write(self, data: Buffer) -> None: mypy.types.write_type_opt(data, self.tuple_type) mypy.types.write_type_opt(data, self.typeddict_type) write_flags(data, self, TypeInfo.FLAGS) - write_str(data, json.dumps(self.metadata)) + write_json(data, self.metadata) if self.slots is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) write_str_list(data, sorted(self.slots)) write_str_list(data, self.deletable_attributes) mypy.types.write_type_opt(data, self.self_type) if self.dataclass_transform_spec is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.dataclass_transform_spec.write(data) write_str_opt(data, self.deprecated) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeInfo: @@ -3963,7 +3996,8 @@ def read(cls, data: Buffer) -> TypeInfo: ti.type_vars = read_str_list(data) ti.has_param_spec_type = read_bool(data) ti.bases = [] - for _ in range(read_int(data)): + assert read_tag(data) == LIST_GEN + for _ in range(read_int_bare(data)): assert read_tag(data) == mypy.types.INSTANCE ti.bases.append(mypy.types.Instance.read(data)) # NOTE: ti.mro will be set in the fixup phase based on these @@ -3978,34 +4012,37 @@ def read(cls, data: Buffer) -> TypeInfo: # rechecked, it can tell that the mro has changed. ti._mro_refs = read_str_list(data) ti._promote = cast(list[mypy.types.ProperType], mypy.types.read_type_list(data)) - if read_bool(data): - assert read_tag(data) == mypy.types.INSTANCE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE ti.alt_promote = mypy.types.Instance.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.INSTANCE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE ti.declared_metaclass = mypy.types.Instance.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.INSTANCE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.INSTANCE ti.metaclass_type = mypy.types.Instance.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.TUPLE_TYPE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TUPLE_TYPE ti.tuple_type = mypy.types.TupleType.read(data) - if read_bool(data): - assert read_tag(data) == mypy.types.TYPED_DICT_TYPE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TYPED_DICT_TYPE ti.typeddict_type = mypy.types.TypedDictType.read(data) read_flags(data, ti, TypeInfo.FLAGS) - metadata = read_str(data) - if metadata != "{}": - ti.metadata = json.loads(metadata) - if read_bool(data): - ti.slots = set(read_str_list(data)) + ti.metadata = read_json(data) + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == LIST_STR + ti.slots = {read_str_bare(data) for _ in range(read_int_bare(data))} ti.deletable_attributes = read_str_list(data) - if read_bool(data): - assert read_tag(data) == mypy.types.TYPE_VAR_TYPE + if (tag := read_tag(data)) != LITERAL_NONE: + assert tag == mypy.types.TYPE_VAR_TYPE ti.self_type = mypy.types.TypeVarType.read(data) - if read_bool(data): + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == DT_SPEC ti.dataclass_transform_spec = DataclassTransformSpec.read(data) ti.deprecated = read_str_opt(data) + assert read_tag(data) == END_TAG return ti @@ -4290,14 +4327,15 @@ def write(self, data: Buffer) -> None: write_bool(data, self.no_args) write_bool(data, self.normalized) write_bool(data, self.python_3_12_type_alias) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeAlias: fullname = read_str(data) module = read_str(data) target = mypy.types.read_type(data) - alias_tvars = [mypy.types.read_type_var_like(data) for _ in range(read_int(data))] - return TypeAlias( + alias_tvars = mypy.types.read_type_var_likes(data) + ret = TypeAlias( target, fullname, module, @@ -4308,6 +4346,8 @@ def read(cls, data: Buffer) -> TypeAlias: normalized=read_bool(data), python_3_12_type_alias=read_bool(data), ) + assert read_tag(data) == END_TAG + return ret class PlaceholderNode(SymbolNode): @@ -4568,6 +4608,7 @@ def deserialize(cls, data: JsonDict) -> SymbolTableNode: return stnode def write(self, data: Buffer, prefix: str, name: str) -> None: + write_tag(data, SYMBOL_TABLE_NODE) write_int(data, self.kind) write_bool(data, self.module_hidden) write_bool(data, self.module_public) @@ -4595,9 +4636,11 @@ def write(self, data: Buffer, prefix: str, name: str) -> None: if cross_ref is None: assert self.node is not None self.node.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> SymbolTableNode: + assert read_tag(data) == SYMBOL_TABLE_NODE sym = SymbolTableNode(read_int(data), None) sym.module_hidden = read_bool(data) sym.module_public = read_bool(data) @@ -4608,6 +4651,7 @@ def read(cls, data: Buffer) -> SymbolTableNode: sym.node = read_symbol(data) else: sym.cross_ref = cross_ref + assert read_tag(data) == END_TAG return sym @@ -4671,18 +4715,23 @@ def write(self, data: Buffer, fullname: str) -> None: if key == "__builtins__" or value.no_serialize: continue size += 1 - write_int(data, size) + # We intentionally tag SymbolTable as a simple dictionary str -> SymbolTableNode. + write_tag(data, DICT_STR_GEN) + write_int_bare(data, size) for key in sorted(self): value = self[key] if key == "__builtins__" or value.no_serialize: continue - write_str(data, key) + write_str_bare(data, key) value.write(data, fullname, key) @classmethod def read(cls, data: Buffer) -> SymbolTable: - size = read_int(data) - return SymbolTable([(read_str(data), SymbolTableNode.read(data)) for _ in range(size)]) + assert read_tag(data) == DICT_STR_GEN + size = read_int_bare(data) + return SymbolTable( + [(read_str_bare(data), SymbolTableNode.read(data)) for _ in range(size)] + ) class DataclassTransformSpec: @@ -4735,21 +4784,25 @@ def deserialize(cls, data: JsonDict) -> DataclassTransformSpec: ) def write(self, data: Buffer) -> None: + write_tag(data, DT_SPEC) write_bool(data, self.eq_default) write_bool(data, self.order_default) write_bool(data, self.kw_only_default) write_bool(data, self.frozen_default) write_str_list(data, self.field_specifiers) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> DataclassTransformSpec: - return DataclassTransformSpec( + ret = DataclassTransformSpec( eq_default=read_bool(data), order_default=read_bool(data), kw_only_default=read_bool(data), frozen_default=read_bool(data), field_specifiers=tuple(read_str_list(data)), ) + assert read_tag(data) == END_TAG + return ret def get_flags(node: Node, names: list[str]) -> list[str]: @@ -4889,17 +4942,19 @@ def local_definitions( yield from local_definitions(node.names, fullname, node) -MYPY_FILE: Final[Tag] = 0 -OVERLOADED_FUNC_DEF: Final[Tag] = 1 -FUNC_DEF: Final[Tag] = 2 -DECORATOR: Final[Tag] = 3 -VAR: Final[Tag] = 4 -TYPE_VAR_EXPR: Final[Tag] = 5 -PARAM_SPEC_EXPR: Final[Tag] = 6 -TYPE_VAR_TUPLE_EXPR: Final[Tag] = 7 -TYPE_INFO: Final[Tag] = 8 -TYPE_ALIAS: Final[Tag] = 9 -CLASS_DEF: Final[Tag] = 10 +# See docstring for mypy/cache.py for reserved tag ranges. +MYPY_FILE: Final[Tag] = 50 +OVERLOADED_FUNC_DEF: Final[Tag] = 51 +FUNC_DEF: Final[Tag] = 52 +DECORATOR: Final[Tag] = 53 +VAR: Final[Tag] = 54 +TYPE_VAR_EXPR: Final[Tag] = 55 +PARAM_SPEC_EXPR: Final[Tag] = 56 +TYPE_VAR_TUPLE_EXPR: Final[Tag] = 57 +TYPE_INFO: Final[Tag] = 58 +TYPE_ALIAS: Final[Tag] = 59 +CLASS_DEF: Final[Tag] = 60 +SYMBOL_TABLE_NODE: Final[Tag] = 61 def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: @@ -4926,8 +4981,9 @@ def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: assert False, f"Unknown symbol tag {tag}" -def read_overload_part(data: Buffer) -> OverloadPart: - tag = read_tag(data) +def read_overload_part(data: Buffer, tag: Tag | None = None) -> OverloadPart: + if tag is None: + tag = read_tag(data) if tag == DECORATOR: return Decorator.read(data) if tag == FUNC_DEF: diff --git a/mypy/types.py b/mypy/types.py index 426d560c2bf7..6013f4c25298 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -8,9 +8,21 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final, NewType, TypeVar, Union, cast, overload from typing_extensions import Self, TypeAlias as _TypeAlias, TypeGuard +from librt.internal import ( + read_int as read_int_bare, + read_str as read_str_bare, + write_int as write_int_bare, + write_str as write_str_bare, +) + import mypy.nodes from mypy.bogus_type import Bogus from mypy.cache import ( + DICT_STR_GEN, + END_TAG, + EXTRA_ATTRS, + LIST_GEN, + LITERAL_NONE, Buffer, Tag, read_bool, @@ -440,11 +452,13 @@ def write(self, data: Buffer) -> None: write_type_list(data, self.args) assert self.alias is not None write_str(data, self.alias.fullname) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeAliasType: alias = TypeAliasType(None, read_type_list(data)) alias.type_ref = read_str(data) + assert read_tag(data) == END_TAG return alias @@ -724,10 +738,11 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.variance) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarType: - return TypeVarType( + ret = TypeVarType( read_str(data), read_str(data), TypeVarId(read_int(data), namespace=read_str(data)), @@ -736,6 +751,8 @@ def read(cls, data: Buffer) -> TypeVarType: read_type(data), read_int(data), ) + assert read_tag(data) == END_TAG + return ret class ParamSpecFlavor: @@ -876,12 +893,13 @@ def write(self, data: Buffer) -> None: write_int(data, self.flavor) self.upper_bound.write(data) self.default.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ParamSpecType: assert read_tag(data) == PARAMETERS prefix = Parameters.read(data) - return ParamSpecType( + ret = ParamSpecType( read_str(data), read_str(data), TypeVarId(read_int(data), namespace=read_str(data)), @@ -890,6 +908,8 @@ def read(cls, data: Buffer) -> ParamSpecType: read_type(data), prefix=prefix, ) + assert read_tag(data) == END_TAG + return ret class TypeVarTupleType(TypeVarLikeType): @@ -956,12 +976,13 @@ def write(self, data: Buffer) -> None: self.upper_bound.write(data) self.default.write(data) write_int(data, self.min_len) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypeVarTupleType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return TypeVarTupleType( + ret = TypeVarTupleType( read_str(data), read_str(data), TypeVarId(read_int(data), namespace=read_str(data)), @@ -970,6 +991,8 @@ def read(cls, data: Buffer) -> TypeVarTupleType: read_type(data), min_len=read_int(data), ) + assert read_tag(data) == END_TAG + return ret def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_type_var_tuple(self) @@ -1108,15 +1131,18 @@ def write(self, data: Buffer) -> None: write_type_list(data, self.args) write_str_opt(data, self.original_str_expr) write_str_opt(data, self.original_str_fallback) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UnboundType: - return UnboundType( + ret = UnboundType( read_str(data), read_type_list(data), original_str_expr=read_str_opt(data), original_str_fallback=read_str_opt(data), ) + assert read_tag(data) == END_TAG + return ret class CallableArgument(ProperType): @@ -1215,10 +1241,13 @@ def serialize(self) -> JsonDict: def write(self, data: Buffer) -> None: write_tag(data, UNPACK_TYPE) self.type.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UnpackType: - return UnpackType(read_type(data)) + ret = UnpackType(read_type(data)) + assert read_tag(data) == END_TAG + return ret @classmethod def deserialize(cls, data: JsonDict) -> UnpackType: @@ -1326,15 +1355,19 @@ def write(self, data: Buffer) -> None: write_type_opt(data, self.source_any) write_int(data, self.type_of_any) write_str_opt(data, self.missing_import_name) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> AnyType: - if read_bool(data): - assert read_tag(data) == ANY_TYPE + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == ANY_TYPE source_any = AnyType.read(data) else: source_any = None - return AnyType(read_int(data), source_any, read_str_opt(data)) + ret = AnyType(read_int(data), source_any, read_str_opt(data)) + assert read_tag(data) == END_TAG + return ret class UninhabitedType(ProperType): @@ -1384,9 +1417,11 @@ def deserialize(cls, data: JsonDict) -> UninhabitedType: def write(self, data: Buffer) -> None: write_tag(data, UNINHABITED_TYPE) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UninhabitedType: + assert read_tag(data) == END_TAG return UninhabitedType() @@ -1423,9 +1458,11 @@ def deserialize(cls, data: JsonDict) -> NoneType: def write(self, data: Buffer) -> None: write_tag(data, NONE_TYPE) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> NoneType: + assert read_tag(data) == END_TAG return NoneType() def is_singleton_type(self) -> bool: @@ -1478,10 +1515,13 @@ def deserialize(cls, data: JsonDict) -> DeletedType: def write(self, data: Buffer) -> None: write_tag(data, DELETED_TYPE) write_str_opt(data, self.source) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> DeletedType: - return DeletedType(read_str_opt(data)) + ret = DeletedType(read_str_opt(data)) + assert read_tag(data) == END_TAG + return ret # Fake TypeInfo to be used as a placeholder during Instance de-serialization. @@ -1539,13 +1579,17 @@ def deserialize(cls, data: JsonDict) -> ExtraAttrs: ) def write(self, data: Buffer) -> None: + write_tag(data, EXTRA_ATTRS) write_type_map(data, self.attrs) write_str_list(data, sorted(self.immutable)) write_str_opt(data, self.mod_name) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> ExtraAttrs: - return ExtraAttrs(read_type_map(data), set(read_str_list(data)), read_str_opt(data)) + ret = ExtraAttrs(read_type_map(data), set(read_str_list(data)), read_str_opt(data)) + assert read_tag(data) == END_TAG + return ret class Instance(ProperType): @@ -1699,17 +1743,17 @@ def write(self, data: Buffer) -> None: write_tag(data, INSTANCE_OBJECT) else: write_tag(data, INSTANCE_SIMPLE) - write_str(data, type_ref) + write_str_bare(data, type_ref) return write_tag(data, INSTANCE_GENERIC) write_str(data, self.type.fullname) write_type_list(data, self.args) write_type_opt(data, self.last_known_value) if self.extra_attrs is None: - write_bool(data, False) + write_tag(data, LITERAL_NONE) else: - write_bool(data, True) self.extra_attrs.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Instance: @@ -1743,17 +1787,21 @@ def read(cls, data: Buffer) -> Instance: return instance_cache.object_type if tag == INSTANCE_SIMPLE: inst = Instance(NOT_READY, []) - inst.type_ref = read_str(data) + inst.type_ref = read_str_bare(data) return inst assert tag == INSTANCE_GENERIC type_ref = read_str(data) inst = Instance(NOT_READY, read_type_list(data)) inst.type_ref = type_ref - if read_bool(data): - assert read_tag(data) == LITERAL_TYPE + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == LITERAL_TYPE inst.last_known_value = LiteralType.read(data) - if read_bool(data): + tag = read_tag(data) + if tag != LITERAL_NONE: + assert tag == EXTRA_ATTRS inst.extra_attrs = ExtraAttrs.read(data) + assert read_tag(data) == END_TAG return inst def copy_modified( @@ -2057,18 +2105,21 @@ def write(self, data: Buffer) -> None: write_str_opt_list(data, self.arg_names) write_type_list(data, self.variables) write_bool(data, self.imprecise_arg_kinds) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Parameters: - return Parameters( + ret = Parameters( read_type_list(data), # This is a micro-optimization until mypyc gets dedicated enum support. Otherwise, # we would spend ~20% of types deserialization time in Enum.__call__(). [ARG_KINDS[ak] for ak in read_int_list(data)], read_str_opt_list(data), - variables=[read_type_var_like(data) for _ in range(read_int(data))], + variables=read_type_var_likes(data), imprecise_arg_kinds=read_bool(data), ) + assert read_tag(data) == END_TAG + return ret def __hash__(self) -> int: return hash( @@ -2593,19 +2644,20 @@ def write(self, data: Buffer) -> None: write_bool(data, self.from_concatenate) write_bool(data, self.imprecise_arg_kinds) write_bool(data, self.unpack_kwargs) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> CallableType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return CallableType( + ret = CallableType( read_type_list(data), [ARG_KINDS[ak] for ak in read_int_list(data)], read_str_opt_list(data), read_type(data), fallback, name=read_str_opt(data), - variables=[read_type_var_like(data) for _ in range(read_int(data))], + variables=read_type_var_likes(data), is_ellipsis_args=read_bool(data), implicit=read_bool(data), is_bound=read_bool(data), @@ -2615,6 +2667,8 @@ def read(cls, data: Buffer) -> CallableType: imprecise_arg_kinds=read_bool(data), unpack_kwargs=read_bool(data), ) + assert read_tag(data) == END_TAG + return ret # This is a little safety net to prevent reckless special-casing of callables @@ -2694,13 +2748,16 @@ def deserialize(cls, data: JsonDict) -> Overloaded: def write(self, data: Buffer) -> None: write_tag(data, OVERLOADED) write_type_list(data, self.items) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Overloaded: items = [] - for _ in range(read_int(data)): + assert read_tag(data) == LIST_GEN + for _ in range(read_int_bare(data)): assert read_tag(data) == CALLABLE_TYPE items.append(CallableType.read(data)) + assert read_tag(data) == END_TAG return Overloaded(items) @@ -2804,12 +2861,15 @@ def write(self, data: Buffer) -> None: self.partial_fallback.write(data) write_type_list(data, self.items) write_bool(data, self.implicit) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TupleType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return TupleType(read_type_list(data), fallback, implicit=read_bool(data)) + ret = TupleType(read_type_list(data), fallback, implicit=read_bool(data)) + assert read_tag(data) == END_TAG + return ret def copy_modified( self, *, fallback: Instance | None = None, items: list[Type] | None = None @@ -2987,14 +3047,17 @@ def write(self, data: Buffer) -> None: write_type_map(data, self.items) write_str_list(data, sorted(self.required_keys)) write_str_list(data, sorted(self.readonly_keys)) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> TypedDictType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) - return TypedDictType( + ret = TypedDictType( read_type_map(data), set(read_str_list(data)), set(read_str_list(data)), fallback ) + assert read_tag(data) == END_TAG + return ret @property def is_final(self) -> bool: @@ -3248,13 +3311,16 @@ def write(self, data: Buffer) -> None: write_tag(data, LITERAL_TYPE) self.fallback.write(data) write_literal(data, self.value) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> LiteralType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) tag = read_tag(data) - return LiteralType(read_literal(data, tag), fallback) + ret = LiteralType(read_literal(data, tag), fallback) + assert read_tag(data) == END_TAG + return ret def is_singleton_type(self) -> bool: return self.is_enum_literal() or isinstance(self.value, bool) @@ -3361,10 +3427,13 @@ def write(self, data: Buffer) -> None: write_tag(data, UNION_TYPE) write_type_list(data, self.items) write_bool(data, self.uses_pep604_syntax) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> UnionType: - return UnionType(read_type_list(data), uses_pep604_syntax=read_bool(data)) + ret = UnionType(read_type_list(data), uses_pep604_syntax=read_bool(data)) + assert read_tag(data) == END_TAG + return ret class PartialType(ProperType): @@ -3505,10 +3574,13 @@ def deserialize(cls, data: JsonDict) -> Type: def write(self, data: Buffer) -> None: write_tag(data, TYPE_TYPE) self.item.write(data) + write_tag(data, END_TAG) @classmethod def read(cls, data: Buffer) -> Type: - return TypeType.make_normalized(read_type(data)) + ret = TypeType.make_normalized(read_type(data)) + assert read_tag(data) == END_TAG + return ret class PlaceholderType(ProperType): @@ -4172,37 +4244,41 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: return tuple(args) -TYPE_ALIAS_TYPE: Final[Tag] = 1 -TYPE_VAR_TYPE: Final[Tag] = 2 -PARAM_SPEC_TYPE: Final[Tag] = 3 -TYPE_VAR_TUPLE_TYPE: Final[Tag] = 4 -UNBOUND_TYPE: Final[Tag] = 5 -UNPACK_TYPE: Final[Tag] = 6 -ANY_TYPE: Final[Tag] = 7 -UNINHABITED_TYPE: Final[Tag] = 8 -NONE_TYPE: Final[Tag] = 9 -DELETED_TYPE: Final[Tag] = 10 -INSTANCE: Final[Tag] = 11 -CALLABLE_TYPE: Final[Tag] = 12 -OVERLOADED: Final[Tag] = 13 -TUPLE_TYPE: Final[Tag] = 14 -TYPED_DICT_TYPE: Final[Tag] = 15 -LITERAL_TYPE: Final[Tag] = 16 -UNION_TYPE: Final[Tag] = 17 -TYPE_TYPE: Final[Tag] = 18 -PARAMETERS: Final[Tag] = 19 - -INSTANCE_STR: Final[Tag] = 101 -INSTANCE_FUNCTION: Final[Tag] = 102 -INSTANCE_INT: Final[Tag] = 103 -INSTANCE_BOOL: Final[Tag] = 104 -INSTANCE_OBJECT: Final[Tag] = 105 -INSTANCE_SIMPLE: Final[Tag] = 106 -INSTANCE_GENERIC: Final[Tag] = 107 - - -def read_type(data: Buffer) -> Type: - tag = read_tag(data) +# See docstring for mypy/cache.py for reserved tag ranges. +# Instance-related tags. +INSTANCE: Final[Tag] = 80 +INSTANCE_SIMPLE: Final[Tag] = 81 +INSTANCE_GENERIC: Final[Tag] = 82 +INSTANCE_STR: Final[Tag] = 83 +INSTANCE_FUNCTION: Final[Tag] = 84 +INSTANCE_INT: Final[Tag] = 85 +INSTANCE_BOOL: Final[Tag] = 86 +INSTANCE_OBJECT: Final[Tag] = 87 + +# Other type tags. +TYPE_ALIAS_TYPE: Final[Tag] = 100 +TYPE_VAR_TYPE: Final[Tag] = 101 +PARAM_SPEC_TYPE: Final[Tag] = 102 +TYPE_VAR_TUPLE_TYPE: Final[Tag] = 103 +UNBOUND_TYPE: Final[Tag] = 104 +UNPACK_TYPE: Final[Tag] = 105 +ANY_TYPE: Final[Tag] = 106 +UNINHABITED_TYPE: Final[Tag] = 107 +NONE_TYPE: Final[Tag] = 108 +DELETED_TYPE: Final[Tag] = 109 +CALLABLE_TYPE: Final[Tag] = 110 +OVERLOADED: Final[Tag] = 111 +TUPLE_TYPE: Final[Tag] = 112 +TYPED_DICT_TYPE: Final[Tag] = 113 +LITERAL_TYPE: Final[Tag] = 114 +UNION_TYPE: Final[Tag] = 115 +TYPE_TYPE: Final[Tag] = 116 +PARAMETERS: Final[Tag] = 117 + + +def read_type(data: Buffer, tag: Tag | None = None) -> Type: + if tag is None: + tag = read_tag(data) # The branches here are ordered manually by type "popularity". if tag == INSTANCE: return Instance.read(data) @@ -4245,8 +4321,7 @@ def read_type(data: Buffer) -> Type: assert False, f"Unknown type tag {tag}" -def read_function_like(data: Buffer) -> FunctionLike: - tag = read_tag(data) +def read_function_like(data: Buffer, tag: Tag) -> FunctionLike: if tag == CALLABLE_TYPE: return CallableType.read(data) if tag == OVERLOADED: @@ -4254,51 +4329,61 @@ def read_function_like(data: Buffer) -> FunctionLike: assert False, f"Invalid type tag for FunctionLike {tag}" -def read_type_var_like(data: Buffer) -> TypeVarLikeType: - tag = read_tag(data) - if tag == TYPE_VAR_TYPE: - return TypeVarType.read(data) - if tag == PARAM_SPEC_TYPE: - return ParamSpecType.read(data) - if tag == TYPE_VAR_TUPLE_TYPE: - return TypeVarTupleType.read(data) - assert False, f"Invalid type tag for TypeVarLikeType {tag}" +def read_type_var_likes(data: Buffer) -> list[TypeVarLikeType]: + """Specialized version of read_type_list() for lists of type variables.""" + assert read_tag(data) == LIST_GEN + ret: list[TypeVarLikeType] = [] + for _ in range(read_int_bare(data)): + tag = read_tag(data) + if tag == TYPE_VAR_TYPE: + ret.append(TypeVarType.read(data)) + elif tag == PARAM_SPEC_TYPE: + ret.append(ParamSpecType.read(data)) + elif tag == TYPE_VAR_TUPLE_TYPE: + ret.append(TypeVarTupleType.read(data)) + else: + assert False, f"Invalid type tag for TypeVarLikeType {tag}" + return ret def read_type_opt(data: Buffer) -> Type | None: - if read_bool(data): - return read_type(data) - return None + tag = read_tag(data) + if tag == LITERAL_NONE: + return None + return read_type(data, tag) def write_type_opt(data: Buffer, value: Type | None) -> None: if value is not None: - write_bool(data, True) value.write(data) else: - write_bool(data, False) + write_tag(data, LITERAL_NONE) def read_type_list(data: Buffer) -> list[Type]: - size = read_int(data) + assert read_tag(data) == LIST_GEN + size = read_int_bare(data) return [read_type(data) for _ in range(size)] def write_type_list(data: Buffer, value: Sequence[Type]) -> None: - write_int(data, len(value)) + write_tag(data, LIST_GEN) + write_int_bare(data, len(value)) for item in value: item.write(data) def read_type_map(data: Buffer) -> dict[str, Type]: - size = read_int(data) - return {read_str(data): read_type(data) for _ in range(size)} + assert read_tag(data) == DICT_STR_GEN + size = read_int_bare(data) + return {read_str_bare(data): read_type(data) for _ in range(size)} def write_type_map(data: Buffer, value: dict[str, Type]) -> None: - write_int(data, len(value)) + write_tag(data, DICT_STR_GEN) + write_int_bare(data, len(value)) for key in sorted(value): - write_str(data, key) + write_str_bare(data, key) value[key].write(data) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index b37136be0784..8599017c31a8 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -206,6 +206,10 @@ read_bool_internal(PyObject *data) { _CHECK_BUFFER(data, CPY_BOOL_ERROR) _CHECK_READ(data, 1, CPY_BOOL_ERROR) char res = _READ(data, char) + if (unlikely((res != 0) & (res != 1))) { + PyErr_SetString(PyExc_ValueError, "invalid bool value"); + return CPY_BOOL_ERROR; + } return res; } From 3f03755c05f6e8506fb5a0b9b83d1a5f279b377f Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 31 Oct 2025 03:28:00 +0100 Subject: [PATCH 127/183] Do not store deferred NamedTuple fields as redefinitions (#20147) Fixes #17059. When we encounter a field with a Placeholder as a default, do not record it in the new symtable as redefinition - it becomes orphan immediately and remains there, crashing suring serialization. The test cases are both crashing on current master. Ideally we hould emit name-defined in all cases, but it is another unrelated issue (#17610, and likely some others; this is not specific to named tuples - plain classes also allow referencing variables that are not defined yet). --- mypy/semanal_namedtuple.py | 13 +++++++++---- test-data/unit/check-namedtuple.test | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 37a650f1b664..f27c89e34fdf 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -34,6 +34,7 @@ NamedTupleExpr, NameExpr, PassStmt, + PlaceholderNode, RefExpr, Statement, StrExpr, @@ -697,10 +698,14 @@ def save_namedtuple_body(self, named_tuple_info: TypeInfo) -> Iterator[None]: if isinstance(sym.node, (FuncBase, Decorator)) and not sym.plugin_generated: # Keep user-defined methods as is. continue - # Keep existing (user-provided) definitions under mangled names, so they - # get semantically analyzed. - r_key = get_unique_redefinition_name(key, named_tuple_info.names) - named_tuple_info.names[r_key] = sym + # Do not retain placeholders - we'll get back here if they cease to + # be placeholders later. If we keep placeholders alive, they may never + # be reached again, making it to cacheable symtable. + if not isinstance(sym.node, PlaceholderNode): + # Keep existing (user-provided) definitions under mangled names, so they + # get semantically analyzed. + r_key = get_unique_redefinition_name(key, named_tuple_info.names) + named_tuple_info.names[r_key] = sym named_tuple_info.names[key] = value # Helpers diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 45de2a9e50ae..66eb555421f4 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1530,3 +1530,28 @@ class Base: names = [name for name in namespace if fail] # E: Name "fail" is not defined self.n = namedtuple("n", names) # E: NamedTuple type as an attribute is not supported [builtins fixtures/tuple.pyi] + +[case testNamedTupleDefaultValueDefer] +# flags: --debug-serialize +from typing import NamedTuple + +class NT(NamedTuple): + foo: int = UNDEFINED # E: Name "UNDEFINED" is not defined +[builtins fixtures/tuple.pyi] + +[case testNamedTupleDefaultValueDefer2] +# flags: --debug-serialize +from typing import NamedTuple + +class NT(NamedTuple): + foo: int = DEFERRED_INT + +class NT2(NamedTuple): + foo: int = DEFERRED_STR # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +from foo import DEFERRED_INT, DEFERRED_STR + +[file foo.py] +DEFERRED_INT = 1 +DEFERRED_STR = "a" +[builtins fixtures/tuple.pyi] From f64bf112da27265e71419dba264de823c2d5f373 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Oct 2025 10:03:04 +0000 Subject: [PATCH 128/183] Optimize serialization format for 2 and 4 bytes ints (#20120) This is important for serialized ASTs (think line numbers for every node). Also in this PR: * Re-use same integer logic for str/bytes length (it is slightly less optimal, but code re-use is good). * Remove unused field from `Buffer` type. * Make format the same on 32-bit and 64-bit platforms (we still assume little-endian platform). --- mypyc/lib-rt/librt_internal.c | 230 ++++++++++++++++++++----------- mypyc/test-data/run-classes.test | 96 ++++++++++--- 2 files changed, 220 insertions(+), 106 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 8599017c31a8..4f6e138c96f9 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -8,15 +8,23 @@ #include "librt_internal.h" #define START_SIZE 512 -#define MAX_SHORT_INT_TAGGED (255 << 1) -#define MAX_SHORT_LEN 127 -#define LONG_STR_TAG 1 +// See comment in read_int_internal() on motivation for these values. +#define MIN_ONE_BYTE_INT -10 +#define MAX_ONE_BYTE_INT 117 // 2 ** 7 - 1 - 10 +#define MIN_TWO_BYTES_INT -100 +#define MAX_TWO_BYTES_INT 16283 // 2 ** (8 + 6) - 1 - 100 +#define MIN_FOUR_BYTES_INT -10000 +#define MAX_FOUR_BYTES_INT 536860911 // 2 ** (3 * 8 + 5) - 1 - 10000 -#define MIN_SHORT_INT -10 -#define MAX_SHORT_INT 117 -#define MEDIUM_INT_TAG 1 -#define LONG_INT_TAG 3 +#define TWO_BYTES_INT_BIT 1 +#define FOUR_BYTES_INT_BIT 2 +#define LONG_INT_BIT 4 + +#define FOUR_BYTES_INT_TRAILER 3 +// We add one reserved bit here so that we can potentially support +// 8 bytes format in the future. +#define LONG_INT_TRAILER 15 #define CPY_BOOL_ERROR 2 #define CPY_NONE_ERROR 2 @@ -35,13 +43,22 @@ #define _WRITE(data, type, v) *(type *)(((BufferObject *)data)->buf + ((BufferObject *)data)->pos) = v; \ ((BufferObject *)data)->pos += sizeof(type); +#if PY_BIG_ENDIAN +uint16_t reverse_16(uint16_t number) { + return (number << 8) | (number >> 8); +} + +uint32_t reverse_32(uint32_t number) { + return ((number & 0xFF) << 24) | ((number & 0xFF00) << 8) | ((number & 0xFF0000) >> 8) | (number >> 24); +} +#endif + typedef struct { PyObject_HEAD Py_ssize_t pos; Py_ssize_t end; Py_ssize_t size; char *buf; - PyObject *source; } BufferObject; static PyTypeObject BufferType; @@ -259,26 +276,50 @@ write_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname } /* -str format: size followed by UTF-8 bytes - short strings (len <= 127): single byte for size as `(uint8_t)size << 1` - long strings: \x01 followed by size as Py_ssize_t +str format: size as int (see below) followed by UTF-8 bytes */ +static inline CPyTagged +_read_short_int(PyObject *data, uint8_t first) { + uint8_t second; + uint16_t two_more; + if ((first & TWO_BYTES_INT_BIT) == 0) { + // Note we use tagged ints since this function can return an error. + return ((Py_ssize_t)(first >> 1) + MIN_ONE_BYTE_INT) << 1; + } + if ((first & FOUR_BYTES_INT_BIT) == 0) { + _CHECK_READ(data, 1, CPY_INT_TAG) + second = _READ(data, uint8_t) + return ((((Py_ssize_t)second) << 6) + (Py_ssize_t)(first >> 2) + MIN_TWO_BYTES_INT) << 1; + } + // The caller is responsible to verify this is called only for short ints. + _CHECK_READ(data, 3, CPY_INT_TAG) + // TODO: check if compilers emit optimal code for these two reads, and tweak if needed. + second = _READ(data, uint8_t) + two_more = _READ(data, uint16_t) +#if PY_BIG_ENDIAN + two_more = reverse_16(two_more); +#endif + Py_ssize_t higher = (((Py_ssize_t)two_more) << 13) + (((Py_ssize_t)second) << 5); + return (higher + (Py_ssize_t)(first >> 3) + MIN_FOUR_BYTES_INT) << 1; +} + static PyObject* read_str_internal(PyObject *data) { _CHECK_BUFFER(data, NULL) // Read string length. - Py_ssize_t size; _CHECK_READ(data, 1, NULL) uint8_t first = _READ(data, uint8_t) - if (likely(first != LONG_STR_TAG)) { - // Common case: short string (len <= 127). - size = (Py_ssize_t)(first >> 1); - } else { - _CHECK_READ(data, sizeof(CPyTagged), NULL) - size = _READ(data, Py_ssize_t) + if (unlikely(first == LONG_INT_TRAILER)) { + // Fail fast for invalid/tampered data. + PyErr_SetString(PyExc_ValueError, "invalid str size"); + return NULL; } + CPyTagged tagged_size = _read_short_int(data, first); + if (tagged_size == CPY_INT_TAG) + return NULL; + Py_ssize_t size = tagged_size >> 1; // Read string content. char *buf = ((BufferObject *)data)->buf; _CHECK_READ(data, size, NULL) @@ -302,6 +343,35 @@ read_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) return read_str_internal(data); } +// The caller *must* check that real_value is within allowed range (29 bits). +static inline char +_write_short_int(PyObject *data, Py_ssize_t real_value) { + if (real_value >= MIN_ONE_BYTE_INT && real_value <= MAX_ONE_BYTE_INT) { + _CHECK_SIZE(data, 1) + _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_ONE_BYTE_INT) << 1) + ((BufferObject *)data)->end += 1; + } else if (real_value >= MIN_TWO_BYTES_INT && real_value <= MAX_TWO_BYTES_INT) { + _CHECK_SIZE(data, 2) +#if PY_BIG_ENDIAN + uint16_t to_write = ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT; + _WRITE(data, uint16_t, reverse_16(to_write)) +#else + _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT) +#endif + ((BufferObject *)data)->end += 2; + } else { + _CHECK_SIZE(data, 4) +#if PY_BIG_ENDIAN + uint32_t to_write = ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER; + _WRITE(data, uint32_t, reverse_32(to_write)) +#else + _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER) +#endif + ((BufferObject *)data)->end += 4; + } + return CPY_NONE; +} + static char write_str_internal(PyObject *data, PyObject *value) { _CHECK_BUFFER(data, CPY_NONE_ERROR) @@ -311,24 +381,20 @@ write_str_internal(PyObject *data, PyObject *value) { if (unlikely(chunk == NULL)) return CPY_NONE_ERROR; - Py_ssize_t need; // Write string length. - if (likely(size <= MAX_SHORT_LEN)) { - // Common case: short string (len <= 127) store as single byte. - need = size + 1; - _CHECK_SIZE(data, need) - _WRITE(data, uint8_t, (uint8_t)size << 1) + if (likely(size >= MIN_FOUR_BYTES_INT && size <= MAX_FOUR_BYTES_INT)) { + if (_write_short_int(data, size) == CPY_NONE_ERROR) + return CPY_NONE_ERROR; } else { - need = size + sizeof(Py_ssize_t) + 1; - _CHECK_SIZE(data, need) - _WRITE(data, uint8_t, LONG_STR_TAG) - _WRITE(data, Py_ssize_t, size) + PyErr_SetString(PyExc_ValueError, "str too long to serialize"); + return CPY_NONE_ERROR; } // Write string content. + _CHECK_SIZE(data, size) char *buf = ((BufferObject *)data)->buf; memcpy(buf + ((BufferObject *)data)->pos, chunk, size); ((BufferObject *)data)->pos += size; - ((BufferObject *)data)->end += need; + ((BufferObject *)data)->end += size; return CPY_NONE; } @@ -353,9 +419,7 @@ write_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames } /* -bytes format: size followed by bytes - short bytes (len <= 127): single byte for size as `(uint8_t)size << 1` - long bytes: \x01 followed by size as Py_ssize_t +bytes format: size as int (see below) followed by bytes */ static PyObject* @@ -363,16 +427,17 @@ read_bytes_internal(PyObject *data) { _CHECK_BUFFER(data, NULL) // Read length. - Py_ssize_t size; _CHECK_READ(data, 1, NULL) uint8_t first = _READ(data, uint8_t) - if (likely(first != LONG_STR_TAG)) { - // Common case: short bytes (len <= 127). - size = (Py_ssize_t)(first >> 1); - } else { - _CHECK_READ(data, sizeof(CPyTagged), NULL) - size = _READ(data, Py_ssize_t) + if (unlikely(first == LONG_INT_TRAILER)) { + // Fail fast for invalid/tampered data. + PyErr_SetString(PyExc_ValueError, "invalid bytes size"); + return NULL; } + CPyTagged tagged_size = _read_short_int(data, first); + if (tagged_size == CPY_INT_TAG) + return NULL; + Py_ssize_t size = tagged_size >> 1; // Read bytes content. char *buf = ((BufferObject *)data)->buf; _CHECK_READ(data, size, NULL) @@ -405,24 +470,20 @@ write_bytes_internal(PyObject *data, PyObject *value) { return CPY_NONE_ERROR; Py_ssize_t size = PyBytes_GET_SIZE(value); - Py_ssize_t need; // Write length. - if (likely(size <= MAX_SHORT_LEN)) { - // Common case: short bytes (len <= 127) store as single byte. - need = size + 1; - _CHECK_SIZE(data, need) - _WRITE(data, uint8_t, (uint8_t)size << 1) + if (likely(size >= MIN_FOUR_BYTES_INT && size <= MAX_FOUR_BYTES_INT)) { + if (_write_short_int(data, size) == CPY_NONE_ERROR) + return CPY_NONE_ERROR; } else { - need = size + sizeof(Py_ssize_t) + 1; - _CHECK_SIZE(data, need) - _WRITE(data, uint8_t, LONG_STR_TAG) - _WRITE(data, Py_ssize_t, size) + PyErr_SetString(PyExc_ValueError, "bytes too long to serialize"); + return CPY_NONE_ERROR; } // Write bytes content. + _CHECK_SIZE(data, size) char *buf = ((BufferObject *)data)->buf; memcpy(buf + ((BufferObject *)data)->pos, chunk, size); ((BufferObject *)data)->pos += size; - ((BufferObject *)data)->end += need; + ((BufferObject *)data)->end += size; return CPY_NONE; } @@ -455,7 +516,7 @@ static double read_float_internal(PyObject *data) { _CHECK_BUFFER(data, CPY_FLOAT_ERROR) _CHECK_READ(data, sizeof(double), CPY_FLOAT_ERROR) - double res = _READ(data, double); + double res = _READ(data, double) return res; } @@ -505,9 +566,13 @@ write_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam /* int format: - most common values (-10 <= value <= 117): single byte as `(uint8_t)(value + 10) << 1` - medium values (fit in CPyTagged): \x01 followed by CPyTagged value - long values (very rare): \x03 followed by decimal string (see str format) + one byte: last bit 0, 7 bits used + two bytes: last two bits 01, 14 bits used + four bytes: last three bits 011, 29 bits used + everything else: 00001111 followed by serialized string representation + +Note: for fixed size formats we skew ranges towards more positive values, +since negative integers are much more rare. */ static CPyTagged @@ -516,22 +581,17 @@ read_int_internal(PyObject *data) { _CHECK_READ(data, 1, CPY_INT_TAG) uint8_t first = _READ(data, uint8_t) - if ((first & MEDIUM_INT_TAG) == 0) { - // Most common case: int that is small in absolute value. - return ((Py_ssize_t)(first >> 1) + MIN_SHORT_INT) << 1; - } - if (first == MEDIUM_INT_TAG) { - _CHECK_READ(data, sizeof(CPyTagged), CPY_INT_TAG) - CPyTagged ret = _READ(data, CPyTagged) - return ret; + if (likely(first != LONG_INT_TRAILER)) { + return _read_short_int(data, first); } - // People who have literal ints not fitting in size_t should be punished :-) PyObject *str_ret = read_str_internal(data); if (unlikely(str_ret == NULL)) return CPY_INT_TAG; PyObject* ret_long = PyLong_FromUnicodeObject(str_ret, 10); Py_DECREF(str_ret); - return ((CPyTagged)ret_long) | CPY_INT_TAG; + if (ret_long == NULL) + return CPY_INT_TAG; + return CPyTagged_StealFromObject(ret_long); } static PyObject* @@ -549,36 +609,38 @@ read_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) return CPyTagged_StealAsObject(retval); } +static inline char +_write_long_int(PyObject *data, CPyTagged value) { + // TODO(jukka): write a more compact/optimal format for arbitrary length ints. + _CHECK_SIZE(data, 1) + _WRITE(data, uint8_t, LONG_INT_TRAILER) + ((BufferObject *)data)->end += 1; + PyObject* int_value = CPyTagged_AsObject(value); + if (unlikely(int_value == NULL)) + return CPY_NONE_ERROR; + PyObject *str_value = PyObject_Str(int_value); + Py_DECREF(int_value); + if (unlikely(str_value == NULL)) + return CPY_NONE_ERROR; + char res = write_str_internal(data, str_value); + Py_DECREF(str_value); + return res; +} + static char write_int_internal(PyObject *data, CPyTagged value) { _CHECK_BUFFER(data, CPY_NONE_ERROR) if (likely((value & CPY_INT_TAG) == 0)) { Py_ssize_t real_value = CPyTagged_ShortAsSsize_t(value); - if (real_value >= MIN_SHORT_INT && real_value <= MAX_SHORT_INT) { - // Most common case: int that is small in absolute value. - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_SHORT_INT) << 1) - ((BufferObject *)data)->end += 1; + if (likely(real_value >= MIN_FOUR_BYTES_INT && real_value <= MAX_FOUR_BYTES_INT)) { + return _write_short_int(data, real_value); } else { - _CHECK_SIZE(data, sizeof(CPyTagged) + 1) - _WRITE(data, uint8_t, MEDIUM_INT_TAG) - _WRITE(data, CPyTagged, value) - ((BufferObject *)data)->end += sizeof(CPyTagged) + 1; + return _write_long_int(data, value); } } else { - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, LONG_INT_TAG) - ((BufferObject *)data)->end += 1; - PyObject *str_value = PyObject_Str(CPyTagged_LongAsObject(value)); - if (unlikely(str_value == NULL)) - return CPY_NONE_ERROR; - char res = write_str_internal(data, str_value); - Py_DECREF(str_value); - if (unlikely(res == CPY_NONE_ERROR)) - return CPY_NONE_ERROR; + return _write_long_int(data, value); } - return CPY_NONE; } static PyObject* diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index e08f2fd7007d..04bbed78b318 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2738,8 +2738,8 @@ def test_buffer_roundtrip() -> None: write_bytes(b, b"bar") write_bytes(b, b"bar" * 100) write_bytes(b, b"") - write_bytes(b, b"a" * 127) - write_bytes(b, b"a" * 128) + write_bytes(b, b"a" * 117) + write_bytes(b, b"a" * 118) write_float(b, 0.1) write_float(b, -113.0) write_int(b, 0) @@ -2752,8 +2752,9 @@ def test_buffer_roundtrip() -> None: write_int(b, 255) write_int(b, -1) write_int(b, -255) - write_int(b, 1234512344) - write_int(b, 1234512345) + write_int(b, 536860911) + write_int(b, 536860912) + write_int(b, 1234567891) b = Buffer(b.getvalue()) assert read_str(b) == "foo" @@ -2763,8 +2764,8 @@ def test_buffer_roundtrip() -> None: assert read_bytes(b) == b"bar" assert read_bytes(b) == b"bar" * 100 assert read_bytes(b) == b"" - assert read_bytes(b) == b"a" * 127 - assert read_bytes(b) == b"a" * 128 + assert read_bytes(b) == b"a" * 117 + assert read_bytes(b) == b"a" * 118 assert read_float(b) == 0.1 assert read_float(b) == -113.0 assert read_int(b) == 0 @@ -2777,8 +2778,9 @@ def test_buffer_roundtrip() -> None: assert read_int(b) == 255 assert read_int(b) == -1 assert read_int(b) == -255 - assert read_int(b) == 1234512344 - assert read_int(b) == 1234512345 + assert read_int(b) == 536860911 + assert read_int(b) == 536860912 + assert read_int(b) == 1234567891 def test_buffer_int_size() -> None: for i in (-10, -9, 0, 116, 117): @@ -2787,21 +2789,44 @@ def test_buffer_int_size() -> None: assert len(b.getvalue()) == 1 b = Buffer(b.getvalue()) assert read_int(b) == i - for i in (-12345, -12344, -11, 118, 12344, 12345): + for i in (-100, -11, 118, 12344, 16283): b = Buffer() write_int(b, i) - assert len(b.getvalue()) <= 9 # sizeof(size_t) + 1 + assert len(b.getvalue()) == 2 b = Buffer(b.getvalue()) assert read_int(b) == i + for i in (-10000, 16284, 123456789): + b = Buffer() + write_int(b, i) + assert len(b.getvalue()) == 4 + b = Buffer(b.getvalue()) + assert read_int(b) == i + +def test_buffer_int_powers() -> None: + # 0, 1, 2 are tested above + for p in range(2, 9): + b = Buffer() + write_int(b, 1 << p) + write_int(b, -1 << p) + b = Buffer(b.getvalue()) + assert read_int(b) == 1 << p + assert read_int(b) == -1 << p def test_buffer_str_size() -> None: - for s in ("", "a", "a" * 127): + for s in ("", "a", "a" * 117): b = Buffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 1 b = Buffer(b.getvalue()) assert read_str(b) == s + for s in ("a" * 118, "a" * 16283): + b = Buffer() + write_str(b, s) + assert len(b.getvalue()) == len(s) + 2 + b = Buffer(b.getvalue()) + assert read_str(b) == s + [file driver.py] from native import * @@ -2809,6 +2834,7 @@ test_buffer_basic() test_buffer_roundtrip() test_buffer_int_size() test_buffer_str_size() +test_buffer_int_powers() def test_buffer_basic_interpreted() -> None: b = Buffer(b"foo") @@ -2823,8 +2849,8 @@ def test_buffer_roundtrip_interpreted() -> None: write_bytes(b, b"bar") write_bytes(b, b"bar" * 100) write_bytes(b, b"") - write_bytes(b, b"a" * 127) - write_bytes(b, b"a" * 128) + write_bytes(b, b"a" * 117) + write_bytes(b, b"a" * 118) write_float(b, 0.1) write_int(b, 0) write_int(b, 1) @@ -2836,8 +2862,9 @@ def test_buffer_roundtrip_interpreted() -> None: write_int(b, 255) write_int(b, -1) write_int(b, -255) - write_int(b, 1234512344) - write_int(b, 1234512345) + write_int(b, 536860911) + write_int(b, 536860912) + write_int(b, 1234567891) b = Buffer(b.getvalue()) assert read_str(b) == "foo" @@ -2847,8 +2874,8 @@ def test_buffer_roundtrip_interpreted() -> None: assert read_bytes(b) == b"bar" assert read_bytes(b) == b"bar" * 100 assert read_bytes(b) == b"" - assert read_bytes(b) == b"a" * 127 - assert read_bytes(b) == b"a" * 128 + assert read_bytes(b) == b"a" * 117 + assert read_bytes(b) == b"a" * 118 assert read_float(b) == 0.1 assert read_int(b) == 0 assert read_int(b) == 1 @@ -2860,8 +2887,9 @@ def test_buffer_roundtrip_interpreted() -> None: assert read_int(b) == 255 assert read_int(b) == -1 assert read_int(b) == -255 - assert read_int(b) == 1234512344 - assert read_int(b) == 1234512345 + assert read_int(b) == 536860911 + assert read_int(b) == 536860912 + assert read_int(b) == 1234567891 def test_buffer_int_size_interpreted() -> None: for i in (-10, -9, 0, 116, 117): @@ -2870,25 +2898,49 @@ def test_buffer_int_size_interpreted() -> None: assert len(b.getvalue()) == 1 b = Buffer(b.getvalue()) assert read_int(b) == i - for i in (-12345, -12344, -11, 118, 12344, 12345): + for i in (-100, -11, 118, 12344, 16283): b = Buffer() write_int(b, i) - assert len(b.getvalue()) <= 9 # sizeof(size_t) + 1 + assert len(b.getvalue()) == 2 b = Buffer(b.getvalue()) assert read_int(b) == i + for i in (-10000, 16284, 123456789): + b = Buffer() + write_int(b, i) + assert len(b.getvalue()) == 4 + b = Buffer(b.getvalue()) + assert read_int(b) == i + +def test_buffer_int_powers_interpreted() -> None: + # 0, 1, 2 are tested above + for p in range(2, 9): + b = Buffer() + write_int(b, 1 << p) + write_int(b, -1 << p) + b = Buffer(b.getvalue()) + assert read_int(b) == 1 << p + assert read_int(b) == -1 << p def test_buffer_str_size_interpreted() -> None: - for s in ("", "a", "a" * 127): + for s in ("", "a", "a" * 117): b = Buffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 1 b = Buffer(b.getvalue()) assert read_str(b) == s + for s in ("a" * 118, "a" * 16283): + b = Buffer() + write_str(b, s) + assert len(b.getvalue()) == len(s) + 2 + b = Buffer(b.getvalue()) + assert read_str(b) == s + test_buffer_basic_interpreted() test_buffer_roundtrip_interpreted() test_buffer_int_size_interpreted() test_buffer_str_size_interpreted() +test_buffer_int_powers_interpreted() [case testBufferEmpty_librt_internal] from librt.internal import Buffer, write_int, read_int From c52b17afe2273cf4b6aee51c6e8b64f2c81f54e1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 31 Oct 2025 12:32:45 +0000 Subject: [PATCH 129/183] More robust packing of flats in FF cache (#20150) This should fix issues with `librt` not working well on platforms with hardware float support, it should also make cache format endian-independent (since we ask for little-endian format using `1` as last argument independently of current endianness). More details in https://docs.python.org/3/c-api/float.html#pack-functions --- mypyc/lib-rt/librt_internal.c | 20 ++++++++++++++------ mypyc/test-data/run-classes.test | 2 ++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 4f6e138c96f9..eb864619d398 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -509,14 +509,18 @@ write_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam /* float format: - stored as a C double + stored using PyFloat helpers in little-endian format. */ static double read_float_internal(PyObject *data) { _CHECK_BUFFER(data, CPY_FLOAT_ERROR) - _CHECK_READ(data, sizeof(double), CPY_FLOAT_ERROR) - double res = _READ(data, double) + _CHECK_READ(data, 8, CPY_FLOAT_ERROR) + char *buf = ((BufferObject *)data)->buf; + double res = PyFloat_Unpack8(buf + ((BufferObject *)data)->pos, 1); + if (unlikely((res == -1.0) && PyErr_Occurred())) + return CPY_FLOAT_ERROR; + ((BufferObject *)data)->pos += 8; return res; } @@ -538,9 +542,13 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname static char write_float_internal(PyObject *data, double value) { _CHECK_BUFFER(data, CPY_NONE_ERROR) - _CHECK_SIZE(data, sizeof(double)) - _WRITE(data, double, value) - ((BufferObject *)data)->end += sizeof(double); + _CHECK_SIZE(data, 8) + char *buf = ((BufferObject *)data)->buf; + int res = PyFloat_Pack8(value, buf + ((BufferObject *)data)->pos, 1); + if (unlikely(res == -1)) + return CPY_NONE_ERROR; + ((BufferObject *)data)->pos += 8; + ((BufferObject *)data)->end += 8; return CPY_NONE; } diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 04bbed78b318..9c2ba14c0873 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2741,6 +2741,7 @@ def test_buffer_roundtrip() -> None: write_bytes(b, b"a" * 117) write_bytes(b, b"a" * 118) write_float(b, 0.1) + write_float(b, -1.0) write_float(b, -113.0) write_int(b, 0) write_int(b, 1) @@ -2767,6 +2768,7 @@ def test_buffer_roundtrip() -> None: assert read_bytes(b) == b"a" * 117 assert read_bytes(b) == b"a" * 118 assert read_float(b) == 0.1 + assert read_float(b) == -1.0 assert read_float(b) == -113.0 assert read_int(b) == 0 assert read_int(b) == 1 From 72131392483c19a6b8db812f4d4acfdb50a769dd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 31 Oct 2025 16:01:27 +0000 Subject: [PATCH 130/183] Use more efficient serialization format for long integers in cache files (#20151) A long integer (one that doesn't fit in the 4-byte encoding) will now be encoded like this: * initial header byte * short integer (1-4 bytes) encoding the number of bytes of data and sign * variable-length number of data bytes (absolute value of the integer) -- all bits are used For example, a 32-bit integer can now always be encoded using at most 6 bytes (+ type tag). This is optimized for size efficiency, not performance, since large integers are not expected to be a performance bottleneck. Having an efficient format makes it easier to improve performance in the future, however, without changing the encoding. The header byte has a few unused bits which could be used to slightly improve efficiency, but I decided that it's not worth the extra complexity. --- mypyc/lib-rt/librt_internal.c | 108 +++++++++++++++++++++++++++---- mypyc/test-data/run-classes.test | 27 +++++++- 2 files changed, 120 insertions(+), 15 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index eb864619d398..6cae63cfadcb 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -592,14 +592,35 @@ read_int_internal(PyObject *data) { if (likely(first != LONG_INT_TRAILER)) { return _read_short_int(data, first); } - PyObject *str_ret = read_str_internal(data); - if (unlikely(str_ret == NULL)) + + // Long integer encoding -- byte length and sign, followed by a byte array. + + // Read byte length and sign. + _CHECK_READ(data, 1, CPY_INT_TAG) + first = _READ(data, uint8_t) + Py_ssize_t size_and_sign = _read_short_int(data, first); + if (size_and_sign == CPY_INT_TAG) return CPY_INT_TAG; - PyObject* ret_long = PyLong_FromUnicodeObject(str_ret, 10); - Py_DECREF(str_ret); - if (ret_long == NULL) + bool sign = (size_and_sign >> 1) & 1; + Py_ssize_t size = size_and_sign >> 2; + + // Construct an int object from the byte array. + _CHECK_READ(data, size, CPY_INT_TAG) + char *buf = ((BufferObject *)data)->buf; + PyObject *num = _PyLong_FromByteArray( + (unsigned char *)(buf + ((BufferObject *)data)->pos), size, 1, 0); + if (num == NULL) return CPY_INT_TAG; - return CPyTagged_StealFromObject(ret_long); + ((BufferObject *)data)->pos += size; + if (sign) { + PyObject *old = num; + num = PyNumber_Negative(old); + Py_DECREF(old); + if (num == NULL) { + return CPY_INT_TAG; + } + } + return CPyTagged_StealFromObject(num); } static PyObject* @@ -617,22 +638,81 @@ read_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) return CPyTagged_StealAsObject(retval); } + +static inline int hex_to_int(char c) { + if (c >= '0' && c <= '9') + return c - '0'; + else if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + else + return c - 'A' + 10; // Assume valid hex digit +} + static inline char _write_long_int(PyObject *data, CPyTagged value) { - // TODO(jukka): write a more compact/optimal format for arbitrary length ints. _CHECK_SIZE(data, 1) _WRITE(data, uint8_t, LONG_INT_TRAILER) ((BufferObject *)data)->end += 1; + + PyObject *hex_str = NULL; PyObject* int_value = CPyTagged_AsObject(value); if (unlikely(int_value == NULL)) - return CPY_NONE_ERROR; - PyObject *str_value = PyObject_Str(int_value); + goto error; + + hex_str = PyNumber_ToBase(int_value, 16); + if (hex_str == NULL) + goto error; Py_DECREF(int_value); - if (unlikely(str_value == NULL)) - return CPY_NONE_ERROR; - char res = write_str_internal(data, str_value); - Py_DECREF(str_value); - return res; + int_value = NULL; + + const char *str = PyUnicode_AsUTF8(hex_str); + if (str == NULL) + goto error; + Py_ssize_t len = strlen(str); + bool neg; + if (str[0] == '-') { + str++; + len--; + neg = true; + } else { + neg = false; + } + // Skip the 0x hex prefix. + str += 2; + len -= 2; + + // Write bytes encoded length and sign. + Py_ssize_t size = (len + 1) / 2; + Py_ssize_t encoded_size = (size << 1) | neg; + if (encoded_size <= MAX_FOUR_BYTES_INT) { + if (_write_short_int(data, encoded_size) == CPY_NONE_ERROR) + goto error; + } else { + PyErr_SetString(PyExc_ValueError, "int too long to serialize"); + goto error; + } + + // Write absolute integer value as byte array in a variable-length little endian format. + int i; + for (i = len; i > 1; i -= 2) { + if (write_tag_internal( + data, hex_to_int(str[i - 1]) | (hex_to_int(str[i - 2]) << 4)) == CPY_NONE_ERROR) + goto error; + } + // The final byte may correspond to only one hex digit. + if (i == 1) { + if (write_tag_internal(data, hex_to_int(str[i - 1])) == CPY_NONE_ERROR) + goto error; + } + + Py_DECREF(hex_str); + return CPY_NONE; + + error: + + Py_XDECREF(int_value); + Py_XDECREF(hex_str); + return CPY_NONE_ERROR; } static char diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 9c2ba14c0873..b02d10446800 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2806,13 +2806,36 @@ def test_buffer_int_size() -> None: def test_buffer_int_powers() -> None: # 0, 1, 2 are tested above - for p in range(2, 9): + for p in range(2, 200): b = Buffer() write_int(b, 1 << p) + write_int(b, (1 << p) - 1) write_int(b, -1 << p) + write_int(b, (-1 << p) + 1) b = Buffer(b.getvalue()) assert read_int(b) == 1 << p + assert read_int(b) == (1 << p) - 1 assert read_int(b) == -1 << p + assert read_int(b) == (-1 << p) + 1 + +def test_positive_long_int_serialized_bytes() -> None: + b = Buffer() + n = 0x123456789ab + write_int(b, n) + x = b.getvalue() + # Two prefix bytes, followed by little endian encoded integer in variable-length format + assert x == b"\x0f\x2c\xab\x89\x67\x45\x23\x01" + b = Buffer(x) + assert read_int(b) == n + +def test_negative_long_int_serialized_bytes() -> None: + b = Buffer() + n = -0x123456789abcde + write_int(b, n) + x = b.getvalue() + assert x == b"\x0f\x32\xde\xbc\x9a\x78\x56\x34\x12" + b = Buffer(x) + assert read_int(b) == n def test_buffer_str_size() -> None: for s in ("", "a", "a" * 117): @@ -2837,6 +2860,8 @@ test_buffer_roundtrip() test_buffer_int_size() test_buffer_str_size() test_buffer_int_powers() +test_positive_long_int_serialized_bytes() +test_negative_long_int_serialized_bytes() def test_buffer_basic_interpreted() -> None: b = Buffer(b"foo") From c5301092ddf8fefab427f578ee399fbf7b273978 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:35:26 -0700 Subject: [PATCH 131/183] Avoid running tests on macos-13 runner (#20155) --- .github/workflows/test.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de3f8877ee67..07b4b3f03020 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,12 +79,18 @@ jobs: # # allow_failure: true # test_mypyc: true - - name: mypyc runtime tests with py39-macos - python: '3.9.21' - # TODO: macos-13 is the last one to support Python 3.9, change it to macos-latest when updating the Python version - os: macos-13 + - name: mypyc runtime tests with py313-macos + python: '3.13' + os: macos-latest toxenv: py tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py" + + - name: mypyc runtime tests with py313-ubuntu + python: '3.13' + os: ubuntu-latest + toxenv: py + tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py" + # This is broken. See # - https://github.com/python/mypy/issues/17819 # - https://github.com/python/mypy/pull/17822 From d6e9c31bc33d3796ece88a9e07d051101515bafd Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:30:33 -0700 Subject: [PATCH 132/183] Upgrade ruff, black (#20158) --- .pre-commit-config.yaml | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1f4282e4f65b..1466c8e0fda4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,14 +6,14 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 + rev: 25.9.0 hooks: - id: black exclude: '^(test-data/)' - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.4 + rev: v0.14.3 hooks: - - id: ruff + - id: ruff-check args: [--exit-non-zero-on-fix] - repo: https://github.com/python-jsonschema/check-jsonschema rev: 0.32.1 diff --git a/pyproject.toml b/pyproject.toml index f9f6c01b5c1c..42e10967cba2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,7 +107,7 @@ mypyc = [ [tool.black] line-length = 99 -target-version = ["py39", "py310", "py311", "py312", "py313"] +target-version = ["py39", "py310", "py311", "py312", "py313", "py314"] skip-magic-trailing-comma = true force-exclude = ''' ^/mypy/typeshed| From 98d79300d2878dac037f8f468b8f9b0dd197e8c2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:54:14 -0700 Subject: [PATCH 133/183] Sync typeshed (#20157) Source commit: https://github.com/python/typeshed/commit/bf7214784877c52638844c065360d4814fae4c65 --- mypy/typeshed/stdlib/builtins.pyi | 4 ++++ mypy/typeshed/stdlib/cmath.pyi | 2 +- mypy/typeshed/stdlib/contextlib.pyi | 21 +++++++++++++++------ mypy/typeshed/stdlib/enum.pyi | 1 + mypy/typeshed/stdlib/os/__init__.pyi | 3 +++ mypy/typeshed/stdlib/sys/__init__.pyi | 12 ++++++++++++ mypy/typeshed/stdlib/sysconfig.pyi | 8 +++++--- mypy/typeshed/stdlib/turtle.pyi | 8 ++++---- mypy/typeshed/stdlib/zlib.pyi | 4 ++-- 9 files changed, 47 insertions(+), 16 deletions(-) diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index ddf81db181bf..e03a92ce3d91 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -1960,6 +1960,10 @@ class BaseException: def __new__(cls, *args: Any, **kwds: Any) -> Self: ... def __setstate__(self, state: dict[str, Any] | None, /) -> None: ... def with_traceback(self, tb: TracebackType | None, /) -> Self: ... + # Necessary for security-focused static analyzers (e.g, pysa) + # See https://github.com/python/typeshed/pull/14900 + def __str__(self) -> str: ... # noqa: Y029 + def __repr__(self) -> str: ... # noqa: Y029 if sys.version_info >= (3, 11): # only present after add_note() is called __notes__: list[str] diff --git a/mypy/typeshed/stdlib/cmath.pyi b/mypy/typeshed/stdlib/cmath.pyi index a08addcf5438..aed4c63862fe 100644 --- a/mypy/typeshed/stdlib/cmath.pyi +++ b/mypy/typeshed/stdlib/cmath.pyi @@ -23,7 +23,7 @@ def exp(z: _C, /) -> complex: ... def isclose(a: _C, b: _C, *, rel_tol: SupportsFloat = 1e-09, abs_tol: SupportsFloat = 0.0) -> bool: ... def isinf(z: _C, /) -> bool: ... def isnan(z: _C, /) -> bool: ... -def log(x: _C, base: _C = ..., /) -> complex: ... +def log(z: _C, base: _C = ..., /) -> complex: ... def log10(z: _C, /) -> complex: ... def phase(z: _C, /) -> float: ... def polar(z: _C, /) -> tuple[float, float]: ... diff --git a/mypy/typeshed/stdlib/contextlib.pyi b/mypy/typeshed/stdlib/contextlib.pyi index 383a1b7f334b..221102ee2395 100644 --- a/mypy/typeshed/stdlib/contextlib.pyi +++ b/mypy/typeshed/stdlib/contextlib.pyi @@ -4,7 +4,7 @@ from _typeshed import FileDescriptorOrPath, Unused from abc import ABC, abstractmethod from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Iterator from types import TracebackType -from typing import IO, Any, Generic, Protocol, TypeVar, overload, runtime_checkable, type_check_only +from typing import Any, Generic, Protocol, TypeVar, overload, runtime_checkable, type_check_only from typing_extensions import ParamSpec, Self, TypeAlias __all__ = [ @@ -30,7 +30,6 @@ if sys.version_info >= (3, 11): _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) -_T_io = TypeVar("_T_io", bound=IO[str] | None) _ExitT_co = TypeVar("_ExitT_co", covariant=True, bound=bool | None, default=bool | None) _F = TypeVar("_F", bound=Callable[..., Any]) _G_co = TypeVar("_G_co", bound=Generator[Any, Any, Any] | AsyncGenerator[Any, Any], covariant=True) @@ -141,14 +140,24 @@ class suppress(AbstractContextManager[None, bool]): self, exctype: type[BaseException] | None, excinst: BaseException | None, exctb: TracebackType | None ) -> bool: ... -class _RedirectStream(AbstractContextManager[_T_io, None]): - def __init__(self, new_target: _T_io) -> None: ... +# This is trying to describe what is needed for (most?) uses +# of `redirect_stdout` and `redirect_stderr`. +# https://github.com/python/typeshed/issues/14903 +@type_check_only +class _SupportsRedirect(Protocol): + def write(self, s: str, /) -> int: ... + def flush(self) -> None: ... + +_SupportsRedirectT = TypeVar("_SupportsRedirectT", bound=_SupportsRedirect | None) + +class _RedirectStream(AbstractContextManager[_SupportsRedirectT, None]): + def __init__(self, new_target: _SupportsRedirectT) -> None: ... def __exit__( self, exctype: type[BaseException] | None, excinst: BaseException | None, exctb: TracebackType | None ) -> None: ... -class redirect_stdout(_RedirectStream[_T_io]): ... -class redirect_stderr(_RedirectStream[_T_io]): ... +class redirect_stdout(_RedirectStream[_SupportsRedirectT]): ... +class redirect_stderr(_RedirectStream[_SupportsRedirectT]): ... class _BaseExitStack(Generic[_ExitT_co]): def enter_context(self, cm: AbstractContextManager[_T, _ExitT_co]) -> _T: ... diff --git a/mypy/typeshed/stdlib/enum.pyi b/mypy/typeshed/stdlib/enum.pyi index 4ac860f5e611..c131c9392393 100644 --- a/mypy/typeshed/stdlib/enum.pyi +++ b/mypy/typeshed/stdlib/enum.pyi @@ -309,6 +309,7 @@ if sys.version_info >= (3, 11): def global_enum(cls: _EnumerationT, update_str: bool = False) -> _EnumerationT: ... def global_enum_repr(self: Enum) -> str: ... def global_flag_repr(self: Flag) -> str: ... + def show_flag_values(value: int) -> list[int]: ... if sys.version_info >= (3, 12): # The body of the class is the same, but the base classes are different. diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index 71c79dfac399..580452739f7f 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -729,6 +729,9 @@ environ: _Environ[str] if sys.platform != "win32": environb: _Environ[bytes] +if sys.version_info >= (3, 14): + def reload_environ() -> None: ... + if sys.version_info >= (3, 11) or sys.platform != "win32": EX_OK: Final[int] diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi index 7807b0eab01f..97e65d3094aa 100644 --- a/mypy/typeshed/stdlib/sys/__init__.pyi +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -354,6 +354,13 @@ else: def _current_frames() -> dict[int, FrameType]: ... def _getframe(depth: int = 0, /) -> FrameType: ... +# documented -- see https://docs.python.org/3/library/sys.html#sys._current_exceptions +if sys.version_info >= (3, 12): + def _current_exceptions() -> dict[int, BaseException | None]: ... + +else: + def _current_exceptions() -> dict[int, OptExcInfo]: ... + if sys.version_info >= (3, 12): def _getframemodulename(depth: int = 0) -> str | None: ... @@ -366,6 +373,10 @@ if sys.version_info >= (3, 11): def exception() -> BaseException | None: ... def exit(status: _ExitCode = None, /) -> NoReturn: ... + +if sys.platform == "android": # noqa: Y008 + def getandroidapilevel() -> int: ... + def getallocatedblocks() -> int: ... def getdefaultencoding() -> str: ... @@ -501,3 +512,4 @@ if sys.version_info >= (3, 12): if sys.version_info >= (3, 14): def is_remote_debug_enabled() -> bool: ... def remote_exec(pid: int, script: StrOrBytesPath) -> None: ... + def _is_immortal(op: object, /) -> bool: ... diff --git a/mypy/typeshed/stdlib/sysconfig.pyi b/mypy/typeshed/stdlib/sysconfig.pyi index 807a979050e8..c6419222df97 100644 --- a/mypy/typeshed/stdlib/sysconfig.pyi +++ b/mypy/typeshed/stdlib/sysconfig.pyi @@ -1,6 +1,6 @@ import sys from typing import IO, Any, Literal, overload -from typing_extensions import deprecated +from typing_extensions import LiteralString, deprecated __all__ = [ "get_config_h_filename", @@ -28,8 +28,10 @@ def get_config_vars(arg: str, /, *args: str) -> list[Any]: ... def get_scheme_names() -> tuple[str, ...]: ... if sys.version_info >= (3, 10): - def get_default_scheme() -> str: ... - def get_preferred_scheme(key: Literal["prefix", "home", "user"]) -> str: ... + def get_default_scheme() -> LiteralString: ... + def get_preferred_scheme(key: Literal["prefix", "home", "user"]) -> LiteralString: ... + # Documented -- see https://docs.python.org/3/library/sysconfig.html#sysconfig._get_preferred_schemes + def _get_preferred_schemes() -> dict[Literal["prefix", "home", "user"], LiteralString]: ... def get_path_names() -> tuple[str, ...]: ... def get_path(name: str, scheme: str = ..., vars: dict[str, Any] | None = None, expand: bool = True) -> str: ... diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index 9b9b329bd74b..b5f536d0e28e 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -266,7 +266,7 @@ class TurtleScreen(TurtleScreenBase): def window_height(self) -> int: ... def getcanvas(self) -> Canvas: ... def getshapes(self) -> list[str]: ... - def onclick(self, fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... + def onclick(self, fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... def onkey(self, fun: Callable[[], object], key: str) -> None: ... def listen(self, xdummy: float | None = None, ydummy: float | None = None) -> None: ... def ontimer(self, fun: Callable[[], object], t: int = 0) -> None: ... @@ -561,7 +561,7 @@ def window_width() -> int: ... def window_height() -> int: ... def getcanvas() -> Canvas: ... def getshapes() -> list[str]: ... -def onclick(fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... +def onclick(fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... def onkey(fun: Callable[[], object], key: str) -> None: ... def listen(xdummy: float | None = None, ydummy: float | None = None) -> None: ... def ontimer(fun: Callable[[], object], t: int = 0) -> None: ... @@ -776,8 +776,8 @@ def getturtle() -> Turtle: ... getpen = getturtle -def onrelease(fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... -def ondrag(fun: Callable[[float, float], object], btn: int = 1, add: Any | None = None) -> None: ... +def onrelease(fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... +def ondrag(fun: Callable[[float, float], object], btn: int = 1, add: bool | None = None) -> None: ... def undo() -> None: ... turtlesize = shapesize diff --git a/mypy/typeshed/stdlib/zlib.pyi b/mypy/typeshed/stdlib/zlib.pyi index 4e410fdd18ad..d5998cab90fe 100644 --- a/mypy/typeshed/stdlib/zlib.pyi +++ b/mypy/typeshed/stdlib/zlib.pyi @@ -26,8 +26,8 @@ Z_RLE: Final = 3 Z_SYNC_FLUSH: Final = 2 Z_TREES: Final = 6 -if sys.version_info >= (3, 14) and sys.platform == "win32": - # Available when zlib was built with zlib-ng, usually only on Windows +if sys.version_info >= (3, 14): + # Available when zlib was built with zlib-ng ZLIBNG_VERSION: Final[str] class error(Exception): ... From 92101f3ec406534cb6be14bef7eb99b7e8fc5348 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 1 Nov 2025 00:57:29 +0000 Subject: [PATCH 134/183] Force-discard cache if cache format changed (#20152) If either low-level (i.e. `librt`) or high-level cache format changes, discard the cache. Note I intentionally don't use `librt` to read/write the first two bytes of cache meta, se we are 100% sure we can always read them. --- mypy-requirements.txt | 2 +- mypy/build.py | 18 ++++++++++++++---- mypy/cache.py | 6 ++++++ pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 622a8c3f3613..7c83178ae1eb 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.3.0 +librt>=0.4.0 diff --git a/mypy/build.py b/mypy/build.py index 0058fb7eaaa0..0b78f879c547 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -28,8 +28,10 @@ from typing import TYPE_CHECKING, Any, Callable, ClassVar, Final, NoReturn, TextIO, TypedDict from typing_extensions import TypeAlias as _TypeAlias +from librt.internal import cache_version + import mypy.semanal_main -from mypy.cache import Buffer, CacheMeta +from mypy.cache import CACHE_VERSION, Buffer, CacheMeta from mypy.checker import TypeChecker from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error @@ -1334,12 +1336,18 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No return None t1 = time.time() if isinstance(meta, bytes): - data_io = Buffer(meta) + # If either low-level buffer format or high-level cache layout changed, we + # cannot use the cache files, even with --skip-version-check. + # TODO: switch to something like librt.internal.read_byte() if this is slow. + if meta[0] != cache_version() or meta[1] != CACHE_VERSION: + manager.log(f"Metadata abandoned for {id}: incompatible cache format") + return None + data_io = Buffer(meta[2:]) m = CacheMeta.read(data_io, data_file) else: m = CacheMeta.deserialize(meta, data_file) if m is None: - manager.log(f"Metadata abandoned for {id}: attributes are missing") + manager.log(f"Metadata abandoned for {id}: cannot deserialize data") return None t2 = time.time() manager.add_stats( @@ -1671,7 +1679,9 @@ def write_cache_meta(meta: CacheMeta, manager: BuildManager, meta_file: str) -> if manager.options.fixed_format_cache: data_io = Buffer() meta.write(data_io) - meta_bytes = data_io.getvalue() + # Prefix with both low- and high-level cache format versions for future validation. + # TODO: switch to something like librt.internal.write_byte() if this is slow. + meta_bytes = bytes([cache_version(), CACHE_VERSION]) + data_io.getvalue() else: meta_dict = meta.serialize() meta_bytes = json_dumps(meta_dict, manager.options.debug_cache) diff --git a/mypy/cache.py b/mypy/cache.py index 0d2db67fac94..900815b9f7e7 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -40,6 +40,9 @@ serialization. The write method should write both class tag and end tag. The read method conventionally *does not* read the start tag (to simplify logic for unions). Known exceptions are MypyFile.read() and SymbolTableNode.read(), since those two never appear in a union. + +If any of these details change, or if the structure of CacheMeta changes please +bump CACHE_VERSION below. """ from __future__ import annotations @@ -65,6 +68,9 @@ ) from mypy_extensions import u8 +# High-level cache layout format +CACHE_VERSION: Final = 0 + class CacheMeta: """Class representing cache metadata for a module.""" diff --git a/pyproject.toml b/pyproject.toml index 42e10967cba2..0de739be9b55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.3.0", + "librt>=0.4.0", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.3.0", + "librt>=0.4.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index b9ff4ffe085b..126abd7149e6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.3.0 +librt==0.4.0 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From d31d5260a33f9a748986baab6ef56187d85f953d Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 1 Nov 2025 02:53:40 +0100 Subject: [PATCH 135/183] Cleanup fastparse (#20159) Remove some unused code in mypy/fastparse.py. --- mypy/fastparse.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 276e183a6bf0..c5e4589ec025 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -144,9 +144,6 @@ def ast3_parse( ) -NamedExpr = ast3.NamedExpr -Constant = ast3.Constant - if sys.version_info >= (3, 10): Match = ast3.Match MatchValue = ast3.MatchValue @@ -957,7 +954,7 @@ def do_func_def( # for ellipsis arg if ( len(func_type_ast.argtypes) == 1 - and isinstance(func_type_ast.argtypes[0], Constant) + and isinstance(func_type_ast.argtypes[0], ast3.Constant) and func_type_ast.argtypes[0].value is Ellipsis ): if n.returns: @@ -1492,7 +1489,7 @@ def visit_Continue(self, n: ast3.Continue) -> ContinueStmt: # --- expr --- - def visit_NamedExpr(self, n: NamedExpr) -> AssignmentExpr: + def visit_NamedExpr(self, n: ast3.NamedExpr) -> AssignmentExpr: s = AssignmentExpr(self.visit(n.target), self.visit(n.value)) return self.set_line(s, n) @@ -1648,8 +1645,8 @@ def visit_Call(self, n: Call) -> CallExpr: ) return self.set_line(e, n) - # Constant(object value) -- a constant, in Python 3.8. - def visit_Constant(self, n: Constant) -> Any: + # Constant(object value) + def visit_Constant(self, n: ast3.Constant) -> Any: val = n.value e: Any = None if val is None: @@ -1773,8 +1770,6 @@ def visit_Tuple(self, n: ast3.Tuple) -> TupleExpr: e = TupleExpr(self.translate_expr_list(n.elts)) return self.set_line(e, n) - # --- slice --- - # Slice(expr? lower, expr? upper, expr? step) def visit_Slice(self, n: ast3.Slice) -> SliceExpr: e = SliceExpr(self.visit(n.lower), self.visit(n.upper), self.visit(n.step)) @@ -2030,9 +2025,9 @@ def translate_argument_list(self, l: Sequence[ast3.expr]) -> TypeList: return TypeList([self.visit(e) for e in l], line=self.line) def _extract_argument_name(self, n: ast3.expr) -> str | None: - if isinstance(n, Constant) and isinstance(n.value, str): + if isinstance(n, ast3.Constant) and isinstance(n.value, str): return n.value.strip() - elif isinstance(n, Constant) and n.value is None: + elif isinstance(n, ast3.Constant) and n.value is None: return None self.fail( message_registry.ARG_NAME_EXPECTED_STRING_LITERAL.format(type(n).__name__), @@ -2058,7 +2053,7 @@ def visit_BinOp(self, n: ast3.BinOp) -> Type: uses_pep604_syntax=True, ) - def visit_Constant(self, n: Constant) -> Type: + def visit_Constant(self, n: ast3.Constant) -> Type: val = n.value if val is None: # None is a type. @@ -2114,16 +2109,10 @@ def numeric_type(self, value: object, n: AST) -> Type: numeric_value, type_name, line=self.line, column=getattr(n, "col_offset", -1) ) - def visit_Index(self, n: ast3.Index) -> Type: - # cast for mypyc's benefit on Python 3.9 - value = self.visit(cast(Any, n).value) - assert isinstance(value, Type) - return value - def visit_Slice(self, n: ast3.Slice) -> Type: return self.invalid_type(n, note="did you mean to use ',' instead of ':' ?") - # Subscript(expr value, expr slice, expr_context ctx) # Python 3.9 and later + # Subscript(expr value, expr slice, expr_context ctx) def visit_Subscript(self, n: ast3.Subscript) -> Type: empty_tuple_index = False if isinstance(n.slice, ast3.Tuple): From 698e910c29c485419c5d5a1e6d99be0630b9d496 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 1 Nov 2025 19:46:18 -0600 Subject: [PATCH 136/183] [nit] command_line.rst: the standard format for multiple choice has no space (#19868) Removing this spaces causes this documentation to match the other documentation on the page, and also in `--help` There are no tests for this change. I manually verified that the link to the option currently is https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-enable-incomplete-feature, so changing the arguments text does not necessitate adding a new anchor in to preserve old inbound links to this section. --- docs/source/command_line.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index d667fa0ff727..79dd68a84b28 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1159,7 +1159,7 @@ format into the specified directory. Enabling incomplete/experimental features ***************************************** -.. option:: --enable-incomplete-feature {PreciseTupleTypes, InlineTypedDict} +.. option:: --enable-incomplete-feature {PreciseTupleTypes,InlineTypedDict} Some features may require several mypy releases to implement, for example due to their complexity, potential for backwards incompatibility, or From 3618369b1263116804255a6af4cbd93abb7c69aa Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 1 Nov 2025 19:49:10 -0600 Subject: [PATCH 137/183] Update kinds_of_types.rst: keep old anchor for #no-strict-optional (#19828) Addresses https://github.com/python/mypy/pull/19252#issuecomment-2953310853 > support this old anchor, for people on older mypy versions This way, when people get the old documentation link, it will continue to go to the right place. I have looked at the documentation that gets generated locally to see if everything still works. It does. In our vast panoply of documentation, we now have two very similar refs, `no-strict-optional` and `no_strict_optional` (used in https://mypy.readthedocs.io/en/stable/common_issues.html#no-errors-reported-for-obviously-wrong-code to link to https://mypy.readthedocs.io/en/stable/command_line.html#no-strict-optional) but the software handles them fine, without getting confused. --- docs/source/kinds_of_types.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index 8e721c0fb321..23ebc14e8670 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -294,6 +294,7 @@ isn't supported by the runtime with some limitations, if you use def f(x: int | str) -> None: # OK on Python 3.7 and later ... +.. _no-strict-optional: .. _strict_optional: Optional types and the None type From 7aed6962621a53e4c31301a5cb07a8aa3547da55 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:40:54 +0100 Subject: [PATCH 138/183] Stubtest: check `_value_` for ellipsis-valued stub enum members (#19760) Currently stubtest allows unsound definitions: ```python # a.pyi from enum import Enum class E(Enum): _value_: str FOO = ... ``` ```python # a.py from enum import Enum class E(Enum): FOO = 0 ``` This PR teaches `stubtest` that `_value_` attribute ([spec](https://typing.python.org/en/latest/spec/enums.html#member-values)) should be used as a fallback in such case. --- mypy/stubtest.py | 18 +++++++++++++++--- mypy/test/teststubtest.py | 24 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 99404dbe52ab..ada56a2489fe 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1274,6 +1274,7 @@ def verify_var( yield Error(object_path, "is read-only at runtime but not in the stub", stub, runtime) runtime_type = get_mypy_type_of_runtime_value(runtime, type_context=stub.type) + note = "" if ( runtime_type is not None and stub.type is not None @@ -1286,17 +1287,28 @@ def verify_var( runtime_type = get_mypy_type_of_runtime_value(runtime.value) if runtime_type is not None and is_subtype_helper(runtime_type, stub.type): should_error = False - # We always allow setting the stub value to ... + # We always allow setting the stub value to Ellipsis (...), but use + # _value_ type as a fallback if given. If a member is ... and _value_ + # type is given, all runtime types should be assignable to _value_. proper_type = mypy.types.get_proper_type(stub.type) if ( isinstance(proper_type, mypy.types.Instance) and proper_type.type.fullname in mypy.types.ELLIPSIS_TYPE_NAMES ): - should_error = False + value_t = stub.info.get("_value_") + if value_t is None or value_t.type is None or runtime_type is None: + should_error = False + elif is_subtype_helper(runtime_type, value_t.type): + should_error = False + else: + note = " (incompatible '_value_')" if should_error: yield Error( - object_path, f"variable differs from runtime type {runtime_type}", stub, runtime + object_path, + f"variable differs from runtime type {runtime_type}{note}", + stub, + runtime, ) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index dfbde217e82f..4bec5daf3ffb 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1483,6 +1483,30 @@ class HasEmptySlots: """, error=None, ) + yield Case( + stub=""" + class HasCompatibleValue(enum.Enum): + _value_: str + FOO = ... + """, + runtime=""" + class HasCompatibleValue(enum.Enum): + FOO = "foo" + """, + error=None, + ) + yield Case( + stub=""" + class HasIncompatibleValue(enum.Enum): + _value_: int + FOO = ... + """, + runtime=""" + class HasIncompatibleValue(enum.Enum): + FOO = "foo" + """, + error="HasIncompatibleValue.FOO", + ) @collect_cases def test_decorator(self) -> Iterator[Case]: From 3174d3f1d4e04d101384f1632e65ecc7911f20a0 Mon Sep 17 00:00:00 2001 From: Theodore Ando Date: Mon, 3 Nov 2025 01:54:26 -0600 Subject: [PATCH 139/183] Use pretty_callable more often for callable expressions (#20128) Fixes #5490 Uses pretty_callable for formatting Callable expressions that would otherwise be formatted with complex Args/VarArgs. Avoids pretty_callable for things that would be formatted with only positional args such as `Callable[[X, ..., Y], Z]` --- mypy/messages.py | 26 ++++++++++++- test-data/unit/check-assert-type-fail.test | 2 +- test-data/unit/check-callable.test | 6 +-- test-data/unit/check-functions.test | 38 +++++++++---------- test-data/unit/check-functools.test | 16 ++++---- test-data/unit/check-incremental.test | 2 +- test-data/unit/check-inference.test | 14 +++---- .../unit/check-parameter-specification.test | 8 ++-- test-data/unit/check-protocols.test | 34 ++++++++--------- test-data/unit/check-python311.test | 2 +- test-data/unit/check-statements.test | 2 +- test-data/unit/check-typevar-tuple.test | 26 ++++++------- test-data/unit/check-varargs.test | 2 +- test-data/unit/pythoneval.test | 2 +- 14 files changed, 102 insertions(+), 78 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index c6378c264757..a9e8ee2e43ab 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2530,6 +2530,15 @@ def quote_type_string(type_string: str) -> str: return f'"{type_string}"' +def should_format_arg_as_type(arg_kind: ArgKind, arg_name: str | None, verbosity: int) -> bool: + """ + Determine whether a function argument should be formatted as its Type or with name. + """ + return (arg_kind == ARG_POS and arg_name is None) or ( + verbosity == 0 and arg_kind.is_positional() + ) + + def format_callable_args( arg_types: list[Type], arg_kinds: list[ArgKind], @@ -2540,7 +2549,7 @@ def format_callable_args( """Format a bunch of Callable arguments into a string""" arg_strings = [] for arg_name, arg_type, arg_kind in zip(arg_names, arg_types, arg_kinds): - if arg_kind == ARG_POS and arg_name is None or verbosity == 0 and arg_kind.is_positional(): + if should_format_arg_as_type(arg_kind, arg_name, verbosity): arg_strings.append(format(arg_type)) else: constructor = ARG_CONSTRUCTOR_NAMES[arg_kind] @@ -2558,13 +2567,18 @@ def format_type_inner( options: Options, fullnames: set[str] | None, module_names: bool = False, + use_pretty_callable: bool = True, ) -> str: """ Convert a type to a relatively short string suitable for error messages. Args: + typ: type to be formatted verbosity: a coarse grained control on the verbosity of the type + options: Options object controlling formatting fullnames: a set of names that should be printed in full + module_names: whether to show module names for module types + use_pretty_callable: use pretty_callable to format Callable types. """ def format(typ: Type) -> str: @@ -2761,6 +2775,16 @@ def format_literal_value(typ: LiteralType) -> str: param_spec = func.param_spec() if param_spec is not None: return f"Callable[{format(param_spec)}, {return_type}]" + + # Use pretty format (def-style) for complex signatures with named, optional, or star args. + # Use compact Callable[[...], ...] only for signatures with all simple positional args. + if use_pretty_callable: + if any( + not should_format_arg_as_type(kind, name, verbosity) + for kind, name in zip(func.arg_kinds, func.arg_names) + ): + return pretty_callable(func, options) + args = format_callable_args( func.arg_types, func.arg_kinds, func.arg_names, format, verbosity ) diff --git a/test-data/unit/check-assert-type-fail.test b/test-data/unit/check-assert-type-fail.test index 514650649641..98aae0ba6b32 100644 --- a/test-data/unit/check-assert-type-fail.test +++ b/test-data/unit/check-assert-type-fail.test @@ -30,7 +30,7 @@ def f(si: arr.array[int]): [case testAssertTypeFailCallableArgKind] from typing import assert_type, Callable def myfunc(arg: int) -> None: pass -assert_type(myfunc, Callable[[int], None]) # E: Expression is of type "Callable[[Arg(int, 'arg')], None]", not "Callable[[int], None]" +assert_type(myfunc, Callable[[int], None]) # E: Expression is of type "def myfunc(arg: int) -> None", not "Callable[[int], None]" [case testAssertTypeOverload] from typing import assert_type, overload diff --git a/test-data/unit/check-callable.test b/test-data/unit/check-callable.test index 23db0bf50a4e..0157ff3d2c53 100644 --- a/test-data/unit/check-callable.test +++ b/test-data/unit/check-callable.test @@ -654,13 +654,13 @@ class Call(Protocol): def f1() -> None: ... a1: Call = f1 # E: Incompatible types in assignment (expression has type "Callable[[], None]", variable has type "Call") \ - # N: "Call.__call__" has type "Callable[[Arg(int, 'x'), VarArg(Any), KwArg(Any)], None]" + # N: "Call.__call__" has type "def __call__(self, x: int, *args: Any, **kwargs: Any) -> None" def f2(x: str) -> None: ... a2: Call = f2 # E: Incompatible types in assignment (expression has type "Callable[[str], None]", variable has type "Call") \ - # N: "Call.__call__" has type "Callable[[Arg(int, 'x'), VarArg(Any), KwArg(Any)], None]" + # N: "Call.__call__" has type "def __call__(self, x: int, *args: Any, **kwargs: Any) -> None" def f3(y: int) -> None: ... a3: Call = f3 # E: Incompatible types in assignment (expression has type "Callable[[int], None]", variable has type "Call") \ - # N: "Call.__call__" has type "Callable[[Arg(int, 'x'), VarArg(Any), KwArg(Any)], None]" + # N: "Call.__call__" has type "def __call__(self, x: int, *args: Any, **kwargs: Any) -> None" def f4(x: int) -> None: ... a4: Call = f4 diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 7fa34a398ea0..1882f235f7e3 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -107,30 +107,30 @@ if int(): [case testSubtypingFunctionsDoubleCorrespondence] def l(x) -> None: ... def r(__x, *, x) -> None: ... -r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "Callable[[Any, NamedArg(Any, 'x')], None]") +r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "def r(Any, /, *, x: Any) -> None") [case testSubtypingFunctionsDoubleCorrespondenceNamedOptional] def l(x) -> None: ... def r(__x, *, x = 1) -> None: ... -r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "Callable[[Any, DefaultNamedArg(Any, 'x')], None]") +r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "def r(Any, /, *, x: Any = ...) -> None") [case testSubtypingFunctionsDoubleCorrespondenceBothNamedOptional] def l(x = 1) -> None: ... def r(__x, *, x = 1) -> None: ... -r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "Callable[[Any, DefaultNamedArg(Any, 'x')], None]") +r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "def r(Any, /, *, x: Any = ...) -> None") [case testSubtypingFunctionsTrivialSuffixRequired] def l(__x) -> None: ... def r(x, *args, **kwargs) -> None: ... -r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "Callable[[Arg(Any, 'x'), VarArg(Any), KwArg(Any)], None]") +r = l # E: Incompatible types in assignment (expression has type "Callable[[Any], None]", variable has type "def r(x: Any, *args: Any, **kwargs: Any) -> None") [builtins fixtures/dict.pyi] [case testSubtypingFunctionsTrivialSuffixOptional] def l(__x = 1) -> None: ... def r(x = 1, *args, **kwargs) -> None: ... -r = l # E: Incompatible types in assignment (expression has type "Callable[[DefaultArg(Any)], None]", variable has type "Callable[[DefaultArg(Any, 'x'), VarArg(Any), KwArg(Any)], None]") +r = l # E: Incompatible types in assignment (expression has type "def l(Any = ..., /) -> None", variable has type "def r(x: Any = ..., *args: Any, **kwargs: Any) -> None") [builtins fixtures/dict.pyi] [case testSubtypingFunctionsRequiredLeftArgNotPresent] @@ -170,13 +170,13 @@ if int(): if int(): ff_nonames = f_nonames # reset if int(): - ff = ff_nonames # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]") + ff = ff_nonames # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "def f(a: int, b: str) -> None") if int(): ff = f # reset if int(): - gg = ff # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]", variable has type "Callable[[Arg(int, 'a'), DefaultArg(str, 'b')], None]") + gg = ff # E: Incompatible types in assignment (expression has type "def f(a: int, b: str) -> None", variable has type "def g(a: int, b: str = ...) -> None") if int(): - gg = hh # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'aa'), DefaultArg(str, 'b')], None]", variable has type "Callable[[Arg(int, 'a'), DefaultArg(str, 'b')], None]") + gg = hh # E: Incompatible types in assignment (expression has type "def h(aa: int, b: str = ...) -> None", variable has type "def g(a: int, b: str = ...) -> None") [case testSubtypingFunctionsArgsKwargs] from typing import Any, Callable @@ -245,7 +245,7 @@ gg = g if int(): ff = g if int(): - gg = f # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]") + gg = f # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "def g(a: int, b: str) -> None") [case testLackOfNamesFastparse] def f(__a: int, __b: str) -> None: pass @@ -257,7 +257,7 @@ gg = g if int(): ff = g if int(): - gg = f # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "Callable[[Arg(int, 'a'), Arg(str, 'b')], None]") + gg = f # E: Incompatible types in assignment (expression has type "Callable[[int, str], None]", variable has type "def g(a: int, b: str) -> None") [case testFunctionTypeCompatibilityWithOtherTypes] # flags: --no-strict-optional @@ -2016,12 +2016,12 @@ def isf_unnamed(__i: int, __s: str) -> str: int_str_fun = isf int_str_fun = isf_unnamed -int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type "Callable[[int, str], str]", variable has type "Callable[[int, Arg(str, 's')], str]") +int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type "Callable[[int, str], str]", variable has type "def (int, /, s: str) -> str") int_opt_str_fun = iosf int_str_fun = iosf -int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'ii'), Arg(str, 'ss')], str]", variable has type "Callable[[int, DefaultArg(str)], str]") +int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type "def isf(ii: int, ss: str) -> str", variable has type "def (int, str = ..., /) -> str") -int_named_str_fun = isf # E: Incompatible types in assignment (expression has type "Callable[[Arg(int, 'ii'), Arg(str, 'ss')], str]", variable has type "Callable[[int, Arg(str, 's')], str]") +int_named_str_fun = isf # E: Incompatible types in assignment (expression has type "def isf(ii: int, ss: str) -> str", variable has type "def (int, /, s: str) -> str") int_named_str_fun = iosf [builtins fixtures/dict.pyi] @@ -2076,7 +2076,7 @@ def g4(*, y: int) -> str: pass f(g1) f(g2) f(g3) -f(g4) # E: Argument 1 to "f" has incompatible type "Callable[[NamedArg(int, 'y')], str]"; expected "Callable[..., int]" +f(g4) # E: Argument 1 to "f" has incompatible type "def g4(*, y: int) -> str"; expected "Callable[..., int]" [case testCallableWithArbitraryArgsSubtypingWithGenericFunc] from typing import Callable, TypeVar @@ -2238,7 +2238,7 @@ def g(x, y): pass def h(x): pass def j(y) -> Any: pass f = h -f = j # E: Incompatible types in assignment (expression has type "Callable[[Arg(Any, 'y')], Any]", variable has type "Callable[[Arg(Any, 'x')], Any]") +f = j # E: Incompatible types in assignment (expression has type "def j(y: Any) -> Any", variable has type "def f(x: Any) -> Any") f = g # E: Incompatible types in assignment (expression has type "Callable[[Any, Any], Any]", variable has type "Callable[[Any], Any]") [case testRedefineFunction2] @@ -3531,7 +3531,7 @@ def decorator(f: Callable[P, None]) -> Callable[[Callable[P, A]], None]: def key(x: int) -> None: ... def fn_b(b: int) -> B: ... -decorator(key)(fn_b) # E: Argument 1 has incompatible type "Callable[[Arg(int, 'b')], B]"; expected "Callable[[Arg(int, 'x')], A]" +decorator(key)(fn_b) # E: Argument 1 has incompatible type "def fn_b(b: int) -> B"; expected "def (x: int) -> A" def decorator2(f: Callable[P, None]) -> Callable[ [Callable[P, Awaitable[None]]], @@ -3542,7 +3542,7 @@ def decorator2(f: Callable[P, None]) -> Callable[ def key2(x: int) -> None: ... -@decorator2(key2) # E: Argument 1 has incompatible type "Callable[[Arg(int, 'y')], Coroutine[Any, Any, None]]"; expected "Callable[[Arg(int, 'x')], Awaitable[None]]" +@decorator2(key2) # E: Argument 1 has incompatible type "def foo2(y: int) -> Coroutine[Any, Any, None]"; expected "def (x: int) -> Awaitable[None]" async def foo2(y: int) -> None: ... @@ -3552,7 +3552,7 @@ class Parent: class Child(Parent): method_without: Callable[[], "Child"] - method_with: Callable[[str], "Child"] # E: Incompatible types in assignment (expression has type "Callable[[str], Child]", base class "Parent" defined the type as "Callable[[Arg(str, 'param')], Parent]") + method_with: Callable[[str], "Child"] # E: Incompatible types in assignment (expression has type "Callable[[str], Child]", base class "Parent" defined the type as "def method_with(self, param: str) -> Parent") [builtins fixtures/tuple.pyi] [case testDistinctFormattingUnion] @@ -3562,7 +3562,7 @@ from mypy_extensions import Arg def f(x: Callable[[Arg(int, 'x')], None]) -> None: pass y: Callable[[Union[int, str]], None] -f(y) # E: Argument 1 to "f" has incompatible type "Callable[[Union[int, str]], None]"; expected "Callable[[Arg(int, 'x')], None]" +f(y) # E: Argument 1 to "f" has incompatible type "Callable[[Union[int, str]], None]"; expected "def (x: int) -> None" [builtins fixtures/tuple.pyi] [case testAbstractOverloadsWithoutImplementationAllowed] diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index fa2cacda275d..650928b1a5ed 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -162,7 +162,7 @@ def takes_callable_int(f: Callable[..., int]) -> None: ... def takes_callable_str(f: Callable[..., str]) -> None: ... takes_callable_int(p1) takes_callable_str(p1) # E: Argument 1 to "takes_callable_str" has incompatible type "partial[int]"; expected "Callable[..., str]" \ - # N: "partial[int].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], int]" + # N: "partial[int].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> int" p2 = functools.partial(foo, 1) p2("a") # OK @@ -386,7 +386,7 @@ q: partial[bool] = partial(generic, resulting_type=str) # E: Argument "resultin pc: Callable[..., str] = partial(generic, resulting_type=str) qc: Callable[..., bool] = partial(generic, resulting_type=str) # E: Incompatible types in assignment (expression has type "partial[str]", variable has type "Callable[..., bool]") \ - # N: "partial[str].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], str]" + # N: "partial[str].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> str" [builtins fixtures/tuple.pyi] [case testFunctoolsPartialNestedPartial] @@ -697,11 +697,11 @@ use_int_callable(partial(func_b, b="")) use_func_callable(partial(func_b, b="")) use_int_callable(partial(func_c, b="")) use_func_callable(partial(func_c, b="")) -use_int_callable(partial(func_fn, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[Callable[[VarArg(Any), KwArg(Any)], Any]]"; expected "Callable[[int], int]" \ - # N: "partial[Callable[[VarArg(Any), KwArg(Any)], Any]].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], Callable[[VarArg(Any), KwArg(Any)], Any]]" +use_int_callable(partial(func_fn, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[def (*Any, **Any) -> Any]"; expected "Callable[[int], int]" \ + # N: "partial[def (*Any, **Any) -> Any].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> def (*Any, **Any) -> Any" use_func_callable(partial(func_fn, b="")) -use_int_callable(partial(func_fn_unpack, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[Callable[[VarArg(Any)], Any]]"; expected "Callable[[int], int]" \ - # N: "partial[Callable[[VarArg(Any)], Any]].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], Callable[[VarArg(Any)], Any]]" +use_int_callable(partial(func_fn_unpack, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[def (*Any) -> Any]"; expected "Callable[[int], int]" \ + # N: "partial[def (*Any) -> Any].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> def (*Any) -> Any" use_func_callable(partial(func_fn_unpack, b="")) # But we should not erase typevars that aren't bound by function @@ -714,7 +714,7 @@ def outer_b(arg: Tb) -> None: reveal_type(partial(inner, b="")) # N: Revealed type is "functools.partial[Tb`-1]" use_int_callable(partial(inner, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[Tb]"; expected "Callable[[int], int]" \ - # N: "partial[Tb].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], Tb]" + # N: "partial[Tb].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> Tb" def outer_c(arg: Tc) -> None: @@ -724,5 +724,5 @@ def outer_c(arg: Tc) -> None: reveal_type(partial(inner, b="")) # N: Revealed type is "functools.partial[builtins.int]" \ # N: Revealed type is "functools.partial[builtins.str]" use_int_callable(partial(inner, b="")) # E: Argument 1 to "use_int_callable" has incompatible type "partial[str]"; expected "Callable[[int], int]" \ - # N: "partial[str].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], str]" + # N: "partial[str].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> str" [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 94f65a950062..5fbaa4f2c904 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6950,7 +6950,7 @@ p3 = functools.partial(foo, b="a") [out] tmp/a.py:8: note: Revealed type is "functools.partial[builtins.int]" tmp/a.py:13: error: Argument 1 to "takes_callable_str" has incompatible type "partial[int]"; expected "Callable[..., str]" -tmp/a.py:13: note: "partial[int].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], int]" +tmp/a.py:13: note: "partial[int].__call__" has type "def __call__(__self, *args: Any, **kwargs: Any) -> int" tmp/a.py:18: error: Argument 1 to "foo" has incompatible type "int"; expected "str" tmp/a.py:19: error: Too many arguments for "foo" tmp/a.py:19: error: Argument 1 to "foo" has incompatible type "int"; expected "str" diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 9ed9c5e9ec78..17f79dbcb663 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1109,7 +1109,7 @@ def f(*, x: int) -> int: ... def g(*, y: int) -> int: ... def h(*, x: int) -> int: ... -list_1 = [f, g] # E: List item 0 has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[NamedArg(int, 'y')], int]" +list_1 = [f, g] # E: List item 0 has incompatible type "def f(*, x: int) -> int"; expected "def g(*, y: int) -> int" list_2 = [f, h] [builtins fixtures/list.pyi] @@ -1434,7 +1434,7 @@ from typing import Callable def f(a: Callable[..., None] = lambda *a, **k: None): pass -def g(a: Callable[..., None] = lambda *a, **k: 1): # E: Incompatible default for argument "a" (default has type "Callable[[VarArg(Any), KwArg(Any)], int]", argument has type "Callable[..., None]") +def g(a: Callable[..., None] = lambda *a, **k: 1): # E: Incompatible default for argument "a" (default has type "def (*a: Any, **k: Any) -> int", argument has type "Callable[..., None]") pass [builtins fixtures/dict.pyi] @@ -3769,7 +3769,7 @@ def f(x: Call[T]) -> Tuple[T, T]: ... def g(__x: str) -> None: pass reveal_type(f(g)) # N: Revealed type is "tuple[Never, Never]" \ # E: Argument 1 to "f" has incompatible type "Callable[[str], None]"; expected "Call[Never]" \ - # N: "Call[Never].__call__" has type "Callable[[NamedArg(Never, 'x')], None]" + # N: "Call[Never].__call__" has type "def __call__(self, *, x: Never) -> None" [builtins fixtures/list.pyi] [case testCallableInferenceAgainstCallableNamedVsPosOnly] @@ -3785,7 +3785,7 @@ def f(x: Call[T]) -> Tuple[T, T]: ... def g(*, x: str) -> None: pass reveal_type(f(g)) # N: Revealed type is "tuple[Never, Never]" \ - # E: Argument 1 to "f" has incompatible type "Callable[[NamedArg(str, 'x')], None]"; expected "Call[Never]" \ + # E: Argument 1 to "f" has incompatible type "def g(*, x: str) -> None"; expected "Call[Never]" \ # N: "Call[Never].__call__" has type "Callable[[Never], None]" [builtins fixtures/list.pyi] @@ -3802,7 +3802,7 @@ def f(x: Call[T]) -> Tuple[T, T]: ... def g(**x: str) -> None: pass reveal_type(f(g)) # N: Revealed type is "tuple[Never, Never]" \ - # E: Argument 1 to "f" has incompatible type "Callable[[KwArg(str)], None]"; expected "Call[Never]" \ + # E: Argument 1 to "f" has incompatible type "def g(**x: str) -> None"; expected "Call[Never]" \ # N: "Call[Never].__call__" has type "Callable[[Never], None]" [builtins fixtures/list.pyi] @@ -3819,8 +3819,8 @@ def f(x: Call[T]) -> Tuple[T, T]: ... def g(*args: str) -> None: pass reveal_type(f(g)) # N: Revealed type is "tuple[Never, Never]" \ - # E: Argument 1 to "f" has incompatible type "Callable[[VarArg(str)], None]"; expected "Call[Never]" \ - # N: "Call[Never].__call__" has type "Callable[[NamedArg(Never, 'x')], None]" + # E: Argument 1 to "f" has incompatible type "def g(*args: str) -> None"; expected "Call[Never]" \ + # N: "Call[Never].__call__" has type "def __call__(self, *, x: Never) -> None" [builtins fixtures/list.pyi] [case testInferenceAgainstTypeVarActualBound] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 2b4f92c7c819..bffd34782f51 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -443,7 +443,7 @@ reveal_type(register(lambda: f(1))) # N: Revealed type is "def ()" reveal_type(register(lambda x: f(x), x=1)) # N: Revealed type is "def (x: Literal[1]?)" register(lambda x: f(x)) # E: Cannot infer type of lambda \ # E: Argument 1 to "register" has incompatible type "Callable[[Any], None]"; expected "Callable[[], None]" -register(lambda x: f(x), y=1) # E: Argument 1 to "register" has incompatible type "Callable[[Arg(int, 'x')], None]"; expected "Callable[[Arg(int, 'y')], None]" +register(lambda x: f(x), y=1) # E: Argument 1 to "register" has incompatible type "def (x: int) -> None"; expected "def (y: int) -> None" reveal_type(register(lambda x: f(x), 1)) # N: Revealed type is "def (Literal[1]?)" reveal_type(register(lambda x, y: g(x, y), 1, "a")) # N: Revealed type is "def (Literal[1]?, Literal['a']?)" reveal_type(register(lambda x, y: g(x, y), 1, y="a")) # N: Revealed type is "def (Literal[1]?, y: Literal['a']?)" @@ -623,10 +623,10 @@ def expects_int_first(x: Callable[Concatenate[int, P], int]) -> None: ... # N: This is likely because "one" has named arguments: "x". Consider marking them positional-only def one(x: str) -> int: ... -@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[int, NamedArg(int, 'x')], int]" +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "def two(*, x: int) -> int"; expected "def (int, /, *, x: int) -> int" def two(*, x: int) -> int: ... -@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[KwArg(int)], int]"; expected "Callable[[int, KwArg(int)], int]" +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "def three(**kwargs: int) -> int"; expected "def (int, /, **kwargs: int) -> int" def three(**kwargs: int) -> int: ... @expects_int_first # Accepted @@ -2154,7 +2154,7 @@ reveal_type(submit( # N: Revealed type is "__main__.Result" backend="asyncio", )) submit( - run, # E: Argument 1 to "submit" has incompatible type "Callable[[Callable[[], R], VarArg(object), DefaultNamedArg(str, 'backend')], R]"; expected "Callable[[Callable[[], Result], int], Result]" + run, # E: Argument 1 to "submit" has incompatible type "def [R] run(func: Callable[[], R], *args: object, backend: str = ...) -> R"; expected "Callable[[Callable[[], Result], int], Result]" run_portal, backend=int(), ) diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index e7971cd5b5d8..fd7f0c3449da 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1907,7 +1907,7 @@ reveal_type(apply_gen(Add5())) # N: Revealed type is "builtins.int" def apply_str(f: Callable[[str], int], x: str) -> int: return f(x) apply_str(Add5(), 'a') # E: Argument 1 to "apply_str" has incompatible type "Add5"; expected "Callable[[str], int]" \ - # N: "Add5.__call__" has type "Callable[[Arg(int, 'x')], int]" + # N: "Add5.__call__" has type "def __call__(self, x: int) -> int" [builtins fixtures/isinstancelist.pyi] [case testMoreComplexCallableStructuralSubtyping] @@ -1923,10 +1923,10 @@ class Bad1: class Bad2: def __call__(self, y: int, *rest: str) -> int: pass call_soon(Good()) -call_soon(Bad1()) # E: Argument 1 to "call_soon" has incompatible type "Bad1"; expected "Callable[[int, VarArg(str)], int]" \ - # N: "Bad1.__call__" has type "Callable[[Arg(int, 'x'), VarArg(int)], int]" -call_soon(Bad2()) # E: Argument 1 to "call_soon" has incompatible type "Bad2"; expected "Callable[[int, VarArg(str)], int]" \ - # N: "Bad2.__call__" has type "Callable[[Arg(int, 'y'), VarArg(str)], int]" +call_soon(Bad1()) # E: Argument 1 to "call_soon" has incompatible type "Bad1"; expected "def (x: int, *str) -> int" \ + # N: "Bad1.__call__" has type "def __call__(self, x: int, *rest: int) -> int" +call_soon(Bad2()) # E: Argument 1 to "call_soon" has incompatible type "Bad2"; expected "def (x: int, *str) -> int" \ + # N: "Bad2.__call__" has type "def __call__(self, y: int, *rest: str) -> int" [builtins fixtures/isinstancelist.pyi] [case testStructuralSupportForPartial] @@ -2486,8 +2486,8 @@ def func(caller: Caller) -> None: pass func(call) -func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[int, VarArg(str)], None]"; expected "Caller" \ - # N: "Caller.__call__" has type "Callable[[Arg(str, 'x'), VarArg(int)], None]" +func(bad) # E: Argument 1 to "func" has incompatible type "def bad(x: int, *args: str) -> None"; expected "Caller" \ + # N: "Caller.__call__" has type "def __call__(self, x: str, *args: int) -> None" [builtins fixtures/tuple.pyi] [out] @@ -2525,7 +2525,7 @@ def func(caller: Caller) -> None: func(call) func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[int], int]"; expected "Caller" \ - # N: "Caller.__call__" has type "Callable[[Arg(T, 'x')], T]" + # N: "Caller.__call__" has type "def [T] __call__(self, x: T) -> T" [builtins fixtures/tuple.pyi] [out] @@ -2546,7 +2546,7 @@ def func(caller: Caller) -> None: func(call) func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[T], tuple[T, T]]"; expected "Caller" \ - # N: "Caller.__call__" has type "Callable[[Arg(int, 'x')], int]" + # N: "Caller.__call__" has type "def __call__(self, x: int) -> int" [builtins fixtures/tuple.pyi] [out] @@ -2586,8 +2586,8 @@ class Caller(Protocol): def bad(x: int, *args: str) -> None: pass -cb: Caller = bad # E: Incompatible types in assignment (expression has type "Callable[[int, VarArg(str)], None]", variable has type "Caller") \ - # N: "Caller.__call__" has type "Callable[[Arg(str, 'x'), VarArg(int)], None]" +cb: Caller = bad # E: Incompatible types in assignment (expression has type "def bad(x: int, *args: str) -> None", variable has type "Caller") \ + # N: "Caller.__call__" has type "def __call__(self, x: str, *args: int) -> None" [builtins fixtures/tuple.pyi] [out] @@ -2614,7 +2614,7 @@ def anon(caller: CallerAnon) -> None: func(call) func(bad) # E: Argument 1 to "func" has incompatible type "Callable[[str], None]"; expected "Caller" \ - # N: "Caller.__call__" has type "Callable[[Arg(str, 'x')], None]" + # N: "Caller.__call__" has type "def __call__(self, x: str) -> None" anon(bad) [out] @@ -2638,7 +2638,7 @@ b: Bad func(a) func(b) # E: Argument 1 to "func" has incompatible type "Bad"; expected "One" \ - # N: "One.__call__" has type "Callable[[Arg(str, 'x')], None]" + # N: "One.__call__" has type "def __call__(self, x: str) -> None" [out] [case testJoinProtocolCallback] @@ -3610,7 +3610,7 @@ test(C) # E: Argument 1 to "test" has incompatible type "type[C]"; expected "P" # N: def __call__(x: int, y: int) -> Any \ # N: Got: \ # N: def __init__(x: int, y: str) -> C \ - # N: "P.__call__" has type "Callable[[Arg(int, 'x'), Arg(int, 'y')], Any]" + # N: "P.__call__" has type "def __call__(self, x: int, y: int) -> Any" [case testProtocolClassObjectPureCallback] from typing import Any, ClassVar, Protocol @@ -3632,7 +3632,7 @@ test(C) # E: Argument 1 to "test" has incompatible type "type[C]"; expected "P" # N: def __call__(x: int, y: int) -> Any \ # N: Got: \ # N: def __init__(x: int, y: str) -> C \ - # N: "P.__call__" has type "Callable[[Arg(int, 'x'), Arg(int, 'y')], Any]" + # N: "P.__call__" has type "def __call__(self, x: int, y: int) -> Any" [builtins fixtures/type.pyi] [case testProtocolClassObjectCallableError] @@ -3655,7 +3655,7 @@ p: P = C # E: Incompatible types in assignment (expression has type "type[C]", # N: def __call__(app: int) -> Callable[[str], None] \ # N: Got: \ # N: def __init__(app: str) -> C \ - # N: "P.__call__" has type "Callable[[Arg(int, 'app')], Callable[[str], None]]" + # N: "P.__call__" has type "def __call__(self, app: int) -> Callable[[str], None]" [builtins fixtures/type.pyi] @@ -3814,7 +3814,7 @@ def f_good(t: S) -> S: return t g: C = f_bad # E: Incompatible types in assignment (expression has type "Callable[[int], int]", variable has type "C") \ - # N: "C.__call__" has type "Callable[[Arg(T, 't')], T]" + # N: "C.__call__" has type "def [T] __call__(self, t: T) -> T" g = f_good # OK [case testModuleAsProtocolImplementation] diff --git a/test-data/unit/check-python311.test b/test-data/unit/check-python311.test index 09c8d6082365..c2a0bb09810a 100644 --- a/test-data/unit/check-python311.test +++ b/test-data/unit/check-python311.test @@ -157,7 +157,7 @@ Alias1 = Callable[[*Ts], int] # E: Variable "__main__.Ts" is not valid as a typ x1: Alias1[int] # E: Bad number of arguments for type alias, expected 0, given 1 reveal_type(x1) # N: Revealed type is "def (*Any) -> builtins.int" x1 = good -x1 = bad # E: Incompatible types in assignment (expression has type "Callable[[VarArg(int), NamedArg(int, 'y')], int]", variable has type "Callable[[VarArg(Any)], int]") +x1 = bad # E: Incompatible types in assignment (expression has type "def bad(*x: int, y: int) -> int", variable has type "def (*Any) -> int") Alias2 = Callable[[*T], int] # E: "T" cannot be unpacked (must be tuple or TypeVarTuple) x2: Alias2[int] diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 9ab68b32472d..658bee76ef0d 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -2358,6 +2358,6 @@ describe(CAny()) describe(C()) describe(CNone()) describe(CWrong()) # E: Argument 1 to "describe" has incompatible type "CWrong"; expected "Callable[[], None]" \ - # N: "CWrong.__call__" has type "Callable[[Arg(int, 'x')], None]" + # N: "CWrong.__call__" has type "def __call__(self, x: int) -> None" describe(f) [builtins fixtures/isinstancelist.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 927a4f037a4a..cb5029ee4e6d 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -518,15 +518,15 @@ call(target=func, args=(0, 'foo')) call(target=func, args=('bar', 'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[str, str], None]" call(target=func, args=(True, 'foo', 0)) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[bool, str, int], None]" call(target=func, args=(0, 0, 'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int, int, str], None]" -call(target=func, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[VarArg(int)], None]" +call(target=func, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "def (*int) -> None" # NOTE: This behavior may be a bit contentious, it is maybe inconsistent with our handling of # PEP646 but consistent with our handling of callable constraints. -call(target=func2, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, int], None]"; expected "Callable[[VarArg(int)], None]" +call(target=func2, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, int], None]"; expected "def (*int) -> None" call(target=func3, args=vargs) call(target=func3, args=(0,1)) -call(target=func3, args=(0,'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[VarArg(int)], None]"; expected "Callable[[int, str], None]" -call(target=func3, args=vargs_str) # E: Argument "target" to "call" has incompatible type "Callable[[VarArg(int)], None]"; expected "Callable[[VarArg(str)], None]" +call(target=func3, args=(0,'foo')) # E: Argument "target" to "call" has incompatible type "def func3(*args: int) -> None"; expected "Callable[[int, str], None]" +call(target=func3, args=vargs_str) # E: Argument "target" to "call" has incompatible type "def func3(*args: int) -> None"; expected "def (*str) -> None" [builtins fixtures/tuple.pyi] [case testTypeVarTuplePep646CallableWithPrefixSuffix] @@ -1903,7 +1903,7 @@ def foo3(func: Callable[[int, Unpack[Args2]], T], *args: Unpack[Args2]) -> T: return submit2(func, 1, *args) def foo_bad(func: Callable[[Unpack[Args2]], T], *args: Unpack[Args2]) -> T: - return submit2(func, 1, *args) # E: Argument 1 to "submit2" has incompatible type "Callable[[VarArg(Unpack[Args2])], T]"; expected "Callable[[int, VarArg(Unpack[Args2])], T]" + return submit2(func, 1, *args) # E: Argument 1 to "submit2" has incompatible type "def (*Unpack[Args2]) -> T"; expected "def (int, /, *Unpack[Args2]) -> T" [builtins fixtures/tuple.pyi] [case testTypeVarTupleParamSpecInteraction] @@ -2321,8 +2321,8 @@ higher_order(good2) higher_order(ok1) higher_order(ok2) -higher_order(bad1) # E: Argument 1 to "higher_order" has incompatible type "Callable[[NamedArg(str, 'd')], int]"; expected "Callable[[VarArg(Any)], Any]" -higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "Callable[[KwArg(None)], None]"; expected "Callable[[VarArg(Any)], Any]" +higher_order(bad1) # E: Argument 1 to "higher_order" has incompatible type "def bad1(*, d: str) -> int"; expected "def (*Any) -> Any" +higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "def bad2(**kwargs: None) -> None"; expected "def (*Any) -> Any" [builtins fixtures/tuple.pyi] [case testAliasToCallableWithUnpack2] @@ -2338,10 +2338,10 @@ def bad3(*, d: str) -> int: ... def bad4(**kwargs: None) -> None: ... higher_order(good) -higher_order(bad1) # E: Argument 1 to "higher_order" has incompatible type "Callable[[str, int], None]"; expected "Callable[[int, str, VarArg(Unpack[tuple[Unpack[tuple[Any, ...]], int]])], Any]" -higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "Callable[[bytes, VarArg(int)], str]"; expected "Callable[[int, str, VarArg(Unpack[tuple[Unpack[tuple[Any, ...]], int]])], Any]" -higher_order(bad3) # E: Argument 1 to "higher_order" has incompatible type "Callable[[NamedArg(str, 'd')], int]"; expected "Callable[[int, str, VarArg(Unpack[tuple[Unpack[tuple[Any, ...]], int]])], Any]" -higher_order(bad4) # E: Argument 1 to "higher_order" has incompatible type "Callable[[KwArg(None)], None]"; expected "Callable[[int, str, VarArg(Unpack[tuple[Unpack[tuple[Any, ...]], int]])], Any]" +higher_order(bad1) # E: Argument 1 to "higher_order" has incompatible type "Callable[[str, int], None]"; expected "def (int, str, /, *Unpack[tuple[Unpack[tuple[Any, ...]], int]]) -> Any" +higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "def bad2(c: bytes, *args: int) -> str"; expected "def (int, str, /, *Unpack[tuple[Unpack[tuple[Any, ...]], int]]) -> Any" +higher_order(bad3) # E: Argument 1 to "higher_order" has incompatible type "def bad3(*, d: str) -> int"; expected "def (int, str, /, *Unpack[tuple[Unpack[tuple[Any, ...]], int]]) -> Any" +higher_order(bad4) # E: Argument 1 to "higher_order" has incompatible type "def bad4(**kwargs: None) -> None"; expected "def (int, str, /, *Unpack[tuple[Unpack[tuple[Any, ...]], int]]) -> Any" [builtins fixtures/tuple.pyi] [case testAliasToCallableWithUnpackInvalid] @@ -2357,7 +2357,7 @@ Alias = Callable[[Unpack[T]], int] # E: "T" cannot be unpacked (must be tuple o x: Alias[int] reveal_type(x) # N: Revealed type is "def (*Any) -> builtins.int" x = good -x = bad # E: Incompatible types in assignment (expression has type "Callable[[VarArg(int), NamedArg(int, 'y')], int]", variable has type "Callable[[VarArg(Any)], int]") +x = bad # E: Incompatible types in assignment (expression has type "def bad(*x: int, y: int) -> int", variable has type "def (*Any) -> int") [builtins fixtures/tuple.pyi] [case testTypeVarTupleInvariant] @@ -2443,7 +2443,7 @@ class CM(Generic[R]): ... def cm(fn: Callable[P, List[R]]) -> Callable[P, CM[R]]: ... Ts = TypeVarTuple("Ts") -@cm # E: Argument 1 to "cm" has incompatible type "Callable[[VarArg(Unpack[Ts])], tuple[Unpack[Ts]]]"; expected "Callable[[VarArg(Never)], list[Never]]" +@cm # E: Argument 1 to "cm" has incompatible type "def [Ts`-1] test(*args: Unpack[Ts]) -> tuple[Unpack[Ts]]"; expected "def (*args: Never) -> list[Never]" def test(*args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: ... reveal_type(test) # N: Revealed type is "def (*args: Never) -> __main__.CM[Never]" diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index 680021a166f2..3b80b9e8829a 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -660,7 +660,7 @@ x: Callable[[int], None] def f(*x: int) -> None: pass def g(*x: str) -> None: pass x = f -x = g # E: Incompatible types in assignment (expression has type "Callable[[VarArg(str)], None]", variable has type "Callable[[int], None]") +x = g # E: Incompatible types in assignment (expression has type "def g(*x: str) -> None", variable has type "Callable[[int], None]") [builtins fixtures/list.pyi] [out] diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 2069d082df17..7dfcf7447b61 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -616,7 +616,7 @@ def f(*args: str) -> str: return args[0] map(f, ['x']) map(f, [1]) [out] -_program.py:4: error: Argument 1 to "map" has incompatible type "Callable[[VarArg(str)], str]"; expected "Callable[[int], str]" +_program.py:4: error: Argument 1 to "map" has incompatible type "def f(*args: str) -> str"; expected "Callable[[int], str]" [case testMapStr] import typing From 630a1439f2f3226f20ef744c32ba3f03bda58ecb Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 3 Nov 2025 01:54:05 -0800 Subject: [PATCH 140/183] Run ubuntu mypyc tests on 3.10 (#20169) from Emma https://github.com/python/mypy/pull/20155#issuecomment-3475902175 --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 07b4b3f03020..6fe825748073 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,8 +85,8 @@ jobs: toxenv: py tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py" - - name: mypyc runtime tests with py313-ubuntu - python: '3.13' + - name: mypyc runtime tests with py310-ubuntu + python: '3.10' os: ubuntu-latest toxenv: py tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py" From 2809328d9f86f4d3c434998d0a2338931362bec0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 3 Nov 2025 10:25:14 +0000 Subject: [PATCH 141/183] Consistently raise ValueError on corrupted cache data and test more (#20153) Test random and arbitrary cache data. Deserialization should fail in a predictable manner. --- mypyc/lib-rt/librt_internal.c | 14 +++++++ mypyc/test-data/run-classes.test | 68 ++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 6cae63cfadcb..eaf451eff22b 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -319,6 +319,11 @@ read_str_internal(PyObject *data) { CPyTagged tagged_size = _read_short_int(data, first); if (tagged_size == CPY_INT_TAG) return NULL; + if ((Py_ssize_t)tagged_size < 0) { + // Fail fast for invalid/tampered data. + PyErr_SetString(PyExc_ValueError, "invalid str size"); + return NULL; + } Py_ssize_t size = tagged_size >> 1; // Read string content. char *buf = ((BufferObject *)data)->buf; @@ -437,6 +442,11 @@ read_bytes_internal(PyObject *data) { CPyTagged tagged_size = _read_short_int(data, first); if (tagged_size == CPY_INT_TAG) return NULL; + if ((Py_ssize_t)tagged_size < 0) { + // Fail fast for invalid/tampered data. + PyErr_SetString(PyExc_ValueError, "invalid bytes size"); + return NULL; + } Py_ssize_t size = tagged_size >> 1; // Read bytes content. char *buf = ((BufferObject *)data)->buf; @@ -601,6 +611,10 @@ read_int_internal(PyObject *data) { Py_ssize_t size_and_sign = _read_short_int(data, first); if (size_and_sign == CPY_INT_TAG) return CPY_INT_TAG; + if ((Py_ssize_t)size_and_sign < 0) { + PyErr_SetString(PyExc_ValueError, "invalid int data"); + return CPY_INT_TAG; + } bool sign = (size_and_sign >> 1) & 1; Py_ssize_t size = size_and_sign >> 2; diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index b02d10446800..0805da184e1a 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -5359,3 +5359,71 @@ def test_deletable_attr() -> None: assert i.del_counter == 1 test_deletable_attr() + +[case testBufferCorruptedData_librt_internal] +from librt.internal import ( + Buffer, read_bool, read_str, read_float, read_int, read_tag, read_bytes +) +from random import randbytes + +def check(data: bytes) -> None: + b = Buffer(data) + try: + while True: + read_bool(b) + except ValueError: + pass + b = Buffer(data) + read_tag(b) # Always succeeds + try: + while True: + read_int(b) + except ValueError: + pass + b = Buffer(data) + try: + while True: + read_str(b) + except ValueError: + pass + b = Buffer(data) + try: + while True: + read_bytes(b) + except ValueError: + pass + b = Buffer(data) + try: + while True: + read_float(b) + except ValueError: + pass + +import time + +def test_read_corrupted_data() -> None: + # Test various deterministic byte sequences (1 to 4 bytes). + t0 = time.time() + for a in range(256): + check(bytes([a])) + for a in range(256): + for b in range(256): + check(bytes([a, b])) + for a in range(32): + for b in range(48): + for c in range(48): + check(bytes([a, b, c])) + for a in range(32): + for b in (0, 5, 17, 34): + for c in (0, 5, 17, 34): + for d in (0, 5, 17, 34): + check(bytes([a, b, c, d])) + # Also test some random data. + for i in range(20000): + data = randbytes(16) + try: + check(data) + except BaseException as e: + print("RANDOMIZED TEST FAILURE -- please open an issue with the following context:") + print(">>>", e, data) + raise From 1b7e717ecc56cd13d76bc110a1db2796e8b3c918 Mon Sep 17 00:00:00 2001 From: David Foster Date: Mon, 3 Nov 2025 02:53:50 -0800 Subject: [PATCH 142/183] [PEP 747] Recognize TypeForm[T] type and values (#9773) (#19596) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _(This PR replaces an earlier draft of the same feature: https://github.com/python/mypy/pull/18690 )_ Feedback from @JukkaL integrated since the last PR, by commit title: * Apply feedback: Change MAYBE_UNRECOGNIZED_STR_TYPEFORM from unaccompanied note to standalone error * Apply feedback: Refactor extract save/restore of SemanticAnalyzer state to a new context manager * Apply feedback: Suppress SyntaxWarnings when parsing strings as types at the _most-targeted_ location * Apply feedback: Add TypeForm profiling counters to SemanticAnalyzer and the --dump-build-stats option * Increase efficiency of quick rejection heuristic from 85.8% -> 99.6% in SemanticAnalyzer.try_parse_as_type_expression() * Apply feedback: Recognize assignment to union of TypeForm with non-TypeForm * Apply feedback: Alter primitives.pyi fixture rather than tuple.pyi and dict.pyi Feedback NOT integrated, with rationale: * ✖️ Add tests related to recursive types * Recursive cases are already well-covered by tests related to TypeType (is_type_form=False). * I _did_ find an infinite recursion bug affecting garden-variety `Type[...]`, which I can fix in a separate PR. * ✖️ Define `TypeForm(...)` in value contexts as a regular function like `Callable[[TypeForm[T]], TypeForm[T]]` rather than as a special expression node (TypeFormExpr). * The special expression node allows mypy to print out _better error messages_ when a user puts an invalid type expression inside `TypeForm(...)`. See case 4 of testTypeFormExpression in check-typeform.test There is one commit unrelated to the core function of this PR that could be split to a separate PR: * Allow TypeAlias and PlaceholderNode to be stringified/printed Closes #9773 --- _(Most of the following description is copied from the original PR, **except for the text in bold**)_ Implements the [TypeForm PEP 747](https://peps.python.org/pep-0747/), as an opt-in feature enabled by the CLI flag `--enable-incomplete-feature=TypeForm`. Implementation approach: * The `TypeForm[T]` is represented as a type using the existing `TypeType` class, with an `is_type_form=True` constructor parameter. `Type[C]` continues to be represented using `TypeType`, but with `is_type_form=False` (the default). * Recognizing a type expression literal such as `int | str` requires parsing an `Expression` as a type expression. Only the SemanticAnalyzer pass has the ability to parse **arbitrary** type expressions **(including stringified annotations)**, using `SemanticAnalyzer.expr_to_analyzed_type()`. **(I've extended the `TypeChecker` pass to parse all kinds of type expressions except stringified annotations, using the new `TypeCheckerAsSemanticAnalyzer` adapter.)** * Therefore during the SemanticAnalyzer pass, at certain syntactic locations (i.e. assignment r-values, callable arguments, returned expressions), the analyzer tries to parse the `Expression` it is looking at using `try_parse_as_type_expression()` - a new function - and stores the result (a `Type`) in `{IndexExpr, OpExpr, StrExpr}.as_type` - a new attribute. * During the later TypeChecker pass, when looking at an `Expression` to determine its type, if the expression is in a type context that expects some kind of `TypeForm[...]` and the expression was successfully parsed as a type expression by the earlier SemanticAnalyzer pass **(or can be parsed as a type expression immediately during the type checker pass)**, the expression will be given the type `TypeForm[expr.as_type]` rather than using the regular type inference rules for a value expression. * Key relationships between `TypeForm[T]`, `Type[C]`, and `object` types are defined in the visitors powering `is_subtype`, `join_types`, and `meet_types`. * The `TypeForm(T)` expression is recognized as a `TypeFormExpr` and has the return type `TypeForm[T]`. * The new test suite in `check-typeform.test` is a good reference to the expected behaviors for operations that interact with `TypeForm` in some way. Controversial parts of this PR, in @davidfstr 's opinion: * Type form literals **containing stringified annotations** are only recognized in certain syntactic locations (and not ALL possible locations). Namely they are recognized as (1) assignment r-values, (2) callable expression arguments, and (3) as returned expressions, but nowhere else. For example they aren't recognized in expressions like `dict_with_typx_keys[int | str]`. **Attempting to use stringified annotations in other locations will emit a MAYBE_UNRECOGNIZED_STR_TYPEFORM error.** * The existing `TypeType` class is now used to represent BOTH the `Type[T]` and `TypeForm[T]` types, rather than introducing a distinct subclass of `Type` to represent the `TypeForm[T]` type. This was done to simplify logic that manipulates both `Type[T]` and `TypeForm[T]` values, since they are both manipulated in very similar ways. * The "normalized" form of `TypeForm[X | Y]` - as returned by `TypeType.make_normalized()` - is just `TypeForm[X | Y]` rather than `TypeForm[X] | TypeForm[Y]`, differing from the normalization behavior of `Type[X | Y]`. --- docs/source/error_code_list.rst | 59 ++ misc/analyze_typeform_stats.py | 91 ++ mypy/checker.py | 114 ++- mypy/checkexpr.py | 124 ++- mypy/copytype.py | 2 +- mypy/erasetype.py | 4 +- mypy/errorcodes.py | 5 + mypy/evalexpr.py | 3 + mypy/expandtype.py | 2 +- mypy/fastparse.py | 19 +- mypy/join.py | 6 +- mypy/literals.py | 4 + mypy/meet.py | 18 +- mypy/messages.py | 6 +- mypy/mixedtraverser.py | 5 + mypy/nodes.py | 49 +- mypy/options.py | 3 +- mypy/semanal.py | 257 +++++- mypy/semanal_main.py | 11 + mypy/server/astdiff.py | 2 +- mypy/server/astmerge.py | 5 + mypy/server/deps.py | 5 + mypy/server/subexpr.py | 5 + mypy/strconv.py | 9 + mypy/subtypes.py | 72 +- mypy/traverser.py | 44 + mypy/treetransform.py | 4 + mypy/type_visitor.py | 4 +- mypy/typeanal.py | 23 +- mypy/typeops.py | 2 +- mypy/types.py | 55 +- mypy/visitor.py | 7 + mypyc/irbuild/visitor.py | 4 + test-data/unit/check-fastparse.test | 15 + test-data/unit/check-typeform.test | 842 ++++++++++++++++++ test-data/unit/fixtures/primitives.pyi | 4 + test-data/unit/fixtures/typing-full.pyi | 1 + test-data/unit/lib-stub/typing_extensions.pyi | 2 + 38 files changed, 1814 insertions(+), 73 deletions(-) create mode 100644 misc/analyze_typeform_stats.py create mode 100644 test-data/unit/check-typeform.test diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index 229230eda4ca..d4e2c83323ac 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -1286,6 +1286,65 @@ type must be a subtype of the original type:: def g(x: object) -> TypeIs[str]: # OK ... +.. _code-maybe-unrecognized-str-typeform: + +String appears in a context which expects a TypeForm [maybe-unrecognized-str-typeform] +-------------------------------------------------------------------------------------- + +TypeForm literals may contain string annotations: + +.. code-block:: python + + typx1: TypeForm = str | None + typx2: TypeForm = 'str | None' # OK + typx3: TypeForm = 'str' | None # OK + +However TypeForm literals containing a string annotation can only be recognized +by mypy in the following locations: + +.. code-block:: python + + typx_var: TypeForm = 'str | None' # assignment r-value + + def func(typx_param: TypeForm) -> TypeForm: + return 'str | None' # returned expression + + func('str | None') # callable's argument + +If you try to use a string annotation in some other location +which expects a TypeForm, the string value will always be treated as a ``str`` +even if a ``TypeForm`` would be more appropriate and this error code +will be generated: + +.. code-block:: python + + # Error: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. [maybe-unrecognized-str-typeform] + # Error: List item 0 has incompatible type "str"; expected "TypeForm[Any]" [list-item] + list_of_typx: list[TypeForm] = ['str | None', float] + +Fix the error by surrounding the entire type with ``TypeForm(...)``: + +.. code-block:: python + + list_of_typx: list[TypeForm] = [TypeForm('str | None'), float] # OK + +Similarly, if you try to use a string literal in a location which expects a +TypeForm, this error code will be generated: + +.. code-block:: python + + dict_of_typx = {'str_or_none': TypeForm(str | None)} + # Error: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. [maybe-unrecognized-str-typeform] + list_of_typx: list[TypeForm] = [dict_of_typx['str_or_none']] + +Fix the error by adding ``# type: ignore[maybe-unrecognized-str-typeform]`` +to the line with the string literal: + +.. code-block:: python + + dict_of_typx = {'str_or_none': TypeForm(str | None)} + list_of_typx: list[TypeForm] = [dict_of_typx['str_or_none']] # type: ignore[maybe-unrecognized-str-typeform] + .. _code-misc: Miscellaneous checks [misc] diff --git a/misc/analyze_typeform_stats.py b/misc/analyze_typeform_stats.py new file mode 100644 index 000000000000..0a540610bc62 --- /dev/null +++ b/misc/analyze_typeform_stats.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +""" +Analyze TypeForm parsing efficiency from mypy build stats. + +Usage: + python3 analyze_typeform_stats.py '' + python3 -m mypy --dump-build-stats file.py 2>&1 | python3 analyze_typeform_stats.py + +Example output: + TypeForm Expression Parsing Statistics: + ================================================== + Total calls to SA.try_parse_as_type_expression: 14,555 + Quick rejections (no full parse): 14,255 + Full parses attempted: 300 + - Successful: 248 + - Failed: 52 + + Efficiency Metrics: + - Quick rejection rate: 97.9% + - Full parse rate: 2.1% + - Full parse success rate: 82.7% + - Overall success rate: 1.7% + + Performance Implications: + - Expensive failed full parses: 52 (0.4% of all calls) + +See also: + - mypy/semanal.py: SemanticAnalyzer.try_parse_as_type_expression() + - mypy/semanal.py: DEBUG_TYPE_EXPRESSION_FULL_PARSE_FAILURES +""" + +import re +import sys + + +def analyze_stats(output: str) -> None: + """Parse mypy stats output and calculate TypeForm parsing efficiency.""" + + # Extract the three counters + total_match = re.search(r"type_expression_parse_count:\s*(\d+)", output) + success_match = re.search(r"type_expression_full_parse_success_count:\s*(\d+)", output) + failure_match = re.search(r"type_expression_full_parse_failure_count:\s*(\d+)", output) + + if not (total_match and success_match and failure_match): + print("Error: Could not find all required counters in output") + return + + total = int(total_match.group(1)) + successes = int(success_match.group(1)) + failures = int(failure_match.group(1)) + + full_parses = successes + failures + + print("TypeForm Expression Parsing Statistics:") + print("=" * 50) + print(f"Total calls to SA.try_parse_as_type_expression: {total:,}") + print(f"Quick rejections (no full parse): {total - full_parses:,}") + print(f"Full parses attempted: {full_parses:,}") + print(f" - Successful: {successes:,}") + print(f" - Failed: {failures:,}") + if total > 0: + print() + print("Efficiency Metrics:") + print(f" - Quick rejection rate: {((total - full_parses) / total * 100):.1f}%") + print(f" - Full parse rate: {(full_parses / total * 100):.1f}%") + print(f" - Full parse success rate: {(successes / full_parses * 100):.1f}%") + print(f" - Overall success rate: {(successes / total * 100):.1f}%") + print() + print("Performance Implications:") + print( + f" - Expensive failed full parses: {failures:,} ({(failures / total * 100):.1f}% of all calls)" + ) + + +if __name__ == "__main__": + if len(sys.argv) == 1: + # Read from stdin + output = sys.stdin.read() + elif len(sys.argv) == 2: + # Read from command line argument + output = sys.argv[1] + else: + print("Usage: python3 analyze_typeform_stats.py [mypy_output_with_stats]") + print("Examples:") + print( + " python3 -m mypy --dump-build-stats file.py 2>&1 | python3 analyze_typeform_stats.py" + ) + print(" python3 analyze_typeform_stats.py 'output_string'") + sys.exit(1) + + analyze_stats(output) diff --git a/mypy/checker.py b/mypy/checker.py index 63e128f78310..f4746bc0c886 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -155,6 +155,7 @@ from mypy.scope import Scope from mypy.semanal import is_trivial_body, refers_to_fullname, set_callable_name from mypy.semanal_enum import ENUM_BASES, ENUM_SPECIAL_PROPS +from mypy.semanal_shared import SemanticAnalyzerCoreInterface from mypy.sharedparse import BINARY_MAGIC_METHODS from mypy.state import state from mypy.subtypes import ( @@ -346,6 +347,8 @@ class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi): tscope: Scope scope: CheckerScope + # Innermost enclosing type + type: TypeInfo | None # Stack of function return types return_types: list[Type] # Flags; true for dynamically typed functions @@ -423,6 +426,7 @@ def __init__( self.scope = CheckerScope(tree) self.binder = ConditionalTypeBinder(options) self.globals = tree.names + self.type = None self.return_types = [] self.dynamic_funcs = [] self.partial_types = [] @@ -2661,7 +2665,11 @@ def visit_class_def(self, defn: ClassDef) -> None: self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn) if not can_have_shared_disjoint_base(typ.bases): self.fail(message_registry.INCOMPATIBLE_DISJOINT_BASES.format(typ.name), defn) - with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True): + with ( + self.tscope.class_scope(defn.info), + self.enter_partial_types(is_class=True), + self.enter_class(defn.info), + ): old_binder = self.binder self.binder = ConditionalTypeBinder(self.options) with self.binder.top_frame_context(): @@ -2729,6 +2737,15 @@ def visit_class_def(self, defn: ClassDef) -> None: self.check_enum(defn) infer_class_variances(defn.info) + @contextmanager + def enter_class(self, type: TypeInfo) -> Iterator[None]: + original_type = self.type + self.type = type + try: + yield + finally: + self.type = original_type + def check_final_deletable(self, typ: TypeInfo) -> None: # These checks are only for mypyc. Only perform some checks that are easier # to implement here than in mypyc. @@ -8023,7 +8040,9 @@ def add_any_attribute_to_type(self, typ: Type, name: str) -> Type: fallback = typ.fallback.copy_with_extra_attr(name, any_type) return typ.copy_modified(fallback=fallback) if isinstance(typ, TypeType) and isinstance(typ.item, Instance): - return TypeType.make_normalized(self.add_any_attribute_to_type(typ.item, name)) + return TypeType.make_normalized( + self.add_any_attribute_to_type(typ.item, name), is_type_form=typ.is_type_form + ) if isinstance(typ, TypeVarType): return typ.copy_modified( upper_bound=self.add_any_attribute_to_type(typ.upper_bound, name), @@ -8151,6 +8170,97 @@ def visit_global_decl(self, o: GlobalDecl, /) -> None: return None +class TypeCheckerAsSemanticAnalyzer(SemanticAnalyzerCoreInterface): + """ + Adapts TypeChecker to the SemanticAnalyzerCoreInterface, + allowing most type expressions to be parsed during the TypeChecker pass. + + See ExpressionChecker.try_parse_as_type_expression() to understand how this + class is used. + """ + + _chk: TypeChecker + _names: dict[str, SymbolTableNode] + did_fail: bool + + def __init__(self, chk: TypeChecker, names: dict[str, SymbolTableNode]) -> None: + self._chk = chk + self._names = names + self.did_fail = False + + def lookup_qualified( + self, name: str, ctx: Context, suppress_errors: bool = False + ) -> SymbolTableNode | None: + sym = self._names.get(name) + # All names being looked up should have been previously gathered, + # even if the related SymbolTableNode does not refer to a valid SymbolNode + assert sym is not None, name + return sym + + def lookup_fully_qualified(self, fullname: str, /) -> SymbolTableNode: + ret = self.lookup_fully_qualified_or_none(fullname) + assert ret is not None, fullname + return ret + + def lookup_fully_qualified_or_none(self, fullname: str, /) -> SymbolTableNode | None: + try: + return self._chk.lookup_qualified(fullname) + except KeyError: + return None + + def fail( + self, + msg: str, + ctx: Context, + serious: bool = False, + *, + blocker: bool = False, + code: ErrorCode | None = None, + ) -> None: + self.did_fail = True + + def note(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: + pass + + def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool: + if feature not in self._chk.options.enable_incomplete_feature: + self.fail("__ignored__", ctx) + return False + return True + + def record_incomplete_ref(self) -> None: + pass + + def defer(self, debug_context: Context | None = None, force_progress: bool = False) -> None: + pass + + def is_incomplete_namespace(self, fullname: str) -> bool: + return False + + @property + def final_iteration(self) -> bool: + return True + + def is_future_flag_set(self, flag: str) -> bool: + return self._chk.tree.is_future_flag_set(flag) + + @property + def is_stub_file(self) -> bool: + return self._chk.tree.is_stub + + def is_func_scope(self) -> bool: + # Return arbitrary value. + # + # This method is currently only used to decide whether to pair + # a fail() message with a note() message or not. Both of those + # message types are ignored. + return False + + @property + def type(self) -> TypeInfo | None: + return self._chk.type + + class CollectArgTypeVarTypes(TypeTraverserVisitor): """Collects the non-nested argument types in a set.""" diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 3eb54579a050..a06af690f8db 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -27,6 +27,7 @@ freshen_all_functions_type_vars, freshen_function_type_vars, ) +from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type from mypy.infer import ArgumentInferContext, infer_function_type_arguments, infer_type_arguments from mypy.literals import literal from mypy.maptype import map_instance_to_supertype @@ -43,6 +44,7 @@ LITERAL_TYPE, REVEAL_LOCALS, REVEAL_TYPE, + UNBOUND_IMPORTED, ArgKind, AssertTypeExpr, AssignmentExpr, @@ -68,11 +70,13 @@ LambdaExpr, ListComprehension, ListExpr, + MaybeTypeExpression, MemberExpr, MypyFile, NamedTupleExpr, NameExpr, NewTypeExpr, + NotParsed, OpExpr, OverloadedFuncDef, ParamSpecExpr, @@ -87,12 +91,14 @@ StrExpr, SuperExpr, SymbolNode, + SymbolTableNode, TempNode, TupleExpr, TypeAlias, TypeAliasExpr, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeInfo, TypeVarExpr, TypeVarLikeExpr, @@ -101,6 +107,7 @@ Var, YieldExpr, YieldFromExpr, + get_member_expr_fullname, ) from mypy.options import PRECISE_TUPLE_TYPES from mypy.plugin import ( @@ -119,8 +126,14 @@ is_subtype, non_method_protocol_members, ) -from mypy.traverser import has_await_expression +from mypy.traverser import ( + all_name_and_member_expressions, + has_await_expression, + has_str_expression, +) +from mypy.tvar_scope import TypeVarLikeScope from mypy.typeanal import ( + TypeAnalyser, check_for_explicit_any, fix_instance, has_any_from_unimported_type, @@ -4692,6 +4705,10 @@ def visit_cast_expr(self, expr: CastExpr) -> Type: ) return target_type + def visit_type_form_expr(self, expr: TypeFormExpr) -> Type: + typ = expr.type + return TypeType.make_normalized(typ, line=typ.line, column=typ.column, is_type_form=True) + def visit_assert_type_expr(self, expr: AssertTypeExpr) -> Type: source_type = self.accept( expr.expr, @@ -6018,6 +6035,7 @@ def accept( old_is_callee = self.is_callee self.is_callee = is_callee try: + p_type_context = get_proper_type(type_context) if allow_none_return and isinstance(node, CallExpr): typ = self.visit_call_expr(node, allow_none_return=True) elif allow_none_return and isinstance(node, YieldFromExpr): @@ -6026,6 +6044,37 @@ def accept( typ = self.visit_conditional_expr(node, allow_none_return=True) elif allow_none_return and isinstance(node, AwaitExpr): typ = self.visit_await_expr(node, allow_none_return=True) + + elif ( + isinstance(p_type_context, TypeType) + and p_type_context.is_type_form + and (node_as_type := self.try_parse_as_type_expression(node)) is not None + ): + typ = TypeType.make_normalized( + node_as_type, + line=node_as_type.line, + column=node_as_type.column, + is_type_form=True, + ) # r-value type, when interpreted as a type expression + elif ( + isinstance(p_type_context, UnionType) + and any( + isinstance(p_item := get_proper_type(item), TypeType) and p_item.is_type_form + for item in p_type_context.items + ) + and (node_as_type := self.try_parse_as_type_expression(node)) is not None + ): + typ1 = TypeType.make_normalized( + node_as_type, + line=node_as_type.line, + column=node_as_type.column, + is_type_form=True, + ) + if is_subtype(typ1, p_type_context): + typ = typ1 # r-value type, when interpreted as a type expression + else: + typ2 = node.accept(self) + typ = typ2 # r-value type, when interpreted as a value expression # Deeply nested generic calls can deteriorate performance dramatically. # Although in most cases caching makes little difference, in worst case # it avoids exponential complexity. @@ -6047,7 +6096,7 @@ def accept( else: typ = self.accept_maybe_cache(node, type_context=type_context) else: - typ = node.accept(self) + typ = node.accept(self) # r-value type, when interpreted as a value expression except Exception as err: report_internal_error( err, self.chk.errors.file, node.line, self.chk.errors, self.chk.options @@ -6379,6 +6428,77 @@ def has_abstract_type(self, caller_type: ProperType, callee_type: ProperType) -> and not self.chk.allow_abstract_call ) + def try_parse_as_type_expression(self, maybe_type_expr: Expression) -> Type | None: + """Try to parse a value Expression as a type expression. + If success then return the type that it spells. + If fails then return None. + + A value expression that is parsable as a type expression may be used + where a TypeForm is expected to represent the spelled type. + + Unlike SemanticAnalyzer.try_parse_as_type_expression() + (used in the earlier SemanticAnalyzer pass), this function can only + recognize type expressions which contain no string annotations.""" + if not isinstance(maybe_type_expr, MaybeTypeExpression): + return None + + # Check whether has already been parsed as a type expression + # by SemanticAnalyzer.try_parse_as_type_expression(), + # perhaps containing a string annotation + if ( + isinstance(maybe_type_expr, (StrExpr, IndexExpr, OpExpr)) + and maybe_type_expr.as_type != NotParsed.VALUE + ): + return maybe_type_expr.as_type + + # If is potentially a type expression containing a string annotation, + # don't try to parse it because there isn't enough information + # available to the TypeChecker pass to resolve string annotations + if has_str_expression(maybe_type_expr): + self.chk.fail( + "TypeForm containing a string annotation cannot be recognized here. " + "Surround with TypeForm(...) to recognize.", + maybe_type_expr, + code=codes.MAYBE_UNRECOGNIZED_STR_TYPEFORM, + ) + return None + + # Collect symbols targeted by NameExprs and MemberExprs, + # to be looked up by TypeAnalyser when binding the + # UnboundTypes corresponding to those expressions. + (name_exprs, member_exprs) = all_name_and_member_expressions(maybe_type_expr) + sym_for_name = {e.name: SymbolTableNode(UNBOUND_IMPORTED, e.node) for e in name_exprs} | { + e_name: SymbolTableNode(UNBOUND_IMPORTED, e.node) + for e in member_exprs + if (e_name := get_member_expr_fullname(e)) is not None + } + + chk_sem = mypy.checker.TypeCheckerAsSemanticAnalyzer(self.chk, sym_for_name) + tpan = TypeAnalyser( + chk_sem, + # NOTE: Will never need to lookup type vars in this scope because + # SemanticAnalyzer.try_parse_as_type_expression() will have + # already recognized any type var referenced in a NameExpr. + # String annotations (which may also reference type vars) + # can't be resolved in the TypeChecker pass anyway. + TypeVarLikeScope(), # empty scope + self.plugin, + self.chk.options, + self.chk.tree, + self.chk.is_typeshed_stub, + ) + + try: + typ1 = expr_to_unanalyzed_type( + maybe_type_expr, self.chk.options, self.chk.is_typeshed_stub + ) + typ2 = typ1.accept(tpan) + if chk_sem.did_fail: + return None + return typ2 + except TypeTranslationError: + return None + def has_any_type(t: Type, ignore_in_type_obj: bool = False) -> bool: """Whether t contains an Any type""" diff --git a/mypy/copytype.py b/mypy/copytype.py index ecb1a89759b6..a890431a1772 100644 --- a/mypy/copytype.py +++ b/mypy/copytype.py @@ -122,7 +122,7 @@ def visit_overloaded(self, t: Overloaded) -> ProperType: def visit_type_type(self, t: TypeType) -> ProperType: # Use cast since the type annotations in TypeType are imprecise. - return self.copy_common(t, TypeType(cast(Any, t.item))) + return self.copy_common(t, TypeType(cast(Any, t.item), is_type_form=t.is_type_form)) def visit_type_alias_type(self, t: TypeAliasType) -> ProperType: assert False, "only ProperTypes supported" diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 6645bcf916d9..500d8fd5ae08 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -134,7 +134,9 @@ def visit_union_type(self, t: UnionType) -> ProperType: return make_simplified_union(erased_items) def visit_type_type(self, t: TypeType) -> ProperType: - return TypeType.make_normalized(t.item.accept(self), line=t.line) + return TypeType.make_normalized( + t.item.accept(self), line=t.line, is_type_form=t.is_type_form + ) def visit_type_alias_type(self, t: TypeAliasType) -> ProperType: raise RuntimeError("Type aliases should be expanded before accepting this visitor") diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index fbfa572b9439..785b6166b18b 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -269,6 +269,11 @@ def __hash__(self) -> int: default_enabled=False, ) METACLASS: Final = ErrorCode("metaclass", "Ensure that metaclass is valid", "General") +MAYBE_UNRECOGNIZED_STR_TYPEFORM: Final = ErrorCode( + "maybe-unrecognized-str-typeform", + "Error when a string is used where a TypeForm is expected but a string annotation cannot be recognized", + "General", +) # Syntax errors are often blocking. SYNTAX: Final = ErrorCode("syntax", "Report syntax errors", "General") diff --git a/mypy/evalexpr.py b/mypy/evalexpr.py index e39c5840d47a..218d50e37ec3 100644 --- a/mypy/evalexpr.py +++ b/mypy/evalexpr.py @@ -75,6 +75,9 @@ def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr) -> object: def visit_cast_expr(self, o: mypy.nodes.CastExpr) -> object: return o.expr.accept(self) + def visit_type_form_expr(self, o: mypy.nodes.TypeFormExpr) -> object: + return UNKNOWN + def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr) -> object: return o.expr.accept(self) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index be1c2dd91e77..891ea4d89a80 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -521,7 +521,7 @@ def visit_type_type(self, t: TypeType) -> Type: # union of instances or Any). Sadly we can't report errors # here yet. item = t.item.accept(self) - return TypeType.make_normalized(item) + return TypeType.make_normalized(item, is_type_form=t.is_type_form) def visit_type_alias_type(self, t: TypeAliasType) -> Type: # Target of the type alias cannot contain type variables (not bound by the type diff --git a/mypy/fastparse.py b/mypy/fastparse.py index c5e4589ec025..0e7b418d0375 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -135,13 +135,18 @@ def ast3_parse( source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION ) -> AST: - return ast3.parse( - source, - filename, - mode, - type_comments=True, # This works the magic - feature_version=feature_version, - ) + # Ignore warnings that look like: + # :1: SyntaxWarning: invalid escape sequence '\.' + # because `source` could be anything, including literals like r'(re\.match)' + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) + return ast3.parse( + source, + filename, + mode, + type_comments=True, # This works the magic + feature_version=feature_version, + ) if sys.version_info >= (3, 10): diff --git a/mypy/join.py b/mypy/join.py index 099df02680f0..0822ddbfd89a 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -637,7 +637,11 @@ def visit_partial_type(self, t: PartialType) -> ProperType: def visit_type_type(self, t: TypeType) -> ProperType: if isinstance(self.s, TypeType): - return TypeType.make_normalized(join_types(t.item, self.s.item), line=t.line) + return TypeType.make_normalized( + join_types(t.item, self.s.item), + line=t.line, + is_type_form=self.s.is_type_form or t.is_type_form, + ) elif isinstance(self.s, Instance) and self.s.type.fullname == "builtins.type": return self.s else: diff --git a/mypy/literals.py b/mypy/literals.py index 5b0c46f4bee8..fd17e0471440 100644 --- a/mypy/literals.py +++ b/mypy/literals.py @@ -48,6 +48,7 @@ TypeAliasExpr, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeVarExpr, TypeVarTupleExpr, UnaryExpr, @@ -244,6 +245,9 @@ def visit_slice_expr(self, e: SliceExpr) -> None: def visit_cast_expr(self, e: CastExpr) -> None: return None + def visit_type_form_expr(self, e: TypeFormExpr) -> None: + return None + def visit_assert_type_expr(self, e: AssertTypeExpr) -> None: return None diff --git a/mypy/meet.py b/mypy/meet.py index 1cb291ff90d5..42229f9b23c1 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -187,12 +187,24 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: elif isinstance(narrowed, TypeVarType) and is_subtype(narrowed.upper_bound, declared): return narrowed elif isinstance(declared, TypeType) and isinstance(narrowed, TypeType): - return TypeType.make_normalized(narrow_declared_type(declared.item, narrowed.item)) + return TypeType.make_normalized( + narrow_declared_type(declared.item, narrowed.item), + is_type_form=declared.is_type_form and narrowed.is_type_form, + ) elif ( isinstance(declared, TypeType) and isinstance(narrowed, Instance) and narrowed.type.is_metaclass() ): + if declared.is_type_form: + # The declared TypeForm[T] after narrowing must be a kind of + # type object at least as narrow as Type[T] + return narrow_declared_type( + TypeType.make_normalized( + declared.item, line=declared.line, column=declared.column, is_type_form=False + ), + original_narrowed, + ) # We'd need intersection types, so give up. return original_declared elif isinstance(declared, Instance): @@ -1115,7 +1127,9 @@ def visit_type_type(self, t: TypeType) -> ProperType: if isinstance(self.s, TypeType): typ = self.meet(t.item, self.s.item) if not isinstance(typ, NoneType): - typ = TypeType.make_normalized(typ, line=t.line) + typ = TypeType.make_normalized( + typ, line=t.line, is_type_form=self.s.is_type_form and t.is_type_form + ) return typ elif isinstance(self.s, Instance) and self.s.type.fullname == "builtins.type": return t diff --git a/mypy/messages.py b/mypy/messages.py index a9e8ee2e43ab..68b9b83572f1 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2756,7 +2756,11 @@ def format_literal_value(typ: LiteralType) -> str: elif isinstance(typ, UninhabitedType): return "Never" elif isinstance(typ, TypeType): - return f"type[{format(typ.item)}]" + if typ.is_type_form: + type_name = "TypeForm" + else: + type_name = "type" + return f"{type_name}[{format(typ.item)}]" elif isinstance(typ, FunctionLike): func = typ if func.is_type_obj(): diff --git a/mypy/mixedtraverser.py b/mypy/mixedtraverser.py index f47d762934bc..39fba49cf3c7 100644 --- a/mypy/mixedtraverser.py +++ b/mypy/mixedtraverser.py @@ -15,6 +15,7 @@ TypeAliasStmt, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeVarExpr, Var, WithStmt, @@ -109,6 +110,10 @@ def visit_cast_expr(self, o: CastExpr, /) -> None: super().visit_cast_expr(o) o.type.accept(self) + def visit_type_form_expr(self, o: TypeFormExpr, /) -> None: + super().visit_type_form_expr(o) + o.type.accept(self) + def visit_assert_type_expr(self, o: AssertTypeExpr, /) -> None: super().visit_assert_type_expr(o) o.type.accept(self) diff --git a/mypy/nodes.py b/mypy/nodes.py index 6cf984e5a218..539995ce9229 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -61,6 +61,11 @@ from mypy.patterns import Pattern +@unique +class NotParsed(Enum): + VALUE = "NotParsed" + + class Context: """Base type for objects that are valid as error message locations.""" @@ -2005,15 +2010,20 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T: class StrExpr(Expression): """String literal""" - __slots__ = ("value",) + __slots__ = ("value", "as_type") __match_args__ = ("value",) value: str # '' by default + # If this value expression can also be parsed as a valid type expression, + # represents the type denoted by the type expression. + # None means "is not a type expression". + as_type: NotParsed | mypy.types.Type | None def __init__(self, value: str) -> None: super().__init__() self.value = value + self.as_type = NotParsed.VALUE def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_str_expr(self) @@ -2314,7 +2324,7 @@ class IndexExpr(Expression): Also wraps type application such as List[int] as a special form. """ - __slots__ = ("base", "index", "method_type", "analyzed") + __slots__ = ("base", "index", "method_type", "analyzed", "as_type") __match_args__ = ("base", "index") @@ -2325,6 +2335,10 @@ class IndexExpr(Expression): # If not None, this is actually semantically a type application # Class[type, ...] or a type alias initializer. analyzed: TypeApplication | TypeAliasExpr | None + # If this value expression can also be parsed as a valid type expression, + # represents the type denoted by the type expression. + # None means "is not a type expression". + as_type: NotParsed | mypy.types.Type | None def __init__(self, base: Expression, index: Expression) -> None: super().__init__() @@ -2332,6 +2346,7 @@ def __init__(self, base: Expression, index: Expression) -> None: self.index = index self.method_type = None self.analyzed = None + self.as_type = NotParsed.VALUE def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_index_expr(self) @@ -2389,6 +2404,7 @@ class OpExpr(Expression): "right_always", "right_unreachable", "analyzed", + "as_type", ) __match_args__ = ("left", "op", "right") @@ -2404,6 +2420,10 @@ class OpExpr(Expression): right_unreachable: bool # Used for expressions that represent a type "X | Y" in some contexts analyzed: TypeAliasExpr | None + # If this value expression can also be parsed as a valid type expression, + # represents the type denoted by the type expression. + # None means "is not a type expression". + as_type: NotParsed | mypy.types.Type | None def __init__( self, op: str, left: Expression, right: Expression, analyzed: TypeAliasExpr | None = None @@ -2416,11 +2436,19 @@ def __init__( self.right_always = False self.right_unreachable = False self.analyzed = analyzed + self.as_type = NotParsed.VALUE def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_op_expr(self) +# Expression subtypes that could represent the root of a valid type expression. +# +# May have an "as_type" attribute to hold the type for a type expression parsed +# during the SemanticAnalyzer pass. +MaybeTypeExpression = (IndexExpr, MemberExpr, NameExpr, OpExpr, StrExpr) + + class ComparisonExpr(Expression): """Comparison expression (e.g. a < b > c < d).""" @@ -2498,6 +2526,23 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_cast_expr(self) +class TypeFormExpr(Expression): + """TypeForm(type) expression.""" + + __slots__ = ("type",) + + __match_args__ = ("type",) + + type: mypy.types.Type + + def __init__(self, typ: mypy.types.Type) -> None: + super().__init__() + self.type = typ + + def accept(self, visitor: ExpressionVisitor[T]) -> T: + return visitor.visit_type_form_expr(self) + + class AssertTypeExpr(Expression): """Represents a typing.assert_type(expr, type) call.""" diff --git a/mypy/options.py b/mypy/options.py index 209759763a5a..39490c9f0bee 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -83,7 +83,8 @@ class BuildType: PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax" INLINE_TYPEDDICT: Final = "InlineTypedDict" -INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT)) +TYPE_FORM: Final = "TypeForm" +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT, TYPE_FORM)) COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, NEW_GENERIC_SYNTAX)) diff --git a/mypy/semanal.py b/mypy/semanal.py index d7b50bd09496..1fdea22e8962 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -50,10 +50,11 @@ from __future__ import annotations +import re from collections.abc import Collection, Iterable, Iterator from contextlib import contextmanager from typing import Any, Callable, Final, TypeVar, cast -from typing_extensions import TypeAlias as _TypeAlias, TypeGuard +from typing_extensions import TypeAlias as _TypeAlias, TypeGuard, assert_never from mypy import errorcodes as codes, message_registry from mypy.constant_fold import constant_fold_expr @@ -136,6 +137,7 @@ ListExpr, Lvalue, MatchStmt, + MaybeTypeExpression, MemberExpr, MypyFile, NamedTupleExpr, @@ -172,6 +174,7 @@ TypeAliasStmt, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeInfo, TypeParam, TypeVarExpr, @@ -190,7 +193,7 @@ type_aliases_source_versions, typing_extensions_aliases, ) -from mypy.options import Options +from mypy.options import TYPE_FORM, Options from mypy.patterns import ( AsPattern, ClassPattern, @@ -311,6 +314,13 @@ T = TypeVar("T") +# Whether to print diagnostic information for failed full parses +# in SemanticAnalyzer.try_parse_as_type_expression(). +# +# See also: misc/analyze_typeform_stats.py +DEBUG_TYPE_EXPRESSION_FULL_PARSE_FAILURES: Final = False + + FUTURE_IMPORTS: Final = { "__future__.nested_scopes": "nested_scopes", "__future__.generators": "generators", @@ -342,6 +352,22 @@ Tag: _TypeAlias = int +# Matches two words separated by whitespace, where each word lacks +# any symbols which have special meaning in a type expression. +# +# Any string literal matching this common pattern cannot be a valid +# type expression and can be ignored quickly when attempting to parse a +# string literal as a type expression. +_MULTIPLE_WORDS_NONTYPE_RE = re.compile(r'\s*[^\s.\'"|\[]+\s+[^\s.\'"|\[]') + +# Matches any valid Python identifier, including identifiers with Unicode characters. +# +# [^\d\W] = word character that is not a digit +# \w = word character +# \Z = match end of string; does not allow a trailing \n, unlike $ +_IDENTIFIER_RE = re.compile(r"^[^\d\W]\w*\Z", re.UNICODE) + + class SemanticAnalyzer( NodeVisitor[None], SemanticAnalyzerInterface, SemanticAnalyzerPluginInterface ): @@ -503,6 +529,11 @@ def __init__( self._function_type: Instance | None = None self._object_type: Instance | None = None + # TypeForm profiling counters + self.type_expression_parse_count: int = 0 # Total try_parse_as_type_expression calls + self.type_expression_full_parse_success_count: int = 0 # Successful full parses + self.type_expression_full_parse_failure_count: int = 0 # Failed full parses + # mypyc doesn't properly handle implementing an abstractproperty # with a regular attribute so we make them properties @property @@ -3272,6 +3303,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.store_final_status(s) self.check_classvar(s) self.process_type_annotation(s) + self.analyze_rvalue_as_type_form(s) self.apply_dynamic_class_hook(s) if not s.type: self.process_module_assignment(s.lvalues, s.rvalue, s) @@ -3607,6 +3639,10 @@ def analyze_lvalues(self, s: AssignmentStmt) -> None: has_explicit_value=has_explicit_value, ) + def analyze_rvalue_as_type_form(self, s: AssignmentStmt) -> None: + if TYPE_FORM in self.options.enable_incomplete_feature: + self.try_parse_as_type_expression(s.rvalue) + def apply_dynamic_class_hook(self, s: AssignmentStmt) -> None: if not isinstance(s.rvalue, CallExpr): return @@ -5337,6 +5373,8 @@ def visit_return_stmt(self, s: ReturnStmt) -> None: self.fail('"return" not allowed in except* block', s, serious=True) if s.expr: s.expr.accept(self) + if TYPE_FORM in self.options.enable_incomplete_feature: + self.try_parse_as_type_expression(s.expr) self.statement = old def visit_raise_stmt(self, s: RaiseStmt) -> None: @@ -5856,10 +5894,31 @@ def visit_call_expr(self, expr: CallExpr) -> None: with self.allow_unbound_tvars_set(): for a in expr.args: a.accept(self) + elif refers_to_fullname(expr.callee, ("typing.TypeForm", "typing_extensions.TypeForm")): + # Special form TypeForm(...). + if not self.check_fixed_args(expr, 1, "TypeForm"): + return + # Translate first argument to an unanalyzed type. + try: + typ = self.expr_to_unanalyzed_type(expr.args[0]) + except TypeTranslationError: + self.fail("TypeForm argument is not a type", expr) + # Suppress future error: "" not callable + expr.analyzed = CastExpr(expr.args[0], AnyType(TypeOfAny.from_error)) + return + # Piggyback TypeFormExpr object to the CallExpr object; it takes + # precedence over the CallExpr semantics. + expr.analyzed = TypeFormExpr(typ) + expr.analyzed.line = expr.line + expr.analyzed.column = expr.column + expr.analyzed.accept(self) else: # Normal call expression. + calculate_type_forms = TYPE_FORM in self.options.enable_incomplete_feature for a in expr.args: a.accept(self) + if calculate_type_forms: + self.try_parse_as_type_expression(a) if ( isinstance(expr.callee, MemberExpr) @@ -6108,6 +6167,11 @@ def visit_cast_expr(self, expr: CastExpr) -> None: if analyzed is not None: expr.type = analyzed + def visit_type_form_expr(self, expr: TypeFormExpr) -> None: + analyzed = self.anal_type(expr.type) + if analyzed is not None: + expr.type = analyzed + def visit_assert_type_expr(self, expr: AssertTypeExpr) -> None: expr.expr.accept(self) analyzed = self.anal_type(expr.type) @@ -7699,6 +7763,195 @@ def visit_pass_stmt(self, o: PassStmt, /) -> None: def visit_singleton_pattern(self, o: SingletonPattern, /) -> None: return None + def try_parse_as_type_expression(self, maybe_type_expr: Expression) -> None: + """Try to parse a value Expression as a type expression. + If success then annotate the Expression with the type that it spells. + If fails then emit no errors and take no further action. + + A value expression that is parsable as a type expression may be used + where a TypeForm is expected to represent the spelled type. + + Unlike ExpressionChecker.try_parse_as_type_expression() + (used in the later TypeChecker pass), this function can recognize + ALL kinds of type expressions, including type expressions containing + string annotations. + + If the provided Expression will be parsable later in + ExpressionChecker.try_parse_as_type_expression(), this function will + skip parsing the Expression to improve performance, because the later + function is called many fewer times (i.e. only lazily in a rare TypeForm + type context) than this function is called (i.e. eagerly for EVERY + expression in certain syntactic positions). + """ + # Count every call to this method for profiling + self.type_expression_parse_count += 1 + + # Bail ASAP if the Expression matches a common pattern that cannot possibly + # be a valid type expression, because this function is called very frequently + if not isinstance(maybe_type_expr, MaybeTypeExpression): + return + # Check types in order from most common to least common, for best performance + if isinstance(maybe_type_expr, (NameExpr, MemberExpr)): + # Defer parsing to the later TypeChecker pass, + # and only lazily in contexts where a TypeForm is expected + return + elif isinstance(maybe_type_expr, StrExpr): + str_value = maybe_type_expr.value # cache + # Filter out string literals with common patterns that could not + # possibly be in a type expression + if _MULTIPLE_WORDS_NONTYPE_RE.match(str_value): + # A common pattern in string literals containing a sentence. + # But cannot be a type expression. + maybe_type_expr.as_type = None + return + # Filter out string literals which look like an identifier but + # cannot be a type expression, for a few common reasons + if _IDENTIFIER_RE.fullmatch(str_value): + sym = self.lookup(str_value, UnboundType(str_value), suppress_errors=True) + if sym is None: + # Does not refer to anything in the local symbol table + maybe_type_expr.as_type = None + return + else: # sym is not None + node = sym.node # cache + if isinstance(node, PlaceholderNode) and not node.becomes_typeinfo: + # Either: + # 1. f'Cannot resolve name "{t.name}" (possible cyclic definition)' + # 2. Reference to an unknown placeholder node. + maybe_type_expr.as_type = None + return + unbound_tvar_or_paramspec = ( + isinstance(node, (TypeVarExpr, TypeVarTupleExpr, ParamSpecExpr)) + and self.tvar_scope.get_binding(sym) is None + ) + if unbound_tvar_or_paramspec: + # Either: + # 1. unbound_tvar: 'Type variable "{}" is unbound' [codes.VALID_TYPE] + # 2. unbound_paramspec: f'ParamSpec "{name}" is unbound' [codes.VALID_TYPE] + maybe_type_expr.as_type = None + return + else: # does not look like an identifier + if '"' in str_value or "'" in str_value: + # Only valid inside a Literal[...] type + if "[" not in str_value: + # Cannot be a Literal[...] type + maybe_type_expr.as_type = None + return + elif str_value == "": + # Empty string is not a valid type + maybe_type_expr.as_type = None + return + elif isinstance(maybe_type_expr, IndexExpr): + if isinstance(maybe_type_expr.base, NameExpr): + if isinstance( + maybe_type_expr.base.node, Var + ) and not self.var_is_typing_special_form(maybe_type_expr.base.node): + # Leftmost part of IndexExpr refers to a Var. Not a valid type. + maybe_type_expr.as_type = None + return + elif isinstance(maybe_type_expr.base, MemberExpr): + next_leftmost = maybe_type_expr.base + while True: + leftmost = next_leftmost.expr + if not isinstance(leftmost, MemberExpr): + break + next_leftmost = leftmost + if isinstance(leftmost, NameExpr): + if isinstance(leftmost.node, Var) and not self.var_is_typing_special_form( + leftmost.node + ): + # Leftmost part of IndexExpr refers to a Var. Not a valid type. + maybe_type_expr.as_type = None + return + else: + # Leftmost part of IndexExpr is not a NameExpr. Not a valid type. + maybe_type_expr.as_type = None + return + else: + # IndexExpr base is neither a NameExpr nor MemberExpr. Not a valid type. + maybe_type_expr.as_type = None + return + elif isinstance(maybe_type_expr, OpExpr): + if maybe_type_expr.op != "|": + # Binary operators other than '|' never spell a valid type + maybe_type_expr.as_type = None + return + else: + assert_never(maybe_type_expr) + + with self.isolated_error_analysis(): + try: + t = self.expr_to_analyzed_type(maybe_type_expr) + if self.errors.is_errors(): + t = None + except TypeTranslationError: + # Not a type expression + t = None + + if DEBUG_TYPE_EXPRESSION_FULL_PARSE_FAILURES and t is None: + original_flushed_files = set(self.errors.flushed_files) # save + try: + errors = self.errors.new_messages() # capture + finally: + self.errors.flushed_files = original_flushed_files # restore + + print( + f"SA.try_parse_as_type_expression: Full parse failure: {maybe_type_expr}, errors={errors!r}" + ) + + # Count full parse attempts for profiling + if t is not None: + self.type_expression_full_parse_success_count += 1 + else: + self.type_expression_full_parse_failure_count += 1 + + maybe_type_expr.as_type = t + + @staticmethod + def var_is_typing_special_form(var: Var) -> bool: + return var.fullname.startswith("typing") and var.fullname in [ + "typing.Annotated", + "typing_extensions.Annotated", + "typing.Callable", + "typing.Literal", + "typing_extensions.Literal", + "typing.Optional", + "typing.TypeGuard", + "typing_extensions.TypeGuard", + "typing.TypeIs", + "typing_extensions.TypeIs", + "typing.Union", + ] + + @contextmanager + def isolated_error_analysis(self) -> Iterator[None]: + """ + Context manager for performing error analysis that should not + affect the main SemanticAnalyzer state. + + Upon entering this context, `self.errors` will start empty. + Within this context, you can analyze expressions for errors. + Upon exiting this context, the original `self.errors` will be restored, + and any errors collected during the analysis will be discarded. + """ + # Save state + original_errors = self.errors + original_num_incomplete_refs = self.num_incomplete_refs + original_progress = self.progress + original_deferred = self.deferred + original_deferral_debug_context_len = len(self.deferral_debug_context) + + self.errors = Errors(Options()) + try: + yield + finally: + # Restore state + self.errors = original_errors + self.num_incomplete_refs = original_num_incomplete_refs + self.progress = original_progress + self.deferred = original_deferred + del self.deferral_debug_context[original_deferral_debug_context_len:] + def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: if isinstance(sig, CallableType): diff --git a/mypy/semanal_main.py b/mypy/semanal_main.py index 7301e9f9b9b3..b2c43e6becb8 100644 --- a/mypy/semanal_main.py +++ b/mypy/semanal_main.py @@ -107,6 +107,17 @@ def semantic_analysis_for_scc(graph: Graph, scc: list[str], errors: Errors) -> N if "builtins" in scc: cleanup_builtin_scc(graph["builtins"]) + # Report TypeForm profiling stats + if len(scc) >= 1: + # Get manager from any state in the SCC (they all share the same manager) + manager = graph[scc[0]].manager + analyzer = manager.semantic_analyzer + manager.add_stats( + type_expression_parse_count=analyzer.type_expression_parse_count, + type_expression_full_parse_success_count=analyzer.type_expression_full_parse_success_count, + type_expression_full_parse_failure_count=analyzer.type_expression_full_parse_failure_count, + ) + def cleanup_builtin_scc(state: State) -> None: """Remove imported names from builtins namespace. diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 25542ce37588..15d472b64886 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -519,7 +519,7 @@ def visit_partial_type(self, typ: PartialType) -> SnapshotItem: raise RuntimeError def visit_type_type(self, typ: TypeType) -> SnapshotItem: - return ("TypeType", snapshot_type(typ.item)) + return ("TypeType", snapshot_type(typ.item), typ.is_type_form) def visit_type_alias_type(self, typ: TypeAliasType) -> SnapshotItem: assert typ.alias is not None diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index cda1d20fb8e4..56f2f935481c 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -75,6 +75,7 @@ SymbolTable, TypeAlias, TypedDictExpr, + TypeFormExpr, TypeInfo, Var, ) @@ -291,6 +292,10 @@ def visit_cast_expr(self, node: CastExpr) -> None: super().visit_cast_expr(node) self.fixup_type(node.type) + def visit_type_form_expr(self, node: TypeFormExpr) -> None: + super().visit_type_form_expr(node) + self.fixup_type(node.type) + def visit_assert_type_expr(self, node: AssertTypeExpr) -> None: super().visit_assert_type_expr(node) self.fixup_type(node.type) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index 076d95e2baf9..ba622329665e 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -125,6 +125,7 @@ class 'mod.Cls'. This can also refer to an attribute inherited from a TypeAliasExpr, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeInfo, TypeVarExpr, UnaryExpr, @@ -766,6 +767,10 @@ def visit_cast_expr(self, e: CastExpr) -> None: super().visit_cast_expr(e) self.add_type_dependencies(e.type) + def visit_type_form_expr(self, e: TypeFormExpr) -> None: + super().visit_type_form_expr(e) + self.add_type_dependencies(e.type) + def visit_assert_type_expr(self, e: AssertTypeExpr) -> None: super().visit_assert_type_expr(e) self.add_type_dependencies(e.type) diff --git a/mypy/server/subexpr.py b/mypy/server/subexpr.py index c94db44445dc..013b936e8b7c 100644 --- a/mypy/server/subexpr.py +++ b/mypy/server/subexpr.py @@ -28,6 +28,7 @@ StarExpr, TupleExpr, TypeApplication, + TypeFormExpr, UnaryExpr, YieldExpr, YieldFromExpr, @@ -122,6 +123,10 @@ def visit_cast_expr(self, e: CastExpr) -> None: self.add(e) super().visit_cast_expr(e) + def visit_type_form_expr(self, e: TypeFormExpr) -> None: + self.add(e) + super().visit_type_form_expr(e) + def visit_assert_type_expr(self, e: AssertTypeExpr) -> None: self.add(e) super().visit_assert_type_expr(e) diff --git a/mypy/strconv.py b/mypy/strconv.py index d1595139572a..168a8bcffdc7 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -213,6 +213,12 @@ def visit_nonlocal_decl(self, o: mypy.nodes.NonlocalDecl) -> str: def visit_decorator(self, o: mypy.nodes.Decorator) -> str: return self.dump([o.var, o.decorators, o.func], o) + def visit_type_alias(self, o: mypy.nodes.TypeAlias, /) -> str: + return self.dump([o.name, o.target, o.alias_tvars, o.no_args], o) + + def visit_placeholder_node(self, o: mypy.nodes.PlaceholderNode, /) -> str: + return self.dump([o.fullname], o) + # Statements def visit_block(self, o: mypy.nodes.Block) -> str: @@ -466,6 +472,9 @@ def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr) -> str: def visit_cast_expr(self, o: mypy.nodes.CastExpr) -> str: return self.dump([o.expr, o.type], o) + def visit_type_form_expr(self, o: mypy.nodes.TypeFormExpr) -> str: + return self.dump([o.type], o) + def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr) -> str: return self.dump([o.expr, o.type], o) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 7da258a827f3..c02ff068560b 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1108,38 +1108,50 @@ def visit_partial_type(self, left: PartialType) -> bool: def visit_type_type(self, left: TypeType) -> bool: right = self.right - if isinstance(right, TypeType): - return self._is_subtype(left.item, right.item) - if isinstance(right, Overloaded) and right.is_type_obj(): - # Same as in other direction: if it's a constructor callable, all - # items should belong to the same class' constructor, so it's enough - # to check one of them. - return self._is_subtype(left, right.items[0]) - if isinstance(right, CallableType): - if self.proper_subtype and not right.is_type_obj(): - # We can't accept `Type[X]` as a *proper* subtype of Callable[P, X] - # since this will break transitivity of subtyping. + if left.is_type_form: + if isinstance(right, TypeType): + if not right.is_type_form: + return False + return self._is_subtype(left.item, right.item) + if isinstance(right, Instance): + if right.type.fullname == "builtins.object": + return True return False - # This is unsound, we don't check the __init__ signature. - return self._is_subtype(left.item, right.ret_type) - if isinstance(right, Instance): - if right.type.fullname in ["builtins.object", "builtins.type"]: - # TODO: Strictly speaking, the type builtins.type is considered equivalent to - # Type[Any]. However, this would break the is_proper_subtype check in - # conditional_types for cases like isinstance(x, type) when the type - # of x is Type[int]. It's unclear what's the right way to address this. - return True - item = left.item - if isinstance(item, TypeVarType): - item = get_proper_type(item.upper_bound) - if isinstance(item, Instance): - if right.type.is_protocol and is_protocol_implementation( - item, right, proper_subtype=self.proper_subtype, class_obj=True - ): + return False + else: # not left.is_type_form + if isinstance(right, TypeType): + return self._is_subtype(left.item, right.item) + if isinstance(right, Overloaded) and right.is_type_obj(): + # Same as in other direction: if it's a constructor callable, all + # items should belong to the same class' constructor, so it's enough + # to check one of them. + return self._is_subtype(left, right.items[0]) + if isinstance(right, CallableType): + if self.proper_subtype and not right.is_type_obj(): + # We can't accept `Type[X]` as a *proper* subtype of Callable[P, X] + # since this will break transitivity of subtyping. + return False + # This is unsound, we don't check the __init__ signature. + return self._is_subtype(left.item, right.ret_type) + + if isinstance(right, Instance): + if right.type.fullname in ["builtins.object", "builtins.type"]: + # TODO: Strictly speaking, the type builtins.type is considered equivalent to + # Type[Any]. However, this would break the is_proper_subtype check in + # conditional_types for cases like isinstance(x, type) when the type + # of x is Type[int]. It's unclear what's the right way to address this. return True - metaclass = item.type.metaclass_type - return metaclass is not None and self._is_subtype(metaclass, right) - return False + item = left.item + if isinstance(item, TypeVarType): + item = get_proper_type(item.upper_bound) + if isinstance(item, Instance): + if right.type.is_protocol and is_protocol_implementation( + item, right, proper_subtype=self.proper_subtype, class_obj=True + ): + return True + metaclass = item.type.metaclass_type + return metaclass is not None and self._is_subtype(metaclass, right) + return False def visit_type_alias_type(self, left: TypeAliasType) -> bool: assert False, f"This should be never called, got {left}" diff --git a/mypy/traverser.py b/mypy/traverser.py index 3c249391bdf8..baf234cc1b25 100644 --- a/mypy/traverser.py +++ b/mypy/traverser.py @@ -76,6 +76,7 @@ TypeAliasStmt, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeVarExpr, TypeVarTupleExpr, UnaryExpr, @@ -289,6 +290,9 @@ def visit_slice_expr(self, o: SliceExpr, /) -> None: def visit_cast_expr(self, o: CastExpr, /) -> None: o.expr.accept(self) + def visit_type_form_expr(self, o: TypeFormExpr, /) -> None: + pass + def visit_assert_type_expr(self, o: AssertTypeExpr, /) -> None: o.expr.accept(self) @@ -737,6 +741,11 @@ def visit_cast_expr(self, o: CastExpr, /) -> None: return super().visit_cast_expr(o) + def visit_type_form_expr(self, o: TypeFormExpr, /) -> None: + if not self.visit(o): + return + super().visit_type_form_expr(o) + def visit_assert_type_expr(self, o: AssertTypeExpr, /) -> None: if not self.visit(o): return @@ -935,6 +944,41 @@ def has_return_statement(fdef: FuncBase) -> bool: return seeker.found +class NameAndMemberCollector(TraverserVisitor): + def __init__(self) -> None: + super().__init__() + self.name_exprs: list[NameExpr] = [] + self.member_exprs: list[MemberExpr] = [] + + def visit_name_expr(self, o: NameExpr, /) -> None: + self.name_exprs.append(o) + super().visit_name_expr(o) + + def visit_member_expr(self, o: MemberExpr, /) -> None: + self.member_exprs.append(o) + super().visit_member_expr(o) + + +def all_name_and_member_expressions(node: Expression) -> tuple[list[NameExpr], list[MemberExpr]]: + v = NameAndMemberCollector() + node.accept(v) + return (v.name_exprs, v.member_exprs) + + +class StringSeeker(TraverserVisitor): + def __init__(self) -> None: + self.found = False + + def visit_str_expr(self, o: StrExpr, /) -> None: + self.found = True + + +def has_str_expression(node: Expression) -> bool: + v = StringSeeker() + node.accept(v) + return v.found + + class FuncCollectorBase(TraverserVisitor): def __init__(self) -> None: self.inside_func = False diff --git a/mypy/treetransform.py b/mypy/treetransform.py index 0abf98a52336..f5af5fb777b5 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -83,6 +83,7 @@ TypeAliasExpr, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeVarExpr, TypeVarTupleExpr, UnaryExpr, @@ -540,6 +541,9 @@ def visit_comparison_expr(self, node: ComparisonExpr) -> ComparisonExpr: def visit_cast_expr(self, node: CastExpr) -> CastExpr: return CastExpr(self.expr(node.expr), self.type(node.type)) + def visit_type_form_expr(self, node: TypeFormExpr) -> TypeFormExpr: + return TypeFormExpr(self.type(node.type)) + def visit_assert_type_expr(self, node: AssertTypeExpr) -> AssertTypeExpr: return AssertTypeExpr(self.expr(node.expr), self.type(node.type)) diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 35846e7c3ddd..1b38481ba000 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -328,7 +328,9 @@ def visit_overloaded(self, t: Overloaded, /) -> Type: return Overloaded(items=items) def visit_type_type(self, t: TypeType, /) -> Type: - return TypeType.make_normalized(t.item.accept(self), line=t.line, column=t.column) + return TypeType.make_normalized( + t.item.accept(self), line=t.line, column=t.column, is_type_form=t.is_type_form + ) @abstractmethod def visit_type_alias_type(self, t: TypeAliasType, /) -> Type: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d7a07c9f48e3..06fa847c5434 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -49,7 +49,7 @@ check_arg_kinds, check_arg_names, ) -from mypy.options import INLINE_TYPEDDICT, Options +from mypy.options import INLINE_TYPEDDICT, TYPE_FORM, Options from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface from mypy.semanal_shared import ( SemanticAnalyzerCoreInterface, @@ -665,6 +665,23 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ self.fail(f'{type_str} can\'t contain "{bad_item_name}"', t, code=codes.VALID_TYPE) item = AnyType(TypeOfAny.from_error) return TypeType.make_normalized(item, line=t.line, column=t.column) + elif fullname in ("typing_extensions.TypeForm", "typing.TypeForm"): + if TYPE_FORM not in self.options.enable_incomplete_feature: + self.fail( + "TypeForm is experimental," + " must be enabled with --enable-incomplete-feature=TypeForm", + t, + ) + if len(t.args) == 0: + any_type = self.get_omitted_any(t) + return TypeType(any_type, line=t.line, column=t.column, is_type_form=True) + if len(t.args) != 1: + type_str = "TypeForm[...]" + self.fail( + type_str + " must have exactly one type argument", t, code=codes.VALID_TYPE + ) + item = self.anal_type(t.args[0]) + return TypeType.make_normalized(item, line=t.line, column=t.column, is_type_form=True) elif fullname == "typing.ClassVar": if self.nesting_level > 0: self.fail( @@ -1400,7 +1417,9 @@ def visit_ellipsis_type(self, t: EllipsisType) -> Type: return AnyType(TypeOfAny.from_error) def visit_type_type(self, t: TypeType) -> Type: - return TypeType.make_normalized(self.anal_type(t.item), line=t.line) + return TypeType.make_normalized( + self.anal_type(t.item), line=t.line, is_type_form=t.is_type_form + ) def visit_placeholder_type(self, t: PlaceholderType) -> Type: n = ( diff --git a/mypy/typeops.py b/mypy/typeops.py index 341c96c08931..d2f9f4da44e4 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -501,7 +501,7 @@ def erase_to_bound(t: Type) -> Type: return t.upper_bound if isinstance(t, TypeType): if isinstance(t.item, TypeVarType): - return TypeType.make_normalized(t.item.upper_bound) + return TypeType.make_normalized(t.item.upper_bound, is_type_form=t.is_type_form) return t diff --git a/mypy/types.py b/mypy/types.py index 6013f4c25298..1c4331022496 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3495,11 +3495,14 @@ def serialize(self) -> JsonDict: class TypeType(ProperType): - """For types like Type[User]. + """For types like Type[User] or TypeForm[User | None]. - This annotates variables that are class objects, constrained by + Type[C] annotates variables that are class objects, constrained by the type argument. See PEP 484 for more details. + TypeForm[T] annotates variables that hold the result of evaluating + a type expression. See PEP 747 for more details. + We may encounter expressions whose values are specific classes; those are represented as callables (possibly overloaded) corresponding to the class's constructor's signature and returning @@ -3522,35 +3525,47 @@ class TypeType(ProperType): assumption). """ - __slots__ = ("item",) + __slots__ = ("item", "is_type_form") # This can't be everything, but it can be a class reference, # a generic class instance, a union, Any, a type variable... item: ProperType + # If True then this TypeType represents a TypeForm[T]. + # If False then this TypeType represents a Type[C]. + is_type_form: bool + def __init__( self, item: Bogus[Instance | AnyType | TypeVarType | TupleType | NoneType | CallableType], *, line: int = -1, column: int = -1, + is_type_form: bool = False, ) -> None: """To ensure Type[Union[A, B]] is always represented as Union[Type[A], Type[B]], item of type UnionType must be handled through make_normalized static method. """ super().__init__(line, column) self.item = item + self.is_type_form = is_type_form @staticmethod - def make_normalized(item: Type, *, line: int = -1, column: int = -1) -> ProperType: + def make_normalized( + item: Type, *, line: int = -1, column: int = -1, is_type_form: bool = False + ) -> ProperType: item = get_proper_type(item) - if isinstance(item, UnionType): - return UnionType.make_union( - [TypeType.make_normalized(union_item) for union_item in item.items], - line=line, - column=column, - ) - return TypeType(item, line=line, column=column) # type: ignore[arg-type] + if is_type_form: + # Don't convert TypeForm[X | Y] to (TypeForm[X] | TypeForm[Y]) + pass + else: + if isinstance(item, UnionType): + return UnionType.make_union( + [TypeType.make_normalized(union_item) for union_item in item.items], + line=line, + column=column, + ) + return TypeType(item, line=line, column=column, is_type_form=is_type_form) # type: ignore[arg-type] def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_type_type(self) @@ -3561,15 +3576,21 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, TypeType): return NotImplemented - return self.item == other.item + return self.item == other.item and self.is_type_form == other.is_type_form def serialize(self) -> JsonDict: - return {".class": "TypeType", "item": self.item.serialize()} + return { + ".class": "TypeType", + "item": self.item.serialize(), + "is_type_form": self.is_type_form, + } @classmethod def deserialize(cls, data: JsonDict) -> Type: assert data[".class"] == "TypeType" - return TypeType.make_normalized(deserialize_type(data["item"])) + return TypeType.make_normalized( + deserialize_type(data["item"]), is_type_form=data["is_type_form"] + ) def write(self, data: Buffer) -> None: write_tag(data, TYPE_TYPE) @@ -3945,7 +3966,11 @@ def visit_ellipsis_type(self, t: EllipsisType, /) -> str: return "..." def visit_type_type(self, t: TypeType, /) -> str: - return f"type[{t.item.accept(self)}]" + if t.is_type_form: + type_name = "TypeForm" + else: + type_name = "type" + return f"{type_name}[{t.item.accept(self)}]" def visit_placeholder_type(self, t: PlaceholderType, /) -> str: return f"" diff --git a/mypy/visitor.py b/mypy/visitor.py index d1b2ca416410..e150788ec3c1 100644 --- a/mypy/visitor.py +++ b/mypy/visitor.py @@ -79,6 +79,10 @@ def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr, /) -> T: def visit_cast_expr(self, o: mypy.nodes.CastExpr, /) -> T: pass + @abstractmethod + def visit_type_form_expr(self, o: mypy.nodes.TypeFormExpr, /) -> T: + pass + @abstractmethod def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr, /) -> T: pass @@ -511,6 +515,9 @@ def visit_comparison_expr(self, o: mypy.nodes.ComparisonExpr, /) -> T: def visit_cast_expr(self, o: mypy.nodes.CastExpr, /) -> T: raise NotImplementedError() + def visit_type_form_expr(self, o: mypy.nodes.TypeFormExpr, /) -> T: + raise NotImplementedError() + def visit_assert_type_expr(self, o: mypy.nodes.AssertTypeExpr, /) -> T: raise NotImplementedError() diff --git a/mypyc/irbuild/visitor.py b/mypyc/irbuild/visitor.py index 05a033c3e6ad..dc81e95a2980 100644 --- a/mypyc/irbuild/visitor.py +++ b/mypyc/irbuild/visitor.py @@ -73,6 +73,7 @@ TypeAliasStmt, TypeApplication, TypedDictExpr, + TypeFormExpr, TypeVarExpr, TypeVarTupleExpr, UnaryExpr, @@ -387,6 +388,9 @@ def visit_var(self, o: Var) -> None: def visit_cast_expr(self, o: CastExpr) -> Value: assert False, "CastExpr should have been handled in CallExpr" + def visit_type_form_expr(self, o: TypeFormExpr) -> Value: + assert False, "TypeFormExpr should have been handled in CallExpr" + def visit_assert_type_expr(self, o: AssertTypeExpr) -> Value: assert False, "AssertTypeExpr should have been handled in CallExpr" diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test index 80d314333ddc..e7417d049442 100644 --- a/test-data/unit/check-fastparse.test +++ b/test-data/unit/check-fastparse.test @@ -324,3 +324,18 @@ class Bla: def call() -> str: pass [builtins fixtures/module.pyi] + +[case testInvalidEscapeSequenceWarningsSuppressed] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# Test that SyntaxWarnings for invalid escape sequences are suppressed +# when parsing potential type expressions containing regex patterns or +# similar strings. Callable arguments are always potential type expressions. +from typing import TypeForm + +def identity(typx: TypeForm) -> TypeForm: + return typx + +# This should not generate SyntaxWarning despite invalid escape sequence +identity(r"re\.match") # E: Argument 1 to "identity" has incompatible type "str"; expected "TypeForm[Any]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-typeform.test b/test-data/unit/check-typeform.test new file mode 100644 index 000000000000..627067903a92 --- /dev/null +++ b/test-data/unit/check-typeform.test @@ -0,0 +1,842 @@ +-- TypeForm Type + +[case testRecognizesUnparameterizedTypeFormInAnnotation] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm = str +reveal_type(typx) # N: Revealed type is "TypeForm[Any]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testRecognizesParameterizedTypeFormInAnnotation] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str] = str +reveal_type(typx) # N: Revealed type is "TypeForm[builtins.str]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expression Location: Assignment + +[case testCanAssignTypeExpressionToTypeFormVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str] = str +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignTypeExpressionToUnionTypeFormVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str | None] = str | None +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCannotAssignTypeExpressionToTypeFormVariableWithIncompatibleItemType] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str] = int # E: Incompatible types in assignment (expression has type "TypeForm[int]", variable has type "TypeForm[str]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignValueExpressionToTypeFormVariableIfValueIsATypeForm1] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx1: TypeForm = str +typx2: TypeForm = typx1 # looks like a type expression: name +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignValueExpressionToTypeFormVariableIfValueIsATypeForm2] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +def identity_tf(x: TypeForm) -> TypeForm: + return x +typx1: TypeForm = str +typx2: TypeForm = identity_tf(typx1) # does not look like a type expression +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCannotAssignValueExpressionToTypeFormVariableIfValueIsNotATypeForm] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +val: int = 42 +typx: TypeForm = val # E: Incompatible types in assignment (expression has type "int", variable has type "TypeForm[Any]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignNoneTypeExpressionToTypeFormVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm = None +reveal_type(typx) # N: Revealed type is "TypeForm[Any]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignTypeExpressionToTypeFormVariableDeclaredEarlier] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Type, TypeForm +typ: Type +typ = int | None # E: Incompatible types in assignment (expression has type "object", variable has type "type[Any]") +typx: TypeForm +typx = int | None +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanAssignTypeExpressionWithStringAnnotationToTypeFormVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str | None] = 'str | None' +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expression Location: Function Parameter + +[case testCanPassTypeExpressionToTypeFormParameterInFunction] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +def is_type(typx: TypeForm) -> bool: + return isinstance(typx, type) +is_type(int | None) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCannotPassTypeExpressionToTypeParameter] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +def is_type(typ: type) -> bool: + return isinstance(typ, type) +is_type(int | None) # E: Argument 1 to "is_type" has incompatible type "object"; expected "type" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanPassTypeExpressionToTypeFormParameterInMethod] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +class C: + def is_type(self, typx: TypeForm) -> bool: + return isinstance(typx, type) +C().is_type(int | None) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanPassTypeExpressionToTypeFormParameterInOverload] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import overload, TypeForm +@overload +def is_type(typx: TypeForm) -> bool: ... +@overload +def is_type(typx: type) -> bool: ... +def is_type(typx): + return isinstance(typx, type) +is_type(int | None) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanPassTypeExpressionToTypeFormParameterInDecorator] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Callable, TypeForm, TypeVar +P = TypeVar('P') +R = TypeVar('R') +def expects_type(typx: TypeForm) -> Callable[[Callable[[P], R]], Callable[[P], R]]: + def wrap(func: Callable[[P], R]) -> Callable[[P], R]: + func.expected_type = typx # type: ignore[attr-defined] + return func + return wrap +@expects_type(int | None) +def sum_ints(x: int | None) -> int: + return (x or 0) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanPassTypeExpressionToTypeFormVarargsParameter] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Callable, ParamSpec, TypeForm, TypeVar +P = ParamSpec('P') +R = TypeVar('R') +def expects_types(*typxs: TypeForm) -> Callable[[Callable[P, R]], Callable[P, R]]: + def wrap(func: Callable[P, R]) -> Callable[P, R]: + func.expected_types = typxs # type: ignore[attr-defined] + return func + return wrap +@expects_types(int | None, int) +def sum_ints(x: int | None, y: int) -> tuple[int, int]: + return ((x or 0), y) +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanPassTypeExpressionWithStringAnnotationToTypeFormParameter] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +def is_type(typx: TypeForm) -> bool: + return isinstance(typx, type) +is_type('int | None') +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expression Location: Return Statement + +[case testCanReturnTypeExpressionInFunctionWithTypeFormReturnType] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +def maybe_int_type() -> TypeForm: + return int | None +reveal_type(maybe_int_type()) # N: Revealed type is "TypeForm[Any]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testCanReturnTypeExpressionWithStringAnnotationInFunctionWithTypeFormReturnType] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +def maybe_int_type() -> TypeForm: + return 'int | None' +reveal_type(maybe_int_type()) # N: Revealed type is "TypeForm[Any]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expression Location: Other + +-- In particular ensure that ExpressionChecker.try_parse_as_type_expression() in +-- the TypeChecker pass is able to parse types correctly even though it doesn't +-- have the same rich context as SemanticAnalyzer.try_parse_as_type_expression(). + +[case testTypeExpressionWithoutStringAnnotationRecognizedInOtherSyntacticLocations] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Dict, List, TypeForm +list_of_typx: List[TypeForm] = [int | str] +dict_with_typx_keys: Dict[TypeForm, int] = { + int | str: 1, + str | None: 2, +} +dict_with_typx_keys[int | str] + 1 +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeExpressionWithStringAnnotationNotRecognizedInOtherSyntacticLocations] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Dict, List, TypeForm +list_of_typx: List[TypeForm] = ['int | str'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: List item 0 has incompatible type "str"; expected "TypeForm[Any]" +dict_with_typx_keys: Dict[TypeForm, int] = { + 'int | str': 1, # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: Dict entry 0 has incompatible type "str": "int"; expected "TypeForm[Any]": "int" + 'str | None': 2, # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: Dict entry 1 has incompatible type "str": "int"; expected "TypeForm[Any]": "int" +} +dict_with_typx_keys['int | str'] += 1 # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: Invalid index type "str" for "dict[TypeForm[Any], int]"; expected type "TypeForm[Any]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testValueExpressionWithStringInTypeFormContextEmitsConservativeWarning] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Any, Dict, List, TypeForm +types: Dict[str, TypeForm] = {'any': Any} +# Ensure warning can be ignored if does not apply. +list_of_typx1: List[TypeForm] = [types['any']] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. +list_of_typx2: List[TypeForm] = [types['any']] # type: ignore[maybe-unrecognized-str-typeform] +# Ensure warning can be fixed using the suggested fix in the warning message. +list_of_typx3: List[TypeForm] = ['Any'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: List item 0 has incompatible type "str"; expected "TypeForm[Any]" +list_of_typx4: List[TypeForm] = [TypeForm('Any')] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] + +[case testSelfRecognizedInOtherSyntacticLocations] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import List, Self, TypeForm +class C: + def foo(self) -> None: + list_of_typx1: List[TypeForm] = [Self] + typx1: TypeForm = Self + typx2: TypeForm = 'Self' +list_of_typx2: List[TypeForm] = [Self] # E: List item 0 has incompatible type "int"; expected "TypeForm[Any]" +typx3: TypeForm = Self # E: Incompatible types in assignment (expression has type "int", variable has type "TypeForm[Any]") +typx4: TypeForm = 'Self' # E: Incompatible types in assignment (expression has type "str", variable has type "TypeForm[Any]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testNameOrDottedNameRecognizedInOtherSyntacticLocations] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +import typing +from typing import List, TypeForm +list_of_typx: List[TypeForm] = [List | typing.Optional[str]] +typx: TypeForm = List | typing.Optional[str] +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testInvalidNameOrDottedNameRecognizedInOtherSyntacticLocations] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import List, TypeForm +list_of_typx1: List[TypeForm] = [NoSuchType] # E: Name "NoSuchType" is not defined +list_of_typx2: List[TypeForm] = [no_such_module.NoSuchType] # E: Name "no_such_module" is not defined +typx1: TypeForm = NoSuchType # E: Name "NoSuchType" is not defined +typx2: TypeForm = no_such_module.NoSuchType # E: Name "no_such_module" is not defined +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expression Context: Union[TypeForm, ] + +[case testAcceptsTypeFormLiteralAssignedToUnionOfTypeFormAndNonStr] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx_or_int1: TypeForm[int | None] | int = int | None # No error; interpret as TypeForm +typx_or_int2: TypeForm[int | None] | int = str | None # E: Incompatible types in assignment (expression has type "object", variable has type "Union[TypeForm[Optional[int]], int]") +typx_or_int3: TypeForm[int | None] | int = 1 +typx_or_int4: TypeForm[int | None] | int = object() # E: Incompatible types in assignment (expression has type "object", variable has type "Union[TypeForm[Optional[int]], int]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testAcceptsTypeFormLiteralAssignedToUnionOfTypeFormAndStr] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx_or_str1: TypeForm[int | None] | str = 'int | None' +typx_or_str2: TypeForm[int | None] | str = 'str | None' # No error; interpret as str +typx_or_str3: TypeForm[int | None] | str = 'hello' +typx_or_str4: TypeForm[int | None] | str = object() # E: Incompatible types in assignment (expression has type "object", variable has type "Union[TypeForm[Optional[int]], str]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testValueExpressionWithStringInTypeFormUnionContextEmitsConservativeWarning1] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import List, TypeForm +list_of_typx1: List[TypeForm[int | None] | str] = ['int | None'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. +list_of_typx2: List[TypeForm[int | None] | str] = ['str | None'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testValueExpressionWithStringInTypeFormUnionContextEmitsConservativeWarning2] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import List, TypeForm +list_of_typx3: List[TypeForm[int | None] | int] = ['int | None'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: List item 0 has incompatible type "str"; expected "Union[TypeForm[Optional[int]], int]" +list_of_typx4: List[TypeForm[str | None] | int] = ['str | None'] # E: TypeForm containing a string annotation cannot be recognized here. Surround with TypeForm(...) to recognize. \ + # E: List item 0 has incompatible type "str"; expected "Union[TypeForm[Optional[str]], int]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Assignability (is_subtype) + +[case testTypeFormToTypeFormAssignability] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] is assignable to TypeForm[T2] iff T1 is assignable to T2. +# - In particular TypeForm[Any] is assignable to TypeForm[Any]. +from typing_extensions import TypeForm +INT_OR_STR_TF: TypeForm[int | str] = int | str +INT_TF: TypeForm[int] = int +STR_TF: TypeForm[str] = str +OBJECT_TF: TypeForm[object] = object +ANY_TF: TypeForm = object +reveal_type(ANY_TF) # N: Revealed type is "TypeForm[Any]" +typx1: TypeForm[int | str] = INT_OR_STR_TF +typx2: TypeForm[int | str] = INT_TF +typx3: TypeForm[int | str] = STR_TF +typx4: TypeForm[int | str] = OBJECT_TF # E: Incompatible types in assignment (expression has type "TypeForm[object]", variable has type "TypeForm[Union[int, str]]") +typx5: TypeForm[int | str] = ANY_TF # no error +typx6: TypeForm[int] = INT_OR_STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[Union[int, str]]", variable has type "TypeForm[int]") +typx7: TypeForm[int] = INT_TF +typx8: TypeForm[int] = STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[str]", variable has type "TypeForm[int]") +typx9: TypeForm[int] = OBJECT_TF # E: Incompatible types in assignment (expression has type "TypeForm[object]", variable has type "TypeForm[int]") +typx10: TypeForm[int] = ANY_TF # no error +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeToTypeFormAssignability] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - Type[C] is assignable to TypeForm[T] iff C is assignable to T. +# - In particular Type[Any] is assignable to TypeForm[Any]. +from typing import Type, TypeForm +INT_T: Type[int] = int +STR_T: Type[str] = str +OBJECT_T: Type[object] = object +ANY_T: Type = object +reveal_type(ANY_T) # N: Revealed type is "type[Any]" +typx1: TypeForm[int | str] = INT_T +typx2: TypeForm[int | str] = STR_T +typx3: TypeForm[int | str] = OBJECT_T # E: Incompatible types in assignment (expression has type "type[object]", variable has type "TypeForm[Union[int, str]]") +typx4: TypeForm[int | str] = ANY_T # no error +typx5: TypeForm[int] = INT_T +typx6: TypeForm[int] = STR_T # E: Incompatible types in assignment (expression has type "type[str]", variable has type "TypeForm[int]") +typx7: TypeForm[int] = OBJECT_T # E: Incompatible types in assignment (expression has type "type[object]", variable has type "TypeForm[int]") +typx8: TypeForm[int] = ANY_T # no error +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeFormToTypeAssignability] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T] is NOT assignable to Type[C]. +# - In particular TypeForm[Any] is NOT assignable to Type[Any]. +from typing import Type, TypeForm +INT_OR_STR_TF: TypeForm[int | str] = int | str +INT_TF: TypeForm[int] = int +STR_TF: TypeForm[str] = str +OBJECT_TF: TypeForm[object] = object +ANY_TF: TypeForm = object +reveal_type(ANY_TF) # N: Revealed type is "TypeForm[Any]" +typ1: Type[int] = INT_OR_STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[Union[int, str]]", variable has type "type[int]") +typ2: Type[int] = INT_TF # E: Incompatible types in assignment (expression has type "TypeForm[int]", variable has type "type[int]") +typ3: Type[int] = STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[str]", variable has type "type[int]") +typ4: Type[int] = OBJECT_TF # E: Incompatible types in assignment (expression has type "TypeForm[object]", variable has type "type[int]") +typ5: Type[int] = ANY_TF # E: Incompatible types in assignment (expression has type "TypeForm[Any]", variable has type "type[int]") +typ6: Type[object] = INT_OR_STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[Union[int, str]]", variable has type "type[object]") +typ7: Type[object] = INT_TF # E: Incompatible types in assignment (expression has type "TypeForm[int]", variable has type "type[object]") +typ8: Type[object] = STR_TF # E: Incompatible types in assignment (expression has type "TypeForm[str]", variable has type "type[object]") +typ9: Type[object] = OBJECT_TF # E: Incompatible types in assignment (expression has type "TypeForm[object]", variable has type "type[object]") +typ10: Type[object] = ANY_TF # E: Incompatible types in assignment (expression has type "TypeForm[Any]", variable has type "type[object]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +# NOTE: This test doesn't involve TypeForm at all, but is still illustrative +# when compared with similarly structured TypeForm-related tests above. +[case testTypeToTypeAssignability] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - Type[C1] is assignable to Type[C2] iff C1 is assignable to C2. +# - In particular Type[Any] is assignable to Type[Any]. +from typing import Type +INT_T: Type[int] = int +STR_T: Type[str] = str +OBJECT_T: Type[object] = object +ANY_T: Type = object +reveal_type(ANY_T) # N: Revealed type is "type[Any]" +typ1: Type[int] = INT_T +typ2: Type[int] = STR_T # E: Incompatible types in assignment (expression has type "type[str]", variable has type "type[int]") +typ3: Type[int] = OBJECT_T # E: Incompatible types in assignment (expression has type "type[object]", variable has type "type[int]") +typ4: Type[int] = ANY_T # no error +typ5: Type[object] = INT_T +typ6: Type[object] = STR_T +typ7: Type[object] = OBJECT_T +typ8: Type[object] = ANY_T # no error +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeFormToObjectAssignability] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T] is assignable to object and Any. +from typing import Any, TypeForm +INT_TF: TypeForm[int] = int +OBJECT_TF: TypeForm[object] = object +ANY_TF: TypeForm = object +reveal_type(ANY_TF) # N: Revealed type is "TypeForm[Any]" +obj1: object = INT_TF +obj2: object = OBJECT_TF +obj3: object = ANY_TF +any1: Any = INT_TF +any2: Any = OBJECT_TF +any3: Any = ANY_TF +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Join (join_types) + +[case testTypeFormToTypeFormJoin] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] join TypeForm[T2] == TypeForm[T1 join T2] +from typing_extensions import TypeForm +class AB: + pass +class A(AB): + pass +class B(AB): + pass +A_TF: TypeForm[A] = A +B_TF: TypeForm[B] = B +reveal_type([A_TF, B_TF][0]) # N: Revealed type is "TypeForm[__main__.AB]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeToTypeFormJoin] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] join Type[T2] == TypeForm[T1 join T2] +from typing import Type, TypeForm +class AB: + pass +class A(AB): + pass +class B(AB): + pass +A_T: Type[A] = A +B_TF: TypeForm[B] = B +reveal_type([A_T, B_TF][0]) # N: Revealed type is "TypeForm[__main__.AB]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeFormToTypeJoin] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] join Type[T2] == TypeForm[T1 join T2] +from typing import Type, TypeForm +class AB: + pass +class A(AB): + pass +class B(AB): + pass +A_TF: TypeForm[A] = A +B_T: Type[B] = B +reveal_type([A_TF, B_T][0]) # N: Revealed type is "TypeForm[__main__.AB]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +# NOTE: This test doesn't involve TypeForm at all, but is still illustrative +# when compared with similarly structured TypeForm-related tests above. +[case testTypeToTypeJoin] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - Type[T1] join Type[T2] == Type[T1 join T2] +from typing import Type, TypeForm +class AB: + pass +class A(AB): + pass +class B(AB): + pass +A_T: Type[A] = A +B_T: Type[B] = B +reveal_type([A_T, B_T][0]) # N: Revealed type is "type[__main__.AB]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Meet (meet_types) + +[case testTypeFormToTypeFormMeet] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] meet TypeForm[T2] == TypeForm[T1 meet T2] +from typing import Callable, TypeForm, TypeVar +class AB: + pass +class A(AB): + pass +class B(AB): + pass +class C(AB): + pass +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass # type: ignore[empty-body] +def g(x: TypeForm[A | B], y: TypeForm[B | C]) -> None: pass +reveal_type(f(g)) # N: Revealed type is "TypeForm[__main__.B]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeToTypeFormMeet] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] meet Type[T2] == Type[T1 meet T2] +from typing import Callable, Type, TypeForm, TypeVar +class AB: + pass +class A(AB): + pass +class B(AB): + pass +class C(AB): + pass +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass # type: ignore[empty-body] +def g(x: Type[B], y: TypeForm[B | C]) -> None: pass +reveal_type(f(g)) # N: Revealed type is "type[__main__.B]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeFormToTypeMeet] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - TypeForm[T1] meet Type[T2] == Type[T1 meet T2] +from typing import Callable, Type, TypeForm, TypeVar +class AB: + pass +class A(AB): + pass +class B(AB): + pass +class C(AB): + pass +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass # type: ignore[empty-body] +def g(x: TypeForm[A | B], y: Type[B]) -> None: pass +reveal_type(f(g)) # N: Revealed type is "type[__main__.B]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +# NOTE: This test doesn't involve TypeForm at all, but is still illustrative +# when compared with similarly structured TypeForm-related tests above. +[case testTypeToTypeMeet] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# - Type[T1] meet Type[T2] == Type[T1 meet T2] +from typing import Callable, Type, TypedDict, TypeForm, TypeVar +class AB(TypedDict): + a: str + b: str +class BC(TypedDict): + b: str + c: str +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass # type: ignore[empty-body] +def g(x: Type[AB], y: Type[BC]) -> None: pass +reveal_type(f(g)) # N: Revealed type is "type[TypedDict({'b': builtins.str, 'c': builtins.str, 'a': builtins.str})]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- TypeForm(...) Expression + +[case testTypeFormExpression] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +tf1 = TypeForm(int | str) +reveal_type(tf1) # N: Revealed type is "TypeForm[Union[builtins.int, builtins.str]]" +tf2 = TypeForm('int | str') +reveal_type(tf2) # N: Revealed type is "TypeForm[Union[builtins.int, builtins.str]]" +tf3: TypeForm = TypeForm(int | str) +reveal_type(tf3) # N: Revealed type is "TypeForm[Any]" +tf4: TypeForm = TypeForm(1) # E: Invalid type: try using Literal[1] instead? +tf5: TypeForm = TypeForm(int) | TypeForm(str) # E: Incompatible types in assignment (expression has type "object", variable has type "TypeForm[Any]") +tf6: TypeForm = TypeForm(TypeForm(int) | TypeForm(str)) # E: TypeForm argument is not a type +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- isinstance + +[case testTypeFormAndTypeIsinstance] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[str] = str +if isinstance(typx, type): + reveal_type(typx) # N: Revealed type is "type[builtins.str]" +else: + reveal_type(typx) # N: Revealed type is "TypeForm[builtins.str]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Variables + +[case testLinkTypeFormToTypeFormWithTypeVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm, TypeVar +T = TypeVar('T') +def as_typeform(typx: TypeForm[T]) -> TypeForm[T]: + return typx +reveal_type(as_typeform(int | str)) # N: Revealed type is "TypeForm[Union[builtins.int, builtins.str]]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testLinkTypeFormToTypeWithTypeVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Type, TypeForm, TypeVar +T = TypeVar('T') +def as_type(typx: TypeForm[T]) -> Type[T] | None: + if isinstance(typx, type): + return typx + else: + return None +reveal_type(as_type(int | str)) # N: Revealed type is "Union[type[builtins.int], type[builtins.str], None]" +reveal_type(as_type(int)) # N: Revealed type is "Union[type[builtins.int], None]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testLinkTypeFormToInstanceWithTypeVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm, TypeVar +T = TypeVar('T') +def as_instance(typx: TypeForm[T]) -> T | None: + if isinstance(typx, type): + return typx() + else: + return None +reveal_type(as_instance(int | str)) # N: Revealed type is "Union[builtins.int, builtins.str, None]" +reveal_type(as_instance(int)) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testLinkTypeFormToTypeIsWithTypeVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm, TypeVar +from typing_extensions import TypeIs +T = TypeVar('T') +def isassignable(value: object, typx: TypeForm[T]) -> TypeIs[T]: + raise BaseException() +count: int | str = 1 +if isassignable(count, int): + reveal_type(count) # N: Revealed type is "builtins.int" +else: + reveal_type(count) # N: Revealed type is "builtins.str" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testLinkTypeFormToTypeGuardWithTypeVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm, TypeVar +from typing_extensions import TypeGuard +T = TypeVar('T') +def isassignable(value: object, typx: TypeForm[T]) -> TypeGuard[T]: + raise BaseException() +count: int | str = 1 +if isassignable(count, int): + reveal_type(count) # N: Revealed type is "builtins.int" +else: + reveal_type(count) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Type Expressions Assignable To TypeForm Variable + +[case testEveryKindOfTypeExpressionIsAssignableToATypeFormVariable] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +# NOTE: Importing Callable from collections.abc also works OK +from typing import ( + Any, Callable, Dict, List, Literal, LiteralString, NoReturn, + Optional, ParamSpec, Self, Type, TypeGuard, TypeVar, Union, +) +from typing_extensions import ( + Annotated, Concatenate, Never, TypeAlias, TypeForm, TypeIs, + TypeVarTuple, Unpack, +) +# +class SomeClass: + pass +SomeTypeAlias: TypeAlias = SomeClass +SomeTypeVar = TypeVar('SomeTypeVar') +Ts = TypeVarTuple('Ts') +IntTuple: TypeAlias = tuple[int, Unpack[Ts]] +P = ParamSpec('P') +R = TypeVar('R') +# +typx: TypeForm +# Begin rules taken from: https://typing.python.org/en/latest/spec/annotations.html#grammar-token-expression-grammar-type_expression +# +typx = Any +class SelfBinder: + def bind_self(self) -> Self: + typx: TypeForm + # (valid only in some contexts) + typx = Self + return self +# +typx = LiteralString +# +typx = NoReturn +# +typx = Never +# +typx = None +# name (where name must refer to a valid in-scope class) +typx = SomeClass +# name (where name must refer to a valid in-scope type alias) +typx = SomeTypeAlias +# name (where name must refer to a valid in-scope TypeVar) +# NOTE: Unbound TypeVar isn't currently accepted as a TypeForm. Is that OK? +typx = SomeTypeVar # E: Incompatible types in assignment (expression has type "TypeVar", variable has type "TypeForm[Any]") +# name '[' type_expression (',' type_expression)* ']' +typx = Dict[str, int] +# (TODO: Add: name '[' unpacked ']') +# (TODO: Add: name '[' type_expression_list (',' type_expression_list)* ']') +# name '[' '(' ')' ']' (denoting specialization with an empty TypeVarTuple) +typx = IntTuple[()] +# '[' expression (',' expression) ']' +typx = Literal[1] +# type_expression '|' type_expression +typx = int | str +# '[' type_expression ']' +typx = Optional[str] +# '[' type_expression (',' type_expression)* ']' +typx = Union[int, str] +# '[' ']' +typx = type[Any] +# '[' name ']' (where name must refer to a valid in-scope class) +typx = type[int] +# (TODO: Add: '[' name ']' (where name must refer to a valid in-scope TypeVar)) +# '[' '...' ',' type_expression ']' +typx = Callable[..., str] +def bind_R(input: R) -> R: + typx: TypeForm + # '[' name ',' type_expression ']' (where name must be a valid in-scope ParamSpec) + typx = Callable[P, R] + # '[' '[' (type_expression ',')+ + # (name | '...') ']' ',' type_expression ']' + # (where name must be a valid in-scope ParamSpec) + typx = Callable[Concatenate[int, P], R] + return input +# '[' '[' maybe_unpacked (',' maybe_unpacked)* ']' ',' type_expression ']' +typx = Callable[[int, str], None] +# '[' '(' ')' ']' (representing an empty tuple) +typx = tuple[()] +# '[' type_expression ',' '...' ']' (representing an arbitrary-length tuple) +typx = tuple[int, ...] +# '[' maybe_unpacked (',' maybe_unpacked)* ']' +typx = tuple[int, str] +# '[' type_expression ',' expression (',' expression)* ']' +typx = Annotated[str, 'uppercase'] +# '[' type_expression ']' (valid only in some contexts) +typx = TypeGuard[List[str]] +# '[' type_expression ']' (valid only in some contexts) +typx = TypeIs[List[str]] +# string_annotation (must evaluate to a valid type_expression) +typx = 'int | str' +# End rules +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + + +-- Misc + +[case testTypeFormHasAllObjectAttributesAndMethods] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +typx: TypeForm[int | str] = int | str +print(typx.__class__) # OK +print(typx.__hash__()) # OK +obj: object = typx +[file builtins.py] +class object: + def __init__(self) -> None: pass + __class__: None + def __hash__(self) -> int: pass +def print(x): + raise BaseException() +class int: pass +class dict: pass +class str: pass +class type: pass +class tuple: pass +class ellipsis: pass +class BaseException: pass +class float: pass +[typing fixtures/typing-full.pyi] + +[case testDottedTypeFormsAreRecognized] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing_extensions import TypeForm +import typing +class C1: + class C2: + pass +typx1: TypeForm[C1.C2] = C1.C2 # OK +typx2: TypeForm[typing.Any] = typing.Any # OK +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +-- mypy already refused to recognize TypeVars in value expressions before +-- the TypeForm feature was introduced. +[case testTypeVarTypeFormsAreOnlyRecognizedInStringAnnotation] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Generic, List, TypeForm, TypeVar +E = TypeVar('E') +class Box(Generic[E]): + def foo(self, e: E) -> None: + list_of_typx: List[TypeForm] = [E] # E: "E" is a type variable and only valid in type context + typx1: TypeForm = E # E: "E" is a type variable and only valid in type context + typx2: TypeForm = 'E' +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] + +[case testIncompleteTypeFormsAreNotRecognized] +# flags: --python-version 3.14 --enable-incomplete-feature=TypeForm +from typing import Optional, TypeForm +typx: TypeForm = Optional # E: Incompatible types in assignment (expression has type "int", variable has type "TypeForm[Any]") +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/fixtures/primitives.pyi b/test-data/unit/fixtures/primitives.pyi index 2f8623c79b9f..98e604e9e81e 100644 --- a/test-data/unit/fixtures/primitives.pyi +++ b/test-data/unit/fixtures/primitives.pyi @@ -13,6 +13,8 @@ class object: class type: def __init__(self, x: object) -> None: pass + # Real implementation returns UnionType + def __or__(self, value: object, /) -> object: pass class int: # Note: this is a simplification of the actual signature @@ -72,3 +74,5 @@ class range(Sequence[int]): def __contains__(self, other: object) -> bool: pass def isinstance(x: object, t: Union[type, Tuple]) -> bool: pass + +class BaseException: pass diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index 1a63deaa727d..87b66c0cd857 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -30,6 +30,7 @@ Protocol = 0 Tuple = 0 _promote = 0 Type = 0 +TypeForm = 0 no_type_check = 0 ClassVar = 0 Final = 0 diff --git a/test-data/unit/lib-stub/typing_extensions.pyi b/test-data/unit/lib-stub/typing_extensions.pyi index 6158a0c9ebbc..71a17a939d41 100644 --- a/test-data/unit/lib-stub/typing_extensions.pyi +++ b/test-data/unit/lib-stub/typing_extensions.pyi @@ -33,6 +33,8 @@ Concatenate: _SpecialForm TypeAlias: _SpecialForm +TypeForm: _SpecialForm + TypeGuard: _SpecialForm TypeIs: _SpecialForm Never: _SpecialForm From 40821bfaac5e42a5f569758b2fc4ea25e563162a Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 3 Nov 2025 13:46:02 +0100 Subject: [PATCH 143/183] Use the fallback for `ModuleSpec` early if it can never be resolved (#20167) Fixes #18237. --- mypy/semanal.py | 10 +++++++++- test-data/unit/cmdline.test | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 1fdea22e8962..a21995fdafa3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -750,7 +750,15 @@ def add_implicit_module_attrs(self, file_node: MypyFile) -> None: else: inst = self.named_type_or_none("importlib.machinery.ModuleSpec") if inst is None: - if self.final_iteration: + if ( + self.final_iteration + or self.options.clone_for_module("importlib.machinery").follow_imports + == "skip" + ): + # If we are not allowed to resolve imports from `importlib.machinery`, + # ModuleSpec will not be available at any iteration. + # Use the fallback earlier. + # (see https://github.com/python/mypy/issues/18237) inst = self.named_type_or_none("builtins.object") assert inst is not None, "Cannot find builtins.object" else: diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 35d7b700b161..bed4134bff72 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1393,3 +1393,21 @@ Ts = TypeVarTuple("Ts") Warning: TypeVarTuple is already enabled by default Warning: Unpack is already enabled by default == Return code: 0 + +[case testImportlibImportCannotBeResolved] +# cmd: mypy a.py +# This test is here because it needs use_builtins_fixtures off. + +[file a.py] +from typing import NamedTuple + +class CodecKey(NamedTuple): + def foo(self) -> "CodecKey": + ... + +[file mypy.ini] +\[mypy] + +\[mypy-importlib.*] +follow_imports = skip +follow_imports_for_stubs = True From 62770f484007a8f7a3ea4dbab0959772b3cf2267 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 3 Nov 2025 04:54:17 -0800 Subject: [PATCH 144/183] Fix type checking of dict type aliases (#20170) Resolves a bad false negative and a false positive Previously, for `D = dict[str, str]` we would require a type annotation for `d = D()` and we would fail to error on `D(x=1)` Fixes #11093, fixes #11246, fixes #13320, fixes #16047 --- mypy/semanal.py | 6 +++++- test-data/unit/check-type-aliases.test | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a21995fdafa3..e55819b898e9 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5888,7 +5888,11 @@ def visit_call_expr(self, expr: CallExpr) -> None: expr.analyzed = PromoteExpr(target) expr.analyzed.line = expr.line expr.analyzed.accept(self) - elif refers_to_fullname(expr.callee, "builtins.dict"): + elif refers_to_fullname(expr.callee, "builtins.dict") and not ( + isinstance(expr.callee, RefExpr) + and isinstance(expr.callee.node, TypeAlias) + and not expr.callee.node.no_args + ): expr.analyzed = self.translate_dict_call(expr) elif refers_to_fullname(expr.callee, "builtins.divmod"): if not self.check_fixed_args(expr, 2, "divmod"): diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 5bbb503a578a..06e223716189 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -1318,3 +1318,17 @@ from typing_extensions import TypeAlias Foo: TypeAlias = ClassVar[int] # E: ClassVar[...] can't be used inside a type alias [builtins fixtures/tuple.pyi] + + +[case testTypeAliasDict] +D = dict[str, int] +d = D() +reveal_type(d) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +reveal_type(D()) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +reveal_type(D(x=1)) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" +reveal_type(D(x="asdf")) # E: No overload variant of "dict" matches argument type "str" \ + # N: Possible overload variants: \ + # N: def __init__(self, **kwargs: int) -> dict[str, int] \ + # N: def __init__(self, arg: Iterable[tuple[str, int]], **kwargs: int) -> dict[str, int] \ + # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] From 904fc779118a7fe9538286de704dc563d20b69b6 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 3 Nov 2025 05:58:26 -0700 Subject: [PATCH 145/183] [docs] Change the InlineTypedDict example (#20172) This way it will be clear that there is no necessary connection between the name of the field and its type. --- docs/source/command_line.rst | 4 ++-- docs/source/typed_dict.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 79dd68a84b28..d6b70c4976dc 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1211,8 +1211,8 @@ List of currently incomplete/experimental features: .. code-block:: python - def test_values() -> {"int": int, "str": str}: - return {"int": 42, "str": "test"} + def test_values() -> {"width": int, "description": str}: + return {"width": 42, "description": "test"} Miscellaneous diff --git a/docs/source/typed_dict.rst b/docs/source/typed_dict.rst index bbb10a12abe8..d42b434b05b2 100644 --- a/docs/source/typed_dict.rst +++ b/docs/source/typed_dict.rst @@ -303,8 +303,8 @@ to use inline TypedDict syntax. For example: .. code-block:: python - def test_values() -> {"int": int, "str": str}: - return {"int": 42, "str": "test"} + def test_values() -> {"width": int, "description": str}: + return {"width": 42, "description": "test"} class Response(TypedDict): status: int From 5bdfe7ef86ea45dc7da0d618104ae93537bc7c6c Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 3 Nov 2025 18:30:11 +0100 Subject: [PATCH 146/183] Respect force-union-syntax flag in error hint (#20165) The error hint for `Need type annotations for ...` should defer to `options.use_union_syntax` when evaluating if `Union` or `|` should be used. This will also make it easier to eventually upgrade all tests to use the PEP 604 syntax. --- mypy/checker.py | 14 +++++--------- mypy/messages.py | 7 +++---- mypy/suggestions.py | 3 ++- mypy/test/testcheck.py | 2 +- test-data/unit/check-inference.test | 2 +- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f4746bc0c886..fedb4e745909 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4449,7 +4449,7 @@ def infer_variable_type( # partial type which will be made more specific later. A partial type # gets generated in assignment like 'x = []' where item type is not known. if name.name != "_" and not self.infer_partial_type(name, lvalue, init_type): - self.msg.need_annotation_for_var(name, context, self.options.python_version) + self.msg.need_annotation_for_var(name, context, self.options) self.set_inference_error_fallback_type(name, lvalue, init_type) elif ( isinstance(lvalue, MemberExpr) @@ -4459,7 +4459,7 @@ def infer_variable_type( and not is_same_type(self.inferred_attribute_types[lvalue.def_var], init_type) ): # Multiple, inconsistent types inferred for an attribute. - self.msg.need_annotation_for_var(name, context, self.options.python_version) + self.msg.need_annotation_for_var(name, context, self.options) name.type = AnyType(TypeOfAny.from_error) else: # Infer type of the target. @@ -4656,9 +4656,7 @@ def check_simple_assignment( rvalue, type_context=lvalue_type, always_allow_any=always_allow_any ) if not is_valid_inferred_type(rvalue_type, self.options) and inferred is not None: - self.msg.need_annotation_for_var( - inferred, context, self.options.python_version - ) + self.msg.need_annotation_for_var(inferred, context, self.options) rvalue_type = rvalue_type.accept(SetNothingToAny()) if ( @@ -7680,7 +7678,7 @@ def enter_partial_types( var.type = NoneType() else: if var not in self.partial_reported and not permissive: - self.msg.need_annotation_for_var(var, context, self.options.python_version) + self.msg.need_annotation_for_var(var, context, self.options) self.partial_reported.add(var) if var.type: fixed = fixup_partial_type(var.type) @@ -7707,9 +7705,7 @@ def handle_partial_var_type( if in_scope: context = partial_types[node] if is_local or not self.options.allow_untyped_globals: - self.msg.need_annotation_for_var( - node, context, self.options.python_version - ) + self.msg.need_annotation_for_var(node, context, self.options) self.partial_reported.add(node) else: # Defer the node -- we might get a better type in the outer scope diff --git a/mypy/messages.py b/mypy/messages.py index 68b9b83572f1..9fdfb748b288 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1803,18 +1803,17 @@ def unimported_type_becomes_any(self, prefix: str, typ: Type, ctx: Context) -> N ) def need_annotation_for_var( - self, node: SymbolNode, context: Context, python_version: tuple[int, int] | None = None + self, node: SymbolNode, context: Context, options: Options | None = None ) -> None: hint = "" - pep604_supported = not python_version or python_version >= (3, 10) # type to recommend the user adds recommended_type = None # Only gives hint if it's a variable declaration and the partial type is a builtin type - if python_version and isinstance(node, Var) and isinstance(node.type, PartialType): + if options and isinstance(node, Var) and isinstance(node.type, PartialType): type_dec = "" if not node.type.type: # partial None - if pep604_supported: + if options.use_or_syntax(): recommended_type = f"{type_dec} | None" else: recommended_type = f"Optional[{type_dec}]" diff --git a/mypy/suggestions.py b/mypy/suggestions.py index 45aa5ade47a4..756cf6ae2ccd 100644 --- a/mypy/suggestions.py +++ b/mypy/suggestions.py @@ -890,7 +890,8 @@ def visit_typeddict_type(self, t: TypedDictType) -> str: def visit_union_type(self, t: UnionType) -> str: if len(t.items) == 2 and is_overlapping_none(t): - return f"Optional[{remove_optional(t).accept(self)}]" + s = remove_optional(t).accept(self) + return f"{s} | None" if self.options.use_or_syntax() else f"Optional[{s}]" else: return super().visit_union_type(t) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index f59cce701ea6..f2b7057d9f20 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -140,7 +140,7 @@ def run_case_once( options.hide_error_codes = False if "abstract" not in testcase.file: options.allow_empty_bodies = not testcase.name.endswith("_no_empty") - if "union-error" not in testcase.file: + if "union-error" not in testcase.file and "Pep604" not in testcase.name: options.force_union_syntax = True if incremental_step and options.incremental: diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 17f79dbcb663..bc4b56e49622 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -3548,7 +3548,7 @@ if x: [builtins fixtures/dict.pyi] [case testSuggestPep604AnnotationForPartialNone] -# flags: --local-partial-types --python-version 3.10 +# flags: --local-partial-types --python-version 3.10 --no-force-union-syntax x = None # E: Need type annotation for "x" (hint: "x: | None = ...") [case testTupleContextFromIterable] From 08a0b0742d75120fe948da7dd3c475135cf9ea2e Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Mon, 3 Nov 2025 22:45:02 +0100 Subject: [PATCH 147/183] Do not cache fast container types inside lambdas (#20166) Fixes #20163. `fast_container_type` uses another layer of expression cache, it also has to be bypassed from within lambdas. --- mypy/checkexpr.py | 10 ++++++++-- test-data/unit/check-inference-context.test | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index a06af690f8db..03ebc5058cee 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -5099,7 +5099,10 @@ def fast_container_type( self.resolved_type[e] = NoneType() return None ct = self.chk.named_generic_type(container_fullname, [vt]) - self.resolved_type[e] = ct + if not self.in_lambda_expr: + # We cannot cache results in lambdas - their bodies can be accepted in + # error-suppressing watchers too early + self.resolved_type[e] = ct return ct def _first_or_join_fast_item(self, items: list[Type]) -> Type | None: @@ -5334,7 +5337,10 @@ def fast_dict_type(self, e: DictExpr) -> Type | None: self.resolved_type[e] = NoneType() return None dt = self.chk.named_generic_type("builtins.dict", [kt, vt]) - self.resolved_type[e] = dt + if not self.in_lambda_expr: + # We cannot cache results in lambdas - their bodies can be accepted in + # error-suppressing watchers too early + self.resolved_type[e] = dt return dt def check_typeddict_literal_in_context( diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index a41ee5f59670..b5b5d778d90f 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -709,6 +709,20 @@ f( A(), r=B()) [builtins fixtures/isinstance.pyi] +[case testLambdaWithFastContainerType] +from collections.abc import Callable +from typing import Never, TypeVar + +T = TypeVar("T") + +def f(a: Callable[[], T]) -> None: ... + +def foo(x: str) -> Never: ... + +f(lambda: [foo(0)]) # E: Argument 1 to "foo" has incompatible type "int"; expected "str" +f(lambda: {"x": foo(0)}) # E: Argument 1 to "foo" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] + -- Overloads + generic functions -- ----------------------------- From 902acd153b0fa04e98ac8bf074c74f0514f9041b Mon Sep 17 00:00:00 2001 From: Frank Dana Date: Mon, 3 Nov 2025 16:47:16 -0500 Subject: [PATCH 148/183] Don't let help formatter line-wrap URLs (#19825) Fixes #19816 This PR adjusts the `AugmentedHelpFormatter` for the command-line `--help` output so that it will no longer wrap/break long URLs in the output. It also (in a separate commit) adds `https://` in front of two URLs that lacked it, so they can be recognized as links by anything parsing the help text. --- mypy/main.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index a44632ed96f9..7d5721851c3d 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -256,10 +256,19 @@ def _fill_text(self, text: str, width: int, indent: str) -> str: if "\n" in text: # Assume we want to manually format the text return super()._fill_text(text, width, indent) - else: - # Assume we want argparse to manage wrapping, indenting, and - # formatting the text for us. - return argparse.HelpFormatter._fill_text(self, text, width, indent) + # Format the text like argparse, but overflow rather than + # breaking long words (like URLs) + text = self._whitespace_matcher.sub(" ", text).strip() + import textwrap + + return textwrap.fill( + text, + width, + initial_indent=indent, + subsequent_indent=indent, + break_on_hyphens=False, + break_long_words=False, + ) # Define pairs of flag prefixes with inverse meaning. @@ -544,10 +553,15 @@ def add_invertible_flag( # Feel free to add subsequent sentences that add additional details. # 3. If you cannot think of a meaningful description for a new group, omit it entirely. # (E.g. see the "miscellaneous" sections). - # 4. The group description should end with a period (unless the last line is a link). If you - # do end the group description with a link, omit the 'http://' prefix. (Some links are too - # long and will break up into multiple lines if we include that prefix, so for consistency - # we omit the prefix on all links.) + # 4. The text of the group description should end with a period, optionally followed + # by a documentation reference (URL). + # 5. If you want to include a documentation reference, place it at the end of the + # description. Feel free to open with a brief reference ("See also:", "For more + # information:", etc.), followed by a space, then the entire URL including + # "https://" scheme identifier and fragment ("#some-target-heading"), if any. + # Do not end with a period (or any other characters not part of the URL). + # URLs longer than the available terminal width will overflow without being + # broken apart. This facilitates both URL detection, and manual copy-pasting. general_group = parser.add_argument_group(title="Optional arguments") general_group.add_argument( @@ -1034,7 +1048,7 @@ def add_invertible_flag( "Mypy caches type information about modules into a cache to " "let you speed up future invocations of mypy. Also see " "mypy's daemon mode: " - "mypy.readthedocs.io/en/stable/mypy_daemon.html#mypy-daemon", + "https://mypy.readthedocs.io/en/stable/mypy_daemon.html#mypy-daemon", ) incremental_group.add_argument( "-i", "--incremental", action="store_true", help=argparse.SUPPRESS @@ -1278,7 +1292,7 @@ def add_invertible_flag( code_group = parser.add_argument_group( title="Running code", description="Specify the code you want to type check. For more details, see " - "mypy.readthedocs.io/en/stable/running_mypy.html#running-mypy", + "https://mypy.readthedocs.io/en/stable/running_mypy.html#running-mypy", ) add_invertible_flag( "--explicit-package-bases", From 3c30736847141e6212701647de95bc0da2728a7f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 3 Nov 2025 18:56:57 -0800 Subject: [PATCH 149/183] Fix errors for raise NotImplemented (#20168) See also discussion on https://github.com/python/typeshed/pull/14966 --- mypy/checker.py | 15 ++++++++++----- mypy/types.py | 2 ++ test-data/unit/check-statements.test | 2 +- test-data/unit/fixtures/notimplemented.pyi | 13 ++++++++++--- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index fedb4e745909..f3a93d1eeda1 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -198,6 +198,7 @@ from mypy.types import ( ANY_STRATEGY, MYPYC_NATIVE_INT_NAMES, + NOT_IMPLEMENTED_TYPE_NAMES, OVERLOAD_NAMES, AnyType, BoolTypeQuery, @@ -4974,10 +4975,7 @@ def check_return_stmt(self, s: ReturnStmt) -> None: ) # Treat NotImplemented as having type Any, consistent with its # definition in typeshed prior to python/typeshed#4222. - if ( - isinstance(typ, Instance) - and typ.type.fullname == "builtins._NotImplementedType" - ): + if isinstance(typ, Instance) and typ.type.fullname in NOT_IMPLEMENTED_TYPE_NAMES: typ = AnyType(TypeOfAny.special_form) if defn.is_async_generator: @@ -5136,7 +5134,14 @@ def type_check_raise(self, e: Expression, s: RaiseStmt, optional: bool = False) # https://github.com/python/mypy/issues/11089 self.expr_checker.check_call(typ, [], [], e) - if isinstance(typ, Instance) and typ.type.fullname == "builtins._NotImplementedType": + if ( + isinstance(typ, Instance) + and typ.type.fullname in {"builtins._NotImplementedType", "types.NotImplementedType"} + ) or ( + isinstance(e, CallExpr) + and isinstance(e.callee, RefExpr) + and e.callee.fullname in {"builtins.NotImplemented"} + ): self.fail( message_registry.INVALID_EXCEPTION.with_additional_msg( '; did you mean "NotImplementedError"?' diff --git a/mypy/types.py b/mypy/types.py index 1c4331022496..7a8343097204 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -200,6 +200,8 @@ ELLIPSIS_TYPE_NAMES: Final = ("builtins.ellipsis", "types.EllipsisType") +NOT_IMPLEMENTED_TYPE_NAMES: Final = ("builtins._NotImplementedType", "types.NotImplementedType") + # A placeholder used for Bogus[...] parameters _dummy: Final[Any] = object() diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 658bee76ef0d..87d015f3de0f 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -523,7 +523,7 @@ if object(): if object(): raise NotImplemented # E: Exception must be derived from BaseException; did you mean "NotImplementedError"? if object(): - raise NotImplemented() # E: NotImplemented? not callable + raise NotImplemented() # E: Exception must be derived from BaseException; did you mean "NotImplementedError"? [builtins fixtures/notimplemented.pyi] [case testTryFinallyStatement] diff --git a/test-data/unit/fixtures/notimplemented.pyi b/test-data/unit/fixtures/notimplemented.pyi index 92edf84a7fd1..c9e58f099477 100644 --- a/test-data/unit/fixtures/notimplemented.pyi +++ b/test-data/unit/fixtures/notimplemented.pyi @@ -10,9 +10,16 @@ class bool: pass class int: pass class str: pass class dict: pass +class tuple: pass +class ellipsis: pass -class _NotImplementedType(Any): - __call__: NotImplemented # type: ignore -NotImplemented: _NotImplementedType +import sys + +if sys.version_info >= (3, 10): # type: ignore + from types import NotImplementedType + NotImplemented: NotImplementedType +else: + class _NotImplementedType(Any): ... + NotImplemented: _NotImplementedType class BaseException: pass From 08dde9fdd70297054efcc58f450ce4390bab9e44 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 4 Nov 2025 17:11:08 +0100 Subject: [PATCH 150/183] refactor: reuse NOT_IMPLEMENTED_TYPE_NAMES (#20178) --- mypy/checker.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f3a93d1eeda1..9f8299e6805d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5134,13 +5134,10 @@ def type_check_raise(self, e: Expression, s: RaiseStmt, optional: bool = False) # https://github.com/python/mypy/issues/11089 self.expr_checker.check_call(typ, [], [], e) - if ( - isinstance(typ, Instance) - and typ.type.fullname in {"builtins._NotImplementedType", "types.NotImplementedType"} - ) or ( + if (isinstance(typ, Instance) and typ.type.fullname in NOT_IMPLEMENTED_TYPE_NAMES) or ( isinstance(e, CallExpr) and isinstance(e.callee, RefExpr) - and e.callee.fullname in {"builtins.NotImplemented"} + and e.callee.fullname == "builtins.NotImplemented" ): self.fail( message_registry.INVALID_EXCEPTION.with_additional_msg( From bed188f7d159da3089bc538b6f8ba31d3e12f3ee Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Tue, 4 Nov 2025 09:31:13 -0700 Subject: [PATCH 151/183] [docs] document --enable-incomplete-feature TypeForm, minimally but sufficiently (#20173) --- docs/source/command_line.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index d6b70c4976dc..5efec6855593 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1159,7 +1159,7 @@ format into the specified directory. Enabling incomplete/experimental features ***************************************** -.. option:: --enable-incomplete-feature {PreciseTupleTypes,InlineTypedDict} +.. option:: --enable-incomplete-feature {PreciseTupleTypes,InlineTypedDict,TypeForm} Some features may require several mypy releases to implement, for example due to their complexity, potential for backwards incompatibility, or @@ -1214,6 +1214,9 @@ List of currently incomplete/experimental features: def test_values() -> {"width": int, "description": str}: return {"width": 42, "description": "test"} +* ``TypeForm``: this feature enables ``TypeForm``, as described in + `PEP 747 – Annotating Type Forms _`. + Miscellaneous ************* From 9ed7bb48050950d670f27190658f46e669272e18 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Wed, 5 Nov 2025 09:37:19 -0700 Subject: [PATCH 152/183] Update duck_type_compatibility.rst: mention strict-bytes & mypy 2.0 (#20121) Seems like a good idea to mention this, while we're on the subject. --- docs/source/duck_type_compatibility.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/duck_type_compatibility.rst b/docs/source/duck_type_compatibility.rst index e801f9251db5..7f4b67503ebe 100644 --- a/docs/source/duck_type_compatibility.rst +++ b/docs/source/duck_type_compatibility.rst @@ -9,6 +9,8 @@ supported for a small set of built-in types: * ``int`` is duck type compatible with ``float`` and ``complex``. * ``float`` is duck type compatible with ``complex``. * ``bytearray`` and ``memoryview`` are duck type compatible with ``bytes``. + (this will be disabled by default in **mypy 2.0**, and currently can be + disabled with :option:`--strict-bytes `.) For example, mypy considers an ``int`` object to be valid whenever a ``float`` object is expected. Thus code like this is nice and clean From 45aa599633e16b714205445ec67b4bcf80739359 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Wed, 5 Nov 2025 17:44:54 +0100 Subject: [PATCH 153/183] Do not abort constructing TypeAlias if only type parameters hold us back. (#20162) Fixes #20135. I am not fully certain that this is the right way. If the type alias can be reasonably constructed and only type parameters prevent that, we can create an "almost" equivalent alias with all placeholder-bearing typevars replaced with their "simple" equivalents with default values/bound/default values. This would cause current iteration to not emit some parameterization errors later, but, since we already defer the alias as a whole, we will recheck all of them again. This obviously adds some pointless work (we check parameterization that will not be used later), but probably is not that big of a deal, recursive aliases are relatively rare in wild. If this turns out to be a bottleneck, we can add a `parameters_ready` flag to aliases and skip the heavy parts if it is False, but I think it isn't necessary now. --- mypy/semanal.py | 56 +++++++++++++++++++++++++++-- mypy/semanal_typeargs.py | 2 +- test-data/unit/check-python312.test | 24 +++++++++++++ 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index e55819b898e9..fe6bd71c1ab9 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5637,13 +5637,20 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: else: incomplete_target = has_placeholder(res) - incomplete_tv = any(has_placeholder(tv) for tv in alias_tvars) - if self.found_incomplete_ref(tag) or incomplete_target or incomplete_tv: + if self.found_incomplete_ref(tag) or incomplete_target: # Since we have got here, we know this must be a type alias (incomplete refs # may appear in nested positions), therefore use becomes_typeinfo=True. self.mark_incomplete(s.name.name, s.value, becomes_typeinfo=True) return + # Now go through all new variables and temporary replace all tvars that still + # refer to some placeholders. We defer the whole alias and will revisit it again, + # as well as all its dependents. + for i, tv in enumerate(alias_tvars): + if has_placeholder(tv): + self.mark_incomplete(s.name.name, s.value, becomes_typeinfo=True) + alias_tvars[i] = self._trivial_typevarlike_like(tv) + self.add_type_alias_deps(depends_on) check_for_explicit_any( res, self.options, self.is_typeshed_stub_file, self.msg, context=s @@ -5677,7 +5684,10 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: ): updated = False if isinstance(existing.node, TypeAlias): - if existing.node.target != res: + if ( + existing.node.target != res + or existing.node.alias_tvars != alias_node.alias_tvars + ): # Copy expansion to the existing alias, this matches how we update base classes # for a TypeInfo _in place_ if there are nested placeholders. existing.node.target = res @@ -5707,6 +5717,46 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: finally: self.pop_type_args(s.type_args) + def _trivial_typevarlike_like(self, tv: TypeVarLikeType) -> TypeVarLikeType: + object_type = self.named_type("builtins.object") + if isinstance(tv, TypeVarType): + return TypeVarType( + tv.name, + tv.fullname, + tv.id, + values=[], + upper_bound=object_type, + default=AnyType(TypeOfAny.from_omitted_generics), + variance=tv.variance, + line=tv.line, + column=tv.column, + ) + elif isinstance(tv, TypeVarTupleType): + tuple_type = self.named_type("builtins.tuple", [object_type]) + return TypeVarTupleType( + tv.name, + tv.fullname, + tv.id, + upper_bound=tuple_type, + tuple_fallback=tuple_type, + default=AnyType(TypeOfAny.from_omitted_generics), + line=tv.line, + column=tv.column, + ) + elif isinstance(tv, ParamSpecType): + return ParamSpecType( + tv.name, + tv.fullname, + tv.id, + flavor=tv.flavor, + upper_bound=object_type, + default=AnyType(TypeOfAny.from_omitted_generics), + line=tv.line, + column=tv.column, + ) + else: + assert False, f"Unknown TypeVarLike: {tv!r}" + # # Expressions # diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index 686e7a57042d..86f8a8700def 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -151,7 +151,7 @@ def validate_args( is_error = False is_invalid = False - for (i, arg), tvar in zip(enumerate(args), type_vars): + for arg, tvar in zip(args, type_vars): context = ctx if arg.line < 0 else arg if isinstance(tvar, TypeVarType): if isinstance(arg, ParamSpecType): diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index be46ff6ee5c0..840a708fecf3 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2057,6 +2057,7 @@ reveal_type(AA.XX) # N: Revealed type is "def () -> __main__.XX" y: B.Y reveal_type(y) # N: Revealed type is "__main__.Y" [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testPEP695TypeAliasRecursiveInvalid] type X = X # E: Cannot resolve name "X" (possible cyclic definition) @@ -2168,3 +2169,26 @@ x: MyTuple[int, str] reveal_type(x[0]) # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] + +[case testPEP695TypeAliasRecursiveInParameterBound] +from typing import Any + +type A1[T: B1] = list[int] +type B1 = None | A1[B1] +x1: A1[B1] +y1: A1[int] # E: Type argument "int" of "A1" must be a subtype of "B1" +z1: A1[None] + +type A2[T: B2] = list[T] +type B2 = None | A2[Any] +x2: A2[B2] +y2: A2[int] # E: Type argument "int" of "A2" must be a subtype of "B2" +z2: A2[None] + +type A3[T: B3] = list[T] +type B3 = None | A3[B3] +x3: A3[B3] +y3: A3[int] # E: Type argument "int" of "A3" must be a subtype of "B3" +z3: A3[None] +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] From f10b462447900ebf9675cd81034340719438cc44 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 6 Nov 2025 12:37:30 +0000 Subject: [PATCH 154/183] Stricter handling of submodules as attributes (#20179) Fixes https://github.com/python/mypy/issues/20174 The idea is quite straightforward: we only allow `foo.bar` without explicit re-export if `foo.bar` was imported in any transitive dependency (and not in some unrelated module). Note: only `import foo.bar` takes effect, effect of using `from foo.bar import ...` is not propagated for two reasons: * It will cost large performance penalty * It is relatively obscure Python feature that may be considered by some as "implementation detail" --- mypy/build.py | 50 ++++++++-------- mypy/nodes.py | 20 +++---- mypy/semanal.py | 48 ++++++++++++++- test-data/unit/check-incremental.test | 84 +++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 34 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 0b78f879c547..e9c50ce6b224 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -603,6 +603,7 @@ def __init__( self.options = options self.version_id = version_id self.modules: dict[str, MypyFile] = {} + self.import_map: dict[str, set[str]] = {} self.missing_modules: set[str] = set() self.fg_deps_meta: dict[str, FgDepMeta] = {} # fg_deps holds the dependencies of every module that has been @@ -623,6 +624,7 @@ def __init__( self.incomplete_namespaces, self.errors, self.plugin, + self.import_map, ) self.all_types: dict[Expression, Type] = {} # Enabled by export_types self.indirection_detector = TypeIndirectionVisitor() @@ -742,6 +744,26 @@ def getmtime(self, path: str) -> int: else: return int(self.metastore.getmtime(path)) + def correct_rel_imp(self, file: MypyFile, imp: ImportFrom | ImportAll) -> str: + """Function to correct for relative imports.""" + file_id = file.fullname + rel = imp.relative + if rel == 0: + return imp.id + if os.path.basename(file.path).startswith("__init__."): + rel -= 1 + if rel != 0: + file_id = ".".join(file_id.split(".")[:-rel]) + new_id = file_id + "." + imp.id if imp.id else file_id + + if not new_id: + self.errors.set_file(file.path, file.name, self.options) + self.errors.report( + imp.line, 0, "No parent module -- cannot perform relative import", blocker=True + ) + + return new_id + def all_imported_modules_in_file(self, file: MypyFile) -> list[tuple[int, str, int]]: """Find all reachable import statements in a file. @@ -750,27 +772,6 @@ def all_imported_modules_in_file(self, file: MypyFile) -> list[tuple[int, str, i Can generate blocking errors on bogus relative imports. """ - - def correct_rel_imp(imp: ImportFrom | ImportAll) -> str: - """Function to correct for relative imports.""" - file_id = file.fullname - rel = imp.relative - if rel == 0: - return imp.id - if os.path.basename(file.path).startswith("__init__."): - rel -= 1 - if rel != 0: - file_id = ".".join(file_id.split(".")[:-rel]) - new_id = file_id + "." + imp.id if imp.id else file_id - - if not new_id: - self.errors.set_file(file.path, file.name, self.options) - self.errors.report( - imp.line, 0, "No parent module -- cannot perform relative import", blocker=True - ) - - return new_id - res: list[tuple[int, str, int]] = [] for imp in file.imports: if not imp.is_unreachable: @@ -785,7 +786,7 @@ def correct_rel_imp(imp: ImportFrom | ImportAll) -> str: ancestors.append(part) res.append((ancestor_pri, ".".join(ancestors), imp.line)) elif isinstance(imp, ImportFrom): - cur_id = correct_rel_imp(imp) + cur_id = self.correct_rel_imp(file, imp) all_are_submodules = True # Also add any imported names that are submodules. pri = import_priority(imp, PRI_MED) @@ -805,7 +806,7 @@ def correct_rel_imp(imp: ImportFrom | ImportAll) -> str: res.append((pri, cur_id, imp.line)) elif isinstance(imp, ImportAll): pri = import_priority(imp, PRI_HIGH) - res.append((pri, correct_rel_imp(imp), imp.line)) + res.append((pri, self.correct_rel_imp(file, imp), imp.line)) # Sort such that module (e.g. foo.bar.baz) comes before its ancestors (e.g. foo # and foo.bar) so that, if FindModuleCache finds the target module in a @@ -2898,6 +2899,9 @@ def dispatch(sources: list[BuildSource], manager: BuildManager, stdout: TextIO) manager.cache_enabled = False graph = load_graph(sources, manager) + for id in graph: + manager.import_map[id] = set(graph[id].dependencies + graph[id].suppressed) + t1 = time.time() manager.add_stats( graph_size=len(graph), diff --git a/mypy/nodes.py b/mypy/nodes.py index 539995ce9229..13ba011eebc0 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5002,27 +5002,27 @@ def local_definitions( SYMBOL_TABLE_NODE: Final[Tag] = 61 -def read_symbol(data: Buffer) -> mypy.nodes.SymbolNode: +def read_symbol(data: Buffer) -> SymbolNode: tag = read_tag(data) # The branches here are ordered manually by type "popularity". if tag == VAR: - return mypy.nodes.Var.read(data) + return Var.read(data) if tag == FUNC_DEF: - return mypy.nodes.FuncDef.read(data) + return FuncDef.read(data) if tag == DECORATOR: - return mypy.nodes.Decorator.read(data) + return Decorator.read(data) if tag == TYPE_INFO: - return mypy.nodes.TypeInfo.read(data) + return TypeInfo.read(data) if tag == OVERLOADED_FUNC_DEF: - return mypy.nodes.OverloadedFuncDef.read(data) + return OverloadedFuncDef.read(data) if tag == TYPE_VAR_EXPR: - return mypy.nodes.TypeVarExpr.read(data) + return TypeVarExpr.read(data) if tag == TYPE_ALIAS: - return mypy.nodes.TypeAlias.read(data) + return TypeAlias.read(data) if tag == PARAM_SPEC_EXPR: - return mypy.nodes.ParamSpecExpr.read(data) + return ParamSpecExpr.read(data) if tag == TYPE_VAR_TUPLE_EXPR: - return mypy.nodes.TypeVarTupleExpr.read(data) + return TypeVarTupleExpr.read(data) assert False, f"Unknown symbol tag {tag}" diff --git a/mypy/semanal.py b/mypy/semanal.py index fe6bd71c1ab9..973a28db0588 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -451,6 +451,7 @@ def __init__( incomplete_namespaces: set[str], errors: Errors, plugin: Plugin, + import_map: dict[str, set[str]], ) -> None: """Construct semantic analyzer. @@ -483,6 +484,7 @@ def __init__( self.loop_depth = [0] self.errors = errors self.modules = modules + self.import_map = import_map self.msg = MessageBuilder(errors, modules) self.missing_modules = missing_modules self.missing_names = [set()] @@ -534,6 +536,16 @@ def __init__( self.type_expression_full_parse_success_count: int = 0 # Successful full parses self.type_expression_full_parse_failure_count: int = 0 # Failed full parses + # Imports of submodules transitively visible from given module. + # This is needed to support patterns like this + # [a.py] + # import b + # import foo + # foo.bar # <- this should work even if bar is not re-exported in foo + # [b.py] + # import foo.bar + self.transitive_submodule_imports: dict[str, set[str]] = {} + # mypyc doesn't properly handle implementing an abstractproperty # with a regular attribute so we make them properties @property @@ -6687,7 +6699,7 @@ def get_module_symbol(self, node: MypyFile, name: str) -> SymbolTableNode | None sym = names.get(name) if not sym: fullname = module + "." + name - if fullname in self.modules: + if fullname in self.modules and self.is_visible_import(module, fullname): sym = SymbolTableNode(GDEF, self.modules[fullname]) elif self.is_incomplete_namespace(module): self.record_incomplete_ref() @@ -6706,6 +6718,40 @@ def get_module_symbol(self, node: MypyFile, name: str) -> SymbolTableNode | None sym = None return sym + def is_visible_import(self, base_id: str, id: str) -> bool: + if id in self.import_map[self.cur_mod_id]: + # Fast path: module is imported locally. + return True + if base_id not in self.transitive_submodule_imports: + # This is a performance optimization for a common pattern. If one module + # in a codebase uses import numpy as np; np.foo.bar, then it is likely that + # other modules use similar pattern as well. So we pre-compute transitive + # dependencies for np, to avoid possible duplicate work in the future. + self.add_transitive_submodule_imports(base_id) + if self.cur_mod_id not in self.transitive_submodule_imports: + self.add_transitive_submodule_imports(self.cur_mod_id) + return id in self.transitive_submodule_imports[self.cur_mod_id] + + def add_transitive_submodule_imports(self, mod_id: str) -> None: + if mod_id not in self.import_map: + return + todo = self.import_map[mod_id] + seen = {mod_id} + result = {mod_id} + while todo: + dep = todo.pop() + if dep in seen: + continue + seen.add(dep) + if "." in dep: + result.add(dep) + if dep in self.transitive_submodule_imports: + result |= self.transitive_submodule_imports[dep] + continue + if dep in self.import_map: + todo |= self.import_map[dep] + self.transitive_submodule_imports[mod_id] = result + def is_missing_module(self, module: str) -> bool: return module in self.missing_modules diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 5fbaa4f2c904..56c9cef80f34 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -7512,3 +7512,87 @@ tmp/impl.py:31: note: Revealed type is "builtins.object" tmp/impl.py:32: note: Revealed type is "Union[builtins.int, builtins.str, lib.Unrelated]" tmp/impl.py:33: note: Revealed type is "builtins.object" tmp/impl.py:34: note: Revealed type is "builtins.object" + +[case testIncrementalAccessSubmoduleWithoutExplicitImport] +import b +import a + +[file a.py] +import pkg + +pkg.submod.foo() + +[file a.py.2] +import pkg + +pkg.submod.foo() +x = 1 + +[file b.py] +import c + +[file c.py] +from pkg import submod + +[file pkg/__init__.pyi] +[file pkg/submod.pyi] +def foo() -> None: pass +[out] +tmp/a.py:3: error: "object" has no attribute "submod" +[out2] +tmp/a.py:3: error: "object" has no attribute "submod" + +[case testIncrementalAccessSubmoduleWithoutExplicitImportNested] +import a + +[file a.py] +import pandas +pandas.core.dtypes + +[file a.py.2] +import pandas +pandas.core.dtypes +# touch + +[file pandas/__init__.py] +import pandas.core.api + +[file pandas/core/__init__.py] +[file pandas/core/api.py] +import pandas.core.dtypes.dtypes + +[file pandas/core/dtypes/__init__.py] +[file pandas/core/dtypes/dtypes.py] +X = 0 +[out] +[out2] + +[case testIncrementalAccessSubmoduleWithoutExplicitImportNestedFrom] +import a + +[file a.py] +import pandas + +# Although this actually works at runtime, we do not support this, since +# this would cause major slowdown for a rare edge case. This test verifies +# that we fail consistently on cold and warm runs. +pandas.core.dtypes + +[file a.py.2] +import pandas +pandas.core.dtypes + +[file pandas/__init__.py] +import pandas.core.api + +[file pandas/core/__init__.py] +[file pandas/core/api.py] +from pandas.core.dtypes.dtypes import X + +[file pandas/core/dtypes/__init__.py] +[file pandas/core/dtypes/dtypes.py] +X = 0 +[out] +tmp/a.py:6: error: "object" has no attribute "dtypes" +[out2] +tmp/a.py:2: error: "object" has no attribute "dtypes" From 66dd2c146be9835902b88c7c9155b9eb82fe6f79 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Fri, 7 Nov 2025 08:44:39 +0100 Subject: [PATCH 155/183] Adjust HTML reports in tests to support newer libxml (#20199) Fixes #20070. I tested this manually in debian:sid container with `libxml2-16` installed via `apt` and `lxml==6.0.2` built against it (`--no-binary`). HTML reports testcases fail on master as reported and pass on my branch. --- mypy/test/helpers.py | 9 +++++++++ test-data/unit/reports.test | 12 ++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 36ad5ad4ec1a..8ff6874e746a 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -444,6 +444,8 @@ def check_test_output_files( if testcase.suite.native_sep and os.path.sep == "\\": normalized_output = [fix_cobertura_filename(line) for line in normalized_output] normalized_output = normalize_error_messages(normalized_output) + if os.path.basename(testcase.file) == "reports.test": + normalized_output = normalize_report_meta(normalized_output) assert_string_arrays_equal( expected_content.splitlines(), normalized_output, @@ -467,6 +469,13 @@ def normalize_file_output(content: list[str], current_abs_path: str) -> list[str return result +def normalize_report_meta(content: list[str]) -> list[str]: + # libxml 2.15 and newer emits the "modern" version of this element. + # Normalize the old style to look the same. + html_meta = '' + return ['' if x == html_meta else x for x in content] + + def find_test_files(pattern: str, exclude: list[str] | None = None) -> list[str]: return [ path.name diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index cce2f7295e3b..6e80683ad957 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -118,7 +118,7 @@ class A(object): [outfile report/html/n.py.html] - + @@ -172,7 +172,7 @@ T = TypeVar('T') [outfile report/html/n.py.html] - + @@ -214,7 +214,7 @@ def bar(x): [outfile report/html/n.py.html] - + @@ -255,7 +255,7 @@ old_stdout = sys.stdout [outfile report/html/n.py.html] - + @@ -487,7 +487,7 @@ DisplayToSource = Callable[[int], int] [outfile report/html/n.py.html] - + @@ -529,7 +529,7 @@ namespace_packages = True [outfile report/html/folder/subfolder/something.py.html] - + From 6986993532d84c3033dc1f28cda1c0c14ee1a55a Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Sun, 9 Nov 2025 00:13:28 +0100 Subject: [PATCH 156/183] Do not assume that args of decorated functions can be cleanly mapped to their nodes (#20203) Fixes #20059. If any non-trivial decorator is present, avoid trying to pick the parameter corresponding to i-th parameter of the callable type. --- mypy/checker.py | 13 +++++++- test-data/unit/check-classes.test | 52 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 9f8299e6805d..07f5c520de95 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2578,7 +2578,18 @@ def erase_override(t: Type) -> Type: continue if not is_subtype(original_arg_type, erase_override(override_arg_type)): context: Context = node - if isinstance(node, FuncDef) and not node.is_property: + if ( + isinstance(node, FuncDef) + and not node.is_property + and ( + not node.is_decorated # fast path + # allow trivial decorators like @classmethod and @override + or not (sym := node.info.get(node.name)) + or not isinstance(sym.node, Decorator) + or not sym.node.decorators + ) + ): + # If there's any decorator, we can no longer map arguments 1:1 reliably. arg_node = node.arguments[i + override.bound()] if arg_node.line != -1: context = arg_node diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index c0b1114db512..0e9d6357af1a 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -585,6 +585,58 @@ class B(A): @dec def f(self) -> int: pass +[case testOverrideWithDecoratorReturningCallable] +from typing import Any, Callable, TypeVar + +class Base: + def get(self, a: str) -> None: ... + +def dec(fn: Any) -> Callable[[Any, int], None]: ... + +class Derived(Base): + @dec + def get(self) -> None: ... # E: Argument 1 of "get" is incompatible with supertype "Base"; supertype defines the argument type as "str" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +[builtins fixtures/tuple.pyi] + +[case testOverrideWithDecoratorReturningCallable2] +# flags: --pretty +from typing import Any, Callable, TypeVar + +_C = TypeVar("_C", bound=Callable[..., Any]) + +def infer_signature(f: _C) -> Callable[[Any], _C]: ... + +class Base: + def get(self, a: str, b: str, c: str) -> None: ... + def post(self, a: str, b: str) -> None: ... + +# Third argument incompatible +def get(self, a: str, b: str, c: int) -> None: ... + +# Second argument incompatible - still should not map to **kwargs +def post(self, a: str, b: int) -> None: ... + +class Derived(Base): + @infer_signature(get) + def get(self, *args: Any, **kwargs: Any) -> None: ... + + @infer_signature(post) + def post(self, *args: Any, **kwargs: Any) -> None: ... +[builtins fixtures/tuple.pyi] +[out] +main:20: error: Argument 3 of "get" is incompatible with supertype "Base"; supertype defines the argument type as "str" + def get(self, *args: Any, **kwargs: Any) -> None: ... + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +main:20: note: This violates the Liskov substitution principle +main:20: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +main:23: error: Argument 2 of "post" is incompatible with supertype "Base"; supertype defines the argument type as "str" + def post(self, *args: Any, **kwargs: Any) -> None: ... + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +main:23: note: This violates the Liskov substitution principle +main:23: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + [case testOverrideWithDecoratorReturningInstance] def dec(f) -> str: pass From 566ba1ed27ccde4d85e736c8fe03d4ac8f744186 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Sun, 9 Nov 2025 20:12:21 -0500 Subject: [PATCH 157/183] chore: fix typo in comment (#20210) Title says it all --- mypyc/transform/flag_elimination.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mypyc/transform/flag_elimination.py b/mypyc/transform/flag_elimination.py index 605e5bc46ae4..c78e60d47cbd 100644 --- a/mypyc/transform/flag_elimination.py +++ b/mypyc/transform/flag_elimination.py @@ -78,10 +78,9 @@ def __init__(self, builder: LowLevelIRBuilder, branch_map: dict[Register, Branch self.branches = set(branch_map.values()) def visit_assign(self, op: Assign) -> None: - old_branch = self.branch_map.get(op.dest) - if old_branch: + if old_branch := self.branch_map.get(op.dest): # Replace assignment with a copy of the old branch, which is in a - # separate basic block. The old branch will be deletecd in visit_branch. + # separate basic block. The old branch will be deleted in visit_branch. new_branch = Branch( op.src, old_branch.true, From ad2b72be21462c98f8b877fe0859123d92d876c0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Nov 2025 16:13:46 +0000 Subject: [PATCH 158/183] Micro-optimize cache primitives in librt.internal (#20194) This includes the following optimizations: * Split Buffer into ReadBuffer and WriteBuffer which are more specialized * Only check buffer types in wrapper functions, not in C primitives * Use pointers instead of integer indexes in the buffer objects This improves the performance of a micro-benchmark that reads integers in a loop by 5%, but this could help more if we'd inline some of the smaller functions (in the future). By making the functions simpler, inlining is more feasible. I'm not sure what's the best way to merge this -- maybe we'll need to have broken master for a while, and then we ca publish a new version of `librt`, and then update mypy to work using the new `librt` version. --- mypy/typeshed/stubs/librt/librt/internal.pyi | 32 +- mypyc/codegen/emit.py | 14 +- mypyc/ir/rtypes.py | 2 +- mypyc/lib-rt/librt_internal.c | 446 ++++++++++++------- mypyc/lib-rt/librt_internal.h | 37 +- mypyc/primitives/misc_ops.py | 23 +- mypyc/test-data/irbuild-classes.test | 42 +- mypyc/test-data/run-classes.test | 196 +++++--- 8 files changed, 517 insertions(+), 275 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index 8654e31c100e..78e7f9caa117 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -1,19 +1,27 @@ from mypy_extensions import u8 +# TODO: Remove Buffer -- right now we have hacky support for BOTH the old and new APIs + class Buffer: def __init__(self, source: bytes = ...) -> None: ... def getvalue(self) -> bytes: ... -def write_bool(data: Buffer, value: bool) -> None: ... -def read_bool(data: Buffer) -> bool: ... -def write_str(data: Buffer, value: str) -> None: ... -def read_str(data: Buffer) -> str: ... -def write_bytes(data: Buffer, value: bytes) -> None: ... -def read_bytes(data: Buffer) -> bytes: ... -def write_float(data: Buffer, value: float) -> None: ... -def read_float(data: Buffer) -> float: ... -def write_int(data: Buffer, value: int) -> None: ... -def read_int(data: Buffer) -> int: ... -def write_tag(data: Buffer, value: u8) -> None: ... -def read_tag(data: Buffer) -> u8: ... +class ReadBuffer: + def __init__(self, source: bytes) -> None: ... + +class WriteBuffer: + def getvalue(self) -> bytes: ... + +def write_bool(data: WriteBuffer | Buffer, value: bool) -> None: ... +def read_bool(data: ReadBuffer | Buffer) -> bool: ... +def write_str(data: WriteBuffer | Buffer, value: str) -> None: ... +def read_str(data: ReadBuffer | Buffer) -> str: ... +def write_bytes(data: WriteBuffer | Buffer, value: bytes) -> None: ... +def read_bytes(data: ReadBuffer | Buffer) -> bytes: ... +def write_float(data: WriteBuffer | Buffer, value: float) -> None: ... +def read_float(data: ReadBuffer | Buffer) -> float: ... +def write_int(data: WriteBuffer | Buffer, value: int) -> None: ... +def read_int(data: ReadBuffer | Buffer) -> int: ... +def write_tag(data: WriteBuffer | Buffer, value: u8) -> None: ... +def read_tag(data: ReadBuffer | Buffer) -> u8: ... def cache_version() -> u8: ... diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 4ef53296ef0d..f2a2271e020e 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -705,13 +705,25 @@ def emit_cast( self.emit_lines(f" {dest} = {src};", "else {") self.emit_cast_error_handler(error, src, dest, typ, raise_exception) self.emit_line("}") - elif is_object_rprimitive(typ) or is_native_rprimitive(typ): + elif is_object_rprimitive(typ): if declare_dest: self.emit_line(f"PyObject *{dest};") self.emit_arg_check(src, dest, typ, "", optional) self.emit_line(f"{dest} = {src};") if optional: self.emit_line("}") + elif is_native_rprimitive(typ): + # Native primitive types have type check functions of form "CPy_Check(...)". + if declare_dest: + self.emit_line(f"PyObject *{dest};") + short_name = typ.name.rsplit(".", 1)[-1] + check = f"(CPy{short_name}_Check({src}))" + if likely: + check = f"(likely{check})" + self.emit_arg_check(src, dest, typ, check, optional) + self.emit_lines(f" {dest} = {src};", "else {") + self.emit_cast_error_handler(error, src, dest, typ, raise_exception) + self.emit_line("}") elif isinstance(typ, RUnion): self.emit_union_cast( src, dest, typ, declare_dest, error, optional, src_type, raise_exception diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 941670ab230d..66b98e5d6398 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -514,7 +514,7 @@ def __hash__(self) -> int: KNOWN_NATIVE_TYPES: Final = { name: RPrimitive(name, is_unboxed=False, is_refcounted=True) - for name in ["librt.internal.Buffer"] + for name in ["librt.internal.WriteBuffer", "librt.internal.ReadBuffer"] } diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index eaf451eff22b..ada2dfeb39a5 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -30,18 +30,30 @@ #define CPY_NONE_ERROR 2 #define CPY_NONE 1 -#define _CHECK_BUFFER(data, err) if (unlikely(_check_buffer(data) == CPY_NONE_ERROR)) \ - return err; -#define _CHECK_SIZE(data, need) if (unlikely(_check_size((BufferObject *)data, need) == CPY_NONE_ERROR)) \ - return CPY_NONE_ERROR; -#define _CHECK_READ(data, size, err) if (unlikely(_check_read((BufferObject *)data, size) == CPY_NONE_ERROR)) \ - return err; - -#define _READ(data, type) *(type *)(((BufferObject *)data)->buf + ((BufferObject *)data)->pos); \ - ((BufferObject *)data)->pos += sizeof(type); - -#define _WRITE(data, type, v) *(type *)(((BufferObject *)data)->buf + ((BufferObject *)data)->pos) = v; \ - ((BufferObject *)data)->pos += sizeof(type); +#define _CHECK_READ_BUFFER(data, err) if (unlikely(_check_read_buffer(data) == CPY_NONE_ERROR)) \ + return err; +#define _CHECK_WRITE_BUFFER(data, err) if (unlikely(_check_write_buffer(data) == CPY_NONE_ERROR)) \ + return err; +#define _CHECK_WRITE(data, need) if (unlikely(_check_size((WriteBufferObject *)data, need) == CPY_NONE_ERROR)) \ + return CPY_NONE_ERROR; +#define _CHECK_READ(data, size, err) if (unlikely(_check_read((ReadBufferObject *)data, size) == CPY_NONE_ERROR)) \ + return err; + +#define _READ(result, data, type) \ + do { \ + *(result) = *(type *)(((ReadBufferObject *)data)->ptr); \ + ((ReadBufferObject *)data)->ptr += sizeof(type); \ + } while (0) + +#define _WRITE(data, type, v) \ + do { \ + *(type *)(((WriteBufferObject *)data)->ptr) = v; \ + ((WriteBufferObject *)data)->ptr += sizeof(type); \ + } while (0) + +// +// ReadBuffer +// #if PY_BIG_ENDIAN uint16_t reverse_16(uint16_t number) { @@ -55,78 +67,59 @@ uint32_t reverse_32(uint32_t number) { typedef struct { PyObject_HEAD - Py_ssize_t pos; - Py_ssize_t end; - Py_ssize_t size; - char *buf; -} BufferObject; + char *ptr; // Current read location in the buffer + char *end; // End of the buffer + PyObject *source; // The object that contains the buffer +} ReadBufferObject; -static PyTypeObject BufferType; +static PyTypeObject ReadBufferType; static PyObject* -Buffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +ReadBuffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - if (type != &BufferType) { - PyErr_SetString(PyExc_TypeError, "Buffer should not be subclassed"); + if (type != &ReadBufferType) { + PyErr_SetString(PyExc_TypeError, "ReadBuffer should not be subclassed"); return NULL; } - BufferObject *self = (BufferObject *)type->tp_alloc(type, 0); + ReadBufferObject *self = (ReadBufferObject *)type->tp_alloc(type, 0); if (self != NULL) { - self->pos = 0; - self->end = 0; - self->size = 0; - self->buf = NULL; + self->source = NULL; + self->ptr = NULL; + self->end = NULL; } return (PyObject *) self; } - static int -Buffer_init_internal(BufferObject *self, PyObject *source) { - if (source) { - if (!PyBytes_Check(source)) { - PyErr_SetString(PyExc_TypeError, "source must be a bytes object"); - return -1; - } - self->end = PyBytes_GET_SIZE(source); - // Allocate at least one byte to simplify resizing logic. - // The original bytes buffer has last null byte, so this is safe. - self->size = self->end + 1; - // This returns a pointer to internal bytes data, so make our own copy. - char *buf = PyBytes_AsString(source); - self->buf = PyMem_Malloc(self->size); - memcpy(self->buf, buf, self->end); - } else { - self->buf = PyMem_Malloc(START_SIZE); - self->size = START_SIZE; +ReadBuffer_init_internal(ReadBufferObject *self, PyObject *source) { + if (!PyBytes_CheckExact(source)) { + PyErr_SetString(PyExc_TypeError, "source must be a bytes object"); + return -1; } + self->source = Py_NewRef(source); + self->ptr = PyBytes_AS_STRING(source); + self->end = self->ptr + PyBytes_GET_SIZE(source); return 0; } static PyObject* -Buffer_internal(PyObject *source) { - BufferObject *self = (BufferObject *)BufferType.tp_alloc(&BufferType, 0); +ReadBuffer_internal(PyObject *source) { + ReadBufferObject *self = (ReadBufferObject *)ReadBufferType.tp_alloc(&ReadBufferType, 0); if (self == NULL) return NULL; - self->pos = 0; - self->end = 0; - self->size = 0; - self->buf = NULL; - if (Buffer_init_internal(self, source) == -1) { + self->ptr = NULL; + self->end = NULL; + self->source = NULL; + if (ReadBuffer_init_internal(self, source) == -1) { Py_DECREF(self); return NULL; } return (PyObject *)self; } -static PyObject* -Buffer_internal_empty(void) { - return Buffer_internal(NULL); -} - static int -Buffer_init(BufferObject *self, PyObject *args, PyObject *kwds) +ReadBuffer_init(ReadBufferObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"source", NULL}; PyObject *source = NULL; @@ -134,53 +127,166 @@ Buffer_init(BufferObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &source)) return -1; - return Buffer_init_internal(self, source); + return ReadBuffer_init_internal(self, source); +} + +static void +ReadBuffer_dealloc(ReadBufferObject *self) +{ + Py_CLEAR(self->source); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyMethodDef ReadBuffer_methods[] = { + {NULL} /* Sentinel */ +}; + +static PyTypeObject ReadBufferType = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "ReadBuffer", + .tp_doc = PyDoc_STR("Mypy cache buffer objects"), + .tp_basicsize = sizeof(ReadBufferObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = ReadBuffer_new, + .tp_init = (initproc) ReadBuffer_init, + .tp_dealloc = (destructor) ReadBuffer_dealloc, + .tp_methods = ReadBuffer_methods, +}; + +// +// WriteBuffer +// + +typedef struct { + PyObject_HEAD + char *buf; // Beginning of the buffer + char *ptr; // Current write location in the buffer + char *end; // End of the buffer +} WriteBufferObject; + +static PyTypeObject WriteBufferType; + +static PyObject* +WriteBuffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (type != &WriteBufferType) { + PyErr_SetString(PyExc_TypeError, "WriteBuffer cannot be subclassed"); + return NULL; + } + + WriteBufferObject *self = (WriteBufferObject *)type->tp_alloc(type, 0); + if (self != NULL) { + self->buf = NULL; + self->ptr = NULL; + self->end = NULL; + } + return (PyObject *)self; +} + +static int +WriteBuffer_init_internal(WriteBufferObject *self) { + Py_ssize_t size = START_SIZE; + self->buf = PyMem_Malloc(size + 1); + if (self->buf == NULL) { + PyErr_NoMemory(); + return -1; + } + self->ptr = self->buf; + self->end = self->buf + size; + return 0; +} + +static PyObject* +WriteBuffer_internal(void) { + WriteBufferObject *self = (WriteBufferObject *)WriteBufferType.tp_alloc(&WriteBufferType, 0); + if (self == NULL) + return NULL; + self->buf = NULL; + self->ptr = NULL; + self->end = NULL; + if (WriteBuffer_init_internal(self) == -1) { + Py_DECREF(self); + return NULL; + } + return (PyObject *)self; +} + +static int +WriteBuffer_init(WriteBufferObject *self, PyObject *args, PyObject *kwds) +{ + if (!PyArg_ParseTuple(args, "")) { + return -1; + } + + if (kwds != NULL && PyDict_Size(kwds) > 0) { + PyErr_SetString(PyExc_TypeError, + "WriteBuffer() takes no keyword arguments"); + return -1; + } + + return WriteBuffer_init_internal(self); } static void -Buffer_dealloc(BufferObject *self) +WriteBuffer_dealloc(WriteBufferObject *self) { PyMem_Free(self->buf); + self->buf = NULL; Py_TYPE(self)->tp_free((PyObject *)self); } static PyObject* -Buffer_getvalue_internal(PyObject *self) +WriteBuffer_getvalue_internal(PyObject *self) { - return PyBytes_FromStringAndSize(((BufferObject *)self)->buf, ((BufferObject *)self)->end); + WriteBufferObject *obj = (WriteBufferObject *)self; + return PyBytes_FromStringAndSize(obj->buf, obj->ptr - obj->buf); } static PyObject* -Buffer_getvalue(BufferObject *self, PyObject *Py_UNUSED(ignored)) +WriteBuffer_getvalue(WriteBufferObject *self, PyObject *Py_UNUSED(ignored)) { - return PyBytes_FromStringAndSize(self->buf, self->end); + return PyBytes_FromStringAndSize(self->buf, self->ptr - self->buf); } -static PyMethodDef Buffer_methods[] = { - {"getvalue", (PyCFunction) Buffer_getvalue, METH_NOARGS, +static PyMethodDef WriteBuffer_methods[] = { + {"getvalue", (PyCFunction) WriteBuffer_getvalue, METH_NOARGS, "Return the buffer content as bytes object" }, {NULL} /* Sentinel */ }; -static PyTypeObject BufferType = { +static PyTypeObject WriteBufferType = { .ob_base = PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "Buffer", + .tp_name = "WriteBuffer", .tp_doc = PyDoc_STR("Mypy cache buffer objects"), - .tp_basicsize = sizeof(BufferObject), + .tp_basicsize = sizeof(WriteBufferObject), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = Buffer_new, - .tp_init = (initproc) Buffer_init, - .tp_dealloc = (destructor) Buffer_dealloc, - .tp_methods = Buffer_methods, + .tp_new = WriteBuffer_new, + .tp_init = (initproc) WriteBuffer_init, + .tp_dealloc = (destructor) WriteBuffer_dealloc, + .tp_methods = WriteBuffer_methods, }; +// ---------- + +static inline char +_check_read_buffer(PyObject *data) { + if (unlikely(Py_TYPE(data) != &ReadBufferType)) { + PyErr_Format( + PyExc_TypeError, "data must be a ReadBuffer object, got %s", Py_TYPE(data)->tp_name + ); + return CPY_NONE_ERROR; + } + return CPY_NONE; +} + static inline char -_check_buffer(PyObject *data) { - if (unlikely(Py_TYPE(data) != &BufferType)) { +_check_write_buffer(PyObject *data) { + if (unlikely(Py_TYPE(data) != &WriteBufferType)) { PyErr_Format( - PyExc_TypeError, "data must be a Buffer object, got %s", Py_TYPE(data)->tp_name + PyExc_TypeError, "data must be a WriteBuffer object, got %s", Py_TYPE(data)->tp_name ); return CPY_NONE_ERROR; } @@ -188,24 +294,28 @@ _check_buffer(PyObject *data) { } static inline char -_check_size(BufferObject *data, Py_ssize_t need) { - Py_ssize_t target = data->pos + need; - if (target <= data->size) +_check_size(WriteBufferObject *data, Py_ssize_t need) { + if (data->end - data->ptr >= need) return CPY_NONE; - do - data->size *= 2; - while (target >= data->size); - data->buf = PyMem_Realloc(data->buf, data->size); + Py_ssize_t index = data->ptr - data->buf; + Py_ssize_t target = index + need; + Py_ssize_t size = data->end - data->buf; + do { + size *= 2; + } while (target >= size); + data->buf = PyMem_Realloc(data->buf, size); if (unlikely(data->buf == NULL)) { PyErr_NoMemory(); return CPY_NONE_ERROR; } + data->ptr = data->buf + index; + data->end = data->buf + size; return CPY_NONE; } static inline char -_check_read(BufferObject *data, Py_ssize_t need) { - if (unlikely(data->pos + need > data->end)) { +_check_read(ReadBufferObject *data, Py_ssize_t need) { + if (unlikely((data->end - data->ptr) < need)) { PyErr_SetString(PyExc_ValueError, "reading past the buffer end"); return CPY_NONE_ERROR; } @@ -220,9 +330,9 @@ bool format: single byte static char read_bool_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_BOOL_ERROR) _CHECK_READ(data, 1, CPY_BOOL_ERROR) - char res = _READ(data, char) + char res; + _READ(&res, data, char); if (unlikely((res != 0) & (res != 1))) { PyErr_SetString(PyExc_ValueError, "invalid bool value"); return CPY_BOOL_ERROR; @@ -238,6 +348,7 @@ read_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) char res = read_bool_internal(data); if (unlikely(res == CPY_BOOL_ERROR)) return NULL; @@ -248,10 +359,8 @@ read_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames static char write_bool_internal(PyObject *data, char value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - _CHECK_SIZE(data, 1) - _WRITE(data, char, value) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, char, value); return CPY_NONE; } @@ -264,6 +373,7 @@ write_bool(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyBool_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a bool"); return NULL; @@ -289,14 +399,14 @@ _read_short_int(PyObject *data, uint8_t first) { } if ((first & FOUR_BYTES_INT_BIT) == 0) { _CHECK_READ(data, 1, CPY_INT_TAG) - second = _READ(data, uint8_t) + _READ(&second, data, uint8_t); return ((((Py_ssize_t)second) << 6) + (Py_ssize_t)(first >> 2) + MIN_TWO_BYTES_INT) << 1; } // The caller is responsible to verify this is called only for short ints. _CHECK_READ(data, 3, CPY_INT_TAG) // TODO: check if compilers emit optimal code for these two reads, and tweak if needed. - second = _READ(data, uint8_t) - two_more = _READ(data, uint16_t) + _READ(&second, data, uint8_t); + _READ(&two_more, data, uint16_t); #if PY_BIG_ENDIAN two_more = reverse_16(two_more); #endif @@ -306,11 +416,10 @@ _read_short_int(PyObject *data, uint8_t first) { static PyObject* read_str_internal(PyObject *data) { - _CHECK_BUFFER(data, NULL) - // Read string length. _CHECK_READ(data, 1, NULL) - uint8_t first = _READ(data, uint8_t) + uint8_t first; + _READ(&first, data, uint8_t); if (unlikely(first == LONG_INT_TRAILER)) { // Fail fast for invalid/tampered data. PyErr_SetString(PyExc_ValueError, "invalid str size"); @@ -326,14 +435,12 @@ read_str_internal(PyObject *data) { } Py_ssize_t size = tagged_size >> 1; // Read string content. - char *buf = ((BufferObject *)data)->buf; + char *ptr = ((ReadBufferObject *)data)->ptr; _CHECK_READ(data, size, NULL) - PyObject *res = PyUnicode_FromStringAndSize( - buf + ((BufferObject *)data)->pos, (Py_ssize_t)size - ); + PyObject *res = PyUnicode_FromStringAndSize(ptr, (Py_ssize_t)size); if (unlikely(res == NULL)) return NULL; - ((BufferObject *)data)->pos += size; + ((ReadBufferObject *)data)->ptr += size; return res; } @@ -345,6 +452,7 @@ read_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) return read_str_internal(data); } @@ -352,35 +460,30 @@ read_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) static inline char _write_short_int(PyObject *data, Py_ssize_t real_value) { if (real_value >= MIN_ONE_BYTE_INT && real_value <= MAX_ONE_BYTE_INT) { - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_ONE_BYTE_INT) << 1) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, uint8_t, (uint8_t)(real_value - MIN_ONE_BYTE_INT) << 1); } else if (real_value >= MIN_TWO_BYTES_INT && real_value <= MAX_TWO_BYTES_INT) { - _CHECK_SIZE(data, 2) + _CHECK_WRITE(data, 2) #if PY_BIG_ENDIAN uint16_t to_write = ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT; _WRITE(data, uint16_t, reverse_16(to_write)) #else - _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT) + _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT); #endif - ((BufferObject *)data)->end += 2; } else { - _CHECK_SIZE(data, 4) + _CHECK_WRITE(data, 4) #if PY_BIG_ENDIAN uint32_t to_write = ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER; _WRITE(data, uint32_t, reverse_32(to_write)) #else - _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER) + _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER); #endif - ((BufferObject *)data)->end += 4; } return CPY_NONE; } static char write_str_internal(PyObject *data, PyObject *value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - Py_ssize_t size; const char *chunk = PyUnicode_AsUTF8AndSize(value, &size); if (unlikely(chunk == NULL)) @@ -395,11 +498,10 @@ write_str_internal(PyObject *data, PyObject *value) { return CPY_NONE_ERROR; } // Write string content. - _CHECK_SIZE(data, size) - char *buf = ((BufferObject *)data)->buf; - memcpy(buf + ((BufferObject *)data)->pos, chunk, size); - ((BufferObject *)data)->pos += size; - ((BufferObject *)data)->end += size; + _CHECK_WRITE(data, size) + char *ptr = ((WriteBufferObject *)data)->ptr; + memcpy(ptr, chunk, size); + ((WriteBufferObject *)data)->ptr += size; return CPY_NONE; } @@ -412,6 +514,7 @@ write_str(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyUnicode_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a str"); return NULL; @@ -429,11 +532,10 @@ bytes format: size as int (see below) followed by bytes static PyObject* read_bytes_internal(PyObject *data) { - _CHECK_BUFFER(data, NULL) - // Read length. _CHECK_READ(data, 1, NULL) - uint8_t first = _READ(data, uint8_t) + uint8_t first; + _READ(&first, data, uint8_t); if (unlikely(first == LONG_INT_TRAILER)) { // Fail fast for invalid/tampered data. PyErr_SetString(PyExc_ValueError, "invalid bytes size"); @@ -449,14 +551,12 @@ read_bytes_internal(PyObject *data) { } Py_ssize_t size = tagged_size >> 1; // Read bytes content. - char *buf = ((BufferObject *)data)->buf; + char *ptr = ((ReadBufferObject *)data)->ptr; _CHECK_READ(data, size, NULL) - PyObject *res = PyBytes_FromStringAndSize( - buf + ((BufferObject *)data)->pos, (Py_ssize_t)size - ); + PyObject *res = PyBytes_FromStringAndSize(ptr, (Py_ssize_t)size); if (unlikely(res == NULL)) return NULL; - ((BufferObject *)data)->pos += size; + ((ReadBufferObject *)data)->ptr += size; return res; } @@ -468,13 +568,12 @@ read_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) return read_bytes_internal(data); } static char write_bytes_internal(PyObject *data, PyObject *value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - const char *chunk = PyBytes_AsString(value); if (unlikely(chunk == NULL)) return CPY_NONE_ERROR; @@ -489,11 +588,10 @@ write_bytes_internal(PyObject *data, PyObject *value) { return CPY_NONE_ERROR; } // Write bytes content. - _CHECK_SIZE(data, size) - char *buf = ((BufferObject *)data)->buf; - memcpy(buf + ((BufferObject *)data)->pos, chunk, size); - ((BufferObject *)data)->pos += size; - ((BufferObject *)data)->end += size; + _CHECK_WRITE(data, size) + char *ptr = ((WriteBufferObject *)data)->ptr; + memcpy(ptr, chunk, size); + ((WriteBufferObject *)data)->ptr += size; return CPY_NONE; } @@ -506,6 +604,7 @@ write_bytes(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyBytes_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a bytes object"); return NULL; @@ -524,13 +623,12 @@ float format: static double read_float_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_FLOAT_ERROR) _CHECK_READ(data, 8, CPY_FLOAT_ERROR) - char *buf = ((BufferObject *)data)->buf; - double res = PyFloat_Unpack8(buf + ((BufferObject *)data)->pos, 1); + char *ptr = ((ReadBufferObject *)data)->ptr; + double res = PyFloat_Unpack8(ptr, 1); if (unlikely((res == -1.0) && PyErr_Occurred())) return CPY_FLOAT_ERROR; - ((BufferObject *)data)->pos += 8; + ((ReadBufferObject *)data)->ptr += 8; return res; } @@ -542,6 +640,7 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) double retval = read_float_internal(data); if (unlikely(retval == CPY_FLOAT_ERROR && PyErr_Occurred())) { return NULL; @@ -551,14 +650,12 @@ read_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwname static char write_float_internal(PyObject *data, double value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - _CHECK_SIZE(data, 8) - char *buf = ((BufferObject *)data)->buf; - int res = PyFloat_Pack8(value, buf + ((BufferObject *)data)->pos, 1); + _CHECK_WRITE(data, 8) + char *ptr = ((WriteBufferObject *)data)->ptr; + int res = PyFloat_Pack8(value, ptr, 1); if (unlikely(res == -1)) return CPY_NONE_ERROR; - ((BufferObject *)data)->pos += 8; - ((BufferObject *)data)->end += 8; + ((WriteBufferObject *)data)->ptr += 8; return CPY_NONE; } @@ -571,6 +668,7 @@ write_float(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnam if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyFloat_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be a float"); return NULL; @@ -595,10 +693,10 @@ since negative integers are much more rare. static CPyTagged read_int_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_INT_TAG) _CHECK_READ(data, 1, CPY_INT_TAG) - uint8_t first = _READ(data, uint8_t) + uint8_t first; + _READ(&first, data, uint8_t); if (likely(first != LONG_INT_TRAILER)) { return _read_short_int(data, first); } @@ -607,7 +705,7 @@ read_int_internal(PyObject *data) { // Read byte length and sign. _CHECK_READ(data, 1, CPY_INT_TAG) - first = _READ(data, uint8_t) + _READ(&first, data, uint8_t); Py_ssize_t size_and_sign = _read_short_int(data, first); if (size_and_sign == CPY_INT_TAG) return CPY_INT_TAG; @@ -620,12 +718,11 @@ read_int_internal(PyObject *data) { // Construct an int object from the byte array. _CHECK_READ(data, size, CPY_INT_TAG) - char *buf = ((BufferObject *)data)->buf; - PyObject *num = _PyLong_FromByteArray( - (unsigned char *)(buf + ((BufferObject *)data)->pos), size, 1, 0); + char *ptr = ((ReadBufferObject *)data)->ptr; + PyObject *num = _PyLong_FromByteArray((unsigned char *)ptr, size, 1, 0); if (num == NULL) return CPY_INT_TAG; - ((BufferObject *)data)->pos += size; + ((ReadBufferObject *)data)->ptr += size; if (sign) { PyObject *old = num; num = PyNumber_Negative(old); @@ -645,6 +742,7 @@ read_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) CPyTagged retval = read_int_internal(data); if (unlikely(retval == CPY_INT_TAG)) { return NULL; @@ -664,9 +762,8 @@ static inline int hex_to_int(char c) { static inline char _write_long_int(PyObject *data, CPyTagged value) { - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, LONG_INT_TRAILER) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, uint8_t, LONG_INT_TRAILER); PyObject *hex_str = NULL; PyObject* int_value = CPyTagged_AsObject(value); @@ -731,8 +828,6 @@ _write_long_int(PyObject *data, CPyTagged value) { static char write_int_internal(PyObject *data, CPyTagged value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - if (likely((value & CPY_INT_TAG) == 0)) { Py_ssize_t real_value = CPyTagged_ShortAsSsize_t(value); if (likely(real_value >= MIN_FOUR_BYTES_INT && real_value <= MAX_FOUR_BYTES_INT)) { @@ -754,6 +849,7 @@ write_int(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) if (unlikely(!PyLong_Check(value))) { PyErr_SetString(PyExc_TypeError, "value must be an int"); return NULL; @@ -773,9 +869,9 @@ integer tag format (0 <= t <= 255): static uint8_t read_tag_internal(PyObject *data) { - _CHECK_BUFFER(data, CPY_LL_UINT_ERROR) _CHECK_READ(data, 1, CPY_LL_UINT_ERROR) - uint8_t ret = _READ(data, uint8_t) + uint8_t ret; + _READ(&ret, data, uint8_t); return ret; } @@ -787,6 +883,7 @@ read_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) if (unlikely(!CPyArg_ParseStackAndKeywordsOneArg(args, nargs, kwnames, &parser, &data))) { return NULL; } + _CHECK_READ_BUFFER(data, NULL) uint8_t retval = read_tag_internal(data); if (unlikely(retval == CPY_LL_UINT_ERROR && PyErr_Occurred())) { return NULL; @@ -796,10 +893,8 @@ read_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames) static char write_tag_internal(PyObject *data, uint8_t value) { - _CHECK_BUFFER(data, CPY_NONE_ERROR) - _CHECK_SIZE(data, 1) - _WRITE(data, uint8_t, value) - ((BufferObject *)data)->end += 1; + _CHECK_WRITE(data, 1) + _WRITE(data, uint8_t, value); return CPY_NONE; } @@ -812,6 +907,7 @@ write_tag(PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames if (unlikely(!CPyArg_ParseStackAndKeywordsSimple(args, nargs, kwnames, &parser, &data, &value))) { return NULL; } + _CHECK_WRITE_BUFFER(data, NULL) uint8_t unboxed = CPyLong_AsUInt8(value); if (unlikely(unboxed == CPY_LL_UINT_ERROR && PyErr_Occurred())) { CPy_TypeError("u8", value); @@ -834,6 +930,16 @@ cache_version(PyObject *self, PyObject *Py_UNUSED(ignored)) { return PyLong_FromLong(cache_version_internal()); } +static PyTypeObject * +ReadBuffer_type_internal(void) { + return &ReadBufferType; // Return borrowed reference +} + +static PyTypeObject * +WriteBuffer_type_internal(void) { + return &WriteBufferType; // Return borrowed reference +}; + static PyMethodDef librt_internal_module_methods[] = { {"write_bool", (PyCFunction)write_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("write a bool")}, {"read_bool", (PyCFunction)read_bool, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("read a bool")}, @@ -859,18 +965,24 @@ NativeInternal_ABI_Version(void) { static int librt_internal_module_exec(PyObject *m) { - if (PyType_Ready(&BufferType) < 0) { + if (PyType_Ready(&ReadBufferType) < 0) { + return -1; + } + if (PyType_Ready(&WriteBufferType) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "ReadBuffer", (PyObject *) &ReadBufferType) < 0) { return -1; } - if (PyModule_AddObjectRef(m, "Buffer", (PyObject *) &BufferType) < 0) { + if (PyModule_AddObjectRef(m, "WriteBuffer", (PyObject *) &WriteBufferType) < 0) { return -1; } // Export mypy internal C API, be careful with the order! - static void *NativeInternal_API[17] = { - (void *)Buffer_internal, - (void *)Buffer_internal_empty, - (void *)Buffer_getvalue_internal, + static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN] = { + (void *)ReadBuffer_internal, + (void *)WriteBuffer_internal, + (void *)WriteBuffer_getvalue_internal, (void *)write_bool_internal, (void *)read_bool_internal, (void *)write_str_internal, @@ -885,6 +997,8 @@ librt_internal_module_exec(PyObject *m) (void *)write_bytes_internal, (void *)read_bytes_internal, (void *)cache_version_internal, + (void *)ReadBuffer_type_internal, + (void *)WriteBuffer_type_internal, }; PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index 1d16e1cb127f..329a0fd68c11 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -1,13 +1,16 @@ #ifndef LIBRT_INTERNAL_H #define LIBRT_INTERNAL_H -#define LIBRT_INTERNAL_ABI_VERSION 0 +#define LIBRT_INTERNAL_ABI_VERSION 1 +#define LIBRT_INTERNAL_API_LEN 19 #ifdef LIBRT_INTERNAL_MODULE -static PyObject *Buffer_internal(PyObject *source); -static PyObject *Buffer_internal_empty(void); -static PyObject *Buffer_getvalue_internal(PyObject *self); +static PyObject *ReadBuffer_internal(PyObject *source); +static PyObject *WriteBuffer_internal(void); +static PyObject *WriteBuffer_getvalue_internal(PyObject *self); +static PyObject *ReadBuffer_internal(PyObject *source); +static PyObject *ReadBuffer_internal_empty(void); static char write_bool_internal(PyObject *data, char value); static char read_bool_internal(PyObject *data); static char write_str_internal(PyObject *data, PyObject *value); @@ -22,14 +25,16 @@ static int NativeInternal_ABI_Version(void); static char write_bytes_internal(PyObject *data, PyObject *value); static PyObject *read_bytes_internal(PyObject *data); static uint8_t cache_version_internal(void); +static PyTypeObject *ReadBuffer_type_internal(void); +static PyTypeObject *WriteBuffer_type_internal(void); #else -static void **NativeInternal_API; +static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN]; -#define Buffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0]) -#define Buffer_internal_empty (*(PyObject* (*)(void)) NativeInternal_API[1]) -#define Buffer_getvalue_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[2]) +#define ReadBuffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0]) +#define WriteBuffer_internal (*(PyObject* (*)(void)) NativeInternal_API[1]) +#define WriteBuffer_getvalue_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[2]) #define write_bool_internal (*(char (*)(PyObject *source, char value)) NativeInternal_API[3]) #define read_bool_internal (*(char (*)(PyObject *source)) NativeInternal_API[4]) #define write_str_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[5]) @@ -44,6 +49,8 @@ static void **NativeInternal_API; #define write_bytes_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[14]) #define read_bytes_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[15]) #define cache_version_internal (*(uint8_t (*)(void)) NativeInternal_API[16]) +#define ReadBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[17]) +#define WriteBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[18]) static int import_librt_internal(void) @@ -52,9 +59,10 @@ import_librt_internal(void) if (mod == NULL) return -1; Py_DECREF(mod); // we import just for the side effect of making the below work. - NativeInternal_API = (void **)PyCapsule_Import("librt.internal._C_API", 0); - if (NativeInternal_API == NULL) + void *capsule = PyCapsule_Import("librt.internal._C_API", 0); + if (capsule == NULL) return -1; + memcpy(NativeInternal_API, capsule, sizeof(NativeInternal_API)); if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) { PyErr_SetString(PyExc_ValueError, "ABI version conflict for librt.internal"); return -1; @@ -63,4 +71,13 @@ import_librt_internal(void) } #endif + +static inline bool CPyReadBuffer_Check(PyObject *obj) { + return Py_TYPE(obj) == ReadBuffer_type_internal(); +} + +static inline bool CPyWriteBuffer_Check(PyObject *obj) { + return Py_TYPE(obj) == WriteBuffer_type_internal(); +} + #endif // LIBRT_INTERNAL_H diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 10f4bc001e29..f685b1cfbcf5 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -333,31 +333,32 @@ error_kind=ERR_NEVER, ) -buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.Buffer"] +write_buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.WriteBuffer"] +read_buffer_rprimitive = KNOWN_NATIVE_TYPES["librt.internal.ReadBuffer"] -# Buffer(source) +# ReadBuffer(source) function_op( - name="librt.internal.Buffer", + name="librt.internal.ReadBuffer", arg_types=[bytes_rprimitive], - return_type=buffer_rprimitive, - c_function_name="Buffer_internal", + return_type=read_buffer_rprimitive, + c_function_name="ReadBuffer_internal", error_kind=ERR_MAGIC, ) -# Buffer() +# WriteBuffer() function_op( - name="librt.internal.Buffer", + name="librt.internal.WriteBuffer", arg_types=[], - return_type=buffer_rprimitive, - c_function_name="Buffer_internal_empty", + return_type=write_buffer_rprimitive, + c_function_name="WriteBuffer_internal", error_kind=ERR_MAGIC, ) method_op( name="getvalue", - arg_types=[buffer_rprimitive], + arg_types=[write_buffer_rprimitive], return_type=bytes_rprimitive, - c_function_name="Buffer_getvalue_internal", + c_function_name="WriteBuffer_getvalue_internal", error_kind=ERR_MAGIC, ) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a8ee7213ef96..0f8ec2b094f0 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1453,7 +1453,7 @@ class TestOverload: from typing import Final from mypy_extensions import u8 from librt.internal import ( - Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, + WriteBuffer, ReadBuffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, cache_version, ) @@ -1462,7 +1462,7 @@ Tag = u8 TAG: Final[Tag] = 1 def foo() -> None: - b = Buffer() + b = WriteBuffer() write_str(b, "foo") write_bytes(b, b"bar") write_bool(b, True) @@ -1470,23 +1470,23 @@ def foo() -> None: write_int(b, 1) write_tag(b, TAG) - b = Buffer(b.getvalue()) - x = read_str(b) - xb = read_bytes(b) - y = read_bool(b) - z = read_float(b) - t = read_int(b) - u = read_tag(b) + rb = ReadBuffer(b.getvalue()) + x = read_str(rb) + xb = read_bytes(rb) + y = read_bool(rb) + z = read_float(rb) + t = read_int(rb) + u = read_tag(rb) v = cache_version() [out] def foo(): - r0, b :: librt.internal.Buffer + r0, b :: librt.internal.WriteBuffer r1 :: str r2 :: None r3 :: bytes r4, r5, r6, r7, r8 :: None r9 :: bytes - r10 :: librt.internal.Buffer + r10, rb :: librt.internal.ReadBuffer r11, x :: str r12, xb :: bytes r13, y :: bool @@ -1494,7 +1494,7 @@ def foo(): r15, t :: int r16, u, r17, v :: u8 L0: - r0 = Buffer_internal_empty() + r0 = WriteBuffer_internal() b = r0 r1 = 'foo' r2 = write_str_internal(b, r1) @@ -1504,20 +1504,20 @@ L0: r6 = write_float_internal(b, 0.1) r7 = write_int_internal(b, 2) r8 = write_tag_internal(b, 1) - r9 = Buffer_getvalue_internal(b) - r10 = Buffer_internal(r9) - b = r10 - r11 = read_str_internal(b) + r9 = WriteBuffer_getvalue_internal(b) + r10 = ReadBuffer_internal(r9) + rb = r10 + r11 = read_str_internal(rb) x = r11 - r12 = read_bytes_internal(b) + r12 = read_bytes_internal(rb) xb = r12 - r13 = read_bool_internal(b) + r13 = read_bool_internal(rb) y = r13 - r14 = read_float_internal(b) + r14 = read_float_internal(rb) z = r14 - r15 = read_int_internal(b) + r15 = read_int_internal(rb) t = r15 - r16 = read_tag_internal(b) + r16 = read_tag_internal(rb) u = r16 r17 = cache_version_internal() v = r17 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 0805da184e1a..2c2eac505797 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2711,14 +2711,18 @@ from native import Player Player.MIN = [case testBufferRoundTrip_librt_internal] -from typing import Final +from __future__ import annotations + +from typing import Final, Any from mypy_extensions import u8 from librt.internal import ( - Buffer, write_bool, read_bool, write_str, read_str, write_float, read_float, + ReadBuffer, WriteBuffer, write_bool, read_bool, write_str, read_str, write_float, read_float, write_int, read_int, write_tag, read_tag, write_bytes, read_bytes, cache_version, ) +from testutil import assertRaises + Tag = u8 TAG_A: Final[Tag] = 33 TAG_B: Final[Tag] = 255 @@ -2726,11 +2730,89 @@ TAG_SPECIAL: Final[Tag] = 239 def test_buffer_basic() -> None: assert cache_version() == 0 - b = Buffer(b"foo") - assert b.getvalue() == b"foo" + w = WriteBuffer() + write_str(w, "foo") + r = ReadBuffer(w.getvalue()) + assert read_str(r) == "foo" + +def test_buffer_grow() -> None: + w = WriteBuffer() + n = 100 * 1000 + for i in range(n): + write_int(w, i & 63) + r = ReadBuffer(w.getvalue()) + for i in range(n): + assert read_int(r) == (i & 63) + with assertRaises(ValueError): + read_int(r) + +def test_buffer_primitive_types() -> None: + a1: Any = WriteBuffer() + w: WriteBuffer = a1 + write_str(w, "foo") + data = w.getvalue() + assert read_str(ReadBuffer(data)) == "foo" + a2: Any = ReadBuffer(b"foo") + with assertRaises(TypeError): + w2: WriteBuffer = a2 + + a3: Any = ReadBuffer(data) + r: ReadBuffer = a3 + assert read_str(r) == "foo" + a4: Any = WriteBuffer() + with assertRaises(TypeError): + r2: ReadBuffer = a4 + +def test_type_check_args_in_write_functions() -> None: + # Test calling wrapper functions with invalid arg types + from librt import internal + alias: Any = internal + w = WriteBuffer() + with assertRaises(TypeError): + alias.write_str(None, "foo") + with assertRaises(TypeError): + alias.write_str(w, None) + with assertRaises(TypeError): + alias.write_bool(None, True) + with assertRaises(TypeError): + alias.write_bool(w, None) + with assertRaises(TypeError): + alias.write_bytes(None, b"foo") + with assertRaises(TypeError): + alias.write_bytes(w, None) + with assertRaises(TypeError): + alias.write_float(None, 1.5) + with assertRaises(TypeError): + alias.write_float(w, None) + with assertRaises(TypeError): + alias.write_int(None, 15) + with assertRaises(TypeError): + alias.write_int(w, None) + with assertRaises(TypeError): + alias.write_tag(None, 15) + with assertRaises(TypeError): + alias.write_tag(w, None) + +def test_type_check_buffer_in_read_functions() -> None: + # Test calling wrapper functions with invalid arg types + from librt import internal + alias: Any = internal + with assertRaises(TypeError): + alias.read_str(None) + with assertRaises(TypeError): + alias.read_bool(None) + with assertRaises(TypeError): + alias.read_bytes(None) + with assertRaises(TypeError): + alias.read_float(None) + with assertRaises(TypeError): + alias.read_int(None) + with assertRaises(TypeError): + alias.read_tag(None) def test_buffer_roundtrip() -> None: - b = Buffer() + b: WriteBuffer | ReadBuffer + b = WriteBuffer() write_str(b, "foo") write_bool(b, True) write_str(b, "bar" * 1000) @@ -2757,7 +2839,7 @@ def test_buffer_roundtrip() -> None: write_int(b, 536860912) write_int(b, 1234567891) - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == "foo" assert read_bool(b) is True assert read_str(b) == "bar" * 1000 @@ -2785,77 +2867,83 @@ def test_buffer_roundtrip() -> None: assert read_int(b) == 1234567891 def test_buffer_int_size() -> None: + b: WriteBuffer | ReadBuffer for i in (-10, -9, 0, 116, 117): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-100, -11, 118, 12344, 16283): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-10000, 16284, 123456789): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 4 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i def test_buffer_int_powers() -> None: # 0, 1, 2 are tested above for p in range(2, 200): - b = Buffer() + b = WriteBuffer() write_int(b, 1 << p) write_int(b, (1 << p) - 1) write_int(b, -1 << p) write_int(b, (-1 << p) + 1) - b = Buffer(b.getvalue()) - assert read_int(b) == 1 << p - assert read_int(b) == (1 << p) - 1 - assert read_int(b) == -1 << p - assert read_int(b) == (-1 << p) + 1 + rb = ReadBuffer(b.getvalue()) + assert read_int(rb) == 1 << p + assert read_int(rb) == (1 << p) - 1 + assert read_int(rb) == -1 << p + assert read_int(rb) == (-1 << p) + 1 def test_positive_long_int_serialized_bytes() -> None: - b = Buffer() + b = WriteBuffer() n = 0x123456789ab write_int(b, n) x = b.getvalue() # Two prefix bytes, followed by little endian encoded integer in variable-length format assert x == b"\x0f\x2c\xab\x89\x67\x45\x23\x01" - b = Buffer(x) - assert read_int(b) == n + rb = ReadBuffer(x) + assert read_int(rb) == n def test_negative_long_int_serialized_bytes() -> None: - b = Buffer() + b = WriteBuffer() n = -0x123456789abcde write_int(b, n) x = b.getvalue() assert x == b"\x0f\x32\xde\xbc\x9a\x78\x56\x34\x12" - b = Buffer(x) - assert read_int(b) == n + rb = ReadBuffer(x) + assert read_int(rb) == n def test_buffer_str_size() -> None: + b: WriteBuffer | ReadBuffer for s in ("", "a", "a" * 117): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s for s in ("a" * 118, "a" * 16283): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s [file driver.py] from native import * test_buffer_basic() +test_buffer_grow() +test_buffer_primitive_types() +test_type_check_args_in_write_functions() +test_type_check_buffer_in_read_functions() test_buffer_roundtrip() test_buffer_int_size() test_buffer_str_size() @@ -2864,11 +2952,13 @@ test_positive_long_int_serialized_bytes() test_negative_long_int_serialized_bytes() def test_buffer_basic_interpreted() -> None: - b = Buffer(b"foo") - assert b.getvalue() == b"foo" + b = WriteBuffer() + write_str(b, "foo") + b = ReadBuffer(b.getvalue()) + assert read_str(b) == "foo" def test_buffer_roundtrip_interpreted() -> None: - b = Buffer() + b = WriteBuffer() write_str(b, "foo") write_bool(b, True) write_str(b, "bar" * 1000) @@ -2893,7 +2983,7 @@ def test_buffer_roundtrip_interpreted() -> None: write_int(b, 536860912) write_int(b, 1234567891) - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == "foo" assert read_bool(b) is True assert read_str(b) == "bar" * 1000 @@ -2920,47 +3010,47 @@ def test_buffer_roundtrip_interpreted() -> None: def test_buffer_int_size_interpreted() -> None: for i in (-10, -9, 0, 116, 117): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-100, -11, 118, 12344, 16283): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i for i in (-10000, 16284, 123456789): - b = Buffer() + b = WriteBuffer() write_int(b, i) assert len(b.getvalue()) == 4 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == i def test_buffer_int_powers_interpreted() -> None: # 0, 1, 2 are tested above for p in range(2, 9): - b = Buffer() + b = WriteBuffer() write_int(b, 1 << p) write_int(b, -1 << p) - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_int(b) == 1 << p assert read_int(b) == -1 << p def test_buffer_str_size_interpreted() -> None: for s in ("", "a", "a" * 117): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 1 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s for s in ("a" * 118, "a" * 16283): - b = Buffer() + b = WriteBuffer() write_str(b, s) assert len(b.getvalue()) == len(s) + 2 - b = Buffer(b.getvalue()) + b = ReadBuffer(b.getvalue()) assert read_str(b) == s test_buffer_basic_interpreted() @@ -2970,12 +3060,12 @@ test_buffer_str_size_interpreted() test_buffer_int_powers_interpreted() [case testBufferEmpty_librt_internal] -from librt.internal import Buffer, write_int, read_int +from librt.internal import WriteBuffer, ReadBuffer, write_int, read_int def test_empty() -> None: - b = Buffer(b"") + b = WriteBuffer() write_int(b, 42) - b1 = Buffer(b.getvalue()) + b1 = ReadBuffer(b.getvalue()) assert read_int(b1) == 42 [case testEnumMethodCalls] @@ -5362,37 +5452,37 @@ test_deletable_attr() [case testBufferCorruptedData_librt_internal] from librt.internal import ( - Buffer, read_bool, read_str, read_float, read_int, read_tag, read_bytes + ReadBuffer, read_bool, read_str, read_float, read_int, read_tag, read_bytes ) from random import randbytes def check(data: bytes) -> None: - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_bool(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) read_tag(b) # Always succeeds try: while True: read_int(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_str(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_bytes(b) except ValueError: pass - b = Buffer(data) + b = ReadBuffer(data) try: while True: read_float(b) From 68d8f9d626f29b20d72cdcb31bb9cf22a80d035d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Nov 2025 17:37:01 +0000 Subject: [PATCH 159/183] Update librt dependency to 0.5.0 (#20214) This new version has breaking API and ABI changes (see https://github.com/python/mypy/pull/20194 for context). This is okay since we haven't made a public release with librt as a dependency yet. (Note that librt is part of the mypy project but distributed separately.) --- mypy-requirements.txt | 2 +- mypy/build.py | 12 +-- mypy/cache.py | 55 +++++----- mypy/exportjson.py | 4 +- mypy/nodes.py | 71 ++++++------- mypy/types.py | 105 ++++++++++--------- mypy/typeshed/stubs/librt/librt/internal.pyi | 30 +++--- pyproject.toml | 4 +- test-requirements.txt | 2 +- 9 files changed, 141 insertions(+), 144 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 7c83178ae1eb..a69d31088e55 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.4.0 +librt>=0.5.0 diff --git a/mypy/build.py b/mypy/build.py index e9c50ce6b224..853e54e445ac 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -31,7 +31,7 @@ from librt.internal import cache_version import mypy.semanal_main -from mypy.cache import CACHE_VERSION, Buffer, CacheMeta +from mypy.cache import CACHE_VERSION, CacheMeta, ReadBuffer, WriteBuffer from mypy.checker import TypeChecker from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error @@ -1343,7 +1343,7 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No if meta[0] != cache_version() or meta[1] != CACHE_VERSION: manager.log(f"Metadata abandoned for {id}: incompatible cache format") return None - data_io = Buffer(meta[2:]) + data_io = ReadBuffer(meta[2:]) m = CacheMeta.read(data_io, data_file) else: m = CacheMeta.deserialize(meta, data_file) @@ -1594,7 +1594,7 @@ def write_cache( # Serialize data and analyze interface if manager.options.fixed_format_cache: - data_io = Buffer() + data_io = WriteBuffer() tree.write(data_io) data_bytes = data_io.getvalue() else: @@ -1678,7 +1678,7 @@ def write_cache_meta(meta: CacheMeta, manager: BuildManager, meta_file: str) -> # Write meta cache file metastore = manager.metastore if manager.options.fixed_format_cache: - data_io = Buffer() + data_io = WriteBuffer() meta.write(data_io) # Prefix with both low- and high-level cache format versions for future validation. # TODO: switch to something like librt.internal.write_byte() if this is slow. @@ -2111,7 +2111,7 @@ def load_tree(self, temporary: bool = False) -> None: t0 = time.time() # TODO: Assert data file wasn't changed. if isinstance(data, bytes): - data_io = Buffer(data) + data_io = ReadBuffer(data) self.tree = MypyFile.read(data_io) else: self.tree = MypyFile.deserialize(data) @@ -2484,7 +2484,7 @@ def write_cache(self) -> tuple[CacheMeta, str] | None: if self.options.debug_serialize: try: if self.manager.options.fixed_format_cache: - data = Buffer() + data = WriteBuffer() self.tree.write(data) else: self.tree.serialize() diff --git a/mypy/cache.py b/mypy/cache.py index 900815b9f7e7..ad12fd96f1fa 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -52,7 +52,8 @@ from typing_extensions import TypeAlias as _TypeAlias from librt.internal import ( - Buffer as Buffer, + ReadBuffer as ReadBuffer, + WriteBuffer as WriteBuffer, read_bool as read_bool, read_bytes as read_bytes_bare, read_float as read_float_bare, @@ -165,7 +166,7 @@ def deserialize(cls, meta: dict[str, Any], data_file: str) -> CacheMeta | None: except (KeyError, ValueError): return None - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_str(data, self.id) write_str(data, self.path) write_int(data, self.mtime) @@ -187,7 +188,7 @@ def write(self, data: Buffer) -> None: write_json_value(data, self.plugin_data) @classmethod - def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: + def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None: try: return CacheMeta( id=read_str(data), @@ -240,7 +241,7 @@ def read(cls, data: Buffer, data_file: str) -> CacheMeta | None: END_TAG: Final[Tag] = 255 -def read_literal(data: Buffer, tag: Tag) -> int | str | bool | float: +def read_literal(data: ReadBuffer, tag: Tag) -> int | str | bool | float: if tag == LITERAL_INT: return read_int_bare(data) elif tag == LITERAL_STR: @@ -256,7 +257,7 @@ def read_literal(data: Buffer, tag: Tag) -> int | str | bool | float: # There is an intentional asymmetry between read and write for literals because # None and/or complex values are only allowed in some contexts but not in others. -def write_literal(data: Buffer, value: int | str | bool | float | complex | None) -> None: +def write_literal(data: WriteBuffer, value: int | str | bool | float | complex | None) -> None: if isinstance(value, bool): write_bool(data, value) elif isinstance(value, int): @@ -276,37 +277,37 @@ def write_literal(data: Buffer, value: int | str | bool | float | complex | None write_tag(data, LITERAL_NONE) -def read_int(data: Buffer) -> int: +def read_int(data: ReadBuffer) -> int: assert read_tag(data) == LITERAL_INT return read_int_bare(data) -def write_int(data: Buffer, value: int) -> None: +def write_int(data: WriteBuffer, value: int) -> None: write_tag(data, LITERAL_INT) write_int_bare(data, value) -def read_str(data: Buffer) -> str: +def read_str(data: ReadBuffer) -> str: assert read_tag(data) == LITERAL_STR return read_str_bare(data) -def write_str(data: Buffer, value: str) -> None: +def write_str(data: WriteBuffer, value: str) -> None: write_tag(data, LITERAL_STR) write_str_bare(data, value) -def read_bytes(data: Buffer) -> bytes: +def read_bytes(data: ReadBuffer) -> bytes: assert read_tag(data) == LITERAL_BYTES return read_bytes_bare(data) -def write_bytes(data: Buffer, value: bytes) -> None: +def write_bytes(data: WriteBuffer, value: bytes) -> None: write_tag(data, LITERAL_BYTES) write_bytes_bare(data, value) -def read_int_opt(data: Buffer) -> int | None: +def read_int_opt(data: ReadBuffer) -> int | None: tag = read_tag(data) if tag == LITERAL_NONE: return None @@ -314,7 +315,7 @@ def read_int_opt(data: Buffer) -> int | None: return read_int_bare(data) -def write_int_opt(data: Buffer, value: int | None) -> None: +def write_int_opt(data: WriteBuffer, value: int | None) -> None: if value is not None: write_tag(data, LITERAL_INT) write_int_bare(data, value) @@ -322,7 +323,7 @@ def write_int_opt(data: Buffer, value: int | None) -> None: write_tag(data, LITERAL_NONE) -def read_str_opt(data: Buffer) -> str | None: +def read_str_opt(data: ReadBuffer) -> str | None: tag = read_tag(data) if tag == LITERAL_NONE: return None @@ -330,7 +331,7 @@ def read_str_opt(data: Buffer) -> str | None: return read_str_bare(data) -def write_str_opt(data: Buffer, value: str | None) -> None: +def write_str_opt(data: WriteBuffer, value: str | None) -> None: if value is not None: write_tag(data, LITERAL_STR) write_str_bare(data, value) @@ -338,52 +339,52 @@ def write_str_opt(data: Buffer, value: str | None) -> None: write_tag(data, LITERAL_NONE) -def read_int_list(data: Buffer) -> list[int]: +def read_int_list(data: ReadBuffer) -> list[int]: assert read_tag(data) == LIST_INT size = read_int_bare(data) return [read_int_bare(data) for _ in range(size)] -def write_int_list(data: Buffer, value: list[int]) -> None: +def write_int_list(data: WriteBuffer, value: list[int]) -> None: write_tag(data, LIST_INT) write_int_bare(data, len(value)) for item in value: write_int_bare(data, item) -def read_str_list(data: Buffer) -> list[str]: +def read_str_list(data: ReadBuffer) -> list[str]: assert read_tag(data) == LIST_STR size = read_int_bare(data) return [read_str_bare(data) for _ in range(size)] -def write_str_list(data: Buffer, value: Sequence[str]) -> None: +def write_str_list(data: WriteBuffer, value: Sequence[str]) -> None: write_tag(data, LIST_STR) write_int_bare(data, len(value)) for item in value: write_str_bare(data, item) -def read_bytes_list(data: Buffer) -> list[bytes]: +def read_bytes_list(data: ReadBuffer) -> list[bytes]: assert read_tag(data) == LIST_BYTES size = read_int_bare(data) return [read_bytes_bare(data) for _ in range(size)] -def write_bytes_list(data: Buffer, value: Sequence[bytes]) -> None: +def write_bytes_list(data: WriteBuffer, value: Sequence[bytes]) -> None: write_tag(data, LIST_BYTES) write_int_bare(data, len(value)) for item in value: write_bytes_bare(data, item) -def read_str_opt_list(data: Buffer) -> list[str | None]: +def read_str_opt_list(data: ReadBuffer) -> list[str | None]: assert read_tag(data) == LIST_GEN size = read_int_bare(data) return [read_str_opt(data) for _ in range(size)] -def write_str_opt_list(data: Buffer, value: list[str | None]) -> None: +def write_str_opt_list(data: WriteBuffer, value: list[str | None]) -> None: write_tag(data, LIST_GEN) write_int_bare(data, len(value)) for item in value: @@ -393,7 +394,7 @@ def write_str_opt_list(data: Buffer, value: list[str | None]) -> None: JsonValue: _TypeAlias = Union[None, int, str, bool, list["JsonValue"], dict[str, "JsonValue"]] -def read_json_value(data: Buffer) -> JsonValue: +def read_json_value(data: ReadBuffer) -> JsonValue: tag = read_tag(data) if tag == LITERAL_NONE: return None @@ -416,7 +417,7 @@ def read_json_value(data: Buffer) -> JsonValue: # Currently tuples are used by mypyc plugin. They will be normalized to # JSON lists after a roundtrip. -def write_json_value(data: Buffer, value: JsonValue | tuple[JsonValue, ...]) -> None: +def write_json_value(data: WriteBuffer, value: JsonValue | tuple[JsonValue, ...]) -> None: if value is None: write_tag(data, LITERAL_NONE) elif isinstance(value, bool): @@ -444,13 +445,13 @@ def write_json_value(data: Buffer, value: JsonValue | tuple[JsonValue, ...]) -> # These are functions for JSON *dictionaries* specifically. Unfortunately, we # must use imprecise types here, because the callers use imprecise types. -def read_json(data: Buffer) -> dict[str, Any]: +def read_json(data: ReadBuffer) -> dict[str, Any]: assert read_tag(data) == DICT_STR_GEN size = read_int_bare(data) return {read_str_bare(data): read_json_value(data) for _ in range(size)} -def write_json(data: Buffer, value: dict[str, Any]) -> None: +def write_json(data: WriteBuffer, value: dict[str, Any]) -> None: write_tag(data, DICT_STR_GEN) write_int_bare(data, len(value)) for key in sorted(value): diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 09945f0ef28f..dfc1cf5abbc6 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -19,7 +19,7 @@ from typing import Any, Union from typing_extensions import TypeAlias as _TypeAlias -from librt.internal import Buffer +from librt.internal import ReadBuffer from mypy.nodes import ( FUNCBASE_FLAGS, @@ -78,7 +78,7 @@ def __init__(self, *, implicit_names: bool = True) -> None: def convert_binary_cache_to_json(data: bytes, *, implicit_names: bool = True) -> Json: - tree = MypyFile.read(Buffer(data)) + tree = MypyFile.read(ReadBuffer(data)) return convert_mypy_file_to_json(tree, Config(implicit_names=implicit_names)) diff --git a/mypy/nodes.py b/mypy/nodes.py index 13ba011eebc0..e7d7e84d5ac2 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -28,8 +28,9 @@ LIST_STR, LITERAL_COMPLEX, LITERAL_NONE, - Buffer, + ReadBuffer, Tag, + WriteBuffer, read_bool, read_int, read_int_list, @@ -285,11 +286,11 @@ def deserialize(cls, data: JsonDict) -> SymbolNode: return method(data) raise NotImplementedError(f"unexpected .class {classname}") - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: raise NotImplementedError(f"Cannot serialize {self.__class__.__name__} instance") @classmethod - def read(cls, data: Buffer) -> SymbolNode: + def read(cls, data: ReadBuffer) -> SymbolNode: raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") @@ -441,7 +442,7 @@ def deserialize(cls, data: JsonDict) -> MypyFile: tree.future_import_flags = set(data["future_import_flags"]) return tree - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, MYPY_FILE) write_str(data, self._fullname) self.names.write(data, self._fullname) @@ -452,7 +453,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> MypyFile: + def read(cls, data: ReadBuffer) -> MypyFile: assert read_tag(data) == MYPY_FILE tree = MypyFile([], []) tree._fullname = read_str(data) @@ -737,7 +738,7 @@ def deserialize(cls, data: JsonDict) -> OverloadedFuncDef: # NOTE: res.info will be set in the fixup phase. return res - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, OVERLOADED_FUNC_DEF) write_tag(data, LIST_GEN) write_int_bare(data, len(self.items)) @@ -755,7 +756,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> OverloadedFuncDef: + def read(cls, data: ReadBuffer) -> OverloadedFuncDef: assert read_tag(data) == LIST_GEN res = OverloadedFuncDef([read_overload_part(data) for _ in range(read_int_bare(data))]) typ = mypy.types.read_type_opt(data) @@ -1052,7 +1053,7 @@ def deserialize(cls, data: JsonDict) -> FuncDef: del ret.min_args return ret - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, FUNC_DEF) write_str(data, self._name) mypy.types.write_type_opt(data, self.type) @@ -1070,7 +1071,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> FuncDef: + def read(cls, data: ReadBuffer) -> FuncDef: name = read_str(data) typ: mypy.types.FunctionLike | None = None tag = read_tag(data) @@ -1168,7 +1169,7 @@ def deserialize(cls, data: JsonDict) -> Decorator: dec.is_overload = data["is_overload"] return dec - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, DECORATOR) self.func.write(data) self.var.write(data) @@ -1176,7 +1177,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Decorator: + def read(cls, data: ReadBuffer) -> Decorator: assert read_tag(data) == FUNC_DEF func = FuncDef.read(data) assert read_tag(data) == VAR @@ -1362,7 +1363,7 @@ def deserialize(cls, data: JsonDict) -> Var: v.final_value = data.get("final_value") return v - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, VAR) write_str(data, self._name) mypy.types.write_type_opt(data, self.type) @@ -1373,7 +1374,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Var: + def read(cls, data: ReadBuffer) -> Var: name = read_str(data) typ = mypy.types.read_type_opt(data) v = Var(name, typ) @@ -1504,7 +1505,7 @@ def deserialize(cls, data: JsonDict) -> ClassDef: res.fullname = data["fullname"] return res - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, CLASS_DEF) write_str(data, self.name) mypy.types.write_type_list(data, self.type_vars) @@ -1512,7 +1513,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> ClassDef: + def read(cls, data: ReadBuffer) -> ClassDef: res = ClassDef(read_str(data), Block([]), mypy.types.read_type_var_likes(data)) res.fullname = read_str(data) assert read_tag(data) == END_TAG @@ -2975,7 +2976,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarExpr: data["variance"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_VAR_EXPR) write_str(data, self._name) write_str(data, self._fullname) @@ -2986,7 +2987,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeVarExpr: + def read(cls, data: ReadBuffer) -> TypeVarExpr: ret = TypeVarExpr( read_str(data), read_str(data), @@ -3028,7 +3029,7 @@ def deserialize(cls, data: JsonDict) -> ParamSpecExpr: data["variance"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, PARAM_SPEC_EXPR) write_str(data, self._name) write_str(data, self._fullname) @@ -3038,7 +3039,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> ParamSpecExpr: + def read(cls, data: ReadBuffer) -> ParamSpecExpr: ret = ParamSpecExpr( read_str(data), read_str(data), @@ -3099,7 +3100,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarTupleExpr: data["variance"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_VAR_TUPLE_EXPR) self.tuple_fallback.write(data) write_str(data, self._name) @@ -3110,7 +3111,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeVarTupleExpr: + def read(cls, data: ReadBuffer) -> TypeVarTupleExpr: assert read_tag(data) == mypy.types.INSTANCE fallback = mypy.types.Instance.read(data) ret = TypeVarTupleExpr( @@ -3994,7 +3995,7 @@ def deserialize(cls, data: JsonDict) -> TypeInfo: ti.deprecated = data.get("deprecated") return ti - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_INFO) self.names.write(data, self.fullname) self.defn.write(data) @@ -4028,7 +4029,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeInfo: + def read(cls, data: ReadBuffer) -> TypeInfo: names = SymbolTable.read(data) assert read_tag(data) == CLASS_DEF defn = ClassDef.read(data) @@ -4363,7 +4364,7 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: python_3_12_type_alias=python_3_12_type_alias, ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_ALIAS) write_str(data, self._fullname) write_str(data, self.module) @@ -4375,7 +4376,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeAlias: + def read(cls, data: ReadBuffer) -> TypeAlias: fullname = read_str(data) module = read_str(data) target = mypy.types.read_type(data) @@ -4652,7 +4653,7 @@ def deserialize(cls, data: JsonDict) -> SymbolTableNode: stnode.plugin_generated = data["plugin_generated"] return stnode - def write(self, data: Buffer, prefix: str, name: str) -> None: + def write(self, data: WriteBuffer, prefix: str, name: str) -> None: write_tag(data, SYMBOL_TABLE_NODE) write_int(data, self.kind) write_bool(data, self.module_hidden) @@ -4684,7 +4685,7 @@ def write(self, data: Buffer, prefix: str, name: str) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> SymbolTableNode: + def read(cls, data: ReadBuffer) -> SymbolTableNode: assert read_tag(data) == SYMBOL_TABLE_NODE sym = SymbolTableNode(read_int(data), None) sym.module_hidden = read_bool(data) @@ -4750,7 +4751,7 @@ def deserialize(cls, data: JsonDict) -> SymbolTable: st[key] = SymbolTableNode.deserialize(value) return st - def write(self, data: Buffer, fullname: str) -> None: + def write(self, data: WriteBuffer, fullname: str) -> None: size = 0 for key, value in self.items(): # Skip __builtins__: it's a reference to the builtins @@ -4771,7 +4772,7 @@ def write(self, data: Buffer, fullname: str) -> None: value.write(data, fullname, key) @classmethod - def read(cls, data: Buffer) -> SymbolTable: + def read(cls, data: ReadBuffer) -> SymbolTable: assert read_tag(data) == DICT_STR_GEN size = read_int_bare(data) return SymbolTable( @@ -4828,7 +4829,7 @@ def deserialize(cls, data: JsonDict) -> DataclassTransformSpec: field_specifiers=tuple(data.get("field_specifiers", [])), ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, DT_SPEC) write_bool(data, self.eq_default) write_bool(data, self.order_default) @@ -4838,7 +4839,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> DataclassTransformSpec: + def read(cls, data: ReadBuffer) -> DataclassTransformSpec: ret = DataclassTransformSpec( eq_default=read_bool(data), order_default=read_bool(data), @@ -4859,12 +4860,12 @@ def set_flags(node: Node, flags: list[str]) -> None: setattr(node, name, True) -def write_flags(data: Buffer, node: SymbolNode, flags: list[str]) -> None: +def write_flags(data: WriteBuffer, node: SymbolNode, flags: list[str]) -> None: for flag in flags: write_bool(data, getattr(node, flag)) -def read_flags(data: Buffer, node: SymbolNode, flags: list[str]) -> None: +def read_flags(data: ReadBuffer, node: SymbolNode, flags: list[str]) -> None: for flag in flags: if read_bool(data): setattr(node, flag, True) @@ -5002,7 +5003,7 @@ def local_definitions( SYMBOL_TABLE_NODE: Final[Tag] = 61 -def read_symbol(data: Buffer) -> SymbolNode: +def read_symbol(data: ReadBuffer) -> SymbolNode: tag = read_tag(data) # The branches here are ordered manually by type "popularity". if tag == VAR: @@ -5026,7 +5027,7 @@ def read_symbol(data: Buffer) -> SymbolNode: assert False, f"Unknown symbol tag {tag}" -def read_overload_part(data: Buffer, tag: Tag | None = None) -> OverloadPart: +def read_overload_part(data: ReadBuffer, tag: Tag | None = None) -> OverloadPart: if tag is None: tag = read_tag(data) if tag == DECORATOR: diff --git a/mypy/types.py b/mypy/types.py index 7a8343097204..056b99cc3f91 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -23,8 +23,9 @@ EXTRA_ATTRS, LIST_GEN, LITERAL_NONE, - Buffer, + ReadBuffer, Tag, + WriteBuffer, read_bool, read_int, read_int_list, @@ -312,11 +313,11 @@ def serialize(self) -> JsonDict | str: def deserialize(cls, data: JsonDict) -> Type: raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: raise NotImplementedError(f"Cannot serialize {self.__class__.__name__} instance") @classmethod - def read(cls, data: Buffer) -> Type: + def read(cls, data: ReadBuffer) -> Type: raise NotImplementedError(f"Cannot deserialize {cls.__name__} instance") def is_singleton_type(self) -> bool: @@ -449,7 +450,7 @@ def deserialize(cls, data: JsonDict) -> TypeAliasType: alias.type_ref = data["type_ref"] return alias - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_ALIAS_TYPE) write_type_list(data, self.args) assert self.alias is not None @@ -457,7 +458,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeAliasType: + def read(cls, data: ReadBuffer) -> TypeAliasType: alias = TypeAliasType(None, read_type_list(data)) alias.type_ref = read_str(data) assert read_tag(data) == END_TAG @@ -730,7 +731,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarType: variance=data["variance"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_VAR_TYPE) write_str(data, self.name) write_str(data, self.fullname) @@ -743,7 +744,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeVarType: + def read(cls, data: ReadBuffer) -> TypeVarType: ret = TypeVarType( read_str(data), read_str(data), @@ -885,7 +886,7 @@ def deserialize(cls, data: JsonDict) -> ParamSpecType: prefix=Parameters.deserialize(data["prefix"]), ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, PARAM_SPEC_TYPE) self.prefix.write(data) write_str(data, self.name) @@ -898,7 +899,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> ParamSpecType: + def read(cls, data: ReadBuffer) -> ParamSpecType: assert read_tag(data) == PARAMETERS prefix = Parameters.read(data) ret = ParamSpecType( @@ -968,7 +969,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarTupleType: min_len=data["min_len"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_VAR_TUPLE_TYPE) self.tuple_fallback.write(data) write_str(data, self.name) @@ -981,7 +982,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypeVarTupleType: + def read(cls, data: ReadBuffer) -> TypeVarTupleType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) ret = TypeVarTupleType( @@ -1127,7 +1128,7 @@ def deserialize(cls, data: JsonDict) -> UnboundType: original_str_fallback=data["expr_fallback"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, UNBOUND_TYPE) write_str(data, self.name) write_type_list(data, self.args) @@ -1136,7 +1137,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> UnboundType: + def read(cls, data: ReadBuffer) -> UnboundType: ret = UnboundType( read_str(data), read_type_list(data), @@ -1240,13 +1241,13 @@ def accept(self, visitor: TypeVisitor[T]) -> T: def serialize(self) -> JsonDict: return {".class": "UnpackType", "type": self.type.serialize()} - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, UNPACK_TYPE) self.type.write(data) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> UnpackType: + def read(cls, data: ReadBuffer) -> UnpackType: ret = UnpackType(read_type(data)) assert read_tag(data) == END_TAG return ret @@ -1352,7 +1353,7 @@ def deserialize(cls, data: JsonDict) -> AnyType: data["missing_import_name"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, ANY_TYPE) write_type_opt(data, self.source_any) write_int(data, self.type_of_any) @@ -1360,7 +1361,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> AnyType: + def read(cls, data: ReadBuffer) -> AnyType: tag = read_tag(data) if tag != LITERAL_NONE: assert tag == ANY_TYPE @@ -1417,12 +1418,12 @@ def deserialize(cls, data: JsonDict) -> UninhabitedType: assert data[".class"] == "UninhabitedType" return UninhabitedType() - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, UNINHABITED_TYPE) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> UninhabitedType: + def read(cls, data: ReadBuffer) -> UninhabitedType: assert read_tag(data) == END_TAG return UninhabitedType() @@ -1458,12 +1459,12 @@ def deserialize(cls, data: JsonDict) -> NoneType: assert data[".class"] == "NoneType" return NoneType() - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, NONE_TYPE) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> NoneType: + def read(cls, data: ReadBuffer) -> NoneType: assert read_tag(data) == END_TAG return NoneType() @@ -1514,13 +1515,13 @@ def deserialize(cls, data: JsonDict) -> DeletedType: assert data[".class"] == "DeletedType" return DeletedType(data["source"]) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, DELETED_TYPE) write_str_opt(data, self.source) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> DeletedType: + def read(cls, data: ReadBuffer) -> DeletedType: ret = DeletedType(read_str_opt(data)) assert read_tag(data) == END_TAG return ret @@ -1580,7 +1581,7 @@ def deserialize(cls, data: JsonDict) -> ExtraAttrs: data["mod_name"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, EXTRA_ATTRS) write_type_map(data, self.attrs) write_str_list(data, sorted(self.immutable)) @@ -1588,7 +1589,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> ExtraAttrs: + def read(cls, data: ReadBuffer) -> ExtraAttrs: ret = ExtraAttrs(read_type_map(data), set(read_str_list(data)), read_str_opt(data)) assert read_tag(data) == END_TAG return ret @@ -1729,7 +1730,7 @@ def deserialize(cls, data: JsonDict | str) -> Instance: inst.extra_attrs = ExtraAttrs.deserialize(data["extra_attrs"]) return inst - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, INSTANCE) if not self.args and not self.last_known_value and not self.extra_attrs: type_ref = self.type.fullname @@ -1758,7 +1759,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Instance: + def read(cls, data: ReadBuffer) -> Instance: tag = read_tag(data) # This is quite verbose, but this is very hot code, so we are not # using dictionary lookups here. @@ -2100,7 +2101,7 @@ def deserialize(cls, data: JsonDict) -> Parameters: imprecise_arg_kinds=data["imprecise_arg_kinds"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, PARAMETERS) write_type_list(data, self.arg_types) write_int_list(data, [int(x.value) for x in self.arg_kinds]) @@ -2110,7 +2111,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Parameters: + def read(cls, data: ReadBuffer) -> Parameters: ret = Parameters( read_type_list(data), # This is a micro-optimization until mypyc gets dedicated enum support. Otherwise, @@ -2629,7 +2630,7 @@ def deserialize(cls, data: JsonDict) -> CallableType: unpack_kwargs=data["unpack_kwargs"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, CALLABLE_TYPE) self.fallback.write(data) write_type_list(data, self.arg_types) @@ -2649,7 +2650,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> CallableType: + def read(cls, data: ReadBuffer) -> CallableType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) ret = CallableType( @@ -2747,13 +2748,13 @@ def deserialize(cls, data: JsonDict) -> Overloaded: assert data[".class"] == "Overloaded" return Overloaded([CallableType.deserialize(t) for t in data["items"]]) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, OVERLOADED) write_type_list(data, self.items) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Overloaded: + def read(cls, data: ReadBuffer) -> Overloaded: items = [] assert read_tag(data) == LIST_GEN for _ in range(read_int_bare(data)): @@ -2858,7 +2859,7 @@ def deserialize(cls, data: JsonDict) -> TupleType: implicit=data["implicit"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TUPLE_TYPE) self.partial_fallback.write(data) write_type_list(data, self.items) @@ -2866,7 +2867,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TupleType: + def read(cls, data: ReadBuffer) -> TupleType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) ret = TupleType(read_type_list(data), fallback, implicit=read_bool(data)) @@ -3043,7 +3044,7 @@ def deserialize(cls, data: JsonDict) -> TypedDictType: Instance.deserialize(data["fallback"]), ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPED_DICT_TYPE) self.fallback.write(data) write_type_map(data, self.items) @@ -3052,7 +3053,7 @@ def write(self, data: Buffer) -> None: write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> TypedDictType: + def read(cls, data: ReadBuffer) -> TypedDictType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) ret = TypedDictType( @@ -3309,14 +3310,14 @@ def deserialize(cls, data: JsonDict) -> LiteralType: assert data[".class"] == "LiteralType" return LiteralType(value=data["value"], fallback=Instance.deserialize(data["fallback"])) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, LITERAL_TYPE) self.fallback.write(data) write_literal(data, self.value) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> LiteralType: + def read(cls, data: ReadBuffer) -> LiteralType: assert read_tag(data) == INSTANCE fallback = Instance.read(data) tag = read_tag(data) @@ -3425,14 +3426,14 @@ def deserialize(cls, data: JsonDict) -> UnionType: uses_pep604_syntax=data["uses_pep604_syntax"], ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, UNION_TYPE) write_type_list(data, self.items) write_bool(data, self.uses_pep604_syntax) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> UnionType: + def read(cls, data: ReadBuffer) -> UnionType: ret = UnionType(read_type_list(data), uses_pep604_syntax=read_bool(data)) assert read_tag(data) == END_TAG return ret @@ -3594,13 +3595,13 @@ def deserialize(cls, data: JsonDict) -> Type: deserialize_type(data["item"]), is_type_form=data["is_type_form"] ) - def write(self, data: Buffer) -> None: + def write(self, data: WriteBuffer) -> None: write_tag(data, TYPE_TYPE) self.item.write(data) write_tag(data, END_TAG) @classmethod - def read(cls, data: Buffer) -> Type: + def read(cls, data: ReadBuffer) -> Type: ret = TypeType.make_normalized(read_type(data)) assert read_tag(data) == END_TAG return ret @@ -4303,7 +4304,7 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: PARAMETERS: Final[Tag] = 117 -def read_type(data: Buffer, tag: Tag | None = None) -> Type: +def read_type(data: ReadBuffer, tag: Tag | None = None) -> Type: if tag is None: tag = read_tag(data) # The branches here are ordered manually by type "popularity". @@ -4348,7 +4349,7 @@ def read_type(data: Buffer, tag: Tag | None = None) -> Type: assert False, f"Unknown type tag {tag}" -def read_function_like(data: Buffer, tag: Tag) -> FunctionLike: +def read_function_like(data: ReadBuffer, tag: Tag) -> FunctionLike: if tag == CALLABLE_TYPE: return CallableType.read(data) if tag == OVERLOADED: @@ -4356,7 +4357,7 @@ def read_function_like(data: Buffer, tag: Tag) -> FunctionLike: assert False, f"Invalid type tag for FunctionLike {tag}" -def read_type_var_likes(data: Buffer) -> list[TypeVarLikeType]: +def read_type_var_likes(data: ReadBuffer) -> list[TypeVarLikeType]: """Specialized version of read_type_list() for lists of type variables.""" assert read_tag(data) == LIST_GEN ret: list[TypeVarLikeType] = [] @@ -4373,40 +4374,40 @@ def read_type_var_likes(data: Buffer) -> list[TypeVarLikeType]: return ret -def read_type_opt(data: Buffer) -> Type | None: +def read_type_opt(data: ReadBuffer) -> Type | None: tag = read_tag(data) if tag == LITERAL_NONE: return None return read_type(data, tag) -def write_type_opt(data: Buffer, value: Type | None) -> None: +def write_type_opt(data: WriteBuffer, value: Type | None) -> None: if value is not None: value.write(data) else: write_tag(data, LITERAL_NONE) -def read_type_list(data: Buffer) -> list[Type]: +def read_type_list(data: ReadBuffer) -> list[Type]: assert read_tag(data) == LIST_GEN size = read_int_bare(data) return [read_type(data) for _ in range(size)] -def write_type_list(data: Buffer, value: Sequence[Type]) -> None: +def write_type_list(data: WriteBuffer, value: Sequence[Type]) -> None: write_tag(data, LIST_GEN) write_int_bare(data, len(value)) for item in value: item.write(data) -def read_type_map(data: Buffer) -> dict[str, Type]: +def read_type_map(data: ReadBuffer) -> dict[str, Type]: assert read_tag(data) == DICT_STR_GEN size = read_int_bare(data) return {read_str_bare(data): read_type(data) for _ in range(size)} -def write_type_map(data: Buffer, value: dict[str, Type]) -> None: +def write_type_map(data: WriteBuffer, value: dict[str, Type]) -> None: write_tag(data, DICT_STR_GEN) write_int_bare(data, len(value)) for key in sorted(value): diff --git a/mypy/typeshed/stubs/librt/librt/internal.pyi b/mypy/typeshed/stubs/librt/librt/internal.pyi index 78e7f9caa117..2969ccfbadda 100644 --- a/mypy/typeshed/stubs/librt/librt/internal.pyi +++ b/mypy/typeshed/stubs/librt/librt/internal.pyi @@ -1,27 +1,21 @@ from mypy_extensions import u8 -# TODO: Remove Buffer -- right now we have hacky support for BOTH the old and new APIs - -class Buffer: - def __init__(self, source: bytes = ...) -> None: ... - def getvalue(self) -> bytes: ... - class ReadBuffer: def __init__(self, source: bytes) -> None: ... class WriteBuffer: def getvalue(self) -> bytes: ... -def write_bool(data: WriteBuffer | Buffer, value: bool) -> None: ... -def read_bool(data: ReadBuffer | Buffer) -> bool: ... -def write_str(data: WriteBuffer | Buffer, value: str) -> None: ... -def read_str(data: ReadBuffer | Buffer) -> str: ... -def write_bytes(data: WriteBuffer | Buffer, value: bytes) -> None: ... -def read_bytes(data: ReadBuffer | Buffer) -> bytes: ... -def write_float(data: WriteBuffer | Buffer, value: float) -> None: ... -def read_float(data: ReadBuffer | Buffer) -> float: ... -def write_int(data: WriteBuffer | Buffer, value: int) -> None: ... -def read_int(data: ReadBuffer | Buffer) -> int: ... -def write_tag(data: WriteBuffer | Buffer, value: u8) -> None: ... -def read_tag(data: ReadBuffer | Buffer) -> u8: ... +def write_bool(data: WriteBuffer, value: bool) -> None: ... +def read_bool(data: ReadBuffer) -> bool: ... +def write_str(data: WriteBuffer, value: str) -> None: ... +def read_str(data: ReadBuffer) -> str: ... +def write_bytes(data: WriteBuffer, value: bytes) -> None: ... +def read_bytes(data: ReadBuffer) -> bytes: ... +def write_float(data: WriteBuffer, value: float) -> None: ... +def read_float(data: ReadBuffer) -> float: ... +def write_int(data: WriteBuffer, value: int) -> None: ... +def read_int(data: ReadBuffer) -> int: ... +def write_tag(data: WriteBuffer, value: u8) -> None: ... +def read_tag(data: ReadBuffer) -> u8: ... def cache_version() -> u8: ... diff --git a/pyproject.toml b/pyproject.toml index 0de739be9b55..42ff3a6ca019 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.4.0", + "librt>=0.5.0", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.4.0", + "librt>=0.5.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index 126abd7149e6..b65b658844d2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.4.0 +librt==0.5.0 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From 7c36b246363ff4f34e5d9130bc6d37bebdece5d4 Mon Sep 17 00:00:00 2001 From: jhance Date: Mon, 10 Nov 2025 15:31:41 -0800 Subject: [PATCH 160/183] Improve error message for librt abi mismatch (#20216) In the case where the abi does not match, it is easier to debug if we show what the expected and actual versions are. --- mypyc/lib-rt/librt_internal.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index 329a0fd68c11..bef9e196d2c1 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -64,7 +64,12 @@ import_librt_internal(void) return -1; memcpy(NativeInternal_API, capsule, sizeof(NativeInternal_API)); if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) { - PyErr_SetString(PyExc_ValueError, "ABI version conflict for librt.internal"); + char err[128]; + snprintf(err, sizeof(err), "ABI version conflict for librt.internal, expected %d, found %d", + LIBRT_INTERNAL_ABI_VERSION, + NativeInternal_ABI_Version() + ); + PyErr_SetString(PyExc_ValueError, err); return -1; } return 0; From c2ee586e56fad3b51117d72912f6c7353114c422 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Nov 2025 14:16:26 +0000 Subject: [PATCH 161/183] [librt] Add `librt.internal` API versioning with backward compat (#20221) This lets us add new API features by adding more functions to the capsule and the module namespace without breaking backward compatibility. We shouldn't use the capsule if the installed version is too old, since we could be calling functions via uninitialized pointers. Note that this is a breaking change in the `librt.internal` ABI. --- mypyc/lib-rt/librt_internal.c | 6 ++++++ mypyc/lib-rt/librt_internal.h | 27 +++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index ada2dfeb39a5..22d54de40f08 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -962,6 +962,11 @@ NativeInternal_ABI_Version(void) { return LIBRT_INTERNAL_ABI_VERSION; } +static int +NativeInternal_API_Version(void) { + return LIBRT_INTERNAL_API_VERSION; +} + static int librt_internal_module_exec(PyObject *m) { @@ -999,6 +1004,7 @@ librt_internal_module_exec(PyObject *m) (void *)cache_version_internal, (void *)ReadBuffer_type_internal, (void *)WriteBuffer_type_internal, + (void *)NativeInternal_API_Version, }; PyObject *c_api_object = PyCapsule_New((void *)NativeInternal_API, "librt.internal._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { diff --git a/mypyc/lib-rt/librt_internal.h b/mypyc/lib-rt/librt_internal.h index bef9e196d2c1..501162a27980 100644 --- a/mypyc/lib-rt/librt_internal.h +++ b/mypyc/lib-rt/librt_internal.h @@ -1,8 +1,19 @@ #ifndef LIBRT_INTERNAL_H #define LIBRT_INTERNAL_H -#define LIBRT_INTERNAL_ABI_VERSION 1 -#define LIBRT_INTERNAL_API_LEN 19 +// ABI version -- only an exact match is compatible. This will only be changed in +// very exceptional cases (likely never) due to strict backward compatibility +// requirements. +#define LIBRT_INTERNAL_ABI_VERSION 2 + +// API version -- more recent versions must maintain backward compatibility, i.e. +// we can add new features but not remove or change existing features (unless +// ABI version is changed, but see the comment above). + #define LIBRT_INTERNAL_API_VERSION 0 + +// Number of functions in the capsule API. If you add a new function, also increase +// LIBRT_INTERNAL_API_VERSION. +#define LIBRT_INTERNAL_API_LEN 20 #ifdef LIBRT_INTERNAL_MODULE @@ -27,6 +38,7 @@ static PyObject *read_bytes_internal(PyObject *data); static uint8_t cache_version_internal(void); static PyTypeObject *ReadBuffer_type_internal(void); static PyTypeObject *WriteBuffer_type_internal(void); +static int NativeInternal_API_Version(void); #else @@ -51,6 +63,7 @@ static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN]; #define cache_version_internal (*(uint8_t (*)(void)) NativeInternal_API[16]) #define ReadBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[17]) #define WriteBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[18]) +#define NativeInternal_API_Version (*(int (*)(void)) NativeInternal_API[19]) static int import_librt_internal(void) @@ -72,6 +85,16 @@ import_librt_internal(void) PyErr_SetString(PyExc_ValueError, err); return -1; } + if (NativeInternal_API_Version() < LIBRT_INTERNAL_API_VERSION) { + char err[128]; + snprintf(err, sizeof(err), + "API version conflict for librt.internal, expected %d or newer, found %d (hint: upgrade librt)", + LIBRT_INTERNAL_API_VERSION, + NativeInternal_API_Version() + ); + PyErr_SetString(PyExc_ValueError, err); + return -1; + } return 0; } From ca2240c1b4e2dd95f016a7df01728c92bce3cb89 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Nov 2025 15:28:50 +0000 Subject: [PATCH 162/183] Update librt dependency to 0.6.0 (#20225) Includes #20221 (which breaks backward compatibility). --- mypy-requirements.txt | 2 +- pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index a69d31088e55..06e0a9bffb1c 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.5.0 +librt>=0.6.0 diff --git a/pyproject.toml b/pyproject.toml index 42ff3a6ca019..336a16c48979 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.5.0", + "librt>=0.6.0", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.5.0", + "librt>=0.6.0", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index b65b658844d2..4d3f644d6bca 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.5.0 +librt==0.6.0 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From 75592ff2866e3d911f7d9a0d1cdca65501dfba5d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 13 Nov 2025 01:54:38 +0000 Subject: [PATCH 163/183] Fix compile on big-endian (#20231) --- mypyc/lib-rt/librt_internal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/librt_internal.c b/mypyc/lib-rt/librt_internal.c index 22d54de40f08..fe18c541c11f 100644 --- a/mypyc/lib-rt/librt_internal.c +++ b/mypyc/lib-rt/librt_internal.c @@ -466,7 +466,7 @@ _write_short_int(PyObject *data, Py_ssize_t real_value) { _CHECK_WRITE(data, 2) #if PY_BIG_ENDIAN uint16_t to_write = ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT; - _WRITE(data, uint16_t, reverse_16(to_write)) + _WRITE(data, uint16_t, reverse_16(to_write)); #else _WRITE(data, uint16_t, ((uint16_t)(real_value - MIN_TWO_BYTES_INT) << 2) | TWO_BYTES_INT_BIT); #endif @@ -474,7 +474,7 @@ _write_short_int(PyObject *data, Py_ssize_t real_value) { _CHECK_WRITE(data, 4) #if PY_BIG_ENDIAN uint32_t to_write = ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER; - _WRITE(data, uint32_t, reverse_32(to_write)) + _WRITE(data, uint32_t, reverse_32(to_write)); #else _WRITE(data, uint32_t, ((uint32_t)(real_value - MIN_FOUR_BYTES_INT) << 3) | FOUR_BYTES_INT_TRAILER); #endif From 568b945c299cb921e937fa21c847004a39993c2e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 13 Nov 2025 11:46:10 +0000 Subject: [PATCH 164/183] Fix crash on recursive tuple with Hashable (#20232) Fixes https://github.com/python/mypy/issues/20227 This is huge pain to reproduce, so I added just _some_ repro as a test. Note the change in `typeops.py` is not required for this fix, but it is a technically correct thing to do that I noticed while looking at this. --- mypy/typeops.py | 8 ++++---- mypy/types.py | 6 +++++- test-data/unit/check-python312.test | 12 ++++++++++++ test-data/unit/fixtures/tuple.pyi | 1 + 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index d2f9f4da44e4..050252eb6205 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -666,15 +666,15 @@ def _remove_redundant_union_items(items: list[Type], keep_erased: bool) -> list[ else: # If not, check if we've seen a supertype of this type for j, tj in enumerate(new_items): - tj = get_proper_type(tj) + proper_tj = get_proper_type(tj) # If tj is an Instance with a last_known_value, do not remove proper_ti # (unless it's an instance with the same last_known_value) if ( - isinstance(tj, Instance) - and tj.last_known_value is not None + isinstance(proper_tj, Instance) + and proper_tj.last_known_value is not None and not ( isinstance(proper_ti, Instance) - and tj.last_known_value == proper_ti.last_known_value + and proper_tj.last_known_value == proper_ti.last_known_value ) ): continue diff --git a/mypy/types.py b/mypy/types.py index 056b99cc3f91..09e4b74bb821 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -4159,7 +4159,11 @@ def flatten_nested_unions( tp = t if isinstance(tp, ProperType) and isinstance(tp, UnionType): flat_items.extend( - flatten_nested_unions(tp.items, handle_type_alias_type=handle_type_alias_type) + flatten_nested_unions( + tp.items, + handle_type_alias_type=handle_type_alias_type, + handle_recursive=handle_recursive, + ) ) else: # Must preserve original aliases when possible. diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 840a708fecf3..c8b306b6bd55 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2192,3 +2192,15 @@ y3: A3[int] # E: Type argument "int" of "A3" must be a subtype of "B3" z3: A3[None] [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] + +[case testPEP695TypeAliasRecursiveTupleUnionNoCrash] +from collections.abc import Hashable + +type HashableArg = int | tuple[Hashable | HashableArg] +x: HashableArg +reveal_type(x) # N: Revealed type is "Union[builtins.int, tuple[Union[typing.Hashable, ...]]]" +if isinstance(x, tuple): + y, = x + reveal_type(y) # N: Revealed type is "Union[typing.Hashable, Union[builtins.int, tuple[Union[typing.Hashable, ...]]]]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/fixtures/tuple.pyi b/test-data/unit/fixtures/tuple.pyi index d01cd0034d26..06dfcf5d0fbc 100644 --- a/test-data/unit/fixtures/tuple.pyi +++ b/test-data/unit/fixtures/tuple.pyi @@ -14,6 +14,7 @@ class type: def __init__(self, *a: object) -> None: pass def __call__(self, *a: object) -> object: pass class tuple(Sequence[_Tco], Generic[_Tco]): + def __hash__(self) -> int: ... def __new__(cls: Type[_T], iterable: Iterable[_Tco] = ...) -> _T: ... def __iter__(self) -> Iterator[_Tco]: pass def __contains__(self, item: object) -> bool: pass From 3d237167e6277731395c34bf459c6003b071bc3c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 13 Nov 2025 13:14:25 +0000 Subject: [PATCH 165/183] [mypyc] Add minimal, experimental librt.base64 module (#20226) The module currently only has a `b64encode` function adapted from CPython. It's only enabled when librt is compiled with experimental features enabled, so that we are free to iterate on this and break backward compatibility until we are ready to declare the module as stable. This also adds a way to define experimental features in `librt` (and mypyc in general, but it's currently only used for `librt`). In follow-up PRs I'm planning to add a more efficient implementation of `b64encode` and add more features to the module, including decoding. I'm not planning to include every feature from the stdlib base64 module, since many of them aren't used very widely. --- mypy/typeshed/stubs/librt/librt/base64.pyi | 1 + mypyc/build.py | 13 +- mypyc/codegen/emitmodule.py | 6 + mypyc/ir/ops.py | 4 + mypyc/irbuild/ll_builder.py | 2 + mypyc/lib-rt/librt_base64.c | 148 +++++++++++++++++++++ mypyc/lib-rt/librt_base64.h | 60 +++++++++ mypyc/lib-rt/setup.py | 5 +- mypyc/options.py | 7 + mypyc/primitives/misc_ops.py | 9 ++ mypyc/primitives/registry.py | 6 + mypyc/test-data/run-base64.test | 52 ++++++++ mypyc/test/test_run.py | 19 ++- 13 files changed, 327 insertions(+), 5 deletions(-) create mode 100644 mypy/typeshed/stubs/librt/librt/base64.pyi create mode 100644 mypyc/lib-rt/librt_base64.c create mode 100644 mypyc/lib-rt/librt_base64.h create mode 100644 mypyc/test-data/run-base64.test diff --git a/mypy/typeshed/stubs/librt/librt/base64.pyi b/mypy/typeshed/stubs/librt/librt/base64.pyi new file mode 100644 index 000000000000..36366f5754ce --- /dev/null +++ b/mypy/typeshed/stubs/librt/librt/base64.pyi @@ -0,0 +1 @@ +def b64encode(s: bytes) -> bytes: ... diff --git a/mypyc/build.py b/mypyc/build.py index 13648911c0b5..2ff9d175947f 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -42,7 +42,7 @@ from mypyc.namegen import exported_name from mypyc.options import CompilerOptions -LIBRT_MODULES = [("librt.internal", "librt_internal.c")] +LIBRT_MODULES = [("librt.internal", "librt_internal.c"), ("librt.base64", "librt_base64.c")] try: # Import setuptools so that it monkey-patch overrides distutils @@ -495,7 +495,9 @@ def mypycify( group_name: str | None = None, log_trace: bool = False, depends_on_librt_internal: bool = False, + depends_on_librt_base64: bool = False, install_librt: bool = False, + experimental_features: bool = False, ) -> list[Extension]: """Main entry point to building using mypyc. @@ -551,6 +553,9 @@ def mypycify( those are build and published on PyPI separately, but during tests, we want to use their development versions (i.e. from current commit). + experimental_features: Enable experimental features (install_librt=True is + also needed if using experimental librt features). These + have no backward compatibility guarantees! """ # Figure out our configuration @@ -565,6 +570,8 @@ def mypycify( group_name=group_name, log_trace=log_trace, depends_on_librt_internal=depends_on_librt_internal, + depends_on_librt_base64=depends_on_librt_base64, + experimental_features=experimental_features, ) # Generate all the actual important C code @@ -607,6 +614,8 @@ def mypycify( ] if log_trace: cflags.append("-DMYPYC_LOG_TRACE") + if experimental_features: + cflags.append("-DMYPYC_EXPERIMENTAL") elif compiler.compiler_type == "msvc": # msvc doesn't have levels, '/O2' is full and '/Od' is disable if opt_level == "0": @@ -633,6 +642,8 @@ def mypycify( cflags += ["/GL-", "/wd9025"] # warning about overriding /GL if log_trace: cflags.append("/DMYPYC_LOG_TRACE") + if experimental_features: + cflags.append("/DMYPYC_EXPERIMENTAL") # If configured to (defaults to yes in multi-file mode), copy the # runtime library in. Otherwise it just gets #included to save on diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index da60e145a790..31fd23229253 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -604,6 +604,8 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line("#include ") if self.compiler_options.depends_on_librt_internal: ext_declarations.emit_line("#include ") + if self.compiler_options.depends_on_librt_base64: + ext_declarations.emit_line("#include ") declarations = Emitter(self.context) declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") @@ -1034,6 +1036,10 @@ def emit_module_exec_func( emitter.emit_line("if (import_librt_internal() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") + if self.compiler_options.depends_on_librt_base64: + emitter.emit_line("if (import_librt_base64() < 0) {") + emitter.emit_line("return -1;") + emitter.emit_line("}") emitter.emit_line("PyObject* modname = NULL;") if self.multi_phase_init: emitter.emit_line(f"{module_static} = module;") diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index ffce529f0756..79a08dc2a9f5 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -707,6 +707,7 @@ def __init__( extra_int_constants: list[tuple[int, RType]], priority: int, is_pure: bool, + experimental: bool, ) -> None: # Each primitive much have a distinct name, but otherwise they are arbitrary. self.name: Final = name @@ -729,6 +730,9 @@ def __init__( self.is_pure: Final = is_pure if is_pure: assert error_kind == ERR_NEVER + # Experimental primitives are not used unless mypyc experimental features are + # explicitly enabled + self.experimental = experimental def __repr__(self) -> str: return f"" diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index d33497d4987b..e19bf54e0744 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -2212,6 +2212,8 @@ def matching_primitive_op( for desc in candidates: if len(desc.arg_types) != len(args): continue + if desc.experimental and not self.options.experimental_features: + continue if all( # formal is not None and # TODO is_subtype(actual.type, formal) diff --git a/mypyc/lib-rt/librt_base64.c b/mypyc/lib-rt/librt_base64.c new file mode 100644 index 000000000000..1c3a6f8d01a5 --- /dev/null +++ b/mypyc/lib-rt/librt_base64.c @@ -0,0 +1,148 @@ +#define PY_SSIZE_T_CLEAN +#include +#include "librt_base64.h" +#include "pythoncapi_compat.h" + +#ifdef MYPYC_EXPERIMENTAL + +// b64encode_internal below is adapted from the CPython 3.14.0 binascii module + +static const unsigned char table_b2a_base64[] = +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BASE64_PAD '=' + +/* Max binary chunk size; limited only by available memory */ +#define BASE64_MAXBIN ((PY_SSIZE_T_MAX - 3) / 2) + +static PyObject * +b64encode_internal(PyObject *obj) { + unsigned char *ascii_data; + const unsigned char *bin_data; + int leftbits = 0; + unsigned char this_ch; + unsigned int leftchar = 0; + Py_ssize_t bin_len, out_len; + PyBytesWriter *writer; + int newline = 0; // TODO + + if (!PyBytes_Check(obj)) { + PyErr_SetString(PyExc_TypeError, "base64() expects a bytes object"); + return NULL; + } + + bin_data = (const unsigned char *)PyBytes_AS_STRING(obj); + bin_len = PyBytes_GET_SIZE(obj); + + assert(bin_len >= 0); + + if ( bin_len > BASE64_MAXBIN ) { + PyErr_SetString(PyExc_ValueError, "Too much data for base64 line"); + return NULL; + } + + /* We're lazy and allocate too much (fixed up later). + "+2" leaves room for up to two pad characters. + Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ + out_len = bin_len*2 + 2; + if (newline) + out_len++; + writer = PyBytesWriter_Create(out_len); + ascii_data = PyBytesWriter_GetData(writer); + if (writer == NULL) + return NULL; + + for( ; bin_len > 0 ; bin_len--, bin_data++ ) { + /* Shift the data into our buffer */ + leftchar = (leftchar << 8) | *bin_data; + leftbits += 8; + + /* See if there are 6-bit groups ready */ + while ( leftbits >= 6 ) { + this_ch = (leftchar >> (leftbits-6)) & 0x3f; + leftbits -= 6; + *ascii_data++ = table_b2a_base64[this_ch]; + } + } + if ( leftbits == 2 ) { + *ascii_data++ = table_b2a_base64[(leftchar&3) << 4]; + *ascii_data++ = BASE64_PAD; + *ascii_data++ = BASE64_PAD; + } else if ( leftbits == 4 ) { + *ascii_data++ = table_b2a_base64[(leftchar&0xf) << 2]; + *ascii_data++ = BASE64_PAD; + } + if (newline) + *ascii_data++ = '\n'; /* Append a courtesy newline */ + + return PyBytesWriter_FinishWithSize(writer, ascii_data - (unsigned char *)PyBytesWriter_GetData(writer)); +} + +static PyObject* +b64encode(PyObject *self, PyObject *const *args, size_t nargs) { + if (nargs != 1) { + PyErr_SetString(PyExc_TypeError, "b64encode() takes exactly one argument"); + return 0; + } + return b64encode_internal(args[0]); +} + +#endif + +static PyMethodDef librt_base64_module_methods[] = { +#ifdef MYPYC_EXPERIMENTAL + {"b64encode", (PyCFunction)b64encode, METH_FASTCALL, PyDoc_STR("Encode bytes-like object using Base64.")}, +#endif + {NULL, NULL, 0, NULL} +}; + +static int +base64_abi_version(void) { + return 0; +} + +static int +base64_api_version(void) { + return 0; +} + +static int +librt_base64_module_exec(PyObject *m) +{ +#ifdef MYPYC_EXPERIMENTAL + // Export mypy internal C API, be careful with the order! + static void *base64_api[LIBRT_BASE64_API_LEN] = { + (void *)base64_abi_version, + (void *)base64_api_version, + (void *)b64encode_internal, + }; + PyObject *c_api_object = PyCapsule_New((void *)base64_api, "librt.base64._C_API", NULL); + if (PyModule_Add(m, "_C_API", c_api_object) < 0) { + return -1; + } +#endif + return 0; +} + +static PyModuleDef_Slot librt_base64_module_slots[] = { + {Py_mod_exec, librt_base64_module_exec}, +#ifdef Py_MOD_GIL_NOT_USED + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL} +}; + +static PyModuleDef librt_base64_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "base64", + .m_doc = "base64 encoding and decoding optimized for mypyc", + .m_size = 0, + .m_methods = librt_base64_module_methods, + .m_slots = librt_base64_module_slots, +}; + +PyMODINIT_FUNC +PyInit_base64(void) +{ + return PyModuleDef_Init(&librt_base64_module); +} diff --git a/mypyc/lib-rt/librt_base64.h b/mypyc/lib-rt/librt_base64.h new file mode 100644 index 000000000000..cc97e54155fd --- /dev/null +++ b/mypyc/lib-rt/librt_base64.h @@ -0,0 +1,60 @@ +#ifndef LIBRT_BASE64_H +#define LIBRT_BASE64_H + +#ifndef MYPYC_EXPERIMENTAL + +static int +import_librt_base64(void) +{ + // All librt.base64 features are experimental for now, so don't set up the API here + return 0; +} + +#else // MYPYC_EXPERIMENTAL + +#define LIBRT_BASE64_ABI_VERSION 0 +#define LIBRT_BASE64_API_VERSION 0 +#define LIBRT_BASE64_API_LEN 3 + +static void *LibRTBase64_API[LIBRT_BASE64_API_LEN]; + +#define LibRTBase64_ABIVersion (*(int (*)(void)) LibRTBase64_API[0]) +#define LibRTBase64_APIVersion (*(int (*)(void)) LibRTBase64_API[1]) +#define LibRTBase64_b64encode_internal (*(PyObject* (*)(PyObject *source)) LibRTBase64_API[2]) + +static int +import_librt_base64(void) +{ + PyObject *mod = PyImport_ImportModule("librt.base64"); + if (mod == NULL) + return -1; + Py_DECREF(mod); // we import just for the side effect of making the below work. + void *capsule = PyCapsule_Import("librt.base64._C_API", 0); + if (capsule == NULL) + return -1; + memcpy(LibRTBase64_API, capsule, sizeof(LibRTBase64_API)); + if (LibRTBase64_ABIVersion() != LIBRT_BASE64_ABI_VERSION) { + char err[128]; + snprintf(err, sizeof(err), "ABI version conflict for librt.base64, expected %d, found %d", + LIBRT_BASE64_ABI_VERSION, + LibRTBase64_ABIVersion() + ); + PyErr_SetString(PyExc_ValueError, err); + return -1; + } + if (LibRTBase64_APIVersion() < LIBRT_BASE64_API_VERSION) { + char err[128]; + snprintf(err, sizeof(err), + "API version conflict for librt.base64, expected %d or newer, found %d (hint: upgrade librt)", + LIBRT_BASE64_API_VERSION, + LibRTBase64_APIVersion() + ); + PyErr_SetString(PyExc_ValueError, err); + return -1; + } + return 0; +} + +#endif // MYPYC_EXPERIMENTAL + +#endif // LIBRT_BASE64_H diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 299b0acd96e7..3c08c7dbb5ee 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -96,6 +96,9 @@ def run(self) -> None: ], include_dirs=["."], extra_compile_args=cflags, - ) + ), + Extension( + "librt.base64", ["librt_base64.c"], include_dirs=["."], extra_compile_args=cflags + ), ] ) diff --git a/mypyc/options.py b/mypyc/options.py index e004a0a52c95..f0290cec4f06 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -18,6 +18,8 @@ def __init__( group_name: str | None = None, log_trace: bool = False, depends_on_librt_internal: bool = False, + depends_on_librt_base64: bool = False, + experimental_features: bool = False, ) -> None: self.strip_asserts = strip_asserts self.multi_file = multi_file @@ -55,3 +57,8 @@ def __init__( # only for mypy itself, third-party code compiled with mypyc should not use # librt.internal. self.depends_on_librt_internal = depends_on_librt_internal + self.depends_on_librt_base64 = depends_on_librt_base64 + # Some experimental features are only available when building librt in + # experimental mode (e.g. use _experimental suffix in librt run test). + # These can't be used with a librt wheel installed from PyPI. + self.experimental_features = experimental_features diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index f685b1cfbcf5..b12ed3a2c523 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -465,3 +465,12 @@ c_function_name="cache_version_internal", error_kind=ERR_NEVER, ) + +function_op( + name="librt.base64.b64encode", + arg_types=[bytes_rprimitive], + return_type=bytes_rprimitive, + c_function_name="LibRTBase64_b64encode_internal", + error_kind=ERR_MAGIC, + experimental=True, +) diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index 3188bc322809..36b9e8d0835c 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -144,6 +144,7 @@ def method_op( extra_int_constants, priority, is_pure=is_pure, + experimental=False, ) ops.append(desc) return desc @@ -162,6 +163,7 @@ def function_op( steals: StealsDescription = False, is_borrowed: bool = False, priority: int = 1, + experimental: bool = False, ) -> PrimitiveDescription: """Define a C function call op that replaces a function call. @@ -190,6 +192,7 @@ def function_op( extra_int_constants=extra_int_constants, priority=priority, is_pure=False, + experimental=experimental, ) ops.append(desc) return desc @@ -236,6 +239,7 @@ def binary_op( extra_int_constants=extra_int_constants, priority=priority, is_pure=False, + experimental=False, ) ops.append(desc) return desc @@ -314,6 +318,7 @@ def custom_primitive_op( extra_int_constants=extra_int_constants, priority=0, is_pure=is_pure, + experimental=False, ) @@ -355,6 +360,7 @@ def unary_op( extra_int_constants=extra_int_constants, priority=priority, is_pure=is_pure, + experimental=False, ) ops.append(desc) return desc diff --git a/mypyc/test-data/run-base64.test b/mypyc/test-data/run-base64.test new file mode 100644 index 000000000000..31997d630596 --- /dev/null +++ b/mypyc/test-data/run-base64.test @@ -0,0 +1,52 @@ +[case testAllBase64Features_librt_base64_experimental] +from typing import Any +import base64 + +from librt.base64 import b64encode + +from testutil import assertRaises + +def test_encode_basic() -> None: + assert b64encode(b"x") == b"eA==" + + with assertRaises(TypeError): + b64encode(bytearray(b"x")) + +def check_encode(b: bytes) -> None: + assert b64encode(b) == getattr(base64, "b64encode")(b) + +def test_encode_different_strings() -> None: + for i in range(256): + check_encode(bytes([i])) + check_encode(bytes([i]) + b"x") + check_encode(bytes([i]) + b"xy") + check_encode(bytes([i]) + b"xyz") + check_encode(bytes([i]) + b"xyza") + check_encode(b"x" + bytes([i])) + check_encode(b"xy" + bytes([i])) + check_encode(b"xyz" + bytes([i])) + check_encode(b"xyza" + bytes([i])) + + b = b"a\x00\xb7" * 1000 + for i in range(1000): + check_encode(b[:i]) + + for b in b"", b"ab", b"bac", b"1234", b"xyz88", b"abc" * 200: + check_encode(b) + +def test_encode_wrapper() -> None: + enc: Any = b64encode + assert enc(b"x") == b"eA==" + + with assertRaises(TypeError): + enc() + + with assertRaises(TypeError): + enc(b"x", b"y") + +[case testBase64FeaturesNotAvailableInNonExperimentalBuild_librt_base64] +# This also ensures librt.base64 can be built without experimental features +import librt.base64 + +def test_b64encode_not_available() -> None: + assert not hasattr(librt.base64, "b64encode") diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 953f61329395..26d8e44b784b 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -73,6 +73,7 @@ "run-weakref.test", "run-python37.test", "run-python38.test", + "run-base64.test", ] if sys.version_info >= (3, 10): @@ -86,7 +87,8 @@ setup(name='test_run_output', ext_modules=mypycify({}, separate={}, skip_cgen_input={!r}, strip_asserts=False, - multi_file={}, opt_level='{}', install_librt={}), + multi_file={}, opt_level='{}', install_librt={}, + experimental_features={}), ) """ @@ -242,13 +244,18 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> # Use _librt_internal to test mypy-specific parts of librt (they have # some special-casing in mypyc), for everything else use _librt suffix. librt_internal = testcase.name.endswith("_librt_internal") - librt = librt_internal or testcase.name.endswith("_librt") + librt_base64 = "_librt_base64" in testcase.name + librt = testcase.name.endswith("_librt") or "_librt_" in testcase.name + # Enable experimental features (local librt build also includes experimental features) + experimental_features = testcase.name.endswith("_experimental") try: compiler_options = CompilerOptions( multi_file=self.multi_file, separate=self.separate, strict_dunder_typing=self.strict_dunder_typing, depends_on_librt_internal=librt_internal, + depends_on_librt_base64=librt_base64, + experimental_features=experimental_features, ) result = emitmodule.parse_and_typecheck( sources=sources, @@ -281,7 +288,13 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> with open(setup_file, "w", encoding="utf-8") as f: f.write( setup_format.format( - module_paths, separate, cfiles, self.multi_file, opt_level, librt + module_paths, + separate, + cfiles, + self.multi_file, + opt_level, + librt, + experimental_features, ) ) From 17f8b3133ace6581942becdd0792a06abc955cbf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 13 Nov 2025 16:31:24 +0000 Subject: [PATCH 166/183] [mypyc] Import librt.base64 capsule automatically if needed (#20233) Allow primitives to specify the capsule they need via module name such as `librt.base64`. This way we can import the capsule automatically only when there are references to the contents of the capsule in the compiled code. Only make the change for `librt.base64`, but we can also do a similar thing for `librt.internal` in a follow-up PR. --- mypyc/analysis/capsule_deps.py | 27 +++++++++++++++++++++ mypyc/build.py | 2 -- mypyc/codegen/emitmodule.py | 9 +++++-- mypyc/ir/module_ir.py | 7 +++++- mypyc/ir/ops.py | 8 +++++++ mypyc/irbuild/ll_builder.py | 2 ++ mypyc/options.py | 2 -- mypyc/primitives/misc_ops.py | 1 + mypyc/primitives/registry.py | 12 ++++++++++ mypyc/test-data/irbuild-base64.test | 37 +++++++++++++++++++++++++++++ mypyc/test-data/run-base64.test | 11 ++++++++- mypyc/test/test_irbuild.py | 1 + mypyc/test/test_run.py | 2 -- mypyc/test/testutil.py | 2 ++ 14 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 mypyc/analysis/capsule_deps.py create mode 100644 mypyc/test-data/irbuild-base64.test diff --git a/mypyc/analysis/capsule_deps.py b/mypyc/analysis/capsule_deps.py new file mode 100644 index 000000000000..ada42ee03f28 --- /dev/null +++ b/mypyc/analysis/capsule_deps.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.ops import CallC, PrimitiveOp + + +def find_implicit_capsule_dependencies(fn: FuncIR) -> set[str] | None: + """Find implicit dependencies on capsules that need to be imported. + + Using primitives or types defined in librt submodules such as "librt.base64" + requires a capsule import. + + Note that a module can depend on a librt module even if it doesn't explicitly + import it, for example via re-exported names or via return types of functions + defined in other modules. + """ + deps: set[str] | None = None + for block in fn.blocks: + for op in block.ops: + # TODO: Also determine implicit type object dependencies (e.g. cast targets) + if isinstance(op, CallC) and op.capsule is not None: + if deps is None: + deps = set() + deps.add(op.capsule) + else: + assert not isinstance(op, PrimitiveOp), "Lowered IR is expected" + return deps diff --git a/mypyc/build.py b/mypyc/build.py index 2ff9d175947f..351aa87b9264 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -495,7 +495,6 @@ def mypycify( group_name: str | None = None, log_trace: bool = False, depends_on_librt_internal: bool = False, - depends_on_librt_base64: bool = False, install_librt: bool = False, experimental_features: bool = False, ) -> list[Extension]: @@ -570,7 +569,6 @@ def mypycify( group_name=group_name, log_trace=log_trace, depends_on_librt_internal=depends_on_librt_internal, - depends_on_librt_base64=depends_on_librt_base64, experimental_features=experimental_features, ) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 31fd23229253..8dd9f750a920 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -27,6 +27,7 @@ from mypy.options import Options from mypy.plugin import Plugin, ReportConfigContext from mypy.util import hash_digest, json_dumps +from mypyc.analysis.capsule_deps import find_implicit_capsule_dependencies from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import Emitter, EmitterContext, HeaderDeclaration, c_array_initializer from mypyc.codegen.emitclass import generate_class, generate_class_reuse, generate_class_type_decl @@ -259,6 +260,10 @@ def compile_scc_to_ir( # Switch to lower abstraction level IR. lower_ir(fn, compiler_options) + # Calculate implicit module dependencies (needed for librt) + capsules = find_implicit_capsule_dependencies(fn) + if capsules is not None: + module.capsules.update(capsules) # Perform optimizations. do_copy_propagation(fn, compiler_options) do_flag_elimination(fn, compiler_options) @@ -604,7 +609,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line("#include ") if self.compiler_options.depends_on_librt_internal: ext_declarations.emit_line("#include ") - if self.compiler_options.depends_on_librt_base64: + if any("librt.base64" in mod.capsules for mod in self.modules.values()): ext_declarations.emit_line("#include ") declarations = Emitter(self.context) @@ -1036,7 +1041,7 @@ def emit_module_exec_func( emitter.emit_line("if (import_librt_internal() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") - if self.compiler_options.depends_on_librt_base64: + if "librt.base64" in module.capsules: emitter.emit_line("if (import_librt_base64() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") diff --git a/mypyc/ir/module_ir.py b/mypyc/ir/module_ir.py index 7d95b48e197e..5aef414490f9 100644 --- a/mypyc/ir/module_ir.py +++ b/mypyc/ir/module_ir.py @@ -30,6 +30,8 @@ def __init__( # These are only visible in the module that defined them, so no need # to serialize. self.type_var_names = type_var_names + # Capsules needed by the module, specified via module names such as "librt.base64" + self.capsules: set[str] = set() def serialize(self) -> JsonDict: return { @@ -38,11 +40,12 @@ def serialize(self) -> JsonDict: "functions": [f.serialize() for f in self.functions], "classes": [c.serialize() for c in self.classes], "final_names": [(k, t.serialize()) for k, t in self.final_names], + "capsules": sorted(self.capsules), } @classmethod def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ModuleIR: - return ModuleIR( + module = ModuleIR( data["fullname"], data["imports"], [ctx.functions[FuncDecl.get_id_from_json(f)] for f in data["functions"]], @@ -50,6 +53,8 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ModuleIR: [(k, deserialize_type(t, ctx)) for k, t in data["final_names"]], [], ) + module.capsules = set(data["capsules"]) + return module def deserialize_modules(data: dict[str, JsonDict], ctx: DeserMaps) -> dict[str, ModuleIR]: diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 79a08dc2a9f5..2153d47e6874 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -708,6 +708,7 @@ def __init__( priority: int, is_pure: bool, experimental: bool, + capsule: str | None, ) -> None: # Each primitive much have a distinct name, but otherwise they are arbitrary. self.name: Final = name @@ -733,6 +734,9 @@ def __init__( # Experimental primitives are not used unless mypyc experimental features are # explicitly enabled self.experimental = experimental + # Capsule that needs to imported and configured to call the primitive + # (name of the target module, e.g. "librt.base64"). + self.capsule = capsule def __repr__(self) -> str: return f"" @@ -1233,6 +1237,7 @@ def __init__( *, is_pure: bool = False, returns_null: bool = False, + capsule: str | None = None, ) -> None: self.error_kind = error_kind super().__init__(line) @@ -1250,6 +1255,9 @@ def __init__( # The function might return a null value that does not indicate # an error. self.returns_null = returns_null + # A capsule from this module must be imported and initialized before calling this + # function (used for C functions exported from librt). Example value: "librt.base64" + self.capsule = capsule if is_pure or returns_null: assert error_kind == ERR_NEVER diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index e19bf54e0744..fd66288dbcc5 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -2075,6 +2075,7 @@ def call_c( var_arg_idx, is_pure=desc.is_pure, returns_null=desc.returns_null, + capsule=desc.capsule, ) ) if desc.is_borrowed: @@ -2159,6 +2160,7 @@ def primitive_op( desc.priority, is_pure=desc.is_pure, returns_null=False, + capsule=desc.capsule, ) return self.call_c(c_desc, args, line, result_type=result_type) diff --git a/mypyc/options.py b/mypyc/options.py index f0290cec4f06..9f16c07dc231 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -18,7 +18,6 @@ def __init__( group_name: str | None = None, log_trace: bool = False, depends_on_librt_internal: bool = False, - depends_on_librt_base64: bool = False, experimental_features: bool = False, ) -> None: self.strip_asserts = strip_asserts @@ -57,7 +56,6 @@ def __init__( # only for mypy itself, third-party code compiled with mypyc should not use # librt.internal. self.depends_on_librt_internal = depends_on_librt_internal - self.depends_on_librt_base64 = depends_on_librt_base64 # Some experimental features are only available when building librt in # experimental mode (e.g. use _experimental suffix in librt run test). # These can't be used with a librt wheel installed from PyPI. diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index b12ed3a2c523..bb225a76acd8 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -473,4 +473,5 @@ c_function_name="LibRTBase64_b64encode_internal", error_kind=ERR_MAGIC, experimental=True, + capsule="librt.base64", ) diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index 36b9e8d0835c..2f66b1915501 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -62,6 +62,7 @@ class CFunctionDescription(NamedTuple): priority: int is_pure: bool returns_null: bool + capsule: str | None # A description for C load operations including LoadGlobal and LoadAddress @@ -100,6 +101,7 @@ def method_op( is_borrowed: bool = False, priority: int = 1, is_pure: bool = False, + capsule: str | None = None, ) -> PrimitiveDescription: """Define a c function call op that replaces a method call. @@ -145,6 +147,7 @@ def method_op( priority, is_pure=is_pure, experimental=False, + capsule=capsule, ) ops.append(desc) return desc @@ -164,6 +167,7 @@ def function_op( is_borrowed: bool = False, priority: int = 1, experimental: bool = False, + capsule: str | None = None, ) -> PrimitiveDescription: """Define a C function call op that replaces a function call. @@ -193,6 +197,7 @@ def function_op( priority=priority, is_pure=False, experimental=experimental, + capsule=capsule, ) ops.append(desc) return desc @@ -212,6 +217,7 @@ def binary_op( steals: StealsDescription = False, is_borrowed: bool = False, priority: int = 1, + capsule: str | None = None, ) -> PrimitiveDescription: """Define a c function call op for a binary operation. @@ -240,6 +246,7 @@ def binary_op( priority=priority, is_pure=False, experimental=False, + capsule=capsule, ) ops.append(desc) return desc @@ -281,6 +288,7 @@ def custom_op( 0, is_pure=is_pure, returns_null=returns_null, + capsule=None, ) @@ -297,6 +305,7 @@ def custom_primitive_op( steals: StealsDescription = False, is_borrowed: bool = False, is_pure: bool = False, + capsule: str | None = None, ) -> PrimitiveDescription: """Define a primitive op that can't be automatically generated based on the AST. @@ -319,6 +328,7 @@ def custom_primitive_op( priority=0, is_pure=is_pure, experimental=False, + capsule=capsule, ) @@ -335,6 +345,7 @@ def unary_op( is_borrowed: bool = False, priority: int = 1, is_pure: bool = False, + capsule: str | None = None, ) -> PrimitiveDescription: """Define a primitive op for an unary operation. @@ -361,6 +372,7 @@ def unary_op( priority=priority, is_pure=is_pure, experimental=False, + capsule=capsule, ) ops.append(desc) return desc diff --git a/mypyc/test-data/irbuild-base64.test b/mypyc/test-data/irbuild-base64.test new file mode 100644 index 000000000000..bc73b7e35431 --- /dev/null +++ b/mypyc/test-data/irbuild-base64.test @@ -0,0 +1,37 @@ +[case testBase64_experimental] +from librt.base64 import b64encode + +def enc(b: bytes) -> bytes: + return b64encode(b) +[out] +def enc(b): + b, r0 :: bytes +L0: + r0 = LibRTBase64_b64encode_internal(b) + return r0 + +[case testBase64ExperimentalDisabled] +from librt.base64 import b64encode + +def enc(b: bytes) -> bytes: + return b64encode(b) +[out] +def enc(b): + b :: bytes + r0 :: dict + r1 :: str + r2 :: object + r3 :: object[1] + r4 :: object_ptr + r5 :: object + r6 :: bytes +L0: + r0 = __main__.globals :: static + r1 = 'b64encode' + r2 = CPyDict_GetItem(r0, r1) + r3 = [b] + r4 = load_address r3 + r5 = PyObject_Vectorcall(r2, r4, 1, 0) + keep_alive b + r6 = cast(bytes, r5) + return r6 diff --git a/mypyc/test-data/run-base64.test b/mypyc/test-data/run-base64.test index 31997d630596..0f9151c2b00b 100644 --- a/mypyc/test-data/run-base64.test +++ b/mypyc/test-data/run-base64.test @@ -1,4 +1,4 @@ -[case testAllBase64Features_librt_base64_experimental] +[case testAllBase64Features_librt_experimental] from typing import Any import base64 @@ -50,3 +50,12 @@ import librt.base64 def test_b64encode_not_available() -> None: assert not hasattr(librt.base64, "b64encode") + +[case testBase64UsedAtTopLevelOnly_librt_experimental] +from librt.base64 import b64encode + +# The only reference to b64encode is at module top level +encoded = b64encode(b"x") + +def test_top_level_only_encode() -> None: + assert encoded == b"eA==" diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index e79cbec392f4..7c248640246d 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -54,6 +54,7 @@ "irbuild-glue-methods.test", "irbuild-math.test", "irbuild-weakref.test", + "irbuild-base64.test", ] if sys.version_info >= (3, 10): diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 26d8e44b784b..6b63a4d546d0 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -244,7 +244,6 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> # Use _librt_internal to test mypy-specific parts of librt (they have # some special-casing in mypyc), for everything else use _librt suffix. librt_internal = testcase.name.endswith("_librt_internal") - librt_base64 = "_librt_base64" in testcase.name librt = testcase.name.endswith("_librt") or "_librt_" in testcase.name # Enable experimental features (local librt build also includes experimental features) experimental_features = testcase.name.endswith("_experimental") @@ -254,7 +253,6 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> separate=self.separate, strict_dunder_typing=self.strict_dunder_typing, depends_on_librt_internal=librt_internal, - depends_on_librt_base64=librt_base64, experimental_features=experimental_features, ) result = emitmodule.parse_and_typecheck( diff --git a/mypyc/test/testutil.py b/mypyc/test/testutil.py index 80a06204bb9d..3e9abc231d9a 100644 --- a/mypyc/test/testutil.py +++ b/mypyc/test/testutil.py @@ -281,4 +281,6 @@ def infer_ir_build_options_from_test_name(name: str) -> CompilerOptions | None: options.python_version = options.capi_version elif "_py" in name or "_Python" in name: assert False, f"Invalid _py* suffix (should be _pythonX_Y): {name}" + if re.search("_experimental(_|$)", name): + options.experimental_features = True return options From 8e2ce962dba5a83128c9ee8f55fc001a8fd4d28d Mon Sep 17 00:00:00 2001 From: KarelKenens <143591762+KarelKenens@users.noreply.github.com> Date: Thu, 13 Nov 2025 19:51:19 +0100 Subject: [PATCH 167/183] Fix annotated with function as type keyword list parameter (#20094) Fixes #20090 --- mypy/exprtotype.py | 61 +++++++++++++++++++++----- test-data/unit/check-python312.test | 19 ++++++++ test-data/unit/check-type-aliases.test | 19 ++++++++ 3 files changed, 87 insertions(+), 12 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 506194a4b285..6fd43c08dbac 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -100,7 +100,9 @@ def expr_to_unanalyzed_type( else: raise TypeTranslationError() elif isinstance(expr, IndexExpr): - base = expr_to_unanalyzed_type(expr.base, options, allow_new_syntax, expr) + base = expr_to_unanalyzed_type( + expr.base, options, allow_new_syntax, expr, lookup_qualified=lookup_qualified + ) if isinstance(base, UnboundType): if base.args: raise TypeTranslationError() @@ -124,9 +126,18 @@ def expr_to_unanalyzed_type( # TODO: this is not the optimal solution as we are basically getting rid # of the Annotation definition and only returning the type information, # losing all the annotations. - return expr_to_unanalyzed_type(args[0], options, allow_new_syntax, expr) + return expr_to_unanalyzed_type( + args[0], options, allow_new_syntax, expr, lookup_qualified=lookup_qualified + ) base.args = tuple( - expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr, allow_unpack=True) + expr_to_unanalyzed_type( + arg, + options, + allow_new_syntax, + expr, + allow_unpack=True, + lookup_qualified=lookup_qualified, + ) for arg in args ) if not base.args: @@ -141,8 +152,12 @@ def expr_to_unanalyzed_type( ): return UnionType( [ - expr_to_unanalyzed_type(expr.left, options, allow_new_syntax), - expr_to_unanalyzed_type(expr.right, options, allow_new_syntax), + expr_to_unanalyzed_type( + expr.left, options, allow_new_syntax, lookup_qualified=lookup_qualified + ), + expr_to_unanalyzed_type( + expr.right, options, allow_new_syntax, lookup_qualified=lookup_qualified + ), ], uses_pep604_syntax=True, ) @@ -178,12 +193,16 @@ def expr_to_unanalyzed_type( if typ is not default_type: # Two types raise TypeTranslationError() - typ = expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr) + typ = expr_to_unanalyzed_type( + arg, options, allow_new_syntax, expr, lookup_qualified=lookup_qualified + ) continue else: raise TypeTranslationError() elif i == 0: - typ = expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr) + typ = expr_to_unanalyzed_type( + arg, options, allow_new_syntax, expr, lookup_qualified=lookup_qualified + ) elif i == 1: name = _extract_argument_name(arg) else: @@ -192,7 +211,14 @@ def expr_to_unanalyzed_type( elif isinstance(expr, ListExpr): return TypeList( [ - expr_to_unanalyzed_type(t, options, allow_new_syntax, expr, allow_unpack=True) + expr_to_unanalyzed_type( + t, + options, + allow_new_syntax, + expr, + allow_unpack=True, + lookup_qualified=lookup_qualified, + ) for t in expr.items ], line=expr.line, @@ -203,7 +229,9 @@ def expr_to_unanalyzed_type( elif isinstance(expr, BytesExpr): return parse_type_string(expr.value, "builtins.bytes", expr.line, expr.column) elif isinstance(expr, UnaryExpr): - typ = expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax) + typ = expr_to_unanalyzed_type( + expr.expr, options, allow_new_syntax, lookup_qualified=lookup_qualified + ) if isinstance(typ, RawExpressionType): if isinstance(typ.literal_value, int): if expr.op == "-": @@ -225,7 +253,10 @@ def expr_to_unanalyzed_type( return EllipsisType(expr.line) elif allow_unpack and isinstance(expr, StarExpr): return UnpackType( - expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax), from_star_syntax=True + expr_to_unanalyzed_type( + expr.expr, options, allow_new_syntax, lookup_qualified=lookup_qualified + ), + from_star_syntax=True, ) elif isinstance(expr, DictExpr): if not expr.items: @@ -236,12 +267,18 @@ def expr_to_unanalyzed_type( if not isinstance(item_name, StrExpr): if item_name is None: extra_items_from.append( - expr_to_unanalyzed_type(value, options, allow_new_syntax, expr) + expr_to_unanalyzed_type( + value, + options, + allow_new_syntax, + expr, + lookup_qualified=lookup_qualified, + ) ) continue raise TypeTranslationError() items[item_name.value] = expr_to_unanalyzed_type( - value, options, allow_new_syntax, expr + value, options, allow_new_syntax, expr, lookup_qualified=lookup_qualified ) result = TypedDictType( items, set(), set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index c8b306b6bd55..c501962967d5 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2170,6 +2170,25 @@ reveal_type(x[0]) # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] +[case testAnnotatedWithCallableAsParameterTypeKeyword] +from typing_extensions import Annotated + +def something() -> None: ... + +type A = list[Annotated[str, something()]] +a: A +reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedWithCallableAsParameterTypeKeywordDeeper] +from typing_extensions import Annotated + +def something() -> None: ... + +type A = list[Annotated[Annotated[str, something()], something()]] +a: A +reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/tuple.pyi] [case testPEP695TypeAliasRecursiveInParameterBound] from typing import Any diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 06e223716189..6923b0d8f006 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -1319,6 +1319,25 @@ from typing_extensions import TypeAlias Foo: TypeAlias = ClassVar[int] # E: ClassVar[...] can't be used inside a type alias [builtins fixtures/tuple.pyi] +[case testAnnotatedWithCallableAsParameterTypeAlias] +from typing_extensions import Annotated, TypeAlias + +def something() -> None: ... + +A: TypeAlias = list[Annotated[str, something()]] +a: A +reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testAnnotatedWithCallableAsParameterTypeAliasDeeper] +from typing_extensions import Annotated, TypeAlias + +def something() -> None: ... + +A: TypeAlias = list[Annotated[Annotated[str, something()], something()]] +a: A +reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/tuple.pyi] [case testTypeAliasDict] D = dict[str, int] From 92864259a75ff91fc8ebf2fe743ff6d7a331519b Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 16 Nov 2025 01:20:10 +0100 Subject: [PATCH 168/183] Sync typeshed (#20241) Source commit: https://github.com/python/typeshed/commit/ebce8d766b41fbf4d83cf47c1297563a9508ff60 --- mypy/typeshed/stdlib/_compression.pyi | 15 ++- mypy/typeshed/stdlib/asyncio/protocols.pyi | 4 +- mypy/typeshed/stdlib/builtins.pyi | 23 ++-- .../stdlib/compression/_common/_streams.pyi | 15 ++- mypy/typeshed/stdlib/html/parser.pyi | 3 +- mypy/typeshed/stdlib/http/client.pyi | 1 + mypy/typeshed/stdlib/imaplib.pyi | 1 + mypy/typeshed/stdlib/importlib/util.pyi | 16 ++- mypy/typeshed/stdlib/locale.pyi | 4 + mypy/typeshed/stdlib/os/__init__.pyi | 27 ++--- mypy/typeshed/stdlib/parser.pyi | 8 +- mypy/typeshed/stdlib/select.pyi | 12 +- mypy/typeshed/stdlib/ssl.pyi | 3 + mypy/typeshed/stdlib/stat.pyi | 109 +++++++++++++++++- mypy/typeshed/stdlib/sys/__init__.pyi | 15 ++- mypy/typeshed/stdlib/sys/_monitoring.pyi | 8 +- mypy/typeshed/stdlib/threading.pyi | 6 +- mypy/typeshed/stdlib/types.pyi | 4 +- mypy/typeshed/stdlib/typing.pyi | 4 - mypy/typeshed/stdlib/unittest/util.pyi | 21 +++- mypy/typeshed/stdlib/uuid.pyi | 11 +- mypy/typeshed/stdlib/winreg.pyi | 6 +- test-data/unit/pythoneval.test | 2 +- 23 files changed, 251 insertions(+), 67 deletions(-) diff --git a/mypy/typeshed/stdlib/_compression.pyi b/mypy/typeshed/stdlib/_compression.pyi index aa67df2ab478..6015bcb13f1c 100644 --- a/mypy/typeshed/stdlib/_compression.pyi +++ b/mypy/typeshed/stdlib/_compression.pyi @@ -1,6 +1,6 @@ # _compression is replaced by compression._common._streams on Python 3.14+ (PEP-784) -from _typeshed import Incomplete, WriteableBuffer +from _typeshed import ReadableBuffer, WriteableBuffer from collections.abc import Callable from io import DEFAULT_BUFFER_SIZE, BufferedIOBase, RawIOBase from typing import Any, Protocol, type_check_only @@ -13,13 +13,24 @@ class _Reader(Protocol): def seekable(self) -> bool: ... def seek(self, n: int, /) -> Any: ... +@type_check_only +class _Decompressor(Protocol): + def decompress(self, data: ReadableBuffer, /, max_length: int = ...) -> bytes: ... + @property + def unused_data(self) -> bytes: ... + @property + def eof(self) -> bool: ... + # `zlib._Decompress` does not have next property, but `DecompressReader` calls it: + # @property + # def needs_input(self) -> bool: ... + class BaseStream(BufferedIOBase): ... class DecompressReader(RawIOBase): def __init__( self, fp: _Reader, - decomp_factory: Callable[..., Incomplete], + decomp_factory: Callable[..., _Decompressor], trailing_error: type[Exception] | tuple[type[Exception], ...] = (), **decomp_args: Any, # These are passed to decomp_factory. ) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/protocols.pyi b/mypy/typeshed/stdlib/asyncio/protocols.pyi index 2c52ad4be410..3a8965f03e29 100644 --- a/mypy/typeshed/stdlib/asyncio/protocols.pyi +++ b/mypy/typeshed/stdlib/asyncio/protocols.pyi @@ -14,7 +14,7 @@ class BaseProtocol: class Protocol(BaseProtocol): # Need annotation or mypy will complain about 'Cannot determine type of "__slots__" in base class' - __slots__: tuple[()] = () + __slots__: tuple[str, ...] = () def data_received(self, data: bytes) -> None: ... def eof_received(self) -> bool | None: ... @@ -35,7 +35,7 @@ class DatagramProtocol(BaseProtocol): def error_received(self, exc: Exception) -> None: ... class SubprocessProtocol(BaseProtocol): - __slots__: tuple[()] = () + __slots__: tuple[str, ...] = () def pipe_data_received(self, fd: int, data: bytes) -> None: ... def pipe_connection_lost(self, fd: int, exc: Exception | None) -> None: ... def process_exited(self) -> None: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index e03a92ce3d91..30dfb9dbb25c 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -42,6 +42,7 @@ from typing import ( # noqa: Y022,UP035 Any, BinaryIO, ClassVar, + Final, Generic, Mapping, MutableMapping, @@ -188,8 +189,9 @@ class type: __bases__: tuple[type, ...] @property def __basicsize__(self) -> int: ... - @property - def __dict__(self) -> types.MappingProxyType[str, Any]: ... # type: ignore[override] + # type.__dict__ is read-only at runtime, but that can't be expressed currently. + # See https://github.com/python/typeshed/issues/11033 for a discussion. + __dict__: Final[types.MappingProxyType[str, Any]] # type: ignore[assignment] @property def __dictoffset__(self) -> int: ... @property @@ -1267,13 +1269,6 @@ class property: def __set__(self, instance: Any, value: Any, /) -> None: ... def __delete__(self, instance: Any, /) -> None: ... -@final -@type_check_only -class _NotImplementedType(Any): - __call__: None - -NotImplemented: _NotImplementedType - def abs(x: SupportsAbs[_T], /) -> _T: ... def all(iterable: Iterable[object], /) -> bool: ... def any(iterable: Iterable[object], /) -> bool: ... @@ -1932,14 +1927,14 @@ def __import__( def __build_class__(func: Callable[[], CellType | Any], name: str, /, *bases: Any, metaclass: Any = ..., **kwds: Any) -> Any: ... if sys.version_info >= (3, 10): - from types import EllipsisType + from types import EllipsisType, NotImplementedType # Backwards compatibility hack for folks who relied on the ellipsis type # existing in typeshed in Python 3.9 and earlier. ellipsis = EllipsisType Ellipsis: EllipsisType - + NotImplemented: NotImplementedType else: # Actually the type of Ellipsis is , but since it's # not exposed anywhere under that name, we make it private here. @@ -1949,6 +1944,12 @@ else: Ellipsis: ellipsis + @final + @type_check_only + class _NotImplementedType(Any): ... + + NotImplemented: _NotImplementedType + @disjoint_base class BaseException: args: tuple[Any, ...] diff --git a/mypy/typeshed/stdlib/compression/_common/_streams.pyi b/mypy/typeshed/stdlib/compression/_common/_streams.pyi index b8463973ec67..96aec24d1c2d 100644 --- a/mypy/typeshed/stdlib/compression/_common/_streams.pyi +++ b/mypy/typeshed/stdlib/compression/_common/_streams.pyi @@ -1,4 +1,4 @@ -from _typeshed import Incomplete, WriteableBuffer +from _typeshed import ReadableBuffer, WriteableBuffer from collections.abc import Callable from io import DEFAULT_BUFFER_SIZE, BufferedIOBase, RawIOBase from typing import Any, Protocol, type_check_only @@ -11,13 +11,24 @@ class _Reader(Protocol): def seekable(self) -> bool: ... def seek(self, n: int, /) -> Any: ... +@type_check_only +class _Decompressor(Protocol): + def decompress(self, data: ReadableBuffer, /, max_length: int = ...) -> bytes: ... + @property + def unused_data(self) -> bytes: ... + @property + def eof(self) -> bool: ... + # `zlib._Decompress` does not have next property, but `DecompressReader` calls it: + # @property + # def needs_input(self) -> bool: ... + class BaseStream(BufferedIOBase): ... class DecompressReader(RawIOBase): def __init__( self, fp: _Reader, - decomp_factory: Callable[..., Incomplete], # Consider backporting changes to _compression + decomp_factory: Callable[..., _Decompressor], # Consider backporting changes to _compression trailing_error: type[Exception] | tuple[type[Exception], ...] = (), **decomp_args: Any, # These are passed to decomp_factory. ) -> None: ... diff --git a/mypy/typeshed/stdlib/html/parser.pyi b/mypy/typeshed/stdlib/html/parser.pyi index 7edd39e8c703..08dc7b936922 100644 --- a/mypy/typeshed/stdlib/html/parser.pyi +++ b/mypy/typeshed/stdlib/html/parser.pyi @@ -9,7 +9,8 @@ class HTMLParser(ParserBase): # Added in Python 3.9.23, 3.10.18, 3.11.13, 3.12.11, 3.13.6 RCDATA_CONTENT_ELEMENTS: Final[tuple[str, ...]] - def __init__(self, *, convert_charrefs: bool = True) -> None: ... + # `scripting` parameter added in Python 3.9.25, 3.10.20, 3.11.15, 3.12.13, 3.13.10, 3.14.1 + def __init__(self, *, convert_charrefs: bool = True, scripting: bool = False) -> None: ... def feed(self, data: str) -> None: ... def close(self) -> None: ... def get_starttag_text(self) -> str | None: ... diff --git a/mypy/typeshed/stdlib/http/client.pyi b/mypy/typeshed/stdlib/http/client.pyi index d259e84e6f2a..1568567d5854 100644 --- a/mypy/typeshed/stdlib/http/client.pyi +++ b/mypy/typeshed/stdlib/http/client.pyi @@ -166,6 +166,7 @@ class HTTPResponse(io.BufferedIOBase, BinaryIO): # type: ignore[misc] # incomp def begin(self) -> None: ... class HTTPConnection: + blocksize: int auto_open: int # undocumented debuglevel: int default_port: int # undocumented diff --git a/mypy/typeshed/stdlib/imaplib.pyi b/mypy/typeshed/stdlib/imaplib.pyi index 536985a592b7..39fd466529bb 100644 --- a/mypy/typeshed/stdlib/imaplib.pyi +++ b/mypy/typeshed/stdlib/imaplib.pyi @@ -26,6 +26,7 @@ class IMAP4: class error(Exception): ... class abort(error): ... class readonly(abort): ... + utf8_enabled: bool mustquote: Pattern[str] debug: int state: str diff --git a/mypy/typeshed/stdlib/importlib/util.pyi b/mypy/typeshed/stdlib/importlib/util.pyi index 05c4d0d1edb3..577d3a667eca 100644 --- a/mypy/typeshed/stdlib/importlib/util.pyi +++ b/mypy/typeshed/stdlib/importlib/util.pyi @@ -12,7 +12,9 @@ from importlib._bootstrap_external import ( spec_from_file_location as spec_from_file_location, ) from importlib.abc import Loader -from typing_extensions import ParamSpec, deprecated +from types import TracebackType +from typing import Literal +from typing_extensions import ParamSpec, Self, deprecated _P = ParamSpec("_P") @@ -44,6 +46,18 @@ class LazyLoader(Loader): def source_hash(source_bytes: ReadableBuffer) -> bytes: ... +if sys.version_info >= (3, 12): + class _incompatible_extension_module_restrictions: + def __init__(self, *, disable_check: bool) -> None: ... + disable_check: bool + old: Literal[-1, 0, 1] # exists only while entered + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + @property + def override(self) -> Literal[-1, 1]: ... # undocumented + if sys.version_info >= (3, 14): __all__ = [ "LazyLoader", diff --git a/mypy/typeshed/stdlib/locale.pyi b/mypy/typeshed/stdlib/locale.pyi index fae9f849b637..80c39a532dc8 100644 --- a/mypy/typeshed/stdlib/locale.pyi +++ b/mypy/typeshed/stdlib/locale.pyi @@ -153,6 +153,10 @@ if sys.version_info < (3, 12): def format_string(f: _str, val: Any, grouping: bool = False, monetary: bool = False) -> _str: ... def currency(val: float | Decimal, symbol: bool = True, grouping: bool = False, international: bool = False) -> _str: ... def delocalize(string: _str) -> _str: ... + +if sys.version_info >= (3, 10): + def localize(string: _str, grouping: bool = False, monetary: bool = False) -> _str: ... + def atof(string: _str, func: Callable[[_str], float] = ...) -> float: ... def atoi(string: _str) -> int: ... def str(val: float) -> _str: ... diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index 580452739f7f..bb0a57153948 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -41,10 +41,22 @@ from typing import ( runtime_checkable, type_check_only, ) -from typing_extensions import Self, TypeAlias, Unpack, deprecated +from typing_extensions import LiteralString, Self, TypeAlias, Unpack, deprecated from . import path as _path +# Re-export common definitions from os.path to reduce duplication +from .path import ( + altsep as altsep, + curdir as curdir, + defpath as defpath, + devnull as devnull, + extsep as extsep, + pardir as pardir, + pathsep as pathsep, + sep as sep, +) + __all__ = [ "F_OK", "O_APPEND", @@ -674,19 +686,8 @@ if sys.platform != "win32": ST_NOSUID: Final[int] ST_RDONLY: Final[int] -curdir: str -pardir: str -sep: str -if sys.platform == "win32": - altsep: str -else: - altsep: str | None -extsep: str -pathsep: str -defpath: str linesep: Literal["\n", "\r\n"] -devnull: str -name: str +name: LiteralString F_OK: Final = 0 R_OK: Final = 4 diff --git a/mypy/typeshed/stdlib/parser.pyi b/mypy/typeshed/stdlib/parser.pyi index 26140c76248a..9b287fcc6529 100644 --- a/mypy/typeshed/stdlib/parser.pyi +++ b/mypy/typeshed/stdlib/parser.pyi @@ -7,8 +7,8 @@ def expr(source: str) -> STType: ... def suite(source: str) -> STType: ... def sequence2st(sequence: Sequence[Any]) -> STType: ... def tuple2st(sequence: Sequence[Any]) -> STType: ... -def st2list(st: STType, line_info: bool = ..., col_info: bool = ...) -> list[Any]: ... -def st2tuple(st: STType, line_info: bool = ..., col_info: bool = ...) -> tuple[Any, ...]: ... +def st2list(st: STType, line_info: bool = False, col_info: bool = False) -> list[Any]: ... +def st2tuple(st: STType, line_info: bool = False, col_info: bool = False) -> tuple[Any, ...]: ... def compilest(st: STType, filename: StrOrBytesPath = ...) -> CodeType: ... def isexpr(st: STType) -> bool: ... def issuite(st: STType) -> bool: ... @@ -21,5 +21,5 @@ class STType: def compile(self, filename: StrOrBytesPath = ...) -> CodeType: ... def isexpr(self) -> bool: ... def issuite(self) -> bool: ... - def tolist(self, line_info: bool = ..., col_info: bool = ...) -> list[Any]: ... - def totuple(self, line_info: bool = ..., col_info: bool = ...) -> tuple[Any, ...]: ... + def tolist(self, line_info: bool = False, col_info: bool = False) -> list[Any]: ... + def totuple(self, line_info: bool = False, col_info: bool = False) -> tuple[Any, ...]: ... diff --git a/mypy/typeshed/stdlib/select.pyi b/mypy/typeshed/stdlib/select.pyi index 587bc75376ef..43a9e4274b23 100644 --- a/mypy/typeshed/stdlib/select.pyi +++ b/mypy/typeshed/stdlib/select.pyi @@ -2,8 +2,8 @@ import sys from _typeshed import FileDescriptorLike from collections.abc import Iterable from types import TracebackType -from typing import Any, ClassVar, Final, final -from typing_extensions import Self +from typing import Any, ClassVar, Final, TypeVar, final +from typing_extensions import Never, Self if sys.platform != "win32": PIPE_BUF: Final[int] @@ -31,9 +31,13 @@ if sys.platform != "win32": def unregister(self, fd: FileDescriptorLike, /) -> None: ... def poll(self, timeout: float | None = None, /) -> list[tuple[int, int]]: ... +_R = TypeVar("_R", default=Never) +_W = TypeVar("_W", default=Never) +_X = TypeVar("_X", default=Never) + def select( - rlist: Iterable[Any], wlist: Iterable[Any], xlist: Iterable[Any], timeout: float | None = None, / -) -> tuple[list[Any], list[Any], list[Any]]: ... + rlist: Iterable[_R], wlist: Iterable[_W], xlist: Iterable[_X], timeout: float | None = None, / +) -> tuple[list[_R], list[_W], list[_X]]: ... error = OSError diff --git a/mypy/typeshed/stdlib/ssl.pyi b/mypy/typeshed/stdlib/ssl.pyi index faa98cb39920..aa94fc84255e 100644 --- a/mypy/typeshed/stdlib/ssl.pyi +++ b/mypy/typeshed/stdlib/ssl.pyi @@ -33,6 +33,9 @@ from typing_extensions import Never, Self, TypeAlias, deprecated if sys.version_info >= (3, 13): from _ssl import HAS_PSK as HAS_PSK +if sys.version_info >= (3, 14): + from _ssl import HAS_PHA as HAS_PHA + if sys.version_info < (3, 12): from _ssl import RAND_pseudo_bytes as RAND_pseudo_bytes diff --git a/mypy/typeshed/stdlib/stat.pyi b/mypy/typeshed/stdlib/stat.pyi index face28ab0cbb..6c26080e0665 100644 --- a/mypy/typeshed/stdlib/stat.pyi +++ b/mypy/typeshed/stdlib/stat.pyi @@ -1,7 +1,114 @@ import sys -from _stat import * +from _stat import ( + S_ENFMT as S_ENFMT, + S_IEXEC as S_IEXEC, + S_IFBLK as S_IFBLK, + S_IFCHR as S_IFCHR, + S_IFDIR as S_IFDIR, + S_IFDOOR as S_IFDOOR, + S_IFIFO as S_IFIFO, + S_IFLNK as S_IFLNK, + S_IFMT as S_IFMT, + S_IFPORT as S_IFPORT, + S_IFREG as S_IFREG, + S_IFSOCK as S_IFSOCK, + S_IFWHT as S_IFWHT, + S_IMODE as S_IMODE, + S_IREAD as S_IREAD, + S_IRGRP as S_IRGRP, + S_IROTH as S_IROTH, + S_IRUSR as S_IRUSR, + S_IRWXG as S_IRWXG, + S_IRWXO as S_IRWXO, + S_IRWXU as S_IRWXU, + S_ISBLK as S_ISBLK, + S_ISCHR as S_ISCHR, + S_ISDIR as S_ISDIR, + S_ISDOOR as S_ISDOOR, + S_ISFIFO as S_ISFIFO, + S_ISGID as S_ISGID, + S_ISLNK as S_ISLNK, + S_ISPORT as S_ISPORT, + S_ISREG as S_ISREG, + S_ISSOCK as S_ISSOCK, + S_ISUID as S_ISUID, + S_ISVTX as S_ISVTX, + S_ISWHT as S_ISWHT, + S_IWGRP as S_IWGRP, + S_IWOTH as S_IWOTH, + S_IWRITE as S_IWRITE, + S_IWUSR as S_IWUSR, + S_IXGRP as S_IXGRP, + S_IXOTH as S_IXOTH, + S_IXUSR as S_IXUSR, + SF_APPEND as SF_APPEND, + SF_ARCHIVED as SF_ARCHIVED, + SF_IMMUTABLE as SF_IMMUTABLE, + SF_NOUNLINK as SF_NOUNLINK, + SF_SNAPSHOT as SF_SNAPSHOT, + ST_ATIME as ST_ATIME, + ST_CTIME as ST_CTIME, + ST_DEV as ST_DEV, + ST_GID as ST_GID, + ST_INO as ST_INO, + ST_MODE as ST_MODE, + ST_MTIME as ST_MTIME, + ST_NLINK as ST_NLINK, + ST_SIZE as ST_SIZE, + ST_UID as ST_UID, + UF_APPEND as UF_APPEND, + UF_COMPRESSED as UF_COMPRESSED, + UF_HIDDEN as UF_HIDDEN, + UF_IMMUTABLE as UF_IMMUTABLE, + UF_NODUMP as UF_NODUMP, + UF_NOUNLINK as UF_NOUNLINK, + UF_OPAQUE as UF_OPAQUE, + filemode as filemode, +) from typing import Final +if sys.platform == "win32": + from _stat import ( + IO_REPARSE_TAG_APPEXECLINK as IO_REPARSE_TAG_APPEXECLINK, + IO_REPARSE_TAG_MOUNT_POINT as IO_REPARSE_TAG_MOUNT_POINT, + IO_REPARSE_TAG_SYMLINK as IO_REPARSE_TAG_SYMLINK, + ) + +if sys.version_info >= (3, 13): + from _stat import ( + SF_DATALESS as SF_DATALESS, + SF_FIRMLINK as SF_FIRMLINK, + SF_SETTABLE as SF_SETTABLE, + UF_DATAVAULT as UF_DATAVAULT, + UF_SETTABLE as UF_SETTABLE, + UF_TRACKED as UF_TRACKED, + ) + + if sys.platform == "darwin": + from _stat import SF_SUPPORTED as SF_SUPPORTED, SF_SYNTHETIC as SF_SYNTHETIC + +# _stat.c defines FILE_ATTRIBUTE_* constants conditionally, +# making them available only at runtime on Windows. +# stat.py unconditionally redefines the same FILE_ATTRIBUTE_* constants +# on all platforms. +FILE_ATTRIBUTE_ARCHIVE: Final = 32 +FILE_ATTRIBUTE_COMPRESSED: Final = 2048 +FILE_ATTRIBUTE_DEVICE: Final = 64 +FILE_ATTRIBUTE_DIRECTORY: Final = 16 +FILE_ATTRIBUTE_ENCRYPTED: Final = 16384 +FILE_ATTRIBUTE_HIDDEN: Final = 2 +FILE_ATTRIBUTE_INTEGRITY_STREAM: Final = 32768 +FILE_ATTRIBUTE_NORMAL: Final = 128 +FILE_ATTRIBUTE_NOT_CONTENT_INDEXED: Final = 8192 +FILE_ATTRIBUTE_NO_SCRUB_DATA: Final = 131072 +FILE_ATTRIBUTE_OFFLINE: Final = 4096 +FILE_ATTRIBUTE_READONLY: Final = 1 +FILE_ATTRIBUTE_REPARSE_POINT: Final = 1024 +FILE_ATTRIBUTE_SPARSE_FILE: Final = 512 +FILE_ATTRIBUTE_SYSTEM: Final = 4 +FILE_ATTRIBUTE_TEMPORARY: Final = 256 +FILE_ATTRIBUTE_VIRTUAL: Final = 65536 + if sys.version_info >= (3, 13): # https://github.com/python/cpython/issues/114081#issuecomment-2119017790 SF_RESTRICTED: Final = 0x00080000 diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi index 97e65d3094aa..6abef85dfb7f 100644 --- a/mypy/typeshed/stdlib/sys/__init__.pyi +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -5,7 +5,7 @@ from builtins import object as _object from collections.abc import AsyncGenerator, Callable, Sequence from io import TextIOWrapper from types import FrameType, ModuleType, TracebackType -from typing import Any, Final, Literal, NoReturn, Protocol, TextIO, TypeVar, final, type_check_only +from typing import Any, Final, Literal, NoReturn, Protocol, TextIO, TypeVar, final, overload, type_check_only from typing_extensions import LiteralString, TypeAlias, deprecated _T = TypeVar("_T") @@ -378,13 +378,13 @@ if sys.platform == "android": # noqa: Y008 def getandroidapilevel() -> int: ... def getallocatedblocks() -> int: ... -def getdefaultencoding() -> str: ... +def getdefaultencoding() -> Literal["utf-8"]: ... if sys.platform != "win32": def getdlopenflags() -> int: ... -def getfilesystemencoding() -> str: ... -def getfilesystemencodeerrors() -> str: ... +def getfilesystemencoding() -> LiteralString: ... +def getfilesystemencodeerrors() -> LiteralString: ... def getrefcount(object: Any, /) -> int: ... def getrecursionlimit() -> int: ... def getsizeof(obj: object, default: int = ...) -> int: ... @@ -422,7 +422,12 @@ if sys.platform == "win32": def getwindowsversion() -> _WinVersion: ... -def intern(string: str, /) -> str: ... +@overload +def intern(string: LiteralString, /) -> LiteralString: ... +@overload +def intern(string: str, /) -> str: ... # type: ignore[misc] + +__interactivehook__: Callable[[], object] if sys.version_info >= (3, 13): def _is_gil_enabled() -> bool: ... diff --git a/mypy/typeshed/stdlib/sys/_monitoring.pyi b/mypy/typeshed/stdlib/sys/_monitoring.pyi index 5d231c7a93b3..db799e6f32f8 100644 --- a/mypy/typeshed/stdlib/sys/_monitoring.pyi +++ b/mypy/typeshed/stdlib/sys/_monitoring.pyi @@ -17,6 +17,10 @@ PROFILER_ID: Final = 2 OPTIMIZER_ID: Final = 5 def use_tool_id(tool_id: int, name: str, /) -> None: ... + +if sys.version_info >= (3, 14): + def clear_tool_id(tool_id: int, /) -> None: ... + def free_tool_id(tool_id: int, /) -> None: ... def get_tool(tool_id: int, /) -> str | None: ... @@ -43,10 +47,10 @@ class _events: STOP_ITERATION: Final[int] if sys.version_info >= (3, 14): BRANCH_LEFT: Final[int] - BRANCH_TAKEN: Final[int] + BRANCH_RIGHT: Final[int] @property - @deprecated("Deprecated since Python 3.14. Use `BRANCH_LEFT` or `BRANCH_TAKEN` instead.") + @deprecated("Deprecated since Python 3.14. Use `BRANCH_LEFT` or `BRANCH_RIGHT` instead.") def BRANCH(self) -> int: ... else: diff --git a/mypy/typeshed/stdlib/threading.pyi b/mypy/typeshed/stdlib/threading.pyi index 28fa5267a997..7b0f15bdfa2e 100644 --- a/mypy/typeshed/stdlib/threading.pyi +++ b/mypy/typeshed/stdlib/threading.pyi @@ -1,6 +1,6 @@ import _thread import sys -from _thread import _excepthook, _ExceptHookArgs, get_native_id as get_native_id +from _thread import _ExceptHookArgs, get_native_id as get_native_id from _typeshed import ProfileFunction, TraceFunction from collections.abc import Callable, Iterable, Mapping from contextvars import ContextVar @@ -169,7 +169,9 @@ class Event: def clear(self) -> None: ... def wait(self, timeout: float | None = None) -> bool: ... -excepthook = _excepthook +excepthook: Callable[[_ExceptHookArgs], object] +if sys.version_info >= (3, 10): + __excepthook__: Callable[[_ExceptHookArgs], object] ExceptHookArgs = _ExceptHookArgs class Timer(Thread): diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 649e463ff71f..0293e5cb0b4b 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -717,9 +717,9 @@ if sys.version_info >= (3, 10): @final class EllipsisType: ... - from builtins import _NotImplementedType + @final + class NotImplementedType(Any): ... - NotImplementedType = _NotImplementedType @final class UnionType: @property diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index 2ca65dad4562..e3e5d1ff2842 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -222,10 +222,6 @@ class TypeVar: @property def evaluate_default(self) -> EvaluateFunc | None: ... -# Used for an undocumented mypy feature. Does not exist at runtime. -# Obsolete, use _typeshed._type_checker_internals.promote instead. -_promote = object() - # N.B. Keep this definition in sync with typing_extensions._SpecialForm @final class _SpecialForm(_Final): diff --git a/mypy/typeshed/stdlib/unittest/util.pyi b/mypy/typeshed/stdlib/unittest/util.pyi index 31c830e8268a..763c1478f5e6 100644 --- a/mypy/typeshed/stdlib/unittest/util.pyi +++ b/mypy/typeshed/stdlib/unittest/util.pyi @@ -1,9 +1,26 @@ from collections.abc import MutableSequence, Sequence -from typing import Any, Final, TypeVar +from typing import Any, Final, Literal, Protocol, TypeVar, type_check_only from typing_extensions import TypeAlias +@type_check_only +class _SupportsDunderLT(Protocol): + def __lt__(self, other: Any, /) -> bool: ... + +@type_check_only +class _SupportsDunderGT(Protocol): + def __gt__(self, other: Any, /) -> bool: ... + +@type_check_only +class _SupportsDunderLE(Protocol): + def __le__(self, other: Any, /) -> bool: ... + +@type_check_only +class _SupportsDunderGE(Protocol): + def __ge__(self, other: Any, /) -> bool: ... + _T = TypeVar("_T") _Mismatch: TypeAlias = tuple[_T, _T, int] +_SupportsComparison: TypeAlias = _SupportsDunderLE | _SupportsDunderGE | _SupportsDunderGT | _SupportsDunderLT _MAX_LENGTH: Final = 80 _PLACEHOLDER_LEN: Final = 12 @@ -18,6 +35,6 @@ def safe_repr(obj: object, short: bool = False) -> str: ... def strclass(cls: type) -> str: ... def sorted_list_difference(expected: Sequence[_T], actual: Sequence[_T]) -> tuple[list[_T], list[_T]]: ... def unorderable_list_difference(expected: MutableSequence[_T], actual: MutableSequence[_T]) -> tuple[list[_T], list[_T]]: ... -def three_way_cmp(x: Any, y: Any) -> int: ... +def three_way_cmp(x: _SupportsComparison, y: _SupportsComparison) -> Literal[-1, 0, 1]: ... def _count_diff_all_purpose(actual: Sequence[_T], expected: Sequence[_T]) -> list[_Mismatch[_T]]: ... def _count_diff_hashable(actual: Sequence[_T], expected: Sequence[_T]) -> list[_Mismatch[_T]]: ... diff --git a/mypy/typeshed/stdlib/uuid.pyi b/mypy/typeshed/stdlib/uuid.pyi index 303fb10eaf53..055f4def311c 100644 --- a/mypy/typeshed/stdlib/uuid.pyi +++ b/mypy/typeshed/stdlib/uuid.pyi @@ -1,7 +1,8 @@ import builtins import sys +from _typeshed import Unused from enum import Enum -from typing import Final +from typing import Final, NoReturn from typing_extensions import LiteralString, TypeAlias _FieldsType: TypeAlias = tuple[int, int, int, int, int, int] @@ -13,6 +14,9 @@ class SafeUUID(Enum): class UUID: __slots__ = ("int", "is_safe", "__weakref__") + is_safe: Final[SafeUUID] + int: Final[builtins.int] + def __init__( self, hex: str | None = None, @@ -25,8 +29,6 @@ class UUID: is_safe: SafeUUID = SafeUUID.unknown, ) -> None: ... @property - def is_safe(self) -> SafeUUID: ... - @property def bytes(self) -> builtins.bytes: ... @property def bytes_le(self) -> builtins.bytes: ... @@ -41,8 +43,6 @@ class UUID: @property def hex(self) -> str: ... @property - def int(self) -> builtins.int: ... - @property def node(self) -> builtins.int: ... @property def time(self) -> builtins.int: ... @@ -65,6 +65,7 @@ class UUID: def __gt__(self, other: UUID) -> bool: ... def __ge__(self, other: UUID) -> bool: ... def __hash__(self) -> builtins.int: ... + def __setattr__(self, name: Unused, value: Unused) -> NoReturn: ... def getnode() -> int: ... def uuid1(node: int | None = None, clock_seq: int | None = None) -> UUID: ... diff --git a/mypy/typeshed/stdlib/winreg.pyi b/mypy/typeshed/stdlib/winreg.pyi index 53457112ee96..a654bbcdfb61 100644 --- a/mypy/typeshed/stdlib/winreg.pyi +++ b/mypy/typeshed/stdlib/winreg.pyi @@ -18,13 +18,13 @@ if sys.platform == "win32": def ExpandEnvironmentStrings(string: str, /) -> str: ... def FlushKey(key: _KeyType, /) -> None: ... def LoadKey(key: _KeyType, sub_key: str, file_name: str, /) -> None: ... - def OpenKey(key: _KeyType, sub_key: str, reserved: int = 0, access: int = 131097) -> HKEYType: ... - def OpenKeyEx(key: _KeyType, sub_key: str, reserved: int = 0, access: int = 131097) -> HKEYType: ... + def OpenKey(key: _KeyType, sub_key: str | None, reserved: int = 0, access: int = 131097) -> HKEYType: ... + def OpenKeyEx(key: _KeyType, sub_key: str | None, reserved: int = 0, access: int = 131097) -> HKEYType: ... def QueryInfoKey(key: _KeyType, /) -> tuple[int, int, int]: ... def QueryValue(key: _KeyType, sub_key: str | None, /) -> str: ... def QueryValueEx(key: _KeyType, name: str, /) -> tuple[Any, int]: ... def SaveKey(key: _KeyType, file_name: str, /) -> None: ... - def SetValue(key: _KeyType, sub_key: str, type: int, value: str, /) -> None: ... + def SetValue(key: _KeyType, sub_key: str | None, type: int, value: str, /) -> None: ... @overload # type=REG_DWORD|REG_QWORD def SetValueEx( key: _KeyType, value_name: str | None, reserved: Unused, type: Literal[4, 5], value: int | None, / diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 7dfcf7447b61..8d6a78bfe470 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1948,7 +1948,7 @@ Foo().__dict__ = {} [out] _testInferenceOfDunderDictOnClassObjects.py:2: note: Revealed type is "types.MappingProxyType[builtins.str, Any]" _testInferenceOfDunderDictOnClassObjects.py:3: note: Revealed type is "builtins.dict[builtins.str, Any]" -_testInferenceOfDunderDictOnClassObjects.py:4: error: Property "__dict__" defined in "type" is read-only +_testInferenceOfDunderDictOnClassObjects.py:4: error: Cannot assign to final attribute "__dict__" _testInferenceOfDunderDictOnClassObjects.py:4: error: Incompatible types in assignment (expression has type "dict[Never, Never]", variable has type "MappingProxyType[str, Any]") [case testTypeVarTuple] From 8f922b3d879f31e4c52c1c643de70e27b5720654 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 17 Nov 2025 11:20:52 +0000 Subject: [PATCH 169/183] [mypyc] Use faster base64 encode implementation in librt.base64 (#20237) Vendor optimized base64 implementation from https://github.com/aklomp/base64. This is based on commit 9e8ed65048ff0f703fad3deb03bf66ac7f78a4d7 (May 2025). Enable SIMD on macOS (64-bit ARM only). Other platforms probably use a generic version. I'll look into enabling SIMD more generally in a follow-up PR. A `b64encode` micro-benchmark was up to 11 times faster compared to the stdlib `base64` module (on a MacBook Pro). --- LICENSE | 35 + mypyc/build.py | 63 +- mypyc/lib-rt/base64/arch/avx/codec.c | 68 ++ mypyc/lib-rt/base64/arch/avx/enc_loop_asm.c | 264 +++++ mypyc/lib-rt/base64/arch/avx2/codec.c | 58 + mypyc/lib-rt/base64/arch/avx2/dec_loop.c | 110 ++ mypyc/lib-rt/base64/arch/avx2/dec_reshuffle.c | 34 + mypyc/lib-rt/base64/arch/avx2/enc_loop.c | 89 ++ mypyc/lib-rt/base64/arch/avx2/enc_loop_asm.c | 291 +++++ mypyc/lib-rt/base64/arch/avx2/enc_reshuffle.c | 83 ++ mypyc/lib-rt/base64/arch/avx2/enc_translate.c | 30 + mypyc/lib-rt/base64/arch/avx512/codec.c | 44 + mypyc/lib-rt/base64/arch/avx512/enc_loop.c | 61 + .../arch/avx512/enc_reshuffle_translate.c | 50 + .../lib-rt/base64/arch/generic/32/dec_loop.c | 86 ++ .../lib-rt/base64/arch/generic/32/enc_loop.c | 73 ++ .../lib-rt/base64/arch/generic/64/enc_loop.c | 77 ++ mypyc/lib-rt/base64/arch/generic/codec.c | 41 + mypyc/lib-rt/base64/arch/generic/dec_head.c | 37 + mypyc/lib-rt/base64/arch/generic/dec_tail.c | 91 ++ mypyc/lib-rt/base64/arch/generic/enc_head.c | 24 + mypyc/lib-rt/base64/arch/generic/enc_tail.c | 34 + mypyc/lib-rt/base64/arch/neon32/codec.c | 79 ++ mypyc/lib-rt/base64/arch/neon32/dec_loop.c | 106 ++ mypyc/lib-rt/base64/arch/neon32/enc_loop.c | 170 +++ .../lib-rt/base64/arch/neon32/enc_reshuffle.c | 31 + .../lib-rt/base64/arch/neon32/enc_translate.c | 57 + mypyc/lib-rt/base64/arch/neon64/codec.c | 93 ++ mypyc/lib-rt/base64/arch/neon64/dec_loop.c | 129 +++ mypyc/lib-rt/base64/arch/neon64/enc_loop.c | 66 ++ .../lib-rt/base64/arch/neon64/enc_loop_asm.c | 168 +++ .../lib-rt/base64/arch/neon64/enc_reshuffle.c | 31 + mypyc/lib-rt/base64/arch/sse41/codec.c | 58 + mypyc/lib-rt/base64/arch/sse42/codec.c | 58 + mypyc/lib-rt/base64/arch/ssse3/codec.c | 60 + mypyc/lib-rt/base64/arch/ssse3/dec_loop.c | 173 +++ .../lib-rt/base64/arch/ssse3/dec_reshuffle.c | 33 + mypyc/lib-rt/base64/arch/ssse3/enc_loop.c | 67 ++ mypyc/lib-rt/base64/arch/ssse3/enc_loop_asm.c | 268 +++++ .../lib-rt/base64/arch/ssse3/enc_reshuffle.c | 48 + .../lib-rt/base64/arch/ssse3/enc_translate.c | 33 + mypyc/lib-rt/base64/codec_choose.c | 314 +++++ mypyc/lib-rt/base64/codecs.h | 57 + mypyc/lib-rt/base64/config.h | 33 + mypyc/lib-rt/base64/env.h | 84 ++ mypyc/lib-rt/base64/lib.c | 164 +++ mypyc/lib-rt/base64/libbase64.h | 146 +++ mypyc/lib-rt/base64/tables/table_dec_32bit.h | 393 +++++++ mypyc/lib-rt/base64/tables/table_enc_12bit.h | 1031 +++++++++++++++++ mypyc/lib-rt/base64/tables/tables.c | 40 + mypyc/lib-rt/base64/tables/tables.h | 23 + mypyc/lib-rt/librt_base64.c | 66 +- mypyc/lib-rt/setup.py | 19 +- 53 files changed, 5787 insertions(+), 54 deletions(-) create mode 100644 mypyc/lib-rt/base64/arch/avx/codec.c create mode 100644 mypyc/lib-rt/base64/arch/avx/enc_loop_asm.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/codec.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/dec_loop.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/dec_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/enc_loop_asm.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/enc_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/avx2/enc_translate.c create mode 100644 mypyc/lib-rt/base64/arch/avx512/codec.c create mode 100644 mypyc/lib-rt/base64/arch/avx512/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/avx512/enc_reshuffle_translate.c create mode 100644 mypyc/lib-rt/base64/arch/generic/32/dec_loop.c create mode 100644 mypyc/lib-rt/base64/arch/generic/32/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/generic/64/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/generic/codec.c create mode 100644 mypyc/lib-rt/base64/arch/generic/dec_head.c create mode 100644 mypyc/lib-rt/base64/arch/generic/dec_tail.c create mode 100644 mypyc/lib-rt/base64/arch/generic/enc_head.c create mode 100644 mypyc/lib-rt/base64/arch/generic/enc_tail.c create mode 100644 mypyc/lib-rt/base64/arch/neon32/codec.c create mode 100644 mypyc/lib-rt/base64/arch/neon32/dec_loop.c create mode 100644 mypyc/lib-rt/base64/arch/neon32/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/neon32/enc_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/neon32/enc_translate.c create mode 100644 mypyc/lib-rt/base64/arch/neon64/codec.c create mode 100644 mypyc/lib-rt/base64/arch/neon64/dec_loop.c create mode 100644 mypyc/lib-rt/base64/arch/neon64/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/neon64/enc_loop_asm.c create mode 100644 mypyc/lib-rt/base64/arch/neon64/enc_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/sse41/codec.c create mode 100644 mypyc/lib-rt/base64/arch/sse42/codec.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/codec.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/dec_loop.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/dec_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/enc_loop.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/enc_loop_asm.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/enc_reshuffle.c create mode 100644 mypyc/lib-rt/base64/arch/ssse3/enc_translate.c create mode 100644 mypyc/lib-rt/base64/codec_choose.c create mode 100644 mypyc/lib-rt/base64/codecs.h create mode 100644 mypyc/lib-rt/base64/config.h create mode 100644 mypyc/lib-rt/base64/env.h create mode 100644 mypyc/lib-rt/base64/lib.c create mode 100644 mypyc/lib-rt/base64/libbase64.h create mode 100644 mypyc/lib-rt/base64/tables/table_dec_32bit.h create mode 100644 mypyc/lib-rt/base64/tables/table_enc_12bit.h create mode 100644 mypyc/lib-rt/base64/tables/tables.c create mode 100644 mypyc/lib-rt/base64/tables/tables.h diff --git a/LICENSE b/LICENSE index 55d01ee19ad8..080c5a402dbb 100644 --- a/LICENSE +++ b/LICENSE @@ -227,3 +227,38 @@ FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + += = = = = + +Files under lib-rt/base64 are licensed under the following license. + += = = = = + +Copyright (c) 2005-2007, Nick Galbreath +Copyright (c) 2015-2018, Wojciech Muła +Copyright (c) 2016-2017, Matthieu Darbois +Copyright (c) 2013-2022, Alfred Klomp +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +- Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/mypyc/build.py b/mypyc/build.py index 351aa87b9264..8505a2d95701 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -26,7 +26,7 @@ import sys import time from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, NoReturn, Union, cast +from typing import TYPE_CHECKING, Any, NamedTuple, NoReturn, Union, cast from mypy.build import BuildSource from mypy.errors import CompileError @@ -42,7 +42,52 @@ from mypyc.namegen import exported_name from mypyc.options import CompilerOptions -LIBRT_MODULES = [("librt.internal", "librt_internal.c"), ("librt.base64", "librt_base64.c")] + +class ModDesc(NamedTuple): + module: str + c_files: list[str] + other_files: list[str] + include_dirs: list[str] + + +LIBRT_MODULES = [ + ModDesc("librt.internal", ["librt_internal.c"], [], []), + ModDesc( + "librt.base64", + [ + "librt_base64.c", + "base64/lib.c", + "base64/codec_choose.c", + "base64/tables/tables.c", + "base64/arch/generic/codec.c", + "base64/arch/ssse3/codec.c", + "base64/arch/sse41/codec.c", + "base64/arch/sse42/codec.c", + "base64/arch/avx/codec.c", + "base64/arch/avx2/codec.c", + "base64/arch/avx512/codec.c", + "base64/arch/neon32/codec.c", + "base64/arch/neon64/codec.c", + ], + [ + "base64/arch/generic/32/enc_loop.c", + "base64/arch/generic/64/enc_loop.c", + "base64/arch/generic/32/dec_loop.c", + "base64/arch/generic/enc_head.c", + "base64/arch/generic/enc_tail.c", + "base64/arch/generic/dec_head.c", + "base64/arch/generic/dec_tail.c", + "base64/arch/neon64/dec_loop.c", + "base64/arch/neon64/enc_loop_asm.c", + "base64/codecs.h", + "base64/env.h", + "base64/tables/tables.h", + "base64/tables/table_dec_32bit.h", + "base64/tables/table_enc_12bit.h", + ], + ["base64"], + ), +] try: # Import setuptools so that it monkey-patch overrides distutils @@ -677,17 +722,19 @@ def mypycify( rt_file = os.path.join(build_dir, name) with open(os.path.join(include_dir(), name), encoding="utf-8") as f: write_file(rt_file, f.read()) - for mod, file_name in LIBRT_MODULES: - rt_file = os.path.join(build_dir, file_name) - with open(os.path.join(include_dir(), file_name), encoding="utf-8") as f: - write_file(rt_file, f.read()) + for mod, file_names, addit_files, includes in LIBRT_MODULES: + for file_name in file_names + addit_files: + rt_file = os.path.join(build_dir, file_name) + with open(os.path.join(include_dir(), file_name), encoding="utf-8") as f: + write_file(rt_file, f.read()) extensions.append( get_extension()( mod, sources=[ - os.path.join(build_dir, file) for file in [file_name] + RUNTIME_C_FILES + os.path.join(build_dir, file) for file in file_names + RUNTIME_C_FILES ], - include_dirs=[include_dir()], + include_dirs=[include_dir()] + + [os.path.join(include_dir(), d) for d in includes], extra_compile_args=cflags, ) ) diff --git a/mypyc/lib-rt/base64/arch/avx/codec.c b/mypyc/lib-rt/base64/arch/avx/codec.c new file mode 100644 index 000000000000..8e2ef5c2e724 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx/codec.c @@ -0,0 +1,68 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_AVX +#include + +// Only enable inline assembly on supported compilers and on 64-bit CPUs. +#ifndef BASE64_AVX_USE_ASM +# if (defined(__GNUC__) || defined(__clang__)) && BASE64_WORDSIZE == 64 +# define BASE64_AVX_USE_ASM 1 +# else +# define BASE64_AVX_USE_ASM 0 +# endif +#endif + +#include "../ssse3/dec_reshuffle.c" +#include "../ssse3/dec_loop.c" + +#if BASE64_AVX_USE_ASM +# include "enc_loop_asm.c" +#else +# include "../ssse3/enc_translate.c" +# include "../ssse3/enc_reshuffle.c" +# include "../ssse3/enc_loop.c" +#endif + +#endif // HAVE_AVX + +void +base64_stream_encode_avx BASE64_ENC_PARAMS +{ +#if HAVE_AVX + #include "../generic/enc_head.c" + + // For supported compilers, use a hand-optimized inline assembly + // encoder. Otherwise fall back on the SSSE3 encoder, but compiled with + // AVX flags to generate better optimized AVX code. + +#if BASE64_AVX_USE_ASM + enc_loop_avx(&s, &slen, &o, &olen); +#else + enc_loop_ssse3(&s, &slen, &o, &olen); +#endif + + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_avx BASE64_DEC_PARAMS +{ +#if HAVE_AVX + #include "../generic/dec_head.c" + dec_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/avx/enc_loop_asm.c b/mypyc/lib-rt/base64/arch/avx/enc_loop_asm.c new file mode 100644 index 000000000000..979269af5774 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx/enc_loop_asm.c @@ -0,0 +1,264 @@ +// Apologies in advance for combining the preprocessor with inline assembly, +// two notoriously gnarly parts of C, but it was necessary to avoid a lot of +// code repetition. The preprocessor is used to template large sections of +// inline assembly that differ only in the registers used. If the code was +// written out by hand, it would become very large and hard to audit. + +// Generate a block of inline assembly that loads register R0 from memory. The +// offset at which the register is loaded is set by the given round. +#define LOAD(R0, ROUND) \ + "vlddqu ("#ROUND" * 12)(%[src]), %["R0"] \n\t" + +// Generate a block of inline assembly that deinterleaves and shuffles register +// R0 using preloaded constants. Outputs in R0 and R1. +#define SHUF(R0, R1, R2) \ + "vpshufb %[lut0], %["R0"], %["R1"] \n\t" \ + "vpand %["R1"], %[msk0], %["R2"] \n\t" \ + "vpand %["R1"], %[msk2], %["R1"] \n\t" \ + "vpmulhuw %["R2"], %[msk1], %["R2"] \n\t" \ + "vpmullw %["R1"], %[msk3], %["R1"] \n\t" \ + "vpor %["R1"], %["R2"], %["R1"] \n\t" + +// Generate a block of inline assembly that takes R0 and R1 and translates +// their contents to the base64 alphabet, using preloaded constants. +#define TRAN(R0, R1, R2) \ + "vpsubusb %[n51], %["R1"], %["R0"] \n\t" \ + "vpcmpgtb %[n25], %["R1"], %["R2"] \n\t" \ + "vpsubb %["R2"], %["R0"], %["R0"] \n\t" \ + "vpshufb %["R0"], %[lut1], %["R2"] \n\t" \ + "vpaddb %["R1"], %["R2"], %["R0"] \n\t" + +// Generate a block of inline assembly that stores the given register R0 at an +// offset set by the given round. +#define STOR(R0, ROUND) \ + "vmovdqu %["R0"], ("#ROUND" * 16)(%[dst]) \n\t" + +// Generate a block of inline assembly that generates a single self-contained +// encoder round: fetch the data, process it, and store the result. Then update +// the source and destination pointers. +#define ROUND() \ + LOAD("a", 0) \ + SHUF("a", "b", "c") \ + TRAN("a", "b", "c") \ + STOR("a", 0) \ + "add $12, %[src] \n\t" \ + "add $16, %[dst] \n\t" + +// Define a macro that initiates a three-way interleaved encoding round by +// preloading registers a, b and c from memory. +// The register graph shows which registers are in use during each step, and +// is a visual aid for choosing registers for that step. Symbol index: +// +// + indicates that a register is loaded by that step. +// | indicates that a register is in use and must not be touched. +// - indicates that a register is decommissioned by that step. +// x indicates that a register is used as a temporary by that step. +// V indicates that a register is an input or output to the macro. +// +#define ROUND_3_INIT() /* a b c d e f */ \ + LOAD("a", 0) /* + */ \ + SHUF("a", "d", "e") /* | + x */ \ + LOAD("b", 1) /* | + | */ \ + TRAN("a", "d", "e") /* | | - x */ \ + LOAD("c", 2) /* V V V */ + +// Define a macro that translates, shuffles and stores the input registers A, B +// and C, and preloads registers D, E and F for the next round. +// This macro can be arbitrarily daisy-chained by feeding output registers D, E +// and F back into the next round as input registers A, B and C. The macro +// carefully interleaves memory operations with data operations for optimal +// pipelined performance. + +#define ROUND_3(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + LOAD(D, (ROUND + 3)) /* V V V + */ \ + SHUF(B, E, F) /* | | | | + x */ \ + STOR(A, (ROUND + 0)) /* - | | | | */ \ + TRAN(B, E, F) /* | | | - x */ \ + LOAD(E, (ROUND + 4)) /* | | | + */ \ + SHUF(C, A, F) /* + | | | | x */ \ + STOR(B, (ROUND + 1)) /* | - | | | */ \ + TRAN(C, A, F) /* - | | | x */ \ + LOAD(F, (ROUND + 5)) /* | | | + */ \ + SHUF(D, A, B) /* + x | | | | */ \ + STOR(C, (ROUND + 2)) /* | - | | | */ \ + TRAN(D, A, B) /* - x V V V */ + +// Define a macro that terminates a ROUND_3 macro by taking pre-loaded +// registers D, E and F, and translating, shuffling and storing them. +#define ROUND_3_END(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + SHUF(E, A, B) /* + x V V V */ \ + STOR(D, (ROUND + 3)) /* | - | | */ \ + TRAN(E, A, B) /* - x | | */ \ + SHUF(F, C, D) /* + x | | */ \ + STOR(E, (ROUND + 4)) /* | - | */ \ + TRAN(F, C, D) /* - x | */ \ + STOR(F, (ROUND + 5)) /* - */ + +// Define a type A round. Inputs are a, b, and c, outputs are d, e, and f. +#define ROUND_3_A(ROUND) \ + ROUND_3(ROUND, "a", "b", "c", "d", "e", "f") + +// Define a type B round. Inputs and outputs are swapped with regard to type A. +#define ROUND_3_B(ROUND) \ + ROUND_3(ROUND, "d", "e", "f", "a", "b", "c") + +// Terminating macro for a type A round. +#define ROUND_3_A_LAST(ROUND) \ + ROUND_3_A(ROUND) \ + ROUND_3_END(ROUND, "a", "b", "c", "d", "e", "f") + +// Terminating macro for a type B round. +#define ROUND_3_B_LAST(ROUND) \ + ROUND_3_B(ROUND) \ + ROUND_3_END(ROUND, "d", "e", "f", "a", "b", "c") + +// Suppress clang's warning that the literal string in the asm statement is +// overlong (longer than the ISO-mandated minimum size of 4095 bytes for C99 +// compilers). It may be true, but the goal here is not C99 portability. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverlength-strings" + +static inline void +enc_loop_avx (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + // For a clearer explanation of the algorithm used by this function, + // please refer to the plain (not inline assembly) implementation. This + // function follows the same basic logic. + + if (*slen < 16) { + return; + } + + // Process blocks of 12 bytes at a time. Input is read in blocks of 16 + // bytes, so "reserve" four bytes from the input buffer to ensure that + // we never read beyond the end of the input buffer. + size_t rounds = (*slen - 4) / 12; + + *slen -= rounds * 12; // 12 bytes consumed per round + *olen += rounds * 16; // 16 bytes produced per round + + // Number of times to go through the 36x loop. + size_t loops = rounds / 36; + + // Number of rounds remaining after the 36x loop. + rounds %= 36; + + // Lookup tables. + const __m128i lut0 = _mm_set_epi8( + 10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1); + + const __m128i lut1 = _mm_setr_epi8( + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0); + + // Temporary registers. + __m128i a, b, c, d, e, f; + + __asm__ volatile ( + + // If there are 36 rounds or more, enter a 36x unrolled loop of + // interleaved encoding rounds. The rounds interleave memory + // operations (load/store) with data operations (table lookups, + // etc) to maximize pipeline throughput. + " test %[loops], %[loops] \n\t" + " jz 18f \n\t" + " jmp 36f \n\t" + " \n\t" + ".balign 64 \n\t" + "36: " ROUND_3_INIT() + " " ROUND_3_A( 0) + " " ROUND_3_B( 3) + " " ROUND_3_A( 6) + " " ROUND_3_B( 9) + " " ROUND_3_A(12) + " " ROUND_3_B(15) + " " ROUND_3_A(18) + " " ROUND_3_B(21) + " " ROUND_3_A(24) + " " ROUND_3_B(27) + " " ROUND_3_A_LAST(30) + " add $(12 * 36), %[src] \n\t" + " add $(16 * 36), %[dst] \n\t" + " dec %[loops] \n\t" + " jnz 36b \n\t" + + // Enter an 18x unrolled loop for rounds of 18 or more. + "18: cmp $18, %[rounds] \n\t" + " jl 9f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B(3) + " " ROUND_3_A(6) + " " ROUND_3_B(9) + " " ROUND_3_A_LAST(12) + " sub $18, %[rounds] \n\t" + " add $(12 * 18), %[src] \n\t" + " add $(16 * 18), %[dst] \n\t" + + // Enter a 9x unrolled loop for rounds of 9 or more. + "9: cmp $9, %[rounds] \n\t" + " jl 6f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B_LAST(3) + " sub $9, %[rounds] \n\t" + " add $(12 * 9), %[src] \n\t" + " add $(16 * 9), %[dst] \n\t" + + // Enter a 6x unrolled loop for rounds of 6 or more. + "6: cmp $6, %[rounds] \n\t" + " jl 55f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A_LAST(0) + " sub $6, %[rounds] \n\t" + " add $(12 * 6), %[src] \n\t" + " add $(16 * 6), %[dst] \n\t" + + // Dispatch the remaining rounds 0..5. + "55: cmp $3, %[rounds] \n\t" + " jg 45f \n\t" + " je 3f \n\t" + " cmp $1, %[rounds] \n\t" + " jg 2f \n\t" + " je 1f \n\t" + " jmp 0f \n\t" + + "45: cmp $4, %[rounds] \n\t" + " je 4f \n\t" + + // Block of non-interlaced encoding rounds, which can each + // individually be jumped to. Rounds fall through to the next. + "5: " ROUND() + "4: " ROUND() + "3: " ROUND() + "2: " ROUND() + "1: " ROUND() + "0: \n\t" + + // Outputs (modified). + : [rounds] "+r" (rounds), + [loops] "+r" (loops), + [src] "+r" (*s), + [dst] "+r" (*o), + [a] "=&x" (a), + [b] "=&x" (b), + [c] "=&x" (c), + [d] "=&x" (d), + [e] "=&x" (e), + [f] "=&x" (f) + + // Inputs (not modified). + : [lut0] "x" (lut0), + [lut1] "x" (lut1), + [msk0] "x" (_mm_set1_epi32(0x0FC0FC00)), + [msk1] "x" (_mm_set1_epi32(0x04000040)), + [msk2] "x" (_mm_set1_epi32(0x003F03F0)), + [msk3] "x" (_mm_set1_epi32(0x01000010)), + [n51] "x" (_mm_set1_epi8(51)), + [n25] "x" (_mm_set1_epi8(25)) + + // Clobbers. + : "cc", "memory" + ); +} + +#pragma GCC diagnostic pop diff --git a/mypyc/lib-rt/base64/arch/avx2/codec.c b/mypyc/lib-rt/base64/arch/avx2/codec.c new file mode 100644 index 000000000000..fe9200296914 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/codec.c @@ -0,0 +1,58 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_AVX2 +#include + +// Only enable inline assembly on supported compilers and on 64-bit CPUs. +#ifndef BASE64_AVX2_USE_ASM +# if (defined(__GNUC__) || defined(__clang__)) && BASE64_WORDSIZE == 64 +# define BASE64_AVX2_USE_ASM 1 +# else +# define BASE64_AVX2_USE_ASM 0 +# endif +#endif + +#include "dec_reshuffle.c" +#include "dec_loop.c" + +#if BASE64_AVX2_USE_ASM +# include "enc_loop_asm.c" +#else +# include "enc_translate.c" +# include "enc_reshuffle.c" +# include "enc_loop.c" +#endif + +#endif // HAVE_AVX2 + +void +base64_stream_encode_avx2 BASE64_ENC_PARAMS +{ +#if HAVE_AVX2 + #include "../generic/enc_head.c" + enc_loop_avx2(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_avx2 BASE64_DEC_PARAMS +{ +#if HAVE_AVX2 + #include "../generic/dec_head.c" + dec_loop_avx2(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/avx2/dec_loop.c b/mypyc/lib-rt/base64/arch/avx2/dec_loop.c new file mode 100644 index 000000000000..b8a4ccafd82a --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/dec_loop.c @@ -0,0 +1,110 @@ +static BASE64_FORCE_INLINE int +dec_loop_avx2_inner (const uint8_t **s, uint8_t **o, size_t *rounds) +{ + const __m256i lut_lo = _mm256_setr_epi8( + 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A, + 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A); + + const __m256i lut_hi = _mm256_setr_epi8( + 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10); + + const __m256i lut_roll = _mm256_setr_epi8( + 0, 16, 19, 4, -65, -65, -71, -71, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 16, 19, 4, -65, -65, -71, -71, + 0, 0, 0, 0, 0, 0, 0, 0); + + const __m256i mask_2F = _mm256_set1_epi8(0x2F); + + // Load input: + __m256i str = _mm256_loadu_si256((__m256i *) *s); + + // See the SSSE3 decoder for an explanation of the algorithm. + const __m256i hi_nibbles = _mm256_and_si256(_mm256_srli_epi32(str, 4), mask_2F); + const __m256i lo_nibbles = _mm256_and_si256(str, mask_2F); + const __m256i hi = _mm256_shuffle_epi8(lut_hi, hi_nibbles); + const __m256i lo = _mm256_shuffle_epi8(lut_lo, lo_nibbles); + + if (!_mm256_testz_si256(lo, hi)) { + return 0; + } + + const __m256i eq_2F = _mm256_cmpeq_epi8(str, mask_2F); + const __m256i roll = _mm256_shuffle_epi8(lut_roll, _mm256_add_epi8(eq_2F, hi_nibbles)); + + // Now simply add the delta values to the input: + str = _mm256_add_epi8(str, roll); + + // Reshuffle the input to packed 12-byte output format: + str = dec_reshuffle(str); + + // Store the output: + _mm256_storeu_si256((__m256i *) *o, str); + + *s += 32; + *o += 24; + *rounds -= 1; + + return 1; +} + +static inline void +dec_loop_avx2 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 45) { + return; + } + + // Process blocks of 32 bytes per round. Because 8 extra zero bytes are + // written after the output, ensure that there will be at least 13 + // bytes of input data left to cover the gap. (11 data bytes and up to + // two end-of-string markers.) + size_t rounds = (*slen - 13) / 32; + + *slen -= rounds * 32; // 32 bytes consumed per round + *olen += rounds * 24; // 24 bytes produced per round + + do { + if (rounds >= 8) { + if (dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 4) { + if (dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 2) { + if (dec_loop_avx2_inner(s, o, &rounds) && + dec_loop_avx2_inner(s, o, &rounds)) { + continue; + } + break; + } + dec_loop_avx2_inner(s, o, &rounds); + break; + + } while (rounds > 0); + + // Adjust for any rounds that were skipped: + *slen += rounds * 32; + *olen -= rounds * 24; +} diff --git a/mypyc/lib-rt/base64/arch/avx2/dec_reshuffle.c b/mypyc/lib-rt/base64/arch/avx2/dec_reshuffle.c new file mode 100644 index 000000000000..bc875ce9dd25 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/dec_reshuffle.c @@ -0,0 +1,34 @@ +static BASE64_FORCE_INLINE __m256i +dec_reshuffle (const __m256i in) +{ + // in, lower lane, bits, upper case are most significant bits, lower + // case are least significant bits: + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA + + const __m256i merge_ab_and_bc = _mm256_maddubs_epi16(in, _mm256_set1_epi32(0x01400140)); + // 0000kkkk LLllllll 0000JJJJ JJjjKKKK + // 0000hhhh IIiiiiii 0000GGGG GGggHHHH + // 0000eeee FFffffff 0000DDDD DDddEEEE + // 0000bbbb CCcccccc 0000AAAA AAaaBBBB + + __m256i out = _mm256_madd_epi16(merge_ab_and_bc, _mm256_set1_epi32(0x00011000)); + // 00000000 JJJJJJjj KKKKkkkk LLllllll + // 00000000 GGGGGGgg HHHHhhhh IIiiiiii + // 00000000 DDDDDDdd EEEEeeee FFffffff + // 00000000 AAAAAAaa BBBBbbbb CCcccccc + + // Pack bytes together in each lane: + out = _mm256_shuffle_epi8(out, _mm256_setr_epi8( + 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1, + 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1)); + // 00000000 00000000 00000000 00000000 + // LLllllll KKKKkkkk JJJJJJjj IIiiiiii + // HHHHhhhh GGGGGGgg FFffffff EEEEeeee + // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa + + // Pack lanes: + return _mm256_permutevar8x32_epi32(out, _mm256_setr_epi32(0, 1, 2, 4, 5, 6, -1, -1)); +} diff --git a/mypyc/lib-rt/base64/arch/avx2/enc_loop.c b/mypyc/lib-rt/base64/arch/avx2/enc_loop.c new file mode 100644 index 000000000000..6f4aa0ab580c --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/enc_loop.c @@ -0,0 +1,89 @@ +static BASE64_FORCE_INLINE void +enc_loop_avx2_inner_first (const uint8_t **s, uint8_t **o) +{ + // First load is done at s - 0 to not get a segfault: + __m256i src = _mm256_loadu_si256((__m256i *) *s); + + // Shift by 4 bytes, as required by enc_reshuffle: + src = _mm256_permutevar8x32_epi32(src, _mm256_setr_epi32(0, 0, 1, 2, 3, 4, 5, 6)); + + // Reshuffle, translate, store: + src = enc_reshuffle(src); + src = enc_translate(src); + _mm256_storeu_si256((__m256i *) *o, src); + + // Subsequent loads will be done at s - 4, set pointer for next round: + *s += 20; + *o += 32; +} + +static BASE64_FORCE_INLINE void +enc_loop_avx2_inner (const uint8_t **s, uint8_t **o) +{ + // Load input: + __m256i src = _mm256_loadu_si256((__m256i *) *s); + + // Reshuffle, translate, store: + src = enc_reshuffle(src); + src = enc_translate(src); + _mm256_storeu_si256((__m256i *) *o, src); + + *s += 24; + *o += 32; +} + +static inline void +enc_loop_avx2 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 32) { + return; + } + + // Process blocks of 24 bytes at a time. Because blocks are loaded 32 + // bytes at a time an offset of -4, ensure that there will be at least + // 4 remaining bytes after the last round, so that the final read will + // not pass beyond the bounds of the input buffer: + size_t rounds = (*slen - 4) / 24; + + *slen -= rounds * 24; // 24 bytes consumed per round + *olen += rounds * 32; // 32 bytes produced per round + + // The first loop iteration requires special handling to ensure that + // the read, which is done at an offset, does not underflow the buffer: + enc_loop_avx2_inner_first(s, o); + rounds--; + + while (rounds > 0) { + if (rounds >= 8) { + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_avx2_inner(s, o); + enc_loop_avx2_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_avx2_inner(s, o); + break; + } + + // Add the offset back: + *s += 4; +} diff --git a/mypyc/lib-rt/base64/arch/avx2/enc_loop_asm.c b/mypyc/lib-rt/base64/arch/avx2/enc_loop_asm.c new file mode 100644 index 000000000000..eb775a1d1f03 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/enc_loop_asm.c @@ -0,0 +1,291 @@ +// Apologies in advance for combining the preprocessor with inline assembly, +// two notoriously gnarly parts of C, but it was necessary to avoid a lot of +// code repetition. The preprocessor is used to template large sections of +// inline assembly that differ only in the registers used. If the code was +// written out by hand, it would become very large and hard to audit. + +// Generate a block of inline assembly that loads register R0 from memory. The +// offset at which the register is loaded is set by the given round and a +// constant offset. +#define LOAD(R0, ROUND, OFFSET) \ + "vlddqu ("#ROUND" * 24 + "#OFFSET")(%[src]), %["R0"] \n\t" + +// Generate a block of inline assembly that deinterleaves and shuffles register +// R0 using preloaded constants. Outputs in R0 and R1. +#define SHUF(R0, R1, R2) \ + "vpshufb %[lut0], %["R0"], %["R1"] \n\t" \ + "vpand %["R1"], %[msk0], %["R2"] \n\t" \ + "vpand %["R1"], %[msk2], %["R1"] \n\t" \ + "vpmulhuw %["R2"], %[msk1], %["R2"] \n\t" \ + "vpmullw %["R1"], %[msk3], %["R1"] \n\t" \ + "vpor %["R1"], %["R2"], %["R1"] \n\t" + +// Generate a block of inline assembly that takes R0 and R1 and translates +// their contents to the base64 alphabet, using preloaded constants. +#define TRAN(R0, R1, R2) \ + "vpsubusb %[n51], %["R1"], %["R0"] \n\t" \ + "vpcmpgtb %[n25], %["R1"], %["R2"] \n\t" \ + "vpsubb %["R2"], %["R0"], %["R0"] \n\t" \ + "vpshufb %["R0"], %[lut1], %["R2"] \n\t" \ + "vpaddb %["R1"], %["R2"], %["R0"] \n\t" + +// Generate a block of inline assembly that stores the given register R0 at an +// offset set by the given round. +#define STOR(R0, ROUND) \ + "vmovdqu %["R0"], ("#ROUND" * 32)(%[dst]) \n\t" + +// Generate a block of inline assembly that generates a single self-contained +// encoder round: fetch the data, process it, and store the result. Then update +// the source and destination pointers. +#define ROUND() \ + LOAD("a", 0, -4) \ + SHUF("a", "b", "c") \ + TRAN("a", "b", "c") \ + STOR("a", 0) \ + "add $24, %[src] \n\t" \ + "add $32, %[dst] \n\t" + +// Define a macro that initiates a three-way interleaved encoding round by +// preloading registers a, b and c from memory. +// The register graph shows which registers are in use during each step, and +// is a visual aid for choosing registers for that step. Symbol index: +// +// + indicates that a register is loaded by that step. +// | indicates that a register is in use and must not be touched. +// - indicates that a register is decommissioned by that step. +// x indicates that a register is used as a temporary by that step. +// V indicates that a register is an input or output to the macro. +// +#define ROUND_3_INIT() /* a b c d e f */ \ + LOAD("a", 0, -4) /* + */ \ + SHUF("a", "d", "e") /* | + x */ \ + LOAD("b", 1, -4) /* | + | */ \ + TRAN("a", "d", "e") /* | | - x */ \ + LOAD("c", 2, -4) /* V V V */ + +// Define a macro that translates, shuffles and stores the input registers A, B +// and C, and preloads registers D, E and F for the next round. +// This macro can be arbitrarily daisy-chained by feeding output registers D, E +// and F back into the next round as input registers A, B and C. The macro +// carefully interleaves memory operations with data operations for optimal +// pipelined performance. + +#define ROUND_3(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + LOAD(D, (ROUND + 3), -4) /* V V V + */ \ + SHUF(B, E, F) /* | | | | + x */ \ + STOR(A, (ROUND + 0)) /* - | | | | */ \ + TRAN(B, E, F) /* | | | - x */ \ + LOAD(E, (ROUND + 4), -4) /* | | | + */ \ + SHUF(C, A, F) /* + | | | | x */ \ + STOR(B, (ROUND + 1)) /* | - | | | */ \ + TRAN(C, A, F) /* - | | | x */ \ + LOAD(F, (ROUND + 5), -4) /* | | | + */ \ + SHUF(D, A, B) /* + x | | | | */ \ + STOR(C, (ROUND + 2)) /* | - | | | */ \ + TRAN(D, A, B) /* - x V V V */ + +// Define a macro that terminates a ROUND_3 macro by taking pre-loaded +// registers D, E and F, and translating, shuffling and storing them. +#define ROUND_3_END(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + SHUF(E, A, B) /* + x V V V */ \ + STOR(D, (ROUND + 3)) /* | - | | */ \ + TRAN(E, A, B) /* - x | | */ \ + SHUF(F, C, D) /* + x | | */ \ + STOR(E, (ROUND + 4)) /* | - | */ \ + TRAN(F, C, D) /* - x | */ \ + STOR(F, (ROUND + 5)) /* - */ + +// Define a type A round. Inputs are a, b, and c, outputs are d, e, and f. +#define ROUND_3_A(ROUND) \ + ROUND_3(ROUND, "a", "b", "c", "d", "e", "f") + +// Define a type B round. Inputs and outputs are swapped with regard to type A. +#define ROUND_3_B(ROUND) \ + ROUND_3(ROUND, "d", "e", "f", "a", "b", "c") + +// Terminating macro for a type A round. +#define ROUND_3_A_LAST(ROUND) \ + ROUND_3_A(ROUND) \ + ROUND_3_END(ROUND, "a", "b", "c", "d", "e", "f") + +// Terminating macro for a type B round. +#define ROUND_3_B_LAST(ROUND) \ + ROUND_3_B(ROUND) \ + ROUND_3_END(ROUND, "d", "e", "f", "a", "b", "c") + +// Suppress clang's warning that the literal string in the asm statement is +// overlong (longer than the ISO-mandated minimum size of 4095 bytes for C99 +// compilers). It may be true, but the goal here is not C99 portability. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverlength-strings" + +static inline void +enc_loop_avx2 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + // For a clearer explanation of the algorithm used by this function, + // please refer to the plain (not inline assembly) implementation. This + // function follows the same basic logic. + + if (*slen < 32) { + return; + } + + // Process blocks of 24 bytes at a time. Because blocks are loaded 32 + // bytes at a time an offset of -4, ensure that there will be at least + // 4 remaining bytes after the last round, so that the final read will + // not pass beyond the bounds of the input buffer. + size_t rounds = (*slen - 4) / 24; + + *slen -= rounds * 24; // 24 bytes consumed per round + *olen += rounds * 32; // 32 bytes produced per round + + // Pre-decrement the number of rounds to get the number of rounds + // *after* the first round, which is handled as a special case. + rounds--; + + // Number of times to go through the 36x loop. + size_t loops = rounds / 36; + + // Number of rounds remaining after the 36x loop. + rounds %= 36; + + // Lookup tables. + const __m256i lut0 = _mm256_set_epi8( + 10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1, + 14, 15, 13, 14, 11, 12, 10, 11, 8, 9, 7, 8, 5, 6, 4, 5); + + const __m256i lut1 = _mm256_setr_epi8( + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0, + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0); + + // Temporary registers. + __m256i a, b, c, d, e; + + // Temporary register f doubles as the shift mask for the first round. + __m256i f = _mm256_setr_epi32(0, 0, 1, 2, 3, 4, 5, 6); + + __asm__ volatile ( + + // The first loop iteration requires special handling to ensure + // that the read, which is normally done at an offset of -4, + // does not underflow the buffer. Load the buffer at an offset + // of 0 and permute the input to achieve the same effect. + LOAD("a", 0, 0) + "vpermd %[a], %[f], %[a] \n\t" + + // Perform the standard shuffling and translation steps. + SHUF("a", "b", "c") + TRAN("a", "b", "c") + + // Store the result and increment the source and dest pointers. + "vmovdqu %[a], (%[dst]) \n\t" + "add $24, %[src] \n\t" + "add $32, %[dst] \n\t" + + // If there are 36 rounds or more, enter a 36x unrolled loop of + // interleaved encoding rounds. The rounds interleave memory + // operations (load/store) with data operations (table lookups, + // etc) to maximize pipeline throughput. + " test %[loops], %[loops] \n\t" + " jz 18f \n\t" + " jmp 36f \n\t" + " \n\t" + ".balign 64 \n\t" + "36: " ROUND_3_INIT() + " " ROUND_3_A( 0) + " " ROUND_3_B( 3) + " " ROUND_3_A( 6) + " " ROUND_3_B( 9) + " " ROUND_3_A(12) + " " ROUND_3_B(15) + " " ROUND_3_A(18) + " " ROUND_3_B(21) + " " ROUND_3_A(24) + " " ROUND_3_B(27) + " " ROUND_3_A_LAST(30) + " add $(24 * 36), %[src] \n\t" + " add $(32 * 36), %[dst] \n\t" + " dec %[loops] \n\t" + " jnz 36b \n\t" + + // Enter an 18x unrolled loop for rounds of 18 or more. + "18: cmp $18, %[rounds] \n\t" + " jl 9f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B(3) + " " ROUND_3_A(6) + " " ROUND_3_B(9) + " " ROUND_3_A_LAST(12) + " sub $18, %[rounds] \n\t" + " add $(24 * 18), %[src] \n\t" + " add $(32 * 18), %[dst] \n\t" + + // Enter a 9x unrolled loop for rounds of 9 or more. + "9: cmp $9, %[rounds] \n\t" + " jl 6f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B_LAST(3) + " sub $9, %[rounds] \n\t" + " add $(24 * 9), %[src] \n\t" + " add $(32 * 9), %[dst] \n\t" + + // Enter a 6x unrolled loop for rounds of 6 or more. + "6: cmp $6, %[rounds] \n\t" + " jl 55f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A_LAST(0) + " sub $6, %[rounds] \n\t" + " add $(24 * 6), %[src] \n\t" + " add $(32 * 6), %[dst] \n\t" + + // Dispatch the remaining rounds 0..5. + "55: cmp $3, %[rounds] \n\t" + " jg 45f \n\t" + " je 3f \n\t" + " cmp $1, %[rounds] \n\t" + " jg 2f \n\t" + " je 1f \n\t" + " jmp 0f \n\t" + + "45: cmp $4, %[rounds] \n\t" + " je 4f \n\t" + + // Block of non-interlaced encoding rounds, which can each + // individually be jumped to. Rounds fall through to the next. + "5: " ROUND() + "4: " ROUND() + "3: " ROUND() + "2: " ROUND() + "1: " ROUND() + "0: \n\t" + + // Outputs (modified). + : [rounds] "+r" (rounds), + [loops] "+r" (loops), + [src] "+r" (*s), + [dst] "+r" (*o), + [a] "=&x" (a), + [b] "=&x" (b), + [c] "=&x" (c), + [d] "=&x" (d), + [e] "=&x" (e), + [f] "+x" (f) + + // Inputs (not modified). + : [lut0] "x" (lut0), + [lut1] "x" (lut1), + [msk0] "x" (_mm256_set1_epi32(0x0FC0FC00)), + [msk1] "x" (_mm256_set1_epi32(0x04000040)), + [msk2] "x" (_mm256_set1_epi32(0x003F03F0)), + [msk3] "x" (_mm256_set1_epi32(0x01000010)), + [n51] "x" (_mm256_set1_epi8(51)), + [n25] "x" (_mm256_set1_epi8(25)) + + // Clobbers. + : "cc", "memory" + ); +} + +#pragma GCC diagnostic pop diff --git a/mypyc/lib-rt/base64/arch/avx2/enc_reshuffle.c b/mypyc/lib-rt/base64/arch/avx2/enc_reshuffle.c new file mode 100644 index 000000000000..82c659b39ce7 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/enc_reshuffle.c @@ -0,0 +1,83 @@ +static BASE64_FORCE_INLINE __m256i +enc_reshuffle (const __m256i input) +{ + // Translation of the SSSE3 reshuffling algorithm to AVX2. This one + // works with shifted (4 bytes) input in order to be able to work + // efficiently in the two 128-bit lanes. + + // Input, bytes MSB to LSB: + // 0 0 0 0 x w v u t s r q p o n m + // l k j i h g f e d c b a 0 0 0 0 + + const __m256i in = _mm256_shuffle_epi8(input, _mm256_set_epi8( + 10, 11, 9, 10, + 7, 8, 6, 7, + 4, 5, 3, 4, + 1, 2, 0, 1, + + 14, 15, 13, 14, + 11, 12, 10, 11, + 8, 9, 7, 8, + 5, 6, 4, 5)); + // in, bytes MSB to LSB: + // w x v w + // t u s t + // q r p q + // n o m n + // k l j k + // h i g h + // e f d e + // b c a b + + const __m256i t0 = _mm256_and_si256(in, _mm256_set1_epi32(0x0FC0FC00)); + // bits, upper case are most significant bits, lower case are least + // significant bits. + // 0000wwww XX000000 VVVVVV00 00000000 + // 0000tttt UU000000 SSSSSS00 00000000 + // 0000qqqq RR000000 PPPPPP00 00000000 + // 0000nnnn OO000000 MMMMMM00 00000000 + // 0000kkkk LL000000 JJJJJJ00 00000000 + // 0000hhhh II000000 GGGGGG00 00000000 + // 0000eeee FF000000 DDDDDD00 00000000 + // 0000bbbb CC000000 AAAAAA00 00000000 + + const __m256i t1 = _mm256_mulhi_epu16(t0, _mm256_set1_epi32(0x04000040)); + // 00000000 00wwwwXX 00000000 00VVVVVV + // 00000000 00ttttUU 00000000 00SSSSSS + // 00000000 00qqqqRR 00000000 00PPPPPP + // 00000000 00nnnnOO 00000000 00MMMMMM + // 00000000 00kkkkLL 00000000 00JJJJJJ + // 00000000 00hhhhII 00000000 00GGGGGG + // 00000000 00eeeeFF 00000000 00DDDDDD + // 00000000 00bbbbCC 00000000 00AAAAAA + + const __m256i t2 = _mm256_and_si256(in, _mm256_set1_epi32(0x003F03F0)); + // 00000000 00xxxxxx 000000vv WWWW0000 + // 00000000 00uuuuuu 000000ss TTTT0000 + // 00000000 00rrrrrr 000000pp QQQQ0000 + // 00000000 00oooooo 000000mm NNNN0000 + // 00000000 00llllll 000000jj KKKK0000 + // 00000000 00iiiiii 000000gg HHHH0000 + // 00000000 00ffffff 000000dd EEEE0000 + // 00000000 00cccccc 000000aa BBBB0000 + + const __m256i t3 = _mm256_mullo_epi16(t2, _mm256_set1_epi32(0x01000010)); + // 00xxxxxx 00000000 00vvWWWW 00000000 + // 00uuuuuu 00000000 00ssTTTT 00000000 + // 00rrrrrr 00000000 00ppQQQQ 00000000 + // 00oooooo 00000000 00mmNNNN 00000000 + // 00llllll 00000000 00jjKKKK 00000000 + // 00iiiiii 00000000 00ggHHHH 00000000 + // 00ffffff 00000000 00ddEEEE 00000000 + // 00cccccc 00000000 00aaBBBB 00000000 + + return _mm256_or_si256(t1, t3); + // 00xxxxxx 00wwwwXX 00vvWWWW 00VVVVVV + // 00uuuuuu 00ttttUU 00ssTTTT 00SSSSSS + // 00rrrrrr 00qqqqRR 00ppQQQQ 00PPPPPP + // 00oooooo 00nnnnOO 00mmNNNN 00MMMMMM + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA +} diff --git a/mypyc/lib-rt/base64/arch/avx2/enc_translate.c b/mypyc/lib-rt/base64/arch/avx2/enc_translate.c new file mode 100644 index 000000000000..370da98f596c --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx2/enc_translate.c @@ -0,0 +1,30 @@ +static BASE64_FORCE_INLINE __m256i +enc_translate (const __m256i in) +{ + // A lookup table containing the absolute offsets for all ranges: + const __m256i lut = _mm256_setr_epi8( + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0, + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0); + + // Translate values 0..63 to the Base64 alphabet. There are five sets: + // # From To Abs Index Characters + // 0 [0..25] [65..90] +65 0 ABCDEFGHIJKLMNOPQRSTUVWXYZ + // 1 [26..51] [97..122] +71 1 abcdefghijklmnopqrstuvwxyz + // 2 [52..61] [48..57] -4 [2..11] 0123456789 + // 3 [62] [43] -19 12 + + // 4 [63] [47] -16 13 / + + // Create LUT indices from the input. The index for range #0 is right, + // others are 1 less than expected: + __m256i indices = _mm256_subs_epu8(in, _mm256_set1_epi8(51)); + + // mask is 0xFF (-1) for range #[1..4] and 0x00 for range #0: + const __m256i mask = _mm256_cmpgt_epi8(in, _mm256_set1_epi8(25)); + + // Subtract -1, so add 1 to indices for range #[1..4]. All indices are + // now correct: + indices = _mm256_sub_epi8(indices, mask); + + // Add offsets to input values: + return _mm256_add_epi8(in, _mm256_shuffle_epi8(lut, indices)); +} diff --git a/mypyc/lib-rt/base64/arch/avx512/codec.c b/mypyc/lib-rt/base64/arch/avx512/codec.c new file mode 100644 index 000000000000..98210826a5fe --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx512/codec.c @@ -0,0 +1,44 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_AVX512 +#include + +#include "../avx2/dec_reshuffle.c" +#include "../avx2/dec_loop.c" +#include "enc_reshuffle_translate.c" +#include "enc_loop.c" + +#endif // HAVE_AVX512 + +void +base64_stream_encode_avx512 BASE64_ENC_PARAMS +{ +#if HAVE_AVX512 + #include "../generic/enc_head.c" + enc_loop_avx512(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +// Reuse AVX2 decoding. Not supporting AVX512 at present +int +base64_stream_decode_avx512 BASE64_DEC_PARAMS +{ +#if HAVE_AVX512 + #include "../generic/dec_head.c" + dec_loop_avx2(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/avx512/enc_loop.c b/mypyc/lib-rt/base64/arch/avx512/enc_loop.c new file mode 100644 index 000000000000..cb44696ba31a --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx512/enc_loop.c @@ -0,0 +1,61 @@ +static BASE64_FORCE_INLINE void +enc_loop_avx512_inner (const uint8_t **s, uint8_t **o) +{ + // Load input. + __m512i src = _mm512_loadu_si512((__m512i *) *s); + + // Reshuffle, translate, store. + src = enc_reshuffle_translate(src); + _mm512_storeu_si512((__m512i *) *o, src); + + *s += 48; + *o += 64; +} + +static inline void +enc_loop_avx512 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 64) { + return; + } + + // Process blocks of 48 bytes at a time. Because blocks are loaded 64 + // bytes at a time, ensure that there will be at least 24 remaining + // bytes after the last round, so that the final read will not pass + // beyond the bounds of the input buffer. + size_t rounds = (*slen - 24) / 48; + + *slen -= rounds * 48; // 48 bytes consumed per round + *olen += rounds * 64; // 64 bytes produced per round + + while (rounds > 0) { + if (rounds >= 8) { + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_avx512_inner(s, o); + enc_loop_avx512_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_avx512_inner(s, o); + break; + } +} diff --git a/mypyc/lib-rt/base64/arch/avx512/enc_reshuffle_translate.c b/mypyc/lib-rt/base64/arch/avx512/enc_reshuffle_translate.c new file mode 100644 index 000000000000..ae12b3af2549 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/avx512/enc_reshuffle_translate.c @@ -0,0 +1,50 @@ +// AVX512 algorithm is based on permutevar and multishift. The code is based on +// https://github.com/WojciechMula/base64simd which is under BSD-2 license. + +static BASE64_FORCE_INLINE __m512i +enc_reshuffle_translate (const __m512i input) +{ + // 32-bit input + // [ 0 0 0 0 0 0 0 0|c1 c0 d5 d4 d3 d2 d1 d0| + // b3 b2 b1 b0 c5 c4 c3 c2|a5 a4 a3 a2 a1 a0 b5 b4] + // output order [1, 2, 0, 1] + // [b3 b2 b1 b0 c5 c4 c3 c2|c1 c0 d5 d4 d3 d2 d1 d0| + // a5 a4 a3 a2 a1 a0 b5 b4|b3 b2 b1 b0 c3 c2 c1 c0] + + const __m512i shuffle_input = _mm512_setr_epi32(0x01020001, + 0x04050304, + 0x07080607, + 0x0a0b090a, + 0x0d0e0c0d, + 0x10110f10, + 0x13141213, + 0x16171516, + 0x191a1819, + 0x1c1d1b1c, + 0x1f201e1f, + 0x22232122, + 0x25262425, + 0x28292728, + 0x2b2c2a2b, + 0x2e2f2d2e); + + // Reorder bytes + // [b3 b2 b1 b0 c5 c4 c3 c2|c1 c0 d5 d4 d3 d2 d1 d0| + // a5 a4 a3 a2 a1 a0 b5 b4|b3 b2 b1 b0 c3 c2 c1 c0] + const __m512i in = _mm512_permutexvar_epi8(shuffle_input, input); + + // After multishift a single 32-bit lane has following layout + // [c1 c0 d5 d4 d3 d2 d1 d0|b1 b0 c5 c4 c3 c2 c1 c0| + // a1 a0 b5 b4 b3 b2 b1 b0|d1 d0 a5 a4 a3 a2 a1 a0] + // (a = [10:17], b = [4:11], c = [22:27], d = [16:21]) + + // 48, 54, 36, 42, 16, 22, 4, 10 + const __m512i shifts = _mm512_set1_epi64(0x3036242a1016040alu); + __m512i shuffled_in = _mm512_multishift_epi64_epi8(shifts, in); + + // Translate immediately after reshuffled. + const __m512i lookup = _mm512_loadu_si512(base64_table_enc_6bit); + + // Translation 6-bit values to ASCII. + return _mm512_permutexvar_epi8(shuffled_in, lookup); +} diff --git a/mypyc/lib-rt/base64/arch/generic/32/dec_loop.c b/mypyc/lib-rt/base64/arch/generic/32/dec_loop.c new file mode 100644 index 000000000000..aa290d7e03a8 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/32/dec_loop.c @@ -0,0 +1,86 @@ +static BASE64_FORCE_INLINE int +dec_loop_generic_32_inner (const uint8_t **s, uint8_t **o, size_t *rounds) +{ + const uint32_t str + = base64_table_dec_32bit_d0[(*s)[0]] + | base64_table_dec_32bit_d1[(*s)[1]] + | base64_table_dec_32bit_d2[(*s)[2]] + | base64_table_dec_32bit_d3[(*s)[3]]; + +#if BASE64_LITTLE_ENDIAN + + // LUTs for little-endian set MSB in case of invalid character: + if (str & UINT32_C(0x80000000)) { + return 0; + } +#else + // LUTs for big-endian set LSB in case of invalid character: + if (str & UINT32_C(1)) { + return 0; + } +#endif + // Store the output: + memcpy(*o, &str, sizeof (str)); + + *s += 4; + *o += 3; + *rounds -= 1; + + return 1; +} + +static inline void +dec_loop_generic_32 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 8) { + return; + } + + // Process blocks of 4 bytes per round. Because one extra zero byte is + // written after the output, ensure that there will be at least 4 bytes + // of input data left to cover the gap. (Two data bytes and up to two + // end-of-string markers.) + size_t rounds = (*slen - 4) / 4; + + *slen -= rounds * 4; // 4 bytes consumed per round + *olen += rounds * 3; // 3 bytes produced per round + + do { + if (rounds >= 8) { + if (dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 4) { + if (dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 2) { + if (dec_loop_generic_32_inner(s, o, &rounds) && + dec_loop_generic_32_inner(s, o, &rounds)) { + continue; + } + break; + } + dec_loop_generic_32_inner(s, o, &rounds); + break; + + } while (rounds > 0); + + // Adjust for any rounds that were skipped: + *slen += rounds * 4; + *olen -= rounds * 3; +} diff --git a/mypyc/lib-rt/base64/arch/generic/32/enc_loop.c b/mypyc/lib-rt/base64/arch/generic/32/enc_loop.c new file mode 100644 index 000000000000..b5e6eefd9d43 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/32/enc_loop.c @@ -0,0 +1,73 @@ +static BASE64_FORCE_INLINE void +enc_loop_generic_32_inner (const uint8_t **s, uint8_t **o) +{ + uint32_t src; + + // Load input: + memcpy(&src, *s, sizeof (src)); + + // Reorder to 32-bit big-endian, if not already in that format. The + // workset must be in big-endian, otherwise the shifted bits do not + // carry over properly among adjacent bytes: + src = BASE64_HTOBE32(src); + + // Two indices for the 12-bit lookup table: + const size_t index0 = (src >> 20) & 0xFFFU; + const size_t index1 = (src >> 8) & 0xFFFU; + + // Table lookup and store: + memcpy(*o + 0, base64_table_enc_12bit + index0, 2); + memcpy(*o + 2, base64_table_enc_12bit + index1, 2); + + *s += 3; + *o += 4; +} + +static inline void +enc_loop_generic_32 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 4) { + return; + } + + // Process blocks of 3 bytes at a time. Because blocks are loaded 4 + // bytes at a time, ensure that there will be at least one remaining + // byte after the last round, so that the final read will not pass + // beyond the bounds of the input buffer: + size_t rounds = (*slen - 1) / 3; + + *slen -= rounds * 3; // 3 bytes consumed per round + *olen += rounds * 4; // 4 bytes produced per round + + do { + if (rounds >= 8) { + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_generic_32_inner(s, o); + enc_loop_generic_32_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_generic_32_inner(s, o); + break; + + } while (rounds > 0); +} diff --git a/mypyc/lib-rt/base64/arch/generic/64/enc_loop.c b/mypyc/lib-rt/base64/arch/generic/64/enc_loop.c new file mode 100644 index 000000000000..e6a29cd5ec41 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/64/enc_loop.c @@ -0,0 +1,77 @@ +static BASE64_FORCE_INLINE void +enc_loop_generic_64_inner (const uint8_t **s, uint8_t **o) +{ + uint64_t src; + + // Load input: + memcpy(&src, *s, sizeof (src)); + + // Reorder to 64-bit big-endian, if not already in that format. The + // workset must be in big-endian, otherwise the shifted bits do not + // carry over properly among adjacent bytes: + src = BASE64_HTOBE64(src); + + // Four indices for the 12-bit lookup table: + const size_t index0 = (src >> 52) & 0xFFFU; + const size_t index1 = (src >> 40) & 0xFFFU; + const size_t index2 = (src >> 28) & 0xFFFU; + const size_t index3 = (src >> 16) & 0xFFFU; + + // Table lookup and store: + memcpy(*o + 0, base64_table_enc_12bit + index0, 2); + memcpy(*o + 2, base64_table_enc_12bit + index1, 2); + memcpy(*o + 4, base64_table_enc_12bit + index2, 2); + memcpy(*o + 6, base64_table_enc_12bit + index3, 2); + + *s += 6; + *o += 8; +} + +static inline void +enc_loop_generic_64 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 8) { + return; + } + + // Process blocks of 6 bytes at a time. Because blocks are loaded 8 + // bytes at a time, ensure that there will be at least 2 remaining + // bytes after the last round, so that the final read will not pass + // beyond the bounds of the input buffer: + size_t rounds = (*slen - 2) / 6; + + *slen -= rounds * 6; // 6 bytes consumed per round + *olen += rounds * 8; // 8 bytes produced per round + + do { + if (rounds >= 8) { + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_generic_64_inner(s, o); + enc_loop_generic_64_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_generic_64_inner(s, o); + break; + + } while (rounds > 0); +} diff --git a/mypyc/lib-rt/base64/arch/generic/codec.c b/mypyc/lib-rt/base64/arch/generic/codec.c new file mode 100644 index 000000000000..1a29be7c836d --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/codec.c @@ -0,0 +1,41 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if BASE64_WORDSIZE == 32 +# include "32/enc_loop.c" +#elif BASE64_WORDSIZE == 64 +# include "64/enc_loop.c" +#endif + +#if BASE64_WORDSIZE >= 32 +# include "32/dec_loop.c" +#endif + +void +base64_stream_encode_plain BASE64_ENC_PARAMS +{ + #include "enc_head.c" +#if BASE64_WORDSIZE == 32 + enc_loop_generic_32(&s, &slen, &o, &olen); +#elif BASE64_WORDSIZE == 64 + enc_loop_generic_64(&s, &slen, &o, &olen); +#endif + #include "enc_tail.c" +} + +int +base64_stream_decode_plain BASE64_DEC_PARAMS +{ + #include "dec_head.c" +#if BASE64_WORDSIZE >= 32 + dec_loop_generic_32(&s, &slen, &o, &olen); +#endif + #include "dec_tail.c" +} diff --git a/mypyc/lib-rt/base64/arch/generic/dec_head.c b/mypyc/lib-rt/base64/arch/generic/dec_head.c new file mode 100644 index 000000000000..179a31b63ff4 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/dec_head.c @@ -0,0 +1,37 @@ +int ret = 0; +const uint8_t *s = (const uint8_t *) src; +uint8_t *o = (uint8_t *) out; +uint8_t q; + +// Use local temporaries to avoid cache thrashing: +size_t olen = 0; +size_t slen = srclen; +struct base64_state st; +st.eof = state->eof; +st.bytes = state->bytes; +st.carry = state->carry; + +// If we previously saw an EOF or an invalid character, bail out: +if (st.eof) { + *outlen = 0; + ret = 0; + // If there was a trailing '=' to check, check it: + if (slen && (st.eof == BASE64_AEOF)) { + state->bytes = 0; + state->eof = BASE64_EOF; + ret = ((base64_table_dec_8bit[*s++] == 254) && (slen == 1)) ? 1 : 0; + } + return ret; +} + +// Turn four 6-bit numbers into three bytes: +// out[0] = 11111122 +// out[1] = 22223333 +// out[2] = 33444444 + +// Duff's device again: +switch (st.bytes) +{ + for (;;) + { + case 0: diff --git a/mypyc/lib-rt/base64/arch/generic/dec_tail.c b/mypyc/lib-rt/base64/arch/generic/dec_tail.c new file mode 100644 index 000000000000..e64f7247f3f1 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/dec_tail.c @@ -0,0 +1,91 @@ + if (slen-- == 0) { + ret = 1; + break; + } + if ((q = base64_table_dec_8bit[*s++]) >= 254) { + st.eof = BASE64_EOF; + // Treat character '=' as invalid for byte 0: + break; + } + st.carry = q << 2; + st.bytes++; + + // Deliberate fallthrough: + BASE64_FALLTHROUGH + + case 1: if (slen-- == 0) { + ret = 1; + break; + } + if ((q = base64_table_dec_8bit[*s++]) >= 254) { + st.eof = BASE64_EOF; + // Treat character '=' as invalid for byte 1: + break; + } + *o++ = st.carry | (q >> 4); + st.carry = q << 4; + st.bytes++; + olen++; + + // Deliberate fallthrough: + BASE64_FALLTHROUGH + + case 2: if (slen-- == 0) { + ret = 1; + break; + } + if ((q = base64_table_dec_8bit[*s++]) >= 254) { + st.bytes++; + // When q == 254, the input char is '='. + // Check if next byte is also '=': + if (q == 254) { + if (slen-- != 0) { + st.bytes = 0; + // EOF: + st.eof = BASE64_EOF; + q = base64_table_dec_8bit[*s++]; + ret = ((q == 254) && (slen == 0)) ? 1 : 0; + break; + } + else { + // Almost EOF + st.eof = BASE64_AEOF; + ret = 1; + break; + } + } + // If we get here, there was an error: + break; + } + *o++ = st.carry | (q >> 2); + st.carry = q << 6; + st.bytes++; + olen++; + + // Deliberate fallthrough: + BASE64_FALLTHROUGH + + case 3: if (slen-- == 0) { + ret = 1; + break; + } + if ((q = base64_table_dec_8bit[*s++]) >= 254) { + st.bytes = 0; + st.eof = BASE64_EOF; + // When q == 254, the input char is '='. Return 1 and EOF. + // When q == 255, the input char is invalid. Return 0 and EOF. + ret = ((q == 254) && (slen == 0)) ? 1 : 0; + break; + } + *o++ = st.carry | q; + st.carry = 0; + st.bytes = 0; + olen++; + } +} + +state->eof = st.eof; +state->bytes = st.bytes; +state->carry = st.carry; +*outlen = olen; +return ret; diff --git a/mypyc/lib-rt/base64/arch/generic/enc_head.c b/mypyc/lib-rt/base64/arch/generic/enc_head.c new file mode 100644 index 000000000000..38d60b2c62b5 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/enc_head.c @@ -0,0 +1,24 @@ +// Assume that *out is large enough to contain the output. +// Theoretically it should be 4/3 the length of src. +const uint8_t *s = (const uint8_t *) src; +uint8_t *o = (uint8_t *) out; + +// Use local temporaries to avoid cache thrashing: +size_t olen = 0; +size_t slen = srclen; +struct base64_state st; +st.bytes = state->bytes; +st.carry = state->carry; + +// Turn three bytes into four 6-bit numbers: +// in[0] = 00111111 +// in[1] = 00112222 +// in[2] = 00222233 +// in[3] = 00333333 + +// Duff's device, a for() loop inside a switch() statement. Legal! +switch (st.bytes) +{ + for (;;) + { + case 0: diff --git a/mypyc/lib-rt/base64/arch/generic/enc_tail.c b/mypyc/lib-rt/base64/arch/generic/enc_tail.c new file mode 100644 index 000000000000..cbd573376812 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/generic/enc_tail.c @@ -0,0 +1,34 @@ + if (slen-- == 0) { + break; + } + *o++ = base64_table_enc_6bit[*s >> 2]; + st.carry = (*s++ << 4) & 0x30; + st.bytes++; + olen += 1; + + // Deliberate fallthrough: + BASE64_FALLTHROUGH + + case 1: if (slen-- == 0) { + break; + } + *o++ = base64_table_enc_6bit[st.carry | (*s >> 4)]; + st.carry = (*s++ << 2) & 0x3C; + st.bytes++; + olen += 1; + + // Deliberate fallthrough: + BASE64_FALLTHROUGH + + case 2: if (slen-- == 0) { + break; + } + *o++ = base64_table_enc_6bit[st.carry | (*s >> 6)]; + *o++ = base64_table_enc_6bit[*s++ & 0x3F]; + st.bytes = 0; + olen += 2; + } +} +state->bytes = st.bytes; +state->carry = st.carry; +*outlen = olen; diff --git a/mypyc/lib-rt/base64/arch/neon32/codec.c b/mypyc/lib-rt/base64/arch/neon32/codec.c new file mode 100644 index 000000000000..4a32592b9c46 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon32/codec.c @@ -0,0 +1,79 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#ifdef __arm__ +# if (defined(__ARM_NEON__) || defined(__ARM_NEON)) && HAVE_NEON32 +# define BASE64_USE_NEON32 +# endif +#endif + +#ifdef BASE64_USE_NEON32 +#include + +// Only enable inline assembly on supported compilers. +#if defined(__GNUC__) || defined(__clang__) +#define BASE64_NEON32_USE_ASM +#endif + +static BASE64_FORCE_INLINE uint8x16_t +vqtbl1q_u8 (const uint8x16_t lut, const uint8x16_t indices) +{ + // NEON32 only supports 64-bit wide lookups in 128-bit tables. Emulate + // the NEON64 `vqtbl1q_u8` intrinsic to do 128-bit wide lookups. + uint8x8x2_t lut2; + uint8x8x2_t result; + + lut2.val[0] = vget_low_u8(lut); + lut2.val[1] = vget_high_u8(lut); + + result.val[0] = vtbl2_u8(lut2, vget_low_u8(indices)); + result.val[1] = vtbl2_u8(lut2, vget_high_u8(indices)); + + return vcombine_u8(result.val[0], result.val[1]); +} + +#include "../generic/32/dec_loop.c" +#include "../generic/32/enc_loop.c" +#include "dec_loop.c" +#include "enc_reshuffle.c" +#include "enc_translate.c" +#include "enc_loop.c" + +#endif // BASE64_USE_NEON32 + +// Stride size is so large on these NEON 32-bit functions +// (48 bytes encode, 32 bytes decode) that we inline the +// uint32 codec to stay performant on smaller inputs. + +void +base64_stream_encode_neon32 BASE64_ENC_PARAMS +{ +#ifdef BASE64_USE_NEON32 + #include "../generic/enc_head.c" + enc_loop_neon32(&s, &slen, &o, &olen); + enc_loop_generic_32(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_neon32 BASE64_DEC_PARAMS +{ +#ifdef BASE64_USE_NEON32 + #include "../generic/dec_head.c" + dec_loop_neon32(&s, &slen, &o, &olen); + dec_loop_generic_32(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/neon32/dec_loop.c b/mypyc/lib-rt/base64/arch/neon32/dec_loop.c new file mode 100644 index 000000000000..e4caed7a7d1c --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon32/dec_loop.c @@ -0,0 +1,106 @@ +static BASE64_FORCE_INLINE int +is_nonzero (const uint8x16_t v) +{ + uint64_t u64; + const uint64x2_t v64 = vreinterpretq_u64_u8(v); + const uint32x2_t v32 = vqmovn_u64(v64); + + vst1_u64(&u64, vreinterpret_u64_u32(v32)); + return u64 != 0; +} + +static BASE64_FORCE_INLINE uint8x16_t +delta_lookup (const uint8x16_t v) +{ + const uint8x8_t lut = { + 0, 16, 19, 4, (uint8_t) -65, (uint8_t) -65, (uint8_t) -71, (uint8_t) -71, + }; + + return vcombine_u8( + vtbl1_u8(lut, vget_low_u8(v)), + vtbl1_u8(lut, vget_high_u8(v))); +} + +static BASE64_FORCE_INLINE uint8x16_t +dec_loop_neon32_lane (uint8x16_t *lane) +{ + // See the SSSE3 decoder for an explanation of the algorithm. + const uint8x16_t lut_lo = { + 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A + }; + + const uint8x16_t lut_hi = { + 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10 + }; + + const uint8x16_t mask_0F = vdupq_n_u8(0x0F); + const uint8x16_t mask_2F = vdupq_n_u8(0x2F); + + const uint8x16_t hi_nibbles = vshrq_n_u8(*lane, 4); + const uint8x16_t lo_nibbles = vandq_u8(*lane, mask_0F); + const uint8x16_t eq_2F = vceqq_u8(*lane, mask_2F); + + const uint8x16_t hi = vqtbl1q_u8(lut_hi, hi_nibbles); + const uint8x16_t lo = vqtbl1q_u8(lut_lo, lo_nibbles); + + // Now simply add the delta values to the input: + *lane = vaddq_u8(*lane, delta_lookup(vaddq_u8(eq_2F, hi_nibbles))); + + // Return the validity mask: + return vandq_u8(lo, hi); +} + +static inline void +dec_loop_neon32 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 64) { + return; + } + + // Process blocks of 64 bytes per round. Unlike the SSE codecs, no + // extra trailing zero bytes are written, so it is not necessary to + // reserve extra input bytes: + size_t rounds = *slen / 64; + + *slen -= rounds * 64; // 64 bytes consumed per round + *olen += rounds * 48; // 48 bytes produced per round + + do { + uint8x16x3_t dec; + + // Load 64 bytes and deinterleave: + uint8x16x4_t str = vld4q_u8(*s); + + // Decode each lane, collect a mask of invalid inputs: + const uint8x16_t classified + = dec_loop_neon32_lane(&str.val[0]) + | dec_loop_neon32_lane(&str.val[1]) + | dec_loop_neon32_lane(&str.val[2]) + | dec_loop_neon32_lane(&str.val[3]); + + // Check for invalid input: if any of the delta values are + // zero, fall back on bytewise code to do error checking and + // reporting: + if (is_nonzero(classified)) { + break; + } + + // Compress four bytes into three: + dec.val[0] = vorrq_u8(vshlq_n_u8(str.val[0], 2), vshrq_n_u8(str.val[1], 4)); + dec.val[1] = vorrq_u8(vshlq_n_u8(str.val[1], 4), vshrq_n_u8(str.val[2], 2)); + dec.val[2] = vorrq_u8(vshlq_n_u8(str.val[2], 6), str.val[3]); + + // Interleave and store decoded result: + vst3q_u8(*o, dec); + + *s += 64; + *o += 48; + + } while (--rounds > 0); + + // Adjust for any rounds that were skipped: + *slen += rounds * 64; + *olen -= rounds * 48; +} diff --git a/mypyc/lib-rt/base64/arch/neon32/enc_loop.c b/mypyc/lib-rt/base64/arch/neon32/enc_loop.c new file mode 100644 index 000000000000..2adff48f2bcc --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon32/enc_loop.c @@ -0,0 +1,170 @@ +#ifdef BASE64_NEON32_USE_ASM +static BASE64_FORCE_INLINE void +enc_loop_neon32_inner_asm (const uint8_t **s, uint8_t **o) +{ + // This function duplicates the functionality of enc_loop_neon32_inner, + // but entirely with inline assembly. This gives a significant speedup + // over using NEON intrinsics, which do not always generate very good + // code. The logic of the assembly is directly lifted from the + // intrinsics version, so it can be used as a guide to this code. + + // Temporary registers, used as scratch space. + uint8x16_t tmp0, tmp1, tmp2, tmp3; + uint8x16_t mask0, mask1, mask2, mask3; + + // A lookup table containing the absolute offsets for all ranges. + const uint8x16_t lut = { + 65U, 71U, 252U, 252U, + 252U, 252U, 252U, 252U, + 252U, 252U, 252U, 252U, + 237U, 240U, 0U, 0U + }; + + // Numeric constants. + const uint8x16_t n51 = vdupq_n_u8(51); + const uint8x16_t n25 = vdupq_n_u8(25); + const uint8x16_t n63 = vdupq_n_u8(63); + + __asm__ ( + + // Load 48 bytes and deinterleave. The bytes are loaded to + // hard-coded registers q12, q13 and q14, to ensure that they + // are contiguous. Increment the source pointer. + "vld3.8 {d24, d26, d28}, [%[src]]! \n\t" + "vld3.8 {d25, d27, d29}, [%[src]]! \n\t" + + // Reshuffle the bytes using temporaries. + "vshr.u8 %q[t0], q12, #2 \n\t" + "vshr.u8 %q[t1], q13, #4 \n\t" + "vshr.u8 %q[t2], q14, #6 \n\t" + "vsli.8 %q[t1], q12, #4 \n\t" + "vsli.8 %q[t2], q13, #2 \n\t" + "vand.u8 %q[t1], %q[t1], %q[n63] \n\t" + "vand.u8 %q[t2], %q[t2], %q[n63] \n\t" + "vand.u8 %q[t3], q14, %q[n63] \n\t" + + // t0..t3 are the reshuffled inputs. Create LUT indices. + "vqsub.u8 q12, %q[t0], %q[n51] \n\t" + "vqsub.u8 q13, %q[t1], %q[n51] \n\t" + "vqsub.u8 q14, %q[t2], %q[n51] \n\t" + "vqsub.u8 q15, %q[t3], %q[n51] \n\t" + + // Create the mask for range #0. + "vcgt.u8 %q[m0], %q[t0], %q[n25] \n\t" + "vcgt.u8 %q[m1], %q[t1], %q[n25] \n\t" + "vcgt.u8 %q[m2], %q[t2], %q[n25] \n\t" + "vcgt.u8 %q[m3], %q[t3], %q[n25] \n\t" + + // Subtract -1 to correct the LUT indices. + "vsub.u8 q12, %q[m0] \n\t" + "vsub.u8 q13, %q[m1] \n\t" + "vsub.u8 q14, %q[m2] \n\t" + "vsub.u8 q15, %q[m3] \n\t" + + // Lookup the delta values. + "vtbl.u8 d24, {%q[lut]}, d24 \n\t" + "vtbl.u8 d25, {%q[lut]}, d25 \n\t" + "vtbl.u8 d26, {%q[lut]}, d26 \n\t" + "vtbl.u8 d27, {%q[lut]}, d27 \n\t" + "vtbl.u8 d28, {%q[lut]}, d28 \n\t" + "vtbl.u8 d29, {%q[lut]}, d29 \n\t" + "vtbl.u8 d30, {%q[lut]}, d30 \n\t" + "vtbl.u8 d31, {%q[lut]}, d31 \n\t" + + // Add the delta values. + "vadd.u8 q12, %q[t0] \n\t" + "vadd.u8 q13, %q[t1] \n\t" + "vadd.u8 q14, %q[t2] \n\t" + "vadd.u8 q15, %q[t3] \n\t" + + // Store 64 bytes and interleave. Increment the dest pointer. + "vst4.8 {d24, d26, d28, d30}, [%[dst]]! \n\t" + "vst4.8 {d25, d27, d29, d31}, [%[dst]]! \n\t" + + // Outputs (modified). + : [src] "+r" (*s), + [dst] "+r" (*o), + [t0] "=&w" (tmp0), + [t1] "=&w" (tmp1), + [t2] "=&w" (tmp2), + [t3] "=&w" (tmp3), + [m0] "=&w" (mask0), + [m1] "=&w" (mask1), + [m2] "=&w" (mask2), + [m3] "=&w" (mask3) + + // Inputs (not modified). + : [lut] "w" (lut), + [n25] "w" (n25), + [n51] "w" (n51), + [n63] "w" (n63) + + // Clobbers. + : "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31", + "cc", "memory" + ); +} +#endif + +static BASE64_FORCE_INLINE void +enc_loop_neon32_inner (const uint8_t **s, uint8_t **o) +{ +#ifdef BASE64_NEON32_USE_ASM + enc_loop_neon32_inner_asm(s, o); +#else + // Load 48 bytes and deinterleave: + uint8x16x3_t src = vld3q_u8(*s); + + // Reshuffle: + uint8x16x4_t out = enc_reshuffle(src); + + // Translate reshuffled bytes to the Base64 alphabet: + out = enc_translate(out); + + // Interleave and store output: + vst4q_u8(*o, out); + + *s += 48; + *o += 64; +#endif +} + +static inline void +enc_loop_neon32 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + size_t rounds = *slen / 48; + + *slen -= rounds * 48; // 48 bytes consumed per round + *olen += rounds * 64; // 64 bytes produced per round + + while (rounds > 0) { + if (rounds >= 8) { + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_neon32_inner(s, o); + enc_loop_neon32_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_neon32_inner(s, o); + break; + } +} diff --git a/mypyc/lib-rt/base64/arch/neon32/enc_reshuffle.c b/mypyc/lib-rt/base64/arch/neon32/enc_reshuffle.c new file mode 100644 index 000000000000..fa94d2799be6 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon32/enc_reshuffle.c @@ -0,0 +1,31 @@ +static BASE64_FORCE_INLINE uint8x16x4_t +enc_reshuffle (uint8x16x3_t in) +{ + uint8x16x4_t out; + + // Input: + // in[0] = a7 a6 a5 a4 a3 a2 a1 a0 + // in[1] = b7 b6 b5 b4 b3 b2 b1 b0 + // in[2] = c7 c6 c5 c4 c3 c2 c1 c0 + + // Output: + // out[0] = 00 00 a7 a6 a5 a4 a3 a2 + // out[1] = 00 00 a1 a0 b7 b6 b5 b4 + // out[2] = 00 00 b3 b2 b1 b0 c7 c6 + // out[3] = 00 00 c5 c4 c3 c2 c1 c0 + + // Move the input bits to where they need to be in the outputs. Except + // for the first output, the high two bits are not cleared. + out.val[0] = vshrq_n_u8(in.val[0], 2); + out.val[1] = vshrq_n_u8(in.val[1], 4); + out.val[2] = vshrq_n_u8(in.val[2], 6); + out.val[1] = vsliq_n_u8(out.val[1], in.val[0], 4); + out.val[2] = vsliq_n_u8(out.val[2], in.val[1], 2); + + // Clear the high two bits in the second, third and fourth output. + out.val[1] = vandq_u8(out.val[1], vdupq_n_u8(0x3F)); + out.val[2] = vandq_u8(out.val[2], vdupq_n_u8(0x3F)); + out.val[3] = vandq_u8(in.val[2], vdupq_n_u8(0x3F)); + + return out; +} diff --git a/mypyc/lib-rt/base64/arch/neon32/enc_translate.c b/mypyc/lib-rt/base64/arch/neon32/enc_translate.c new file mode 100644 index 000000000000..ff3d88dd152b --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon32/enc_translate.c @@ -0,0 +1,57 @@ +static BASE64_FORCE_INLINE uint8x16x4_t +enc_translate (const uint8x16x4_t in) +{ + // A lookup table containing the absolute offsets for all ranges: + const uint8x16_t lut = { + 65U, 71U, 252U, 252U, + 252U, 252U, 252U, 252U, + 252U, 252U, 252U, 252U, + 237U, 240U, 0U, 0U + }; + + const uint8x16_t offset = vdupq_n_u8(51); + + uint8x16x4_t indices, mask, delta, out; + + // Translate values 0..63 to the Base64 alphabet. There are five sets: + // # From To Abs Index Characters + // 0 [0..25] [65..90] +65 0 ABCDEFGHIJKLMNOPQRSTUVWXYZ + // 1 [26..51] [97..122] +71 1 abcdefghijklmnopqrstuvwxyz + // 2 [52..61] [48..57] -4 [2..11] 0123456789 + // 3 [62] [43] -19 12 + + // 4 [63] [47] -16 13 / + + // Create LUT indices from input: + // the index for range #0 is right, others are 1 less than expected: + indices.val[0] = vqsubq_u8(in.val[0], offset); + indices.val[1] = vqsubq_u8(in.val[1], offset); + indices.val[2] = vqsubq_u8(in.val[2], offset); + indices.val[3] = vqsubq_u8(in.val[3], offset); + + // mask is 0xFF (-1) for range #[1..4] and 0x00 for range #0: + mask.val[0] = vcgtq_u8(in.val[0], vdupq_n_u8(25)); + mask.val[1] = vcgtq_u8(in.val[1], vdupq_n_u8(25)); + mask.val[2] = vcgtq_u8(in.val[2], vdupq_n_u8(25)); + mask.val[3] = vcgtq_u8(in.val[3], vdupq_n_u8(25)); + + // Subtract -1, so add 1 to indices for range #[1..4], All indices are + // now correct: + indices.val[0] = vsubq_u8(indices.val[0], mask.val[0]); + indices.val[1] = vsubq_u8(indices.val[1], mask.val[1]); + indices.val[2] = vsubq_u8(indices.val[2], mask.val[2]); + indices.val[3] = vsubq_u8(indices.val[3], mask.val[3]); + + // Lookup delta values: + delta.val[0] = vqtbl1q_u8(lut, indices.val[0]); + delta.val[1] = vqtbl1q_u8(lut, indices.val[1]); + delta.val[2] = vqtbl1q_u8(lut, indices.val[2]); + delta.val[3] = vqtbl1q_u8(lut, indices.val[3]); + + // Add delta values: + out.val[0] = vaddq_u8(in.val[0], delta.val[0]); + out.val[1] = vaddq_u8(in.val[1], delta.val[1]); + out.val[2] = vaddq_u8(in.val[2], delta.val[2]); + out.val[3] = vaddq_u8(in.val[3], delta.val[3]); + + return out; +} diff --git a/mypyc/lib-rt/base64/arch/neon64/codec.c b/mypyc/lib-rt/base64/arch/neon64/codec.c new file mode 100644 index 000000000000..70dc463de94f --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon64/codec.c @@ -0,0 +1,93 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_NEON64 +#include + +// Only enable inline assembly on supported compilers. +#if defined(__GNUC__) || defined(__clang__) +#define BASE64_NEON64_USE_ASM +#endif + +static BASE64_FORCE_INLINE uint8x16x4_t +load_64byte_table (const uint8_t *p) +{ +#ifdef BASE64_NEON64_USE_ASM + + // Force the table to be loaded into contiguous registers. GCC will not + // normally allocate contiguous registers for a `uint8x16x4_t'. These + // registers are chosen to not conflict with the ones in the enc loop. + register uint8x16_t t0 __asm__ ("v8"); + register uint8x16_t t1 __asm__ ("v9"); + register uint8x16_t t2 __asm__ ("v10"); + register uint8x16_t t3 __asm__ ("v11"); + + __asm__ ( + "ld1 {%[t0].16b, %[t1].16b, %[t2].16b, %[t3].16b}, [%[src]], #64 \n\t" + : [src] "+r" (p), + [t0] "=w" (t0), + [t1] "=w" (t1), + [t2] "=w" (t2), + [t3] "=w" (t3) + ); + + return (uint8x16x4_t) { + .val[0] = t0, + .val[1] = t1, + .val[2] = t2, + .val[3] = t3, + }; +#else + return vld1q_u8_x4(p); +#endif +} + +#include "../generic/32/dec_loop.c" +#include "../generic/64/enc_loop.c" +#include "dec_loop.c" + +#ifdef BASE64_NEON64_USE_ASM +# include "enc_loop_asm.c" +#else +# include "enc_reshuffle.c" +# include "enc_loop.c" +#endif + +#endif // HAVE_NEON64 + +// Stride size is so large on these NEON 64-bit functions +// (48 bytes encode, 64 bytes decode) that we inline the +// uint64 codec to stay performant on smaller inputs. + +void +base64_stream_encode_neon64 BASE64_ENC_PARAMS +{ +#if HAVE_NEON64 + #include "../generic/enc_head.c" + enc_loop_neon64(&s, &slen, &o, &olen); + enc_loop_generic_64(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_neon64 BASE64_DEC_PARAMS +{ +#if HAVE_NEON64 + #include "../generic/dec_head.c" + dec_loop_neon64(&s, &slen, &o, &olen); + dec_loop_generic_32(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/neon64/dec_loop.c b/mypyc/lib-rt/base64/arch/neon64/dec_loop.c new file mode 100644 index 000000000000..428e0651f8e0 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon64/dec_loop.c @@ -0,0 +1,129 @@ +// The input consists of five valid character sets in the Base64 alphabet, +// which we need to map back to the 6-bit values they represent. +// There are three ranges, two singles, and then there's the rest. +// +// # From To LUT Characters +// 1 [0..42] [255] #1 invalid input +// 2 [43] [62] #1 + +// 3 [44..46] [255] #1 invalid input +// 4 [47] [63] #1 / +// 5 [48..57] [52..61] #1 0..9 +// 6 [58..63] [255] #1 invalid input +// 7 [64] [255] #2 invalid input +// 8 [65..90] [0..25] #2 A..Z +// 9 [91..96] [255] #2 invalid input +// 10 [97..122] [26..51] #2 a..z +// 11 [123..126] [255] #2 invalid input +// (12) Everything else => invalid input + +// The first LUT will use the VTBL instruction (out of range indices are set to +// 0 in destination). +static const uint8_t dec_lut1[] = { + 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, + 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, + 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 255U, 62U, 255U, 255U, 255U, 63U, + 52U, 53U, 54U, 55U, 56U, 57U, 58U, 59U, 60U, 61U, 255U, 255U, 255U, 255U, 255U, 255U, +}; + +// The second LUT will use the VTBX instruction (out of range indices will be +// unchanged in destination). Input [64..126] will be mapped to index [1..63] +// in this LUT. Index 0 means that value comes from LUT #1. +static const uint8_t dec_lut2[] = { + 0U, 255U, 0U, 1U, 2U, 3U, 4U, 5U, 6U, 7U, 8U, 9U, 10U, 11U, 12U, 13U, + 14U, 15U, 16U, 17U, 18U, 19U, 20U, 21U, 22U, 23U, 24U, 25U, 255U, 255U, 255U, 255U, + 255U, 255U, 26U, 27U, 28U, 29U, 30U, 31U, 32U, 33U, 34U, 35U, 36U, 37U, 38U, 39U, + 40U, 41U, 42U, 43U, 44U, 45U, 46U, 47U, 48U, 49U, 50U, 51U, 255U, 255U, 255U, 255U, +}; + +// All input values in range for the first look-up will be 0U in the second +// look-up result. All input values out of range for the first look-up will be +// 0U in the first look-up result. Thus, the two results can be ORed without +// conflicts. +// +// Invalid characters that are in the valid range for either look-up will be +// set to 255U in the combined result. Other invalid characters will just be +// passed through with the second look-up result (using the VTBX instruction). +// Since the second LUT is 64 bytes, those passed-through values are guaranteed +// to have a value greater than 63U. Therefore, valid characters will be mapped +// to the valid [0..63] range and all invalid characters will be mapped to +// values greater than 63. + +static inline void +dec_loop_neon64 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 64) { + return; + } + + // Process blocks of 64 bytes per round. Unlike the SSE codecs, no + // extra trailing zero bytes are written, so it is not necessary to + // reserve extra input bytes: + size_t rounds = *slen / 64; + + *slen -= rounds * 64; // 64 bytes consumed per round + *olen += rounds * 48; // 48 bytes produced per round + + const uint8x16x4_t tbl_dec1 = load_64byte_table(dec_lut1); + const uint8x16x4_t tbl_dec2 = load_64byte_table(dec_lut2); + + do { + const uint8x16_t offset = vdupq_n_u8(63U); + uint8x16x4_t dec1, dec2; + uint8x16x3_t dec; + + // Load 64 bytes and deinterleave: + uint8x16x4_t str = vld4q_u8((uint8_t *) *s); + + // Get indices for second LUT: + dec2.val[0] = vqsubq_u8(str.val[0], offset); + dec2.val[1] = vqsubq_u8(str.val[1], offset); + dec2.val[2] = vqsubq_u8(str.val[2], offset); + dec2.val[3] = vqsubq_u8(str.val[3], offset); + + // Get values from first LUT: + dec1.val[0] = vqtbl4q_u8(tbl_dec1, str.val[0]); + dec1.val[1] = vqtbl4q_u8(tbl_dec1, str.val[1]); + dec1.val[2] = vqtbl4q_u8(tbl_dec1, str.val[2]); + dec1.val[3] = vqtbl4q_u8(tbl_dec1, str.val[3]); + + // Get values from second LUT: + dec2.val[0] = vqtbx4q_u8(dec2.val[0], tbl_dec2, dec2.val[0]); + dec2.val[1] = vqtbx4q_u8(dec2.val[1], tbl_dec2, dec2.val[1]); + dec2.val[2] = vqtbx4q_u8(dec2.val[2], tbl_dec2, dec2.val[2]); + dec2.val[3] = vqtbx4q_u8(dec2.val[3], tbl_dec2, dec2.val[3]); + + // Get final values: + str.val[0] = vorrq_u8(dec1.val[0], dec2.val[0]); + str.val[1] = vorrq_u8(dec1.val[1], dec2.val[1]); + str.val[2] = vorrq_u8(dec1.val[2], dec2.val[2]); + str.val[3] = vorrq_u8(dec1.val[3], dec2.val[3]); + + // Check for invalid input, any value larger than 63: + const uint8x16_t classified + = vorrq_u8( + vorrq_u8(vcgtq_u8(str.val[0], vdupq_n_u8(63)), vcgtq_u8(str.val[1], vdupq_n_u8(63))), + vorrq_u8(vcgtq_u8(str.val[2], vdupq_n_u8(63)), vcgtq_u8(str.val[3], vdupq_n_u8(63))) + ); + + // Check that all bits are zero: + if (vmaxvq_u8(classified) != 0U) { + break; + } + + // Compress four bytes into three: + dec.val[0] = vorrq_u8(vshlq_n_u8(str.val[0], 2), vshrq_n_u8(str.val[1], 4)); + dec.val[1] = vorrq_u8(vshlq_n_u8(str.val[1], 4), vshrq_n_u8(str.val[2], 2)); + dec.val[2] = vorrq_u8(vshlq_n_u8(str.val[2], 6), str.val[3]); + + // Interleave and store decoded result: + vst3q_u8((uint8_t *) *o, dec); + + *s += 64; + *o += 48; + + } while (--rounds > 0); + + // Adjust for any rounds that were skipped: + *slen += rounds * 64; + *olen -= rounds * 48; +} diff --git a/mypyc/lib-rt/base64/arch/neon64/enc_loop.c b/mypyc/lib-rt/base64/arch/neon64/enc_loop.c new file mode 100644 index 000000000000..8bdd08830653 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon64/enc_loop.c @@ -0,0 +1,66 @@ +static BASE64_FORCE_INLINE void +enc_loop_neon64_inner (const uint8_t **s, uint8_t **o, const uint8x16x4_t tbl_enc) +{ + // Load 48 bytes and deinterleave: + uint8x16x3_t src = vld3q_u8(*s); + + // Divide bits of three input bytes over four output bytes: + uint8x16x4_t out = enc_reshuffle(src); + + // The bits have now been shifted to the right locations; + // translate their values 0..63 to the Base64 alphabet. + // Use a 64-byte table lookup: + out.val[0] = vqtbl4q_u8(tbl_enc, out.val[0]); + out.val[1] = vqtbl4q_u8(tbl_enc, out.val[1]); + out.val[2] = vqtbl4q_u8(tbl_enc, out.val[2]); + out.val[3] = vqtbl4q_u8(tbl_enc, out.val[3]); + + // Interleave and store output: + vst4q_u8(*o, out); + + *s += 48; + *o += 64; +} + +static inline void +enc_loop_neon64 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + size_t rounds = *slen / 48; + + *slen -= rounds * 48; // 48 bytes consumed per round + *olen += rounds * 64; // 64 bytes produced per round + + // Load the encoding table: + const uint8x16x4_t tbl_enc = load_64byte_table(base64_table_enc_6bit); + + while (rounds > 0) { + if (rounds >= 8) { + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_neon64_inner(s, o, tbl_enc); + enc_loop_neon64_inner(s, o, tbl_enc); + rounds -= 2; + continue; + } + enc_loop_neon64_inner(s, o, tbl_enc); + break; + } +} diff --git a/mypyc/lib-rt/base64/arch/neon64/enc_loop_asm.c b/mypyc/lib-rt/base64/arch/neon64/enc_loop_asm.c new file mode 100644 index 000000000000..182e9cdf4a17 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon64/enc_loop_asm.c @@ -0,0 +1,168 @@ +// Apologies in advance for combining the preprocessor with inline assembly, +// two notoriously gnarly parts of C, but it was necessary to avoid a lot of +// code repetition. The preprocessor is used to template large sections of +// inline assembly that differ only in the registers used. If the code was +// written out by hand, it would become very large and hard to audit. + +// Generate a block of inline assembly that loads three user-defined registers +// A, B, C from memory and deinterleaves them, post-incrementing the src +// pointer. The register set should be sequential. +#define LOAD(A, B, C) \ + "ld3 {"A".16b, "B".16b, "C".16b}, [%[src]], #48 \n\t" + +// Generate a block of inline assembly that takes three deinterleaved registers +// and shuffles the bytes. The output is in temporary registers t0..t3. +#define SHUF(A, B, C) \ + "ushr %[t0].16b, "A".16b, #2 \n\t" \ + "ushr %[t1].16b, "B".16b, #4 \n\t" \ + "ushr %[t2].16b, "C".16b, #6 \n\t" \ + "sli %[t1].16b, "A".16b, #4 \n\t" \ + "sli %[t2].16b, "B".16b, #2 \n\t" \ + "and %[t1].16b, %[t1].16b, %[n63].16b \n\t" \ + "and %[t2].16b, %[t2].16b, %[n63].16b \n\t" \ + "and %[t3].16b, "C".16b, %[n63].16b \n\t" + +// Generate a block of inline assembly that takes temporary registers t0..t3 +// and translates them to the base64 alphabet, using a table loaded into +// v8..v11. The output is in user-defined registers A..D. +#define TRAN(A, B, C, D) \ + "tbl "A".16b, {v8.16b-v11.16b}, %[t0].16b \n\t" \ + "tbl "B".16b, {v8.16b-v11.16b}, %[t1].16b \n\t" \ + "tbl "C".16b, {v8.16b-v11.16b}, %[t2].16b \n\t" \ + "tbl "D".16b, {v8.16b-v11.16b}, %[t3].16b \n\t" + +// Generate a block of inline assembly that interleaves four registers and +// stores them, post-incrementing the destination pointer. +#define STOR(A, B, C, D) \ + "st4 {"A".16b, "B".16b, "C".16b, "D".16b}, [%[dst]], #64 \n\t" + +// Generate a block of inline assembly that generates a single self-contained +// encoder round: fetch the data, process it, and store the result. +#define ROUND() \ + LOAD("v12", "v13", "v14") \ + SHUF("v12", "v13", "v14") \ + TRAN("v12", "v13", "v14", "v15") \ + STOR("v12", "v13", "v14", "v15") + +// Generate a block of assembly that generates a type A interleaved encoder +// round. It uses registers that were loaded by the previous type B round, and +// in turn loads registers for the next type B round. +#define ROUND_A() \ + SHUF("v2", "v3", "v4") \ + LOAD("v12", "v13", "v14") \ + TRAN("v2", "v3", "v4", "v5") \ + STOR("v2", "v3", "v4", "v5") + +// Type B interleaved encoder round. Same as type A, but register sets swapped. +#define ROUND_B() \ + SHUF("v12", "v13", "v14") \ + LOAD("v2", "v3", "v4") \ + TRAN("v12", "v13", "v14", "v15") \ + STOR("v12", "v13", "v14", "v15") + +// The first type A round needs to load its own registers. +#define ROUND_A_FIRST() \ + LOAD("v2", "v3", "v4") \ + ROUND_A() + +// The last type B round omits the load for the next step. +#define ROUND_B_LAST() \ + SHUF("v12", "v13", "v14") \ + TRAN("v12", "v13", "v14", "v15") \ + STOR("v12", "v13", "v14", "v15") + +// Suppress clang's warning that the literal string in the asm statement is +// overlong (longer than the ISO-mandated minimum size of 4095 bytes for C99 +// compilers). It may be true, but the goal here is not C99 portability. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverlength-strings" + +static inline void +enc_loop_neon64 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + size_t rounds = *slen / 48; + + if (rounds == 0) { + return; + } + + *slen -= rounds * 48; // 48 bytes consumed per round. + *olen += rounds * 64; // 64 bytes produced per round. + + // Number of times to go through the 8x loop. + size_t loops = rounds / 8; + + // Number of rounds remaining after the 8x loop. + rounds %= 8; + + // Temporary registers, used as scratch space. + uint8x16_t tmp0, tmp1, tmp2, tmp3; + + __asm__ volatile ( + + // Load the encoding table into v8..v11. + " ld1 {v8.16b-v11.16b}, [%[tbl]] \n\t" + + // If there are eight rounds or more, enter an 8x unrolled loop + // of interleaved encoding rounds. The rounds interleave memory + // operations (load/store) with data operations to maximize + // pipeline throughput. + " cbz %[loops], 4f \n\t" + + // The SIMD instructions do not touch the flags. + "88: subs %[loops], %[loops], #1 \n\t" + " " ROUND_A_FIRST() + " " ROUND_B() + " " ROUND_A() + " " ROUND_B() + " " ROUND_A() + " " ROUND_B() + " " ROUND_A() + " " ROUND_B_LAST() + " b.ne 88b \n\t" + + // Enter a 4x unrolled loop for rounds of 4 or more. + "4: cmp %[rounds], #4 \n\t" + " b.lt 30f \n\t" + " " ROUND_A_FIRST() + " " ROUND_B() + " " ROUND_A() + " " ROUND_B_LAST() + " sub %[rounds], %[rounds], #4 \n\t" + + // Dispatch the remaining rounds 0..3. + "30: cbz %[rounds], 0f \n\t" + " cmp %[rounds], #2 \n\t" + " b.eq 2f \n\t" + " b.lt 1f \n\t" + + // Block of non-interlaced encoding rounds, which can each + // individually be jumped to. Rounds fall through to the next. + "3: " ROUND() + "2: " ROUND() + "1: " ROUND() + "0: \n\t" + + // Outputs (modified). + : [loops] "+r" (loops), + [src] "+r" (*s), + [dst] "+r" (*o), + [t0] "=&w" (tmp0), + [t1] "=&w" (tmp1), + [t2] "=&w" (tmp2), + [t3] "=&w" (tmp3) + + // Inputs (not modified). + : [rounds] "r" (rounds), + [tbl] "r" (base64_table_enc_6bit), + [n63] "w" (vdupq_n_u8(63)) + + // Clobbers. + : "v2", "v3", "v4", "v5", + "v8", "v9", "v10", "v11", + "v12", "v13", "v14", "v15", + "cc", "memory" + ); +} + +#pragma GCC diagnostic pop diff --git a/mypyc/lib-rt/base64/arch/neon64/enc_reshuffle.c b/mypyc/lib-rt/base64/arch/neon64/enc_reshuffle.c new file mode 100644 index 000000000000..2655df10f3eb --- /dev/null +++ b/mypyc/lib-rt/base64/arch/neon64/enc_reshuffle.c @@ -0,0 +1,31 @@ +static BASE64_FORCE_INLINE uint8x16x4_t +enc_reshuffle (const uint8x16x3_t in) +{ + uint8x16x4_t out; + + // Input: + // in[0] = a7 a6 a5 a4 a3 a2 a1 a0 + // in[1] = b7 b6 b5 b4 b3 b2 b1 b0 + // in[2] = c7 c6 c5 c4 c3 c2 c1 c0 + + // Output: + // out[0] = 00 00 a7 a6 a5 a4 a3 a2 + // out[1] = 00 00 a1 a0 b7 b6 b5 b4 + // out[2] = 00 00 b3 b2 b1 b0 c7 c6 + // out[3] = 00 00 c5 c4 c3 c2 c1 c0 + + // Move the input bits to where they need to be in the outputs. Except + // for the first output, the high two bits are not cleared. + out.val[0] = vshrq_n_u8(in.val[0], 2); + out.val[1] = vshrq_n_u8(in.val[1], 4); + out.val[2] = vshrq_n_u8(in.val[2], 6); + out.val[1] = vsliq_n_u8(out.val[1], in.val[0], 4); + out.val[2] = vsliq_n_u8(out.val[2], in.val[1], 2); + + // Clear the high two bits in the second, third and fourth output. + out.val[1] = vandq_u8(out.val[1], vdupq_n_u8(0x3F)); + out.val[2] = vandq_u8(out.val[2], vdupq_n_u8(0x3F)); + out.val[3] = vandq_u8(in.val[2], vdupq_n_u8(0x3F)); + + return out; +} diff --git a/mypyc/lib-rt/base64/arch/sse41/codec.c b/mypyc/lib-rt/base64/arch/sse41/codec.c new file mode 100644 index 000000000000..c627db5f726d --- /dev/null +++ b/mypyc/lib-rt/base64/arch/sse41/codec.c @@ -0,0 +1,58 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_SSE41 +#include + +// Only enable inline assembly on supported compilers and on 64-bit CPUs. +#ifndef BASE64_SSE41_USE_ASM +# if (defined(__GNUC__) || defined(__clang__)) && BASE64_WORDSIZE == 64 +# define BASE64_SSE41_USE_ASM 1 +# else +# define BASE64_SSE41_USE_ASM 0 +# endif +#endif + +#include "../ssse3/dec_reshuffle.c" +#include "../ssse3/dec_loop.c" + +#if BASE64_SSE41_USE_ASM +# include "../ssse3/enc_loop_asm.c" +#else +# include "../ssse3/enc_translate.c" +# include "../ssse3/enc_reshuffle.c" +# include "../ssse3/enc_loop.c" +#endif + +#endif // HAVE_SSE41 + +void +base64_stream_encode_sse41 BASE64_ENC_PARAMS +{ +#if HAVE_SSE41 + #include "../generic/enc_head.c" + enc_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_sse41 BASE64_DEC_PARAMS +{ +#if HAVE_SSE41 + #include "../generic/dec_head.c" + dec_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/sse42/codec.c b/mypyc/lib-rt/base64/arch/sse42/codec.c new file mode 100644 index 000000000000..2fe4e2997aa1 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/sse42/codec.c @@ -0,0 +1,58 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_SSE42 +#include + +// Only enable inline assembly on supported compilers and on 64-bit CPUs. +#ifndef BASE64_SSE42_USE_ASM +# if (defined(__GNUC__) || defined(__clang__)) && BASE64_WORDSIZE == 64 +# define BASE64_SSE42_USE_ASM 1 +# else +# define BASE64_SSE42_USE_ASM 0 +# endif +#endif + +#include "../ssse3/dec_reshuffle.c" +#include "../ssse3/dec_loop.c" + +#if BASE64_SSE42_USE_ASM +# include "../ssse3/enc_loop_asm.c" +#else +# include "../ssse3/enc_translate.c" +# include "../ssse3/enc_reshuffle.c" +# include "../ssse3/enc_loop.c" +#endif + +#endif // HAVE_SSE42 + +void +base64_stream_encode_sse42 BASE64_ENC_PARAMS +{ +#if HAVE_SSE42 + #include "../generic/enc_head.c" + enc_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_sse42 BASE64_DEC_PARAMS +{ +#if HAVE_SSE42 + #include "../generic/dec_head.c" + dec_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/codec.c b/mypyc/lib-rt/base64/arch/ssse3/codec.c new file mode 100644 index 000000000000..e51b3dfdb167 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/codec.c @@ -0,0 +1,60 @@ +#include +#include +#include + +#include "libbase64.h" +#include "../../tables/tables.h" +#include "../../codecs.h" +#include "config.h" +#include "../../env.h" + +#if HAVE_SSSE3 +#include + +// Only enable inline assembly on supported compilers and on 64-bit CPUs. +// 32-bit CPUs with SSSE3 support, such as low-end Atoms, only have eight XMM +// registers, which is not enough to run the inline assembly. +#ifndef BASE64_SSSE3_USE_ASM +# if (defined(__GNUC__) || defined(__clang__)) && BASE64_WORDSIZE == 64 +# define BASE64_SSSE3_USE_ASM 1 +# else +# define BASE64_SSSE3_USE_ASM 0 +# endif +#endif + +#include "dec_reshuffle.c" +#include "dec_loop.c" + +#if BASE64_SSSE3_USE_ASM +# include "enc_loop_asm.c" +#else +# include "enc_reshuffle.c" +# include "enc_translate.c" +# include "enc_loop.c" +#endif + +#endif // HAVE_SSSE3 + +void +base64_stream_encode_ssse3 BASE64_ENC_PARAMS +{ +#if HAVE_SSSE3 + #include "../generic/enc_head.c" + enc_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/enc_tail.c" +#else + base64_enc_stub(state, src, srclen, out, outlen); +#endif +} + +int +base64_stream_decode_ssse3 BASE64_DEC_PARAMS +{ +#if HAVE_SSSE3 + #include "../generic/dec_head.c" + dec_loop_ssse3(&s, &slen, &o, &olen); + #include "../generic/dec_tail.c" +#else + return base64_dec_stub(state, src, srclen, out, outlen); +#endif +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/dec_loop.c b/mypyc/lib-rt/base64/arch/ssse3/dec_loop.c new file mode 100644 index 000000000000..7ddb73bf8814 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/dec_loop.c @@ -0,0 +1,173 @@ +// The input consists of six character sets in the Base64 alphabet, which we +// need to map back to the 6-bit values they represent. There are three ranges, +// two singles, and then there's the rest. +// +// # From To Add Characters +// 1 [43] [62] +19 + +// 2 [47] [63] +16 / +// 3 [48..57] [52..61] +4 0..9 +// 4 [65..90] [0..25] -65 A..Z +// 5 [97..122] [26..51] -71 a..z +// (6) Everything else => invalid input +// +// We will use lookup tables for character validation and offset computation. +// Remember that 0x2X and 0x0X are the same index for _mm_shuffle_epi8, this +// allows to mask with 0x2F instead of 0x0F and thus save one constant +// declaration (register and/or memory access). +// +// For offsets: +// Perfect hash for lut = ((src >> 4) & 0x2F) + ((src == 0x2F) ? 0xFF : 0x00) +// 0000 = garbage +// 0001 = / +// 0010 = + +// 0011 = 0-9 +// 0100 = A-Z +// 0101 = A-Z +// 0110 = a-z +// 0111 = a-z +// 1000 >= garbage +// +// For validation, here's the table. +// A character is valid if and only if the AND of the 2 lookups equals 0: +// +// hi \ lo 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 1100 1101 1110 1111 +// LUT 0x15 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x13 0x1A 0x1B 0x1B 0x1B 0x1A +// +// 0000 0x10 char NUL SOH STX ETX EOT ENQ ACK BEL BS HT LF VT FF CR SO SI +// andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// +// 0001 0x10 char DLE DC1 DC2 DC3 DC4 NAK SYN ETB CAN EM SUB ESC FS GS RS US +// andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// +// 0010 0x01 char ! " # $ % & ' ( ) * + , - . / +// andlut 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x00 0x01 0x01 0x01 0x00 +// +// 0011 0x02 char 0 1 2 3 4 5 6 7 8 9 : ; < = > ? +// andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x02 0x02 0x02 0x02 0x02 0x02 +// +// 0100 0x04 char @ A B C D E F G H I J K L M N O +// andlut 0x04 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 +// +// 0101 0x08 char P Q R S T U V W X Y Z [ \ ] ^ _ +// andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x08 0x08 0x08 0x08 0x08 +// +// 0110 0x04 char ` a b c d e f g h i j k l m n o +// andlut 0x04 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 +// 0111 0x08 char p q r s t u v w x y z { | } ~ +// andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x08 0x08 0x08 0x08 0x08 +// +// 1000 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1001 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1010 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1011 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1100 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1101 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1110 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 +// 1111 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + +static BASE64_FORCE_INLINE int +dec_loop_ssse3_inner (const uint8_t **s, uint8_t **o, size_t *rounds) +{ + const __m128i lut_lo = _mm_setr_epi8( + 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A); + + const __m128i lut_hi = _mm_setr_epi8( + 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10); + + const __m128i lut_roll = _mm_setr_epi8( + 0, 16, 19, 4, -65, -65, -71, -71, + 0, 0, 0, 0, 0, 0, 0, 0); + + const __m128i mask_2F = _mm_set1_epi8(0x2F); + + // Load input: + __m128i str = _mm_loadu_si128((__m128i *) *s); + + // Table lookups: + const __m128i hi_nibbles = _mm_and_si128(_mm_srli_epi32(str, 4), mask_2F); + const __m128i lo_nibbles = _mm_and_si128(str, mask_2F); + const __m128i hi = _mm_shuffle_epi8(lut_hi, hi_nibbles); + const __m128i lo = _mm_shuffle_epi8(lut_lo, lo_nibbles); + + // Check for invalid input: if any "and" values from lo and hi are not + // zero, fall back on bytewise code to do error checking and reporting: + if (_mm_movemask_epi8(_mm_cmpgt_epi8(_mm_and_si128(lo, hi), _mm_setzero_si128())) != 0) { + return 0; + } + + const __m128i eq_2F = _mm_cmpeq_epi8(str, mask_2F); + const __m128i roll = _mm_shuffle_epi8(lut_roll, _mm_add_epi8(eq_2F, hi_nibbles)); + + // Now simply add the delta values to the input: + str = _mm_add_epi8(str, roll); + + // Reshuffle the input to packed 12-byte output format: + str = dec_reshuffle(str); + + // Store the output: + _mm_storeu_si128((__m128i *) *o, str); + + *s += 16; + *o += 12; + *rounds -= 1; + + return 1; +} + +static inline void +dec_loop_ssse3 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 24) { + return; + } + + // Process blocks of 16 bytes per round. Because 4 extra zero bytes are + // written after the output, ensure that there will be at least 8 bytes + // of input data left to cover the gap. (6 data bytes and up to two + // end-of-string markers.) + size_t rounds = (*slen - 8) / 16; + + *slen -= rounds * 16; // 16 bytes consumed per round + *olen += rounds * 12; // 12 bytes produced per round + + do { + if (rounds >= 8) { + if (dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 4) { + if (dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds)) { + continue; + } + break; + } + if (rounds >= 2) { + if (dec_loop_ssse3_inner(s, o, &rounds) && + dec_loop_ssse3_inner(s, o, &rounds)) { + continue; + } + break; + } + dec_loop_ssse3_inner(s, o, &rounds); + break; + + } while (rounds > 0); + + // Adjust for any rounds that were skipped: + *slen += rounds * 16; + *olen -= rounds * 12; +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/dec_reshuffle.c b/mypyc/lib-rt/base64/arch/ssse3/dec_reshuffle.c new file mode 100644 index 000000000000..d3dd395427ae --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/dec_reshuffle.c @@ -0,0 +1,33 @@ +static BASE64_FORCE_INLINE __m128i +dec_reshuffle (const __m128i in) +{ + // in, bits, upper case are most significant bits, lower case are least significant bits + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA + + const __m128i merge_ab_and_bc = _mm_maddubs_epi16(in, _mm_set1_epi32(0x01400140)); + // 0000kkkk LLllllll 0000JJJJ JJjjKKKK + // 0000hhhh IIiiiiii 0000GGGG GGggHHHH + // 0000eeee FFffffff 0000DDDD DDddEEEE + // 0000bbbb CCcccccc 0000AAAA AAaaBBBB + + const __m128i out = _mm_madd_epi16(merge_ab_and_bc, _mm_set1_epi32(0x00011000)); + // 00000000 JJJJJJjj KKKKkkkk LLllllll + // 00000000 GGGGGGgg HHHHhhhh IIiiiiii + // 00000000 DDDDDDdd EEEEeeee FFffffff + // 00000000 AAAAAAaa BBBBbbbb CCcccccc + + // Pack bytes together: + return _mm_shuffle_epi8(out, _mm_setr_epi8( + 2, 1, 0, + 6, 5, 4, + 10, 9, 8, + 14, 13, 12, + -1, -1, -1, -1)); + // 00000000 00000000 00000000 00000000 + // LLllllll KKKKkkkk JJJJJJjj IIiiiiii + // HHHHhhhh GGGGGGgg FFffffff EEEEeeee + // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/enc_loop.c b/mypyc/lib-rt/base64/arch/ssse3/enc_loop.c new file mode 100644 index 000000000000..9b67b70db1fd --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/enc_loop.c @@ -0,0 +1,67 @@ +static BASE64_FORCE_INLINE void +enc_loop_ssse3_inner (const uint8_t **s, uint8_t **o) +{ + // Load input: + __m128i str = _mm_loadu_si128((__m128i *) *s); + + // Reshuffle: + str = enc_reshuffle(str); + + // Translate reshuffled bytes to the Base64 alphabet: + str = enc_translate(str); + + // Store: + _mm_storeu_si128((__m128i *) *o, str); + + *s += 12; + *o += 16; +} + +static inline void +enc_loop_ssse3 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + if (*slen < 16) { + return; + } + + // Process blocks of 12 bytes at a time. Because blocks are loaded 16 + // bytes at a time, ensure that there will be at least 4 remaining + // bytes after the last round, so that the final read will not pass + // beyond the bounds of the input buffer: + size_t rounds = (*slen - 4) / 12; + + *slen -= rounds * 12; // 12 bytes consumed per round + *olen += rounds * 16; // 16 bytes produced per round + + do { + if (rounds >= 8) { + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + rounds -= 8; + continue; + } + if (rounds >= 4) { + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + rounds -= 4; + continue; + } + if (rounds >= 2) { + enc_loop_ssse3_inner(s, o); + enc_loop_ssse3_inner(s, o); + rounds -= 2; + continue; + } + enc_loop_ssse3_inner(s, o); + break; + + } while (rounds > 0); +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/enc_loop_asm.c b/mypyc/lib-rt/base64/arch/ssse3/enc_loop_asm.c new file mode 100644 index 000000000000..0cdb340a63b7 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/enc_loop_asm.c @@ -0,0 +1,268 @@ +// Apologies in advance for combining the preprocessor with inline assembly, +// two notoriously gnarly parts of C, but it was necessary to avoid a lot of +// code repetition. The preprocessor is used to template large sections of +// inline assembly that differ only in the registers used. If the code was +// written out by hand, it would become very large and hard to audit. + +// Generate a block of inline assembly that loads register R0 from memory. The +// offset at which the register is loaded is set by the given round. +#define LOAD(R0, ROUND) \ + "lddqu ("#ROUND" * 12)(%[src]), %["R0"] \n\t" + +// Generate a block of inline assembly that deinterleaves and shuffles register +// R0 using preloaded constants. Outputs in R0 and R1. +#define SHUF(R0, R1) \ + "pshufb %[lut0], %["R0"] \n\t" \ + "movdqa %["R0"], %["R1"] \n\t" \ + "pand %[msk0], %["R0"] \n\t" \ + "pand %[msk2], %["R1"] \n\t" \ + "pmulhuw %[msk1], %["R0"] \n\t" \ + "pmullw %[msk3], %["R1"] \n\t" \ + "por %["R1"], %["R0"] \n\t" + +// Generate a block of inline assembly that takes R0 and R1 and translates +// their contents to the base64 alphabet, using preloaded constants. +#define TRAN(R0, R1, R2) \ + "movdqa %["R0"], %["R1"] \n\t" \ + "movdqa %["R0"], %["R2"] \n\t" \ + "psubusb %[n51], %["R1"] \n\t" \ + "pcmpgtb %[n25], %["R2"] \n\t" \ + "psubb %["R2"], %["R1"] \n\t" \ + "movdqa %[lut1], %["R2"] \n\t" \ + "pshufb %["R1"], %["R2"] \n\t" \ + "paddb %["R2"], %["R0"] \n\t" + +// Generate a block of inline assembly that stores the given register R0 at an +// offset set by the given round. +#define STOR(R0, ROUND) \ + "movdqu %["R0"], ("#ROUND" * 16)(%[dst]) \n\t" + +// Generate a block of inline assembly that generates a single self-contained +// encoder round: fetch the data, process it, and store the result. Then update +// the source and destination pointers. +#define ROUND() \ + LOAD("a", 0) \ + SHUF("a", "b") \ + TRAN("a", "b", "c") \ + STOR("a", 0) \ + "add $12, %[src] \n\t" \ + "add $16, %[dst] \n\t" + +// Define a macro that initiates a three-way interleaved encoding round by +// preloading registers a, b and c from memory. +// The register graph shows which registers are in use during each step, and +// is a visual aid for choosing registers for that step. Symbol index: +// +// + indicates that a register is loaded by that step. +// | indicates that a register is in use and must not be touched. +// - indicates that a register is decommissioned by that step. +// x indicates that a register is used as a temporary by that step. +// V indicates that a register is an input or output to the macro. +// +#define ROUND_3_INIT() /* a b c d e f */ \ + LOAD("a", 0) /* + */ \ + SHUF("a", "d") /* | + */ \ + LOAD("b", 1) /* | + | */ \ + TRAN("a", "d", "e") /* | | - x */ \ + LOAD("c", 2) /* V V V */ + +// Define a macro that translates, shuffles and stores the input registers A, B +// and C, and preloads registers D, E and F for the next round. +// This macro can be arbitrarily daisy-chained by feeding output registers D, E +// and F back into the next round as input registers A, B and C. The macro +// carefully interleaves memory operations with data operations for optimal +// pipelined performance. + +#define ROUND_3(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + LOAD(D, (ROUND + 3)) /* V V V + */ \ + SHUF(B, E) /* | | | | + */ \ + STOR(A, (ROUND + 0)) /* - | | | | */ \ + TRAN(B, E, F) /* | | | - x */ \ + LOAD(E, (ROUND + 4)) /* | | | + */ \ + SHUF(C, A) /* + | | | | */ \ + STOR(B, (ROUND + 1)) /* | - | | | */ \ + TRAN(C, A, F) /* - | | | x */ \ + LOAD(F, (ROUND + 5)) /* | | | + */ \ + SHUF(D, A) /* + | | | | */ \ + STOR(C, (ROUND + 2)) /* | - | | | */ \ + TRAN(D, A, B) /* - x V V V */ + +// Define a macro that terminates a ROUND_3 macro by taking pre-loaded +// registers D, E and F, and translating, shuffling and storing them. +#define ROUND_3_END(ROUND, A,B,C,D,E,F) /* A B C D E F */ \ + SHUF(E, A) /* + V V V */ \ + STOR(D, (ROUND + 3)) /* | - | | */ \ + TRAN(E, A, B) /* - x | | */ \ + SHUF(F, C) /* + | | */ \ + STOR(E, (ROUND + 4)) /* | - | */ \ + TRAN(F, C, D) /* - x | */ \ + STOR(F, (ROUND + 5)) /* - */ + +// Define a type A round. Inputs are a, b, and c, outputs are d, e, and f. +#define ROUND_3_A(ROUND) \ + ROUND_3(ROUND, "a", "b", "c", "d", "e", "f") + +// Define a type B round. Inputs and outputs are swapped with regard to type A. +#define ROUND_3_B(ROUND) \ + ROUND_3(ROUND, "d", "e", "f", "a", "b", "c") + +// Terminating macro for a type A round. +#define ROUND_3_A_LAST(ROUND) \ + ROUND_3_A(ROUND) \ + ROUND_3_END(ROUND, "a", "b", "c", "d", "e", "f") + +// Terminating macro for a type B round. +#define ROUND_3_B_LAST(ROUND) \ + ROUND_3_B(ROUND) \ + ROUND_3_END(ROUND, "d", "e", "f", "a", "b", "c") + +// Suppress clang's warning that the literal string in the asm statement is +// overlong (longer than the ISO-mandated minimum size of 4095 bytes for C99 +// compilers). It may be true, but the goal here is not C99 portability. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverlength-strings" + +static inline void +enc_loop_ssse3 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) +{ + // For a clearer explanation of the algorithm used by this function, + // please refer to the plain (not inline assembly) implementation. This + // function follows the same basic logic. + + if (*slen < 16) { + return; + } + + // Process blocks of 12 bytes at a time. Input is read in blocks of 16 + // bytes, so "reserve" four bytes from the input buffer to ensure that + // we never read beyond the end of the input buffer. + size_t rounds = (*slen - 4) / 12; + + *slen -= rounds * 12; // 12 bytes consumed per round + *olen += rounds * 16; // 16 bytes produced per round + + // Number of times to go through the 36x loop. + size_t loops = rounds / 36; + + // Number of rounds remaining after the 36x loop. + rounds %= 36; + + // Lookup tables. + const __m128i lut0 = _mm_set_epi8( + 10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1); + + const __m128i lut1 = _mm_setr_epi8( + 65, 71, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -19, -16, 0, 0); + + // Temporary registers. + __m128i a, b, c, d, e, f; + + __asm__ volatile ( + + // If there are 36 rounds or more, enter a 36x unrolled loop of + // interleaved encoding rounds. The rounds interleave memory + // operations (load/store) with data operations (table lookups, + // etc) to maximize pipeline throughput. + " test %[loops], %[loops] \n\t" + " jz 18f \n\t" + " jmp 36f \n\t" + " \n\t" + ".balign 64 \n\t" + "36: " ROUND_3_INIT() + " " ROUND_3_A( 0) + " " ROUND_3_B( 3) + " " ROUND_3_A( 6) + " " ROUND_3_B( 9) + " " ROUND_3_A(12) + " " ROUND_3_B(15) + " " ROUND_3_A(18) + " " ROUND_3_B(21) + " " ROUND_3_A(24) + " " ROUND_3_B(27) + " " ROUND_3_A_LAST(30) + " add $(12 * 36), %[src] \n\t" + " add $(16 * 36), %[dst] \n\t" + " dec %[loops] \n\t" + " jnz 36b \n\t" + + // Enter an 18x unrolled loop for rounds of 18 or more. + "18: cmp $18, %[rounds] \n\t" + " jl 9f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B(3) + " " ROUND_3_A(6) + " " ROUND_3_B(9) + " " ROUND_3_A_LAST(12) + " sub $18, %[rounds] \n\t" + " add $(12 * 18), %[src] \n\t" + " add $(16 * 18), %[dst] \n\t" + + // Enter a 9x unrolled loop for rounds of 9 or more. + "9: cmp $9, %[rounds] \n\t" + " jl 6f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A(0) + " " ROUND_3_B_LAST(3) + " sub $9, %[rounds] \n\t" + " add $(12 * 9), %[src] \n\t" + " add $(16 * 9), %[dst] \n\t" + + // Enter a 6x unrolled loop for rounds of 6 or more. + "6: cmp $6, %[rounds] \n\t" + " jl 55f \n\t" + " " ROUND_3_INIT() + " " ROUND_3_A_LAST(0) + " sub $6, %[rounds] \n\t" + " add $(12 * 6), %[src] \n\t" + " add $(16 * 6), %[dst] \n\t" + + // Dispatch the remaining rounds 0..5. + "55: cmp $3, %[rounds] \n\t" + " jg 45f \n\t" + " je 3f \n\t" + " cmp $1, %[rounds] \n\t" + " jg 2f \n\t" + " je 1f \n\t" + " jmp 0f \n\t" + + "45: cmp $4, %[rounds] \n\t" + " je 4f \n\t" + + // Block of non-interlaced encoding rounds, which can each + // individually be jumped to. Rounds fall through to the next. + "5: " ROUND() + "4: " ROUND() + "3: " ROUND() + "2: " ROUND() + "1: " ROUND() + "0: \n\t" + + // Outputs (modified). + : [rounds] "+r" (rounds), + [loops] "+r" (loops), + [src] "+r" (*s), + [dst] "+r" (*o), + [a] "=&x" (a), + [b] "=&x" (b), + [c] "=&x" (c), + [d] "=&x" (d), + [e] "=&x" (e), + [f] "=&x" (f) + + // Inputs (not modified). + : [lut0] "x" (lut0), + [lut1] "x" (lut1), + [msk0] "x" (_mm_set1_epi32(0x0FC0FC00)), + [msk1] "x" (_mm_set1_epi32(0x04000040)), + [msk2] "x" (_mm_set1_epi32(0x003F03F0)), + [msk3] "x" (_mm_set1_epi32(0x01000010)), + [n51] "x" (_mm_set1_epi8(51)), + [n25] "x" (_mm_set1_epi8(25)) + + // Clobbers. + : "cc", "memory" + ); +} + +#pragma GCC diagnostic pop diff --git a/mypyc/lib-rt/base64/arch/ssse3/enc_reshuffle.c b/mypyc/lib-rt/base64/arch/ssse3/enc_reshuffle.c new file mode 100644 index 000000000000..f9dc949f255d --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/enc_reshuffle.c @@ -0,0 +1,48 @@ +static BASE64_FORCE_INLINE __m128i +enc_reshuffle (__m128i in) +{ + // Input, bytes MSB to LSB: + // 0 0 0 0 l k j i h g f e d c b a + + in = _mm_shuffle_epi8(in, _mm_set_epi8( + 10, 11, 9, 10, + 7, 8, 6, 7, + 4, 5, 3, 4, + 1, 2, 0, 1)); + // in, bytes MSB to LSB: + // k l j k + // h i g h + // e f d e + // b c a b + + const __m128i t0 = _mm_and_si128(in, _mm_set1_epi32(0x0FC0FC00)); + // bits, upper case are most significant bits, lower case are least significant bits + // 0000kkkk LL000000 JJJJJJ00 00000000 + // 0000hhhh II000000 GGGGGG00 00000000 + // 0000eeee FF000000 DDDDDD00 00000000 + // 0000bbbb CC000000 AAAAAA00 00000000 + + const __m128i t1 = _mm_mulhi_epu16(t0, _mm_set1_epi32(0x04000040)); + // 00000000 00kkkkLL 00000000 00JJJJJJ + // 00000000 00hhhhII 00000000 00GGGGGG + // 00000000 00eeeeFF 00000000 00DDDDDD + // 00000000 00bbbbCC 00000000 00AAAAAA + + const __m128i t2 = _mm_and_si128(in, _mm_set1_epi32(0x003F03F0)); + // 00000000 00llllll 000000jj KKKK0000 + // 00000000 00iiiiii 000000gg HHHH0000 + // 00000000 00ffffff 000000dd EEEE0000 + // 00000000 00cccccc 000000aa BBBB0000 + + const __m128i t3 = _mm_mullo_epi16(t2, _mm_set1_epi32(0x01000010)); + // 00llllll 00000000 00jjKKKK 00000000 + // 00iiiiii 00000000 00ggHHHH 00000000 + // 00ffffff 00000000 00ddEEEE 00000000 + // 00cccccc 00000000 00aaBBBB 00000000 + + return _mm_or_si128(t1, t3); + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA +} diff --git a/mypyc/lib-rt/base64/arch/ssse3/enc_translate.c b/mypyc/lib-rt/base64/arch/ssse3/enc_translate.c new file mode 100644 index 000000000000..60d9a42b8a30 --- /dev/null +++ b/mypyc/lib-rt/base64/arch/ssse3/enc_translate.c @@ -0,0 +1,33 @@ +static BASE64_FORCE_INLINE __m128i +enc_translate (const __m128i in) +{ + // A lookup table containing the absolute offsets for all ranges: + const __m128i lut = _mm_setr_epi8( + 65, 71, -4, -4, + -4, -4, -4, -4, + -4, -4, -4, -4, + -19, -16, 0, 0 + ); + + // Translate values 0..63 to the Base64 alphabet. There are five sets: + // # From To Abs Index Characters + // 0 [0..25] [65..90] +65 0 ABCDEFGHIJKLMNOPQRSTUVWXYZ + // 1 [26..51] [97..122] +71 1 abcdefghijklmnopqrstuvwxyz + // 2 [52..61] [48..57] -4 [2..11] 0123456789 + // 3 [62] [43] -19 12 + + // 4 [63] [47] -16 13 / + + // Create LUT indices from the input. The index for range #0 is right, + // others are 1 less than expected: + __m128i indices = _mm_subs_epu8(in, _mm_set1_epi8(51)); + + // mask is 0xFF (-1) for range #[1..4] and 0x00 for range #0: + __m128i mask = _mm_cmpgt_epi8(in, _mm_set1_epi8(25)); + + // Subtract -1, so add 1 to indices for range #[1..4]. All indices are + // now correct: + indices = _mm_sub_epi8(indices, mask); + + // Add offsets to input values: + return _mm_add_epi8(in, _mm_shuffle_epi8(lut, indices)); +} diff --git a/mypyc/lib-rt/base64/codec_choose.c b/mypyc/lib-rt/base64/codec_choose.c new file mode 100644 index 000000000000..74b0aac7b2d2 --- /dev/null +++ b/mypyc/lib-rt/base64/codec_choose.c @@ -0,0 +1,314 @@ +#include +#include +#include +#include +#include + +#include "libbase64.h" +#include "codecs.h" +#include "config.h" +#include "env.h" + +#if (__x86_64__ || __i386__ || _M_X86 || _M_X64) + #define BASE64_X86 + #if (HAVE_SSSE3 || HAVE_SSE41 || HAVE_SSE42 || HAVE_AVX || HAVE_AVX2 || HAVE_AVX512) + #define BASE64_X86_SIMD + #endif +#endif + +#ifdef BASE64_X86 +#ifdef _MSC_VER + #include + #define __cpuid_count(__level, __count, __eax, __ebx, __ecx, __edx) \ + { \ + int info[4]; \ + __cpuidex(info, __level, __count); \ + __eax = info[0]; \ + __ebx = info[1]; \ + __ecx = info[2]; \ + __edx = info[3]; \ + } + #define __cpuid(__level, __eax, __ebx, __ecx, __edx) \ + __cpuid_count(__level, 0, __eax, __ebx, __ecx, __edx) +#else + #include + #if HAVE_AVX512 || HAVE_AVX2 || HAVE_AVX + #if ((__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 2) || (__clang_major__ >= 3)) + static inline uint64_t _xgetbv (uint32_t index) + { + uint32_t eax, edx; + __asm__ __volatile__("xgetbv" : "=a"(eax), "=d"(edx) : "c"(index)); + return ((uint64_t)edx << 32) | eax; + } + #else + #error "Platform not supported" + #endif + #endif +#endif + +#ifndef bit_AVX512vl +#define bit_AVX512vl (1 << 31) +#endif +#ifndef bit_AVX512vbmi +#define bit_AVX512vbmi (1 << 1) +#endif +#ifndef bit_AVX2 +#define bit_AVX2 (1 << 5) +#endif +#ifndef bit_SSSE3 +#define bit_SSSE3 (1 << 9) +#endif +#ifndef bit_SSE41 +#define bit_SSE41 (1 << 19) +#endif +#ifndef bit_SSE42 +#define bit_SSE42 (1 << 20) +#endif +#ifndef bit_AVX +#define bit_AVX (1 << 28) +#endif + +#define bit_XSAVE_XRSTORE (1 << 27) + +#ifndef _XCR_XFEATURE_ENABLED_MASK +#define _XCR_XFEATURE_ENABLED_MASK 0 +#endif + +#define bit_XMM (1 << 1) +#define bit_YMM (1 << 2) +#define bit_OPMASK (1 << 5) +#define bit_ZMM (1 << 6) +#define bit_HIGH_ZMM (1 << 7) + +#define _XCR_XMM_AND_YMM_STATE_ENABLED_BY_OS (bit_XMM | bit_YMM) + +#define _AVX_512_ENABLED_BY_OS (bit_XMM | bit_YMM | bit_OPMASK | bit_ZMM | bit_HIGH_ZMM) + +#endif + +// Function declarations: +#define BASE64_CODEC_FUNCS(arch) \ + extern void base64_stream_encode_ ## arch BASE64_ENC_PARAMS; \ + extern int base64_stream_decode_ ## arch BASE64_DEC_PARAMS; + +BASE64_CODEC_FUNCS(avx512) +BASE64_CODEC_FUNCS(avx2) +BASE64_CODEC_FUNCS(neon32) +BASE64_CODEC_FUNCS(neon64) +BASE64_CODEC_FUNCS(plain) +BASE64_CODEC_FUNCS(ssse3) +BASE64_CODEC_FUNCS(sse41) +BASE64_CODEC_FUNCS(sse42) +BASE64_CODEC_FUNCS(avx) + +static bool +codec_choose_forced (struct codec *codec, int flags) +{ + // If the user wants to use a certain codec, + // always allow it, even if the codec is a no-op. + // For testing purposes. + + if (!(flags & 0xFFFF)) { + return false; + } + + if (flags & BASE64_FORCE_AVX2) { + codec->enc = base64_stream_encode_avx2; + codec->dec = base64_stream_decode_avx2; + return true; + } + if (flags & BASE64_FORCE_NEON32) { + codec->enc = base64_stream_encode_neon32; + codec->dec = base64_stream_decode_neon32; + return true; + } + if (flags & BASE64_FORCE_NEON64) { + codec->enc = base64_stream_encode_neon64; + codec->dec = base64_stream_decode_neon64; + return true; + } + if (flags & BASE64_FORCE_PLAIN) { + codec->enc = base64_stream_encode_plain; + codec->dec = base64_stream_decode_plain; + return true; + } + if (flags & BASE64_FORCE_SSSE3) { + codec->enc = base64_stream_encode_ssse3; + codec->dec = base64_stream_decode_ssse3; + return true; + } + if (flags & BASE64_FORCE_SSE41) { + codec->enc = base64_stream_encode_sse41; + codec->dec = base64_stream_decode_sse41; + return true; + } + if (flags & BASE64_FORCE_SSE42) { + codec->enc = base64_stream_encode_sse42; + codec->dec = base64_stream_decode_sse42; + return true; + } + if (flags & BASE64_FORCE_AVX) { + codec->enc = base64_stream_encode_avx; + codec->dec = base64_stream_decode_avx; + return true; + } + if (flags & BASE64_FORCE_AVX512) { + codec->enc = base64_stream_encode_avx512; + codec->dec = base64_stream_decode_avx512; + return true; + } + return false; +} + +static bool +codec_choose_arm (struct codec *codec) +{ +#if HAVE_NEON64 || ((defined(__ARM_NEON__) || defined(__ARM_NEON)) && HAVE_NEON32) + + // Unfortunately there is no portable way to check for NEON + // support at runtime from userland in the same way that x86 + // has cpuid, so just stick to the compile-time configuration: + + #if HAVE_NEON64 + codec->enc = base64_stream_encode_neon64; + codec->dec = base64_stream_decode_neon64; + #else + codec->enc = base64_stream_encode_neon32; + codec->dec = base64_stream_decode_neon32; + #endif + + return true; + +#else + (void)codec; + return false; +#endif +} + +static bool +codec_choose_x86 (struct codec *codec) +{ +#ifdef BASE64_X86_SIMD + + unsigned int eax, ebx = 0, ecx = 0, edx; + unsigned int max_level; + + #ifdef _MSC_VER + int info[4]; + __cpuidex(info, 0, 0); + max_level = info[0]; + #else + max_level = __get_cpuid_max(0, NULL); + #endif + + #if HAVE_AVX512 || HAVE_AVX2 || HAVE_AVX + // Check for AVX/AVX2/AVX512 support: + // Checking for AVX requires 3 things: + // 1) CPUID indicates that the OS uses XSAVE and XRSTORE instructions + // (allowing saving YMM registers on context switch) + // 2) CPUID indicates support for AVX + // 3) XGETBV indicates the AVX registers will be saved and restored on + // context switch + // + // Note that XGETBV is only available on 686 or later CPUs, so the + // instruction needs to be conditionally run. + if (max_level >= 1) { + __cpuid_count(1, 0, eax, ebx, ecx, edx); + if (ecx & bit_XSAVE_XRSTORE) { + uint64_t xcr_mask; + xcr_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); + if ((xcr_mask & _XCR_XMM_AND_YMM_STATE_ENABLED_BY_OS) == _XCR_XMM_AND_YMM_STATE_ENABLED_BY_OS) { // check multiple bits at once + #if HAVE_AVX512 + if (max_level >= 7 && ((xcr_mask & _AVX_512_ENABLED_BY_OS) == _AVX_512_ENABLED_BY_OS)) { + __cpuid_count(7, 0, eax, ebx, ecx, edx); + if ((ebx & bit_AVX512vl) && (ecx & bit_AVX512vbmi)) { + codec->enc = base64_stream_encode_avx512; + codec->dec = base64_stream_decode_avx512; + return true; + } + } + #endif + #if HAVE_AVX2 + if (max_level >= 7) { + __cpuid_count(7, 0, eax, ebx, ecx, edx); + if (ebx & bit_AVX2) { + codec->enc = base64_stream_encode_avx2; + codec->dec = base64_stream_decode_avx2; + return true; + } + } + #endif + #if HAVE_AVX + __cpuid_count(1, 0, eax, ebx, ecx, edx); + if (ecx & bit_AVX) { + codec->enc = base64_stream_encode_avx; + codec->dec = base64_stream_decode_avx; + return true; + } + #endif + } + } + } + #endif + + #if HAVE_SSE42 + // Check for SSE42 support: + if (max_level >= 1) { + __cpuid(1, eax, ebx, ecx, edx); + if (ecx & bit_SSE42) { + codec->enc = base64_stream_encode_sse42; + codec->dec = base64_stream_decode_sse42; + return true; + } + } + #endif + + #if HAVE_SSE41 + // Check for SSE41 support: + if (max_level >= 1) { + __cpuid(1, eax, ebx, ecx, edx); + if (ecx & bit_SSE41) { + codec->enc = base64_stream_encode_sse41; + codec->dec = base64_stream_decode_sse41; + return true; + } + } + #endif + + #if HAVE_SSSE3 + // Check for SSSE3 support: + if (max_level >= 1) { + __cpuid(1, eax, ebx, ecx, edx); + if (ecx & bit_SSSE3) { + codec->enc = base64_stream_encode_ssse3; + codec->dec = base64_stream_decode_ssse3; + return true; + } + } + #endif + +#else + (void)codec; +#endif + + return false; +} + +void +codec_choose (struct codec *codec, int flags) +{ + // User forced a codec: + if (codec_choose_forced(codec, flags)) { + return; + } + + // Runtime feature detection: + if (codec_choose_arm(codec)) { + return; + } + if (codec_choose_x86(codec)) { + return; + } + codec->enc = base64_stream_encode_plain; + codec->dec = base64_stream_decode_plain; +} diff --git a/mypyc/lib-rt/base64/codecs.h b/mypyc/lib-rt/base64/codecs.h new file mode 100644 index 000000000000..34d54dc8fde9 --- /dev/null +++ b/mypyc/lib-rt/base64/codecs.h @@ -0,0 +1,57 @@ +#include "libbase64.h" + +// Function parameters for encoding functions: +#define BASE64_ENC_PARAMS \ + ( struct base64_state *state \ + , const char *src \ + , size_t srclen \ + , char *out \ + , size_t *outlen \ + ) + +// Function parameters for decoding functions: +#define BASE64_DEC_PARAMS \ + ( struct base64_state *state \ + , const char *src \ + , size_t srclen \ + , char *out \ + , size_t *outlen \ + ) + +// This function is used as a stub when a certain encoder is not compiled in. +// It discards the inputs and returns zero output bytes. +static inline void +base64_enc_stub BASE64_ENC_PARAMS +{ + (void) state; + (void) src; + (void) srclen; + (void) out; + + *outlen = 0; +} + +// This function is used as a stub when a certain decoder is not compiled in. +// It discards the inputs and returns an invalid decoding result. +static inline int +base64_dec_stub BASE64_DEC_PARAMS +{ + (void) state; + (void) src; + (void) srclen; + (void) out; + (void) outlen; + + return -1; +} + +typedef void (* base64_enc_fn) BASE64_ENC_PARAMS; +typedef int (* base64_dec_fn) BASE64_DEC_PARAMS; + +struct codec +{ + base64_enc_fn enc; + base64_dec_fn dec; +}; + +extern void codec_choose (struct codec *, int flags); diff --git a/mypyc/lib-rt/base64/config.h b/mypyc/lib-rt/base64/config.h new file mode 100644 index 000000000000..fd516c4be2d6 --- /dev/null +++ b/mypyc/lib-rt/base64/config.h @@ -0,0 +1,33 @@ +#ifndef BASE64_CONFIG_H +#define BASE64_CONFIG_H + +#define BASE64_WITH_SSSE3 0 +#define HAVE_SSSE3 BASE64_WITH_SSSE3 + +#define BASE64_WITH_SSE41 0 +#define HAVE_SSE41 BASE64_WITH_SSE41 + +#define BASE64_WITH_SSE42 0 +#define HAVE_SSE42 BASE64_WITH_SSE42 + +#define BASE64_WITH_AVX 0 +#define HAVE_AVX BASE64_WITH_AVX + +#define BASE64_WITH_AVX2 0 +#define HAVE_AVX2 BASE64_WITH_AVX2 + +#define BASE64_WITH_AVX512 0 +#define HAVE_AVX512 BASE64_WITH_AVX512 + +#define BASE64_WITH_NEON32 0 +#define HAVE_NEON32 BASE64_WITH_NEON32 + +#if defined(__APPLE__) && defined(__aarch64__) +#define BASE64_WITH_NEON64 1 +#else +#define BASE64_WITH_NEON64 0 +#endif + +#define HAVE_NEON64 BASE64_WITH_NEON64 + +#endif // BASE64_CONFIG_H diff --git a/mypyc/lib-rt/base64/env.h b/mypyc/lib-rt/base64/env.h new file mode 100644 index 000000000000..083706507900 --- /dev/null +++ b/mypyc/lib-rt/base64/env.h @@ -0,0 +1,84 @@ +#ifndef BASE64_ENV_H +#define BASE64_ENV_H + +#include + +// This header file contains macro definitions that describe certain aspects of +// the compile-time environment. Compatibility and portability macros go here. + +// Define machine endianness. This is for GCC: +#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define BASE64_LITTLE_ENDIAN 1 +#else +# define BASE64_LITTLE_ENDIAN 0 +#endif + +// This is for Clang: +#ifdef __LITTLE_ENDIAN__ +# define BASE64_LITTLE_ENDIAN 1 +#endif + +#ifdef __BIG_ENDIAN__ +# define BASE64_LITTLE_ENDIAN 0 +#endif + +// MSVC++ needs intrin.h for _byteswap_uint64 (issue #68): +#if BASE64_LITTLE_ENDIAN && defined(_MSC_VER) +# include +#endif + +// Endian conversion functions: +#if BASE64_LITTLE_ENDIAN +# ifdef _MSC_VER +// Microsoft Visual C++: +# define BASE64_HTOBE32(x) _byteswap_ulong(x) +# define BASE64_HTOBE64(x) _byteswap_uint64(x) +# else +// GCC and Clang: +# define BASE64_HTOBE32(x) __builtin_bswap32(x) +# define BASE64_HTOBE64(x) __builtin_bswap64(x) +# endif +#else +// No conversion needed: +# define BASE64_HTOBE32(x) (x) +# define BASE64_HTOBE64(x) (x) +#endif + +// Detect word size: +#if defined (__x86_64__) +// This also works for the x32 ABI, which has a 64-bit word size. +# define BASE64_WORDSIZE 64 +#elif SIZE_MAX == UINT32_MAX +# define BASE64_WORDSIZE 32 +#elif SIZE_MAX == UINT64_MAX +# define BASE64_WORDSIZE 64 +#else +# error BASE64_WORDSIZE_NOT_DEFINED +#endif + +// End-of-file definitions. +// Almost end-of-file when waiting for the last '=' character: +#define BASE64_AEOF 1 +// End-of-file when stream end has been reached or invalid input provided: +#define BASE64_EOF 2 + +// GCC 7 defaults to issuing a warning for fallthrough in switch statements, +// unless the fallthrough cases are marked with an attribute. As we use +// fallthrough deliberately, define an alias for the attribute: +#if __GNUC__ >= 7 +# define BASE64_FALLTHROUGH __attribute__((fallthrough)); +#else +# define BASE64_FALLTHROUGH +#endif + +// Declare macros to ensure that functions that are intended to be inlined, are +// actually inlined, even when no optimization is applied. A lot of inner loop +// code is factored into separate functions for reasons of readability, but +// that code should always be inlined (and optimized) in the main loop. +#ifdef _MSC_VER +# define BASE64_FORCE_INLINE __forceinline +#else +# define BASE64_FORCE_INLINE inline __attribute__((always_inline)) +#endif + +#endif // BASE64_ENV_H diff --git a/mypyc/lib-rt/base64/lib.c b/mypyc/lib-rt/base64/lib.c new file mode 100644 index 000000000000..0f24d52e9991 --- /dev/null +++ b/mypyc/lib-rt/base64/lib.c @@ -0,0 +1,164 @@ +#include +#include +#ifdef _OPENMP +#include +#endif + +#include "libbase64.h" +#include "tables/tables.h" +#include "codecs.h" +#include "env.h" + +// These static function pointers are initialized once when the library is +// first used, and remain in use for the remaining lifetime of the program. +// The idea being that CPU features don't change at runtime. +static struct codec codec = { NULL, NULL }; + +void +base64_stream_encode_init (struct base64_state *state, int flags) +{ + // If any of the codec flags are set, redo choice: + if (codec.enc == NULL || flags & 0xFF) { + codec_choose(&codec, flags); + } + state->eof = 0; + state->bytes = 0; + state->carry = 0; + state->flags = flags; +} + +void +base64_stream_encode + ( struct base64_state *state + , const char *src + , size_t srclen + , char *out + , size_t *outlen + ) +{ + codec.enc(state, src, srclen, out, outlen); +} + +void +base64_stream_encode_final + ( struct base64_state *state + , char *out + , size_t *outlen + ) +{ + uint8_t *o = (uint8_t *)out; + + if (state->bytes == 1) { + *o++ = base64_table_enc_6bit[state->carry]; + *o++ = '='; + *o++ = '='; + *outlen = 3; + return; + } + if (state->bytes == 2) { + *o++ = base64_table_enc_6bit[state->carry]; + *o++ = '='; + *outlen = 2; + return; + } + *outlen = 0; +} + +void +base64_stream_decode_init (struct base64_state *state, int flags) +{ + // If any of the codec flags are set, redo choice: + if (codec.dec == NULL || flags & 0xFFFF) { + codec_choose(&codec, flags); + } + state->eof = 0; + state->bytes = 0; + state->carry = 0; + state->flags = flags; +} + +int +base64_stream_decode + ( struct base64_state *state + , const char *src + , size_t srclen + , char *out + , size_t *outlen + ) +{ + return codec.dec(state, src, srclen, out, outlen); +} + +#ifdef _OPENMP + + // Due to the overhead of initializing OpenMP and creating a team of + // threads, we require the data length to be larger than a threshold: + #define OMP_THRESHOLD 20000 + + // Conditionally include OpenMP-accelerated codec implementations: + #include "lib_openmp.c" +#endif + +void +base64_encode + ( const char *src + , size_t srclen + , char *out + , size_t *outlen + , int flags + ) +{ + size_t s; + size_t t; + struct base64_state state; + + #ifdef _OPENMP + if (srclen >= OMP_THRESHOLD) { + base64_encode_openmp(src, srclen, out, outlen, flags); + return; + } + #endif + + // Init the stream reader: + base64_stream_encode_init(&state, flags); + + // Feed the whole string to the stream reader: + base64_stream_encode(&state, src, srclen, out, &s); + + // Finalize the stream by writing trailer if any: + base64_stream_encode_final(&state, out + s, &t); + + // Final output length is stream length plus tail: + *outlen = s + t; +} + +int +base64_decode + ( const char *src + , size_t srclen + , char *out + , size_t *outlen + , int flags + ) +{ + int ret; + struct base64_state state; + + #ifdef _OPENMP + if (srclen >= OMP_THRESHOLD) { + return base64_decode_openmp(src, srclen, out, outlen, flags); + } + #endif + + // Init the stream reader: + base64_stream_decode_init(&state, flags); + + // Feed the whole string to the stream reader: + ret = base64_stream_decode(&state, src, srclen, out, outlen); + + // If when decoding a whole block, we're still waiting for input then fail: + if (ret && (state.bytes == 0)) { + return ret; + } + return 0; +} diff --git a/mypyc/lib-rt/base64/libbase64.h b/mypyc/lib-rt/base64/libbase64.h new file mode 100644 index 000000000000..c5908973c5e7 --- /dev/null +++ b/mypyc/lib-rt/base64/libbase64.h @@ -0,0 +1,146 @@ +#ifndef LIBBASE64_H +#define LIBBASE64_H + +#include /* size_t */ + + +#if defined(_WIN32) || defined(__CYGWIN__) +#define BASE64_SYMBOL_IMPORT __declspec(dllimport) +#define BASE64_SYMBOL_EXPORT __declspec(dllexport) +#define BASE64_SYMBOL_PRIVATE + +#elif __GNUC__ >= 4 +#define BASE64_SYMBOL_IMPORT __attribute__ ((visibility ("default"))) +#define BASE64_SYMBOL_EXPORT __attribute__ ((visibility ("default"))) +#define BASE64_SYMBOL_PRIVATE __attribute__ ((visibility ("hidden"))) + +#else +#define BASE64_SYMBOL_IMPORT +#define BASE64_SYMBOL_EXPORT +#define BASE64_SYMBOL_PRIVATE +#endif + +#if defined(BASE64_STATIC_DEFINE) +#define BASE64_EXPORT +#define BASE64_NO_EXPORT + +#else +#if defined(BASE64_EXPORTS) // defined if we are building the shared library +#define BASE64_EXPORT BASE64_SYMBOL_EXPORT + +#else +#define BASE64_EXPORT BASE64_SYMBOL_IMPORT +#endif + +#define BASE64_NO_EXPORT BASE64_SYMBOL_PRIVATE +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + +/* These are the flags that can be passed in the `flags` argument. The values + * below force the use of a given codec, even if that codec is a no-op in the + * current build. Used in testing. Set to 0 for the default behavior, which is + * runtime feature detection on x86, a compile-time fixed codec on ARM, and + * the plain codec on other platforms: */ +#define BASE64_FORCE_AVX2 (1 << 0) +#define BASE64_FORCE_NEON32 (1 << 1) +#define BASE64_FORCE_NEON64 (1 << 2) +#define BASE64_FORCE_PLAIN (1 << 3) +#define BASE64_FORCE_SSSE3 (1 << 4) +#define BASE64_FORCE_SSE41 (1 << 5) +#define BASE64_FORCE_SSE42 (1 << 6) +#define BASE64_FORCE_AVX (1 << 7) +#define BASE64_FORCE_AVX512 (1 << 8) + +struct base64_state { + int eof; + int bytes; + int flags; + unsigned char carry; +}; + +/* Wrapper function to encode a plain string of given length. Output is written + * to *out without trailing zero. Output length in bytes is written to *outlen. + * The buffer in `out` has been allocated by the caller and is at least 4/3 the + * size of the input. See above for `flags`; set to 0 for default operation: */ +void BASE64_EXPORT base64_encode + ( const char *src + , size_t srclen + , char *out + , size_t *outlen + , int flags + ) ; + +/* Call this before calling base64_stream_encode() to init the state. See above + * for `flags`; set to 0 for default operation: */ +void BASE64_EXPORT base64_stream_encode_init + ( struct base64_state *state + , int flags + ) ; + +/* Encodes the block of data of given length at `src`, into the buffer at + * `out`. Caller is responsible for allocating a large enough out-buffer; it + * must be at least 4/3 the size of the in-buffer, but take some margin. Places + * the number of new bytes written into `outlen` (which is set to zero when the + * function starts). Does not zero-terminate or finalize the output. */ +void BASE64_EXPORT base64_stream_encode + ( struct base64_state *state + , const char *src + , size_t srclen + , char *out + , size_t *outlen + ) ; + +/* Finalizes the output begun by previous calls to `base64_stream_encode()`. + * Adds the required end-of-stream markers if appropriate. `outlen` is modified + * and will contain the number of new bytes written at `out` (which will quite + * often be zero). */ +void BASE64_EXPORT base64_stream_encode_final + ( struct base64_state *state + , char *out + , size_t *outlen + ) ; + +/* Wrapper function to decode a plain string of given length. Output is written + * to *out without trailing zero. Output length in bytes is written to *outlen. + * The buffer in `out` has been allocated by the caller and is at least 3/4 the + * size of the input. See above for `flags`, set to 0 for default operation: */ +int BASE64_EXPORT base64_decode + ( const char *src + , size_t srclen + , char *out + , size_t *outlen + , int flags + ) ; + +/* Call this before calling base64_stream_decode() to init the state. See above + * for `flags`; set to 0 for default operation: */ +void BASE64_EXPORT base64_stream_decode_init + ( struct base64_state *state + , int flags + ) ; + +/* Decodes the block of data of given length at `src`, into the buffer at + * `out`. Caller is responsible for allocating a large enough out-buffer; it + * must be at least 3/4 the size of the in-buffer, but take some margin. Places + * the number of new bytes written into `outlen` (which is set to zero when the + * function starts). Does not zero-terminate the output. Returns 1 if all is + * well, and 0 if a decoding error was found, such as an invalid character. + * Returns -1 if the chosen codec is not included in the current build. Used by + * the test harness to check whether a codec is available for testing. */ +int BASE64_EXPORT base64_stream_decode + ( struct base64_state *state + , const char *src + , size_t srclen + , char *out + , size_t *outlen + ) ; + +#ifdef __cplusplus +} +#endif + +#endif /* LIBBASE64_H */ diff --git a/mypyc/lib-rt/base64/tables/table_dec_32bit.h b/mypyc/lib-rt/base64/tables/table_dec_32bit.h new file mode 100644 index 000000000000..f5d951fa79c7 --- /dev/null +++ b/mypyc/lib-rt/base64/tables/table_dec_32bit.h @@ -0,0 +1,393 @@ +#include +#define CHAR62 '+' +#define CHAR63 '/' +#define CHARPAD '=' + + +#if BASE64_LITTLE_ENDIAN + + +/* SPECIAL DECODE TABLES FOR LITTLE ENDIAN (INTEL) CPUS */ + +const uint32_t base64_table_dec_32bit_d0[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x000000f8, 0xffffffff, 0xffffffff, 0xffffffff, 0x000000fc, +0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, +0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, +0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, +0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, +0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, +0x00000064, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, +0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, +0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, +0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, +0x000000c4, 0x000000c8, 0x000000cc, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d1[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x0000e003, 0xffffffff, 0xffffffff, 0xffffffff, 0x0000f003, +0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, +0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, +0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, +0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, +0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, +0x00009001, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, +0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, +0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, +0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, +0x00001003, 0x00002003, 0x00003003, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d2[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00800f00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00c00f00, +0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, +0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, +0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, +0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, +0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, +0x00400600, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, +0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, +0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, +0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, +0x00400c00, 0x00800c00, 0x00c00c00, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d3[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x003e0000, 0xffffffff, 0xffffffff, 0xffffffff, 0x003f0000, +0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, +0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, +0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, +0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, +0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, +0x00190000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, +0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, +0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, +0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, +0x00310000, 0x00320000, 0x00330000, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +#else + + +/* SPECIAL DECODE TABLES FOR BIG ENDIAN (IBM/MOTOROLA/SUN) CPUS */ + +const uint32_t base64_table_dec_32bit_d0[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xf8000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xfc000000, +0xd0000000, 0xd4000000, 0xd8000000, 0xdc000000, 0xe0000000, 0xe4000000, +0xe8000000, 0xec000000, 0xf0000000, 0xf4000000, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x04000000, 0x08000000, 0x0c000000, 0x10000000, 0x14000000, 0x18000000, +0x1c000000, 0x20000000, 0x24000000, 0x28000000, 0x2c000000, 0x30000000, +0x34000000, 0x38000000, 0x3c000000, 0x40000000, 0x44000000, 0x48000000, +0x4c000000, 0x50000000, 0x54000000, 0x58000000, 0x5c000000, 0x60000000, +0x64000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x68000000, 0x6c000000, 0x70000000, 0x74000000, 0x78000000, +0x7c000000, 0x80000000, 0x84000000, 0x88000000, 0x8c000000, 0x90000000, +0x94000000, 0x98000000, 0x9c000000, 0xa0000000, 0xa4000000, 0xa8000000, +0xac000000, 0xb0000000, 0xb4000000, 0xb8000000, 0xbc000000, 0xc0000000, +0xc4000000, 0xc8000000, 0xcc000000, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d1[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x03e00000, 0xffffffff, 0xffffffff, 0xffffffff, 0x03f00000, +0x03400000, 0x03500000, 0x03600000, 0x03700000, 0x03800000, 0x03900000, +0x03a00000, 0x03b00000, 0x03c00000, 0x03d00000, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00100000, 0x00200000, 0x00300000, 0x00400000, 0x00500000, 0x00600000, +0x00700000, 0x00800000, 0x00900000, 0x00a00000, 0x00b00000, 0x00c00000, +0x00d00000, 0x00e00000, 0x00f00000, 0x01000000, 0x01100000, 0x01200000, +0x01300000, 0x01400000, 0x01500000, 0x01600000, 0x01700000, 0x01800000, +0x01900000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x01a00000, 0x01b00000, 0x01c00000, 0x01d00000, 0x01e00000, +0x01f00000, 0x02000000, 0x02100000, 0x02200000, 0x02300000, 0x02400000, +0x02500000, 0x02600000, 0x02700000, 0x02800000, 0x02900000, 0x02a00000, +0x02b00000, 0x02c00000, 0x02d00000, 0x02e00000, 0x02f00000, 0x03000000, +0x03100000, 0x03200000, 0x03300000, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d2[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x000f8000, 0xffffffff, 0xffffffff, 0xffffffff, 0x000fc000, +0x000d0000, 0x000d4000, 0x000d8000, 0x000dc000, 0x000e0000, 0x000e4000, +0x000e8000, 0x000ec000, 0x000f0000, 0x000f4000, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00004000, 0x00008000, 0x0000c000, 0x00010000, 0x00014000, 0x00018000, +0x0001c000, 0x00020000, 0x00024000, 0x00028000, 0x0002c000, 0x00030000, +0x00034000, 0x00038000, 0x0003c000, 0x00040000, 0x00044000, 0x00048000, +0x0004c000, 0x00050000, 0x00054000, 0x00058000, 0x0005c000, 0x00060000, +0x00064000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00068000, 0x0006c000, 0x00070000, 0x00074000, 0x00078000, +0x0007c000, 0x00080000, 0x00084000, 0x00088000, 0x0008c000, 0x00090000, +0x00094000, 0x00098000, 0x0009c000, 0x000a0000, 0x000a4000, 0x000a8000, +0x000ac000, 0x000b0000, 0x000b4000, 0x000b8000, 0x000bc000, 0x000c0000, +0x000c4000, 0x000c8000, 0x000cc000, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +const uint32_t base64_table_dec_32bit_d3[256] = { +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00003e00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00003f00, +0x00003400, 0x00003500, 0x00003600, 0x00003700, 0x00003800, 0x00003900, +0x00003a00, 0x00003b00, 0x00003c00, 0x00003d00, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, +0x00000100, 0x00000200, 0x00000300, 0x00000400, 0x00000500, 0x00000600, +0x00000700, 0x00000800, 0x00000900, 0x00000a00, 0x00000b00, 0x00000c00, +0x00000d00, 0x00000e00, 0x00000f00, 0x00001000, 0x00001100, 0x00001200, +0x00001300, 0x00001400, 0x00001500, 0x00001600, 0x00001700, 0x00001800, +0x00001900, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0x00001a00, 0x00001b00, 0x00001c00, 0x00001d00, 0x00001e00, +0x00001f00, 0x00002000, 0x00002100, 0x00002200, 0x00002300, 0x00002400, +0x00002500, 0x00002600, 0x00002700, 0x00002800, 0x00002900, 0x00002a00, +0x00002b00, 0x00002c00, 0x00002d00, 0x00002e00, 0x00002f00, 0x00003000, +0x00003100, 0x00003200, 0x00003300, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, +0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff +}; + + +#endif diff --git a/mypyc/lib-rt/base64/tables/table_enc_12bit.h b/mypyc/lib-rt/base64/tables/table_enc_12bit.h new file mode 100644 index 000000000000..2bc0d2306875 --- /dev/null +++ b/mypyc/lib-rt/base64/tables/table_enc_12bit.h @@ -0,0 +1,1031 @@ +#include + +const uint16_t base64_table_enc_12bit[] = { +#if BASE64_LITTLE_ENDIAN + 0x4141U, 0x4241U, 0x4341U, 0x4441U, 0x4541U, 0x4641U, 0x4741U, 0x4841U, + 0x4941U, 0x4A41U, 0x4B41U, 0x4C41U, 0x4D41U, 0x4E41U, 0x4F41U, 0x5041U, + 0x5141U, 0x5241U, 0x5341U, 0x5441U, 0x5541U, 0x5641U, 0x5741U, 0x5841U, + 0x5941U, 0x5A41U, 0x6141U, 0x6241U, 0x6341U, 0x6441U, 0x6541U, 0x6641U, + 0x6741U, 0x6841U, 0x6941U, 0x6A41U, 0x6B41U, 0x6C41U, 0x6D41U, 0x6E41U, + 0x6F41U, 0x7041U, 0x7141U, 0x7241U, 0x7341U, 0x7441U, 0x7541U, 0x7641U, + 0x7741U, 0x7841U, 0x7941U, 0x7A41U, 0x3041U, 0x3141U, 0x3241U, 0x3341U, + 0x3441U, 0x3541U, 0x3641U, 0x3741U, 0x3841U, 0x3941U, 0x2B41U, 0x2F41U, + 0x4142U, 0x4242U, 0x4342U, 0x4442U, 0x4542U, 0x4642U, 0x4742U, 0x4842U, + 0x4942U, 0x4A42U, 0x4B42U, 0x4C42U, 0x4D42U, 0x4E42U, 0x4F42U, 0x5042U, + 0x5142U, 0x5242U, 0x5342U, 0x5442U, 0x5542U, 0x5642U, 0x5742U, 0x5842U, + 0x5942U, 0x5A42U, 0x6142U, 0x6242U, 0x6342U, 0x6442U, 0x6542U, 0x6642U, + 0x6742U, 0x6842U, 0x6942U, 0x6A42U, 0x6B42U, 0x6C42U, 0x6D42U, 0x6E42U, + 0x6F42U, 0x7042U, 0x7142U, 0x7242U, 0x7342U, 0x7442U, 0x7542U, 0x7642U, + 0x7742U, 0x7842U, 0x7942U, 0x7A42U, 0x3042U, 0x3142U, 0x3242U, 0x3342U, + 0x3442U, 0x3542U, 0x3642U, 0x3742U, 0x3842U, 0x3942U, 0x2B42U, 0x2F42U, + 0x4143U, 0x4243U, 0x4343U, 0x4443U, 0x4543U, 0x4643U, 0x4743U, 0x4843U, + 0x4943U, 0x4A43U, 0x4B43U, 0x4C43U, 0x4D43U, 0x4E43U, 0x4F43U, 0x5043U, + 0x5143U, 0x5243U, 0x5343U, 0x5443U, 0x5543U, 0x5643U, 0x5743U, 0x5843U, + 0x5943U, 0x5A43U, 0x6143U, 0x6243U, 0x6343U, 0x6443U, 0x6543U, 0x6643U, + 0x6743U, 0x6843U, 0x6943U, 0x6A43U, 0x6B43U, 0x6C43U, 0x6D43U, 0x6E43U, + 0x6F43U, 0x7043U, 0x7143U, 0x7243U, 0x7343U, 0x7443U, 0x7543U, 0x7643U, + 0x7743U, 0x7843U, 0x7943U, 0x7A43U, 0x3043U, 0x3143U, 0x3243U, 0x3343U, + 0x3443U, 0x3543U, 0x3643U, 0x3743U, 0x3843U, 0x3943U, 0x2B43U, 0x2F43U, + 0x4144U, 0x4244U, 0x4344U, 0x4444U, 0x4544U, 0x4644U, 0x4744U, 0x4844U, + 0x4944U, 0x4A44U, 0x4B44U, 0x4C44U, 0x4D44U, 0x4E44U, 0x4F44U, 0x5044U, + 0x5144U, 0x5244U, 0x5344U, 0x5444U, 0x5544U, 0x5644U, 0x5744U, 0x5844U, + 0x5944U, 0x5A44U, 0x6144U, 0x6244U, 0x6344U, 0x6444U, 0x6544U, 0x6644U, + 0x6744U, 0x6844U, 0x6944U, 0x6A44U, 0x6B44U, 0x6C44U, 0x6D44U, 0x6E44U, + 0x6F44U, 0x7044U, 0x7144U, 0x7244U, 0x7344U, 0x7444U, 0x7544U, 0x7644U, + 0x7744U, 0x7844U, 0x7944U, 0x7A44U, 0x3044U, 0x3144U, 0x3244U, 0x3344U, + 0x3444U, 0x3544U, 0x3644U, 0x3744U, 0x3844U, 0x3944U, 0x2B44U, 0x2F44U, + 0x4145U, 0x4245U, 0x4345U, 0x4445U, 0x4545U, 0x4645U, 0x4745U, 0x4845U, + 0x4945U, 0x4A45U, 0x4B45U, 0x4C45U, 0x4D45U, 0x4E45U, 0x4F45U, 0x5045U, + 0x5145U, 0x5245U, 0x5345U, 0x5445U, 0x5545U, 0x5645U, 0x5745U, 0x5845U, + 0x5945U, 0x5A45U, 0x6145U, 0x6245U, 0x6345U, 0x6445U, 0x6545U, 0x6645U, + 0x6745U, 0x6845U, 0x6945U, 0x6A45U, 0x6B45U, 0x6C45U, 0x6D45U, 0x6E45U, + 0x6F45U, 0x7045U, 0x7145U, 0x7245U, 0x7345U, 0x7445U, 0x7545U, 0x7645U, + 0x7745U, 0x7845U, 0x7945U, 0x7A45U, 0x3045U, 0x3145U, 0x3245U, 0x3345U, + 0x3445U, 0x3545U, 0x3645U, 0x3745U, 0x3845U, 0x3945U, 0x2B45U, 0x2F45U, + 0x4146U, 0x4246U, 0x4346U, 0x4446U, 0x4546U, 0x4646U, 0x4746U, 0x4846U, + 0x4946U, 0x4A46U, 0x4B46U, 0x4C46U, 0x4D46U, 0x4E46U, 0x4F46U, 0x5046U, + 0x5146U, 0x5246U, 0x5346U, 0x5446U, 0x5546U, 0x5646U, 0x5746U, 0x5846U, + 0x5946U, 0x5A46U, 0x6146U, 0x6246U, 0x6346U, 0x6446U, 0x6546U, 0x6646U, + 0x6746U, 0x6846U, 0x6946U, 0x6A46U, 0x6B46U, 0x6C46U, 0x6D46U, 0x6E46U, + 0x6F46U, 0x7046U, 0x7146U, 0x7246U, 0x7346U, 0x7446U, 0x7546U, 0x7646U, + 0x7746U, 0x7846U, 0x7946U, 0x7A46U, 0x3046U, 0x3146U, 0x3246U, 0x3346U, + 0x3446U, 0x3546U, 0x3646U, 0x3746U, 0x3846U, 0x3946U, 0x2B46U, 0x2F46U, + 0x4147U, 0x4247U, 0x4347U, 0x4447U, 0x4547U, 0x4647U, 0x4747U, 0x4847U, + 0x4947U, 0x4A47U, 0x4B47U, 0x4C47U, 0x4D47U, 0x4E47U, 0x4F47U, 0x5047U, + 0x5147U, 0x5247U, 0x5347U, 0x5447U, 0x5547U, 0x5647U, 0x5747U, 0x5847U, + 0x5947U, 0x5A47U, 0x6147U, 0x6247U, 0x6347U, 0x6447U, 0x6547U, 0x6647U, + 0x6747U, 0x6847U, 0x6947U, 0x6A47U, 0x6B47U, 0x6C47U, 0x6D47U, 0x6E47U, + 0x6F47U, 0x7047U, 0x7147U, 0x7247U, 0x7347U, 0x7447U, 0x7547U, 0x7647U, + 0x7747U, 0x7847U, 0x7947U, 0x7A47U, 0x3047U, 0x3147U, 0x3247U, 0x3347U, + 0x3447U, 0x3547U, 0x3647U, 0x3747U, 0x3847U, 0x3947U, 0x2B47U, 0x2F47U, + 0x4148U, 0x4248U, 0x4348U, 0x4448U, 0x4548U, 0x4648U, 0x4748U, 0x4848U, + 0x4948U, 0x4A48U, 0x4B48U, 0x4C48U, 0x4D48U, 0x4E48U, 0x4F48U, 0x5048U, + 0x5148U, 0x5248U, 0x5348U, 0x5448U, 0x5548U, 0x5648U, 0x5748U, 0x5848U, + 0x5948U, 0x5A48U, 0x6148U, 0x6248U, 0x6348U, 0x6448U, 0x6548U, 0x6648U, + 0x6748U, 0x6848U, 0x6948U, 0x6A48U, 0x6B48U, 0x6C48U, 0x6D48U, 0x6E48U, + 0x6F48U, 0x7048U, 0x7148U, 0x7248U, 0x7348U, 0x7448U, 0x7548U, 0x7648U, + 0x7748U, 0x7848U, 0x7948U, 0x7A48U, 0x3048U, 0x3148U, 0x3248U, 0x3348U, + 0x3448U, 0x3548U, 0x3648U, 0x3748U, 0x3848U, 0x3948U, 0x2B48U, 0x2F48U, + 0x4149U, 0x4249U, 0x4349U, 0x4449U, 0x4549U, 0x4649U, 0x4749U, 0x4849U, + 0x4949U, 0x4A49U, 0x4B49U, 0x4C49U, 0x4D49U, 0x4E49U, 0x4F49U, 0x5049U, + 0x5149U, 0x5249U, 0x5349U, 0x5449U, 0x5549U, 0x5649U, 0x5749U, 0x5849U, + 0x5949U, 0x5A49U, 0x6149U, 0x6249U, 0x6349U, 0x6449U, 0x6549U, 0x6649U, + 0x6749U, 0x6849U, 0x6949U, 0x6A49U, 0x6B49U, 0x6C49U, 0x6D49U, 0x6E49U, + 0x6F49U, 0x7049U, 0x7149U, 0x7249U, 0x7349U, 0x7449U, 0x7549U, 0x7649U, + 0x7749U, 0x7849U, 0x7949U, 0x7A49U, 0x3049U, 0x3149U, 0x3249U, 0x3349U, + 0x3449U, 0x3549U, 0x3649U, 0x3749U, 0x3849U, 0x3949U, 0x2B49U, 0x2F49U, + 0x414AU, 0x424AU, 0x434AU, 0x444AU, 0x454AU, 0x464AU, 0x474AU, 0x484AU, + 0x494AU, 0x4A4AU, 0x4B4AU, 0x4C4AU, 0x4D4AU, 0x4E4AU, 0x4F4AU, 0x504AU, + 0x514AU, 0x524AU, 0x534AU, 0x544AU, 0x554AU, 0x564AU, 0x574AU, 0x584AU, + 0x594AU, 0x5A4AU, 0x614AU, 0x624AU, 0x634AU, 0x644AU, 0x654AU, 0x664AU, + 0x674AU, 0x684AU, 0x694AU, 0x6A4AU, 0x6B4AU, 0x6C4AU, 0x6D4AU, 0x6E4AU, + 0x6F4AU, 0x704AU, 0x714AU, 0x724AU, 0x734AU, 0x744AU, 0x754AU, 0x764AU, + 0x774AU, 0x784AU, 0x794AU, 0x7A4AU, 0x304AU, 0x314AU, 0x324AU, 0x334AU, + 0x344AU, 0x354AU, 0x364AU, 0x374AU, 0x384AU, 0x394AU, 0x2B4AU, 0x2F4AU, + 0x414BU, 0x424BU, 0x434BU, 0x444BU, 0x454BU, 0x464BU, 0x474BU, 0x484BU, + 0x494BU, 0x4A4BU, 0x4B4BU, 0x4C4BU, 0x4D4BU, 0x4E4BU, 0x4F4BU, 0x504BU, + 0x514BU, 0x524BU, 0x534BU, 0x544BU, 0x554BU, 0x564BU, 0x574BU, 0x584BU, + 0x594BU, 0x5A4BU, 0x614BU, 0x624BU, 0x634BU, 0x644BU, 0x654BU, 0x664BU, + 0x674BU, 0x684BU, 0x694BU, 0x6A4BU, 0x6B4BU, 0x6C4BU, 0x6D4BU, 0x6E4BU, + 0x6F4BU, 0x704BU, 0x714BU, 0x724BU, 0x734BU, 0x744BU, 0x754BU, 0x764BU, + 0x774BU, 0x784BU, 0x794BU, 0x7A4BU, 0x304BU, 0x314BU, 0x324BU, 0x334BU, + 0x344BU, 0x354BU, 0x364BU, 0x374BU, 0x384BU, 0x394BU, 0x2B4BU, 0x2F4BU, + 0x414CU, 0x424CU, 0x434CU, 0x444CU, 0x454CU, 0x464CU, 0x474CU, 0x484CU, + 0x494CU, 0x4A4CU, 0x4B4CU, 0x4C4CU, 0x4D4CU, 0x4E4CU, 0x4F4CU, 0x504CU, + 0x514CU, 0x524CU, 0x534CU, 0x544CU, 0x554CU, 0x564CU, 0x574CU, 0x584CU, + 0x594CU, 0x5A4CU, 0x614CU, 0x624CU, 0x634CU, 0x644CU, 0x654CU, 0x664CU, + 0x674CU, 0x684CU, 0x694CU, 0x6A4CU, 0x6B4CU, 0x6C4CU, 0x6D4CU, 0x6E4CU, + 0x6F4CU, 0x704CU, 0x714CU, 0x724CU, 0x734CU, 0x744CU, 0x754CU, 0x764CU, + 0x774CU, 0x784CU, 0x794CU, 0x7A4CU, 0x304CU, 0x314CU, 0x324CU, 0x334CU, + 0x344CU, 0x354CU, 0x364CU, 0x374CU, 0x384CU, 0x394CU, 0x2B4CU, 0x2F4CU, + 0x414DU, 0x424DU, 0x434DU, 0x444DU, 0x454DU, 0x464DU, 0x474DU, 0x484DU, + 0x494DU, 0x4A4DU, 0x4B4DU, 0x4C4DU, 0x4D4DU, 0x4E4DU, 0x4F4DU, 0x504DU, + 0x514DU, 0x524DU, 0x534DU, 0x544DU, 0x554DU, 0x564DU, 0x574DU, 0x584DU, + 0x594DU, 0x5A4DU, 0x614DU, 0x624DU, 0x634DU, 0x644DU, 0x654DU, 0x664DU, + 0x674DU, 0x684DU, 0x694DU, 0x6A4DU, 0x6B4DU, 0x6C4DU, 0x6D4DU, 0x6E4DU, + 0x6F4DU, 0x704DU, 0x714DU, 0x724DU, 0x734DU, 0x744DU, 0x754DU, 0x764DU, + 0x774DU, 0x784DU, 0x794DU, 0x7A4DU, 0x304DU, 0x314DU, 0x324DU, 0x334DU, + 0x344DU, 0x354DU, 0x364DU, 0x374DU, 0x384DU, 0x394DU, 0x2B4DU, 0x2F4DU, + 0x414EU, 0x424EU, 0x434EU, 0x444EU, 0x454EU, 0x464EU, 0x474EU, 0x484EU, + 0x494EU, 0x4A4EU, 0x4B4EU, 0x4C4EU, 0x4D4EU, 0x4E4EU, 0x4F4EU, 0x504EU, + 0x514EU, 0x524EU, 0x534EU, 0x544EU, 0x554EU, 0x564EU, 0x574EU, 0x584EU, + 0x594EU, 0x5A4EU, 0x614EU, 0x624EU, 0x634EU, 0x644EU, 0x654EU, 0x664EU, + 0x674EU, 0x684EU, 0x694EU, 0x6A4EU, 0x6B4EU, 0x6C4EU, 0x6D4EU, 0x6E4EU, + 0x6F4EU, 0x704EU, 0x714EU, 0x724EU, 0x734EU, 0x744EU, 0x754EU, 0x764EU, + 0x774EU, 0x784EU, 0x794EU, 0x7A4EU, 0x304EU, 0x314EU, 0x324EU, 0x334EU, + 0x344EU, 0x354EU, 0x364EU, 0x374EU, 0x384EU, 0x394EU, 0x2B4EU, 0x2F4EU, + 0x414FU, 0x424FU, 0x434FU, 0x444FU, 0x454FU, 0x464FU, 0x474FU, 0x484FU, + 0x494FU, 0x4A4FU, 0x4B4FU, 0x4C4FU, 0x4D4FU, 0x4E4FU, 0x4F4FU, 0x504FU, + 0x514FU, 0x524FU, 0x534FU, 0x544FU, 0x554FU, 0x564FU, 0x574FU, 0x584FU, + 0x594FU, 0x5A4FU, 0x614FU, 0x624FU, 0x634FU, 0x644FU, 0x654FU, 0x664FU, + 0x674FU, 0x684FU, 0x694FU, 0x6A4FU, 0x6B4FU, 0x6C4FU, 0x6D4FU, 0x6E4FU, + 0x6F4FU, 0x704FU, 0x714FU, 0x724FU, 0x734FU, 0x744FU, 0x754FU, 0x764FU, + 0x774FU, 0x784FU, 0x794FU, 0x7A4FU, 0x304FU, 0x314FU, 0x324FU, 0x334FU, + 0x344FU, 0x354FU, 0x364FU, 0x374FU, 0x384FU, 0x394FU, 0x2B4FU, 0x2F4FU, + 0x4150U, 0x4250U, 0x4350U, 0x4450U, 0x4550U, 0x4650U, 0x4750U, 0x4850U, + 0x4950U, 0x4A50U, 0x4B50U, 0x4C50U, 0x4D50U, 0x4E50U, 0x4F50U, 0x5050U, + 0x5150U, 0x5250U, 0x5350U, 0x5450U, 0x5550U, 0x5650U, 0x5750U, 0x5850U, + 0x5950U, 0x5A50U, 0x6150U, 0x6250U, 0x6350U, 0x6450U, 0x6550U, 0x6650U, + 0x6750U, 0x6850U, 0x6950U, 0x6A50U, 0x6B50U, 0x6C50U, 0x6D50U, 0x6E50U, + 0x6F50U, 0x7050U, 0x7150U, 0x7250U, 0x7350U, 0x7450U, 0x7550U, 0x7650U, + 0x7750U, 0x7850U, 0x7950U, 0x7A50U, 0x3050U, 0x3150U, 0x3250U, 0x3350U, + 0x3450U, 0x3550U, 0x3650U, 0x3750U, 0x3850U, 0x3950U, 0x2B50U, 0x2F50U, + 0x4151U, 0x4251U, 0x4351U, 0x4451U, 0x4551U, 0x4651U, 0x4751U, 0x4851U, + 0x4951U, 0x4A51U, 0x4B51U, 0x4C51U, 0x4D51U, 0x4E51U, 0x4F51U, 0x5051U, + 0x5151U, 0x5251U, 0x5351U, 0x5451U, 0x5551U, 0x5651U, 0x5751U, 0x5851U, + 0x5951U, 0x5A51U, 0x6151U, 0x6251U, 0x6351U, 0x6451U, 0x6551U, 0x6651U, + 0x6751U, 0x6851U, 0x6951U, 0x6A51U, 0x6B51U, 0x6C51U, 0x6D51U, 0x6E51U, + 0x6F51U, 0x7051U, 0x7151U, 0x7251U, 0x7351U, 0x7451U, 0x7551U, 0x7651U, + 0x7751U, 0x7851U, 0x7951U, 0x7A51U, 0x3051U, 0x3151U, 0x3251U, 0x3351U, + 0x3451U, 0x3551U, 0x3651U, 0x3751U, 0x3851U, 0x3951U, 0x2B51U, 0x2F51U, + 0x4152U, 0x4252U, 0x4352U, 0x4452U, 0x4552U, 0x4652U, 0x4752U, 0x4852U, + 0x4952U, 0x4A52U, 0x4B52U, 0x4C52U, 0x4D52U, 0x4E52U, 0x4F52U, 0x5052U, + 0x5152U, 0x5252U, 0x5352U, 0x5452U, 0x5552U, 0x5652U, 0x5752U, 0x5852U, + 0x5952U, 0x5A52U, 0x6152U, 0x6252U, 0x6352U, 0x6452U, 0x6552U, 0x6652U, + 0x6752U, 0x6852U, 0x6952U, 0x6A52U, 0x6B52U, 0x6C52U, 0x6D52U, 0x6E52U, + 0x6F52U, 0x7052U, 0x7152U, 0x7252U, 0x7352U, 0x7452U, 0x7552U, 0x7652U, + 0x7752U, 0x7852U, 0x7952U, 0x7A52U, 0x3052U, 0x3152U, 0x3252U, 0x3352U, + 0x3452U, 0x3552U, 0x3652U, 0x3752U, 0x3852U, 0x3952U, 0x2B52U, 0x2F52U, + 0x4153U, 0x4253U, 0x4353U, 0x4453U, 0x4553U, 0x4653U, 0x4753U, 0x4853U, + 0x4953U, 0x4A53U, 0x4B53U, 0x4C53U, 0x4D53U, 0x4E53U, 0x4F53U, 0x5053U, + 0x5153U, 0x5253U, 0x5353U, 0x5453U, 0x5553U, 0x5653U, 0x5753U, 0x5853U, + 0x5953U, 0x5A53U, 0x6153U, 0x6253U, 0x6353U, 0x6453U, 0x6553U, 0x6653U, + 0x6753U, 0x6853U, 0x6953U, 0x6A53U, 0x6B53U, 0x6C53U, 0x6D53U, 0x6E53U, + 0x6F53U, 0x7053U, 0x7153U, 0x7253U, 0x7353U, 0x7453U, 0x7553U, 0x7653U, + 0x7753U, 0x7853U, 0x7953U, 0x7A53U, 0x3053U, 0x3153U, 0x3253U, 0x3353U, + 0x3453U, 0x3553U, 0x3653U, 0x3753U, 0x3853U, 0x3953U, 0x2B53U, 0x2F53U, + 0x4154U, 0x4254U, 0x4354U, 0x4454U, 0x4554U, 0x4654U, 0x4754U, 0x4854U, + 0x4954U, 0x4A54U, 0x4B54U, 0x4C54U, 0x4D54U, 0x4E54U, 0x4F54U, 0x5054U, + 0x5154U, 0x5254U, 0x5354U, 0x5454U, 0x5554U, 0x5654U, 0x5754U, 0x5854U, + 0x5954U, 0x5A54U, 0x6154U, 0x6254U, 0x6354U, 0x6454U, 0x6554U, 0x6654U, + 0x6754U, 0x6854U, 0x6954U, 0x6A54U, 0x6B54U, 0x6C54U, 0x6D54U, 0x6E54U, + 0x6F54U, 0x7054U, 0x7154U, 0x7254U, 0x7354U, 0x7454U, 0x7554U, 0x7654U, + 0x7754U, 0x7854U, 0x7954U, 0x7A54U, 0x3054U, 0x3154U, 0x3254U, 0x3354U, + 0x3454U, 0x3554U, 0x3654U, 0x3754U, 0x3854U, 0x3954U, 0x2B54U, 0x2F54U, + 0x4155U, 0x4255U, 0x4355U, 0x4455U, 0x4555U, 0x4655U, 0x4755U, 0x4855U, + 0x4955U, 0x4A55U, 0x4B55U, 0x4C55U, 0x4D55U, 0x4E55U, 0x4F55U, 0x5055U, + 0x5155U, 0x5255U, 0x5355U, 0x5455U, 0x5555U, 0x5655U, 0x5755U, 0x5855U, + 0x5955U, 0x5A55U, 0x6155U, 0x6255U, 0x6355U, 0x6455U, 0x6555U, 0x6655U, + 0x6755U, 0x6855U, 0x6955U, 0x6A55U, 0x6B55U, 0x6C55U, 0x6D55U, 0x6E55U, + 0x6F55U, 0x7055U, 0x7155U, 0x7255U, 0x7355U, 0x7455U, 0x7555U, 0x7655U, + 0x7755U, 0x7855U, 0x7955U, 0x7A55U, 0x3055U, 0x3155U, 0x3255U, 0x3355U, + 0x3455U, 0x3555U, 0x3655U, 0x3755U, 0x3855U, 0x3955U, 0x2B55U, 0x2F55U, + 0x4156U, 0x4256U, 0x4356U, 0x4456U, 0x4556U, 0x4656U, 0x4756U, 0x4856U, + 0x4956U, 0x4A56U, 0x4B56U, 0x4C56U, 0x4D56U, 0x4E56U, 0x4F56U, 0x5056U, + 0x5156U, 0x5256U, 0x5356U, 0x5456U, 0x5556U, 0x5656U, 0x5756U, 0x5856U, + 0x5956U, 0x5A56U, 0x6156U, 0x6256U, 0x6356U, 0x6456U, 0x6556U, 0x6656U, + 0x6756U, 0x6856U, 0x6956U, 0x6A56U, 0x6B56U, 0x6C56U, 0x6D56U, 0x6E56U, + 0x6F56U, 0x7056U, 0x7156U, 0x7256U, 0x7356U, 0x7456U, 0x7556U, 0x7656U, + 0x7756U, 0x7856U, 0x7956U, 0x7A56U, 0x3056U, 0x3156U, 0x3256U, 0x3356U, + 0x3456U, 0x3556U, 0x3656U, 0x3756U, 0x3856U, 0x3956U, 0x2B56U, 0x2F56U, + 0x4157U, 0x4257U, 0x4357U, 0x4457U, 0x4557U, 0x4657U, 0x4757U, 0x4857U, + 0x4957U, 0x4A57U, 0x4B57U, 0x4C57U, 0x4D57U, 0x4E57U, 0x4F57U, 0x5057U, + 0x5157U, 0x5257U, 0x5357U, 0x5457U, 0x5557U, 0x5657U, 0x5757U, 0x5857U, + 0x5957U, 0x5A57U, 0x6157U, 0x6257U, 0x6357U, 0x6457U, 0x6557U, 0x6657U, + 0x6757U, 0x6857U, 0x6957U, 0x6A57U, 0x6B57U, 0x6C57U, 0x6D57U, 0x6E57U, + 0x6F57U, 0x7057U, 0x7157U, 0x7257U, 0x7357U, 0x7457U, 0x7557U, 0x7657U, + 0x7757U, 0x7857U, 0x7957U, 0x7A57U, 0x3057U, 0x3157U, 0x3257U, 0x3357U, + 0x3457U, 0x3557U, 0x3657U, 0x3757U, 0x3857U, 0x3957U, 0x2B57U, 0x2F57U, + 0x4158U, 0x4258U, 0x4358U, 0x4458U, 0x4558U, 0x4658U, 0x4758U, 0x4858U, + 0x4958U, 0x4A58U, 0x4B58U, 0x4C58U, 0x4D58U, 0x4E58U, 0x4F58U, 0x5058U, + 0x5158U, 0x5258U, 0x5358U, 0x5458U, 0x5558U, 0x5658U, 0x5758U, 0x5858U, + 0x5958U, 0x5A58U, 0x6158U, 0x6258U, 0x6358U, 0x6458U, 0x6558U, 0x6658U, + 0x6758U, 0x6858U, 0x6958U, 0x6A58U, 0x6B58U, 0x6C58U, 0x6D58U, 0x6E58U, + 0x6F58U, 0x7058U, 0x7158U, 0x7258U, 0x7358U, 0x7458U, 0x7558U, 0x7658U, + 0x7758U, 0x7858U, 0x7958U, 0x7A58U, 0x3058U, 0x3158U, 0x3258U, 0x3358U, + 0x3458U, 0x3558U, 0x3658U, 0x3758U, 0x3858U, 0x3958U, 0x2B58U, 0x2F58U, + 0x4159U, 0x4259U, 0x4359U, 0x4459U, 0x4559U, 0x4659U, 0x4759U, 0x4859U, + 0x4959U, 0x4A59U, 0x4B59U, 0x4C59U, 0x4D59U, 0x4E59U, 0x4F59U, 0x5059U, + 0x5159U, 0x5259U, 0x5359U, 0x5459U, 0x5559U, 0x5659U, 0x5759U, 0x5859U, + 0x5959U, 0x5A59U, 0x6159U, 0x6259U, 0x6359U, 0x6459U, 0x6559U, 0x6659U, + 0x6759U, 0x6859U, 0x6959U, 0x6A59U, 0x6B59U, 0x6C59U, 0x6D59U, 0x6E59U, + 0x6F59U, 0x7059U, 0x7159U, 0x7259U, 0x7359U, 0x7459U, 0x7559U, 0x7659U, + 0x7759U, 0x7859U, 0x7959U, 0x7A59U, 0x3059U, 0x3159U, 0x3259U, 0x3359U, + 0x3459U, 0x3559U, 0x3659U, 0x3759U, 0x3859U, 0x3959U, 0x2B59U, 0x2F59U, + 0x415AU, 0x425AU, 0x435AU, 0x445AU, 0x455AU, 0x465AU, 0x475AU, 0x485AU, + 0x495AU, 0x4A5AU, 0x4B5AU, 0x4C5AU, 0x4D5AU, 0x4E5AU, 0x4F5AU, 0x505AU, + 0x515AU, 0x525AU, 0x535AU, 0x545AU, 0x555AU, 0x565AU, 0x575AU, 0x585AU, + 0x595AU, 0x5A5AU, 0x615AU, 0x625AU, 0x635AU, 0x645AU, 0x655AU, 0x665AU, + 0x675AU, 0x685AU, 0x695AU, 0x6A5AU, 0x6B5AU, 0x6C5AU, 0x6D5AU, 0x6E5AU, + 0x6F5AU, 0x705AU, 0x715AU, 0x725AU, 0x735AU, 0x745AU, 0x755AU, 0x765AU, + 0x775AU, 0x785AU, 0x795AU, 0x7A5AU, 0x305AU, 0x315AU, 0x325AU, 0x335AU, + 0x345AU, 0x355AU, 0x365AU, 0x375AU, 0x385AU, 0x395AU, 0x2B5AU, 0x2F5AU, + 0x4161U, 0x4261U, 0x4361U, 0x4461U, 0x4561U, 0x4661U, 0x4761U, 0x4861U, + 0x4961U, 0x4A61U, 0x4B61U, 0x4C61U, 0x4D61U, 0x4E61U, 0x4F61U, 0x5061U, + 0x5161U, 0x5261U, 0x5361U, 0x5461U, 0x5561U, 0x5661U, 0x5761U, 0x5861U, + 0x5961U, 0x5A61U, 0x6161U, 0x6261U, 0x6361U, 0x6461U, 0x6561U, 0x6661U, + 0x6761U, 0x6861U, 0x6961U, 0x6A61U, 0x6B61U, 0x6C61U, 0x6D61U, 0x6E61U, + 0x6F61U, 0x7061U, 0x7161U, 0x7261U, 0x7361U, 0x7461U, 0x7561U, 0x7661U, + 0x7761U, 0x7861U, 0x7961U, 0x7A61U, 0x3061U, 0x3161U, 0x3261U, 0x3361U, + 0x3461U, 0x3561U, 0x3661U, 0x3761U, 0x3861U, 0x3961U, 0x2B61U, 0x2F61U, + 0x4162U, 0x4262U, 0x4362U, 0x4462U, 0x4562U, 0x4662U, 0x4762U, 0x4862U, + 0x4962U, 0x4A62U, 0x4B62U, 0x4C62U, 0x4D62U, 0x4E62U, 0x4F62U, 0x5062U, + 0x5162U, 0x5262U, 0x5362U, 0x5462U, 0x5562U, 0x5662U, 0x5762U, 0x5862U, + 0x5962U, 0x5A62U, 0x6162U, 0x6262U, 0x6362U, 0x6462U, 0x6562U, 0x6662U, + 0x6762U, 0x6862U, 0x6962U, 0x6A62U, 0x6B62U, 0x6C62U, 0x6D62U, 0x6E62U, + 0x6F62U, 0x7062U, 0x7162U, 0x7262U, 0x7362U, 0x7462U, 0x7562U, 0x7662U, + 0x7762U, 0x7862U, 0x7962U, 0x7A62U, 0x3062U, 0x3162U, 0x3262U, 0x3362U, + 0x3462U, 0x3562U, 0x3662U, 0x3762U, 0x3862U, 0x3962U, 0x2B62U, 0x2F62U, + 0x4163U, 0x4263U, 0x4363U, 0x4463U, 0x4563U, 0x4663U, 0x4763U, 0x4863U, + 0x4963U, 0x4A63U, 0x4B63U, 0x4C63U, 0x4D63U, 0x4E63U, 0x4F63U, 0x5063U, + 0x5163U, 0x5263U, 0x5363U, 0x5463U, 0x5563U, 0x5663U, 0x5763U, 0x5863U, + 0x5963U, 0x5A63U, 0x6163U, 0x6263U, 0x6363U, 0x6463U, 0x6563U, 0x6663U, + 0x6763U, 0x6863U, 0x6963U, 0x6A63U, 0x6B63U, 0x6C63U, 0x6D63U, 0x6E63U, + 0x6F63U, 0x7063U, 0x7163U, 0x7263U, 0x7363U, 0x7463U, 0x7563U, 0x7663U, + 0x7763U, 0x7863U, 0x7963U, 0x7A63U, 0x3063U, 0x3163U, 0x3263U, 0x3363U, + 0x3463U, 0x3563U, 0x3663U, 0x3763U, 0x3863U, 0x3963U, 0x2B63U, 0x2F63U, + 0x4164U, 0x4264U, 0x4364U, 0x4464U, 0x4564U, 0x4664U, 0x4764U, 0x4864U, + 0x4964U, 0x4A64U, 0x4B64U, 0x4C64U, 0x4D64U, 0x4E64U, 0x4F64U, 0x5064U, + 0x5164U, 0x5264U, 0x5364U, 0x5464U, 0x5564U, 0x5664U, 0x5764U, 0x5864U, + 0x5964U, 0x5A64U, 0x6164U, 0x6264U, 0x6364U, 0x6464U, 0x6564U, 0x6664U, + 0x6764U, 0x6864U, 0x6964U, 0x6A64U, 0x6B64U, 0x6C64U, 0x6D64U, 0x6E64U, + 0x6F64U, 0x7064U, 0x7164U, 0x7264U, 0x7364U, 0x7464U, 0x7564U, 0x7664U, + 0x7764U, 0x7864U, 0x7964U, 0x7A64U, 0x3064U, 0x3164U, 0x3264U, 0x3364U, + 0x3464U, 0x3564U, 0x3664U, 0x3764U, 0x3864U, 0x3964U, 0x2B64U, 0x2F64U, + 0x4165U, 0x4265U, 0x4365U, 0x4465U, 0x4565U, 0x4665U, 0x4765U, 0x4865U, + 0x4965U, 0x4A65U, 0x4B65U, 0x4C65U, 0x4D65U, 0x4E65U, 0x4F65U, 0x5065U, + 0x5165U, 0x5265U, 0x5365U, 0x5465U, 0x5565U, 0x5665U, 0x5765U, 0x5865U, + 0x5965U, 0x5A65U, 0x6165U, 0x6265U, 0x6365U, 0x6465U, 0x6565U, 0x6665U, + 0x6765U, 0x6865U, 0x6965U, 0x6A65U, 0x6B65U, 0x6C65U, 0x6D65U, 0x6E65U, + 0x6F65U, 0x7065U, 0x7165U, 0x7265U, 0x7365U, 0x7465U, 0x7565U, 0x7665U, + 0x7765U, 0x7865U, 0x7965U, 0x7A65U, 0x3065U, 0x3165U, 0x3265U, 0x3365U, + 0x3465U, 0x3565U, 0x3665U, 0x3765U, 0x3865U, 0x3965U, 0x2B65U, 0x2F65U, + 0x4166U, 0x4266U, 0x4366U, 0x4466U, 0x4566U, 0x4666U, 0x4766U, 0x4866U, + 0x4966U, 0x4A66U, 0x4B66U, 0x4C66U, 0x4D66U, 0x4E66U, 0x4F66U, 0x5066U, + 0x5166U, 0x5266U, 0x5366U, 0x5466U, 0x5566U, 0x5666U, 0x5766U, 0x5866U, + 0x5966U, 0x5A66U, 0x6166U, 0x6266U, 0x6366U, 0x6466U, 0x6566U, 0x6666U, + 0x6766U, 0x6866U, 0x6966U, 0x6A66U, 0x6B66U, 0x6C66U, 0x6D66U, 0x6E66U, + 0x6F66U, 0x7066U, 0x7166U, 0x7266U, 0x7366U, 0x7466U, 0x7566U, 0x7666U, + 0x7766U, 0x7866U, 0x7966U, 0x7A66U, 0x3066U, 0x3166U, 0x3266U, 0x3366U, + 0x3466U, 0x3566U, 0x3666U, 0x3766U, 0x3866U, 0x3966U, 0x2B66U, 0x2F66U, + 0x4167U, 0x4267U, 0x4367U, 0x4467U, 0x4567U, 0x4667U, 0x4767U, 0x4867U, + 0x4967U, 0x4A67U, 0x4B67U, 0x4C67U, 0x4D67U, 0x4E67U, 0x4F67U, 0x5067U, + 0x5167U, 0x5267U, 0x5367U, 0x5467U, 0x5567U, 0x5667U, 0x5767U, 0x5867U, + 0x5967U, 0x5A67U, 0x6167U, 0x6267U, 0x6367U, 0x6467U, 0x6567U, 0x6667U, + 0x6767U, 0x6867U, 0x6967U, 0x6A67U, 0x6B67U, 0x6C67U, 0x6D67U, 0x6E67U, + 0x6F67U, 0x7067U, 0x7167U, 0x7267U, 0x7367U, 0x7467U, 0x7567U, 0x7667U, + 0x7767U, 0x7867U, 0x7967U, 0x7A67U, 0x3067U, 0x3167U, 0x3267U, 0x3367U, + 0x3467U, 0x3567U, 0x3667U, 0x3767U, 0x3867U, 0x3967U, 0x2B67U, 0x2F67U, + 0x4168U, 0x4268U, 0x4368U, 0x4468U, 0x4568U, 0x4668U, 0x4768U, 0x4868U, + 0x4968U, 0x4A68U, 0x4B68U, 0x4C68U, 0x4D68U, 0x4E68U, 0x4F68U, 0x5068U, + 0x5168U, 0x5268U, 0x5368U, 0x5468U, 0x5568U, 0x5668U, 0x5768U, 0x5868U, + 0x5968U, 0x5A68U, 0x6168U, 0x6268U, 0x6368U, 0x6468U, 0x6568U, 0x6668U, + 0x6768U, 0x6868U, 0x6968U, 0x6A68U, 0x6B68U, 0x6C68U, 0x6D68U, 0x6E68U, + 0x6F68U, 0x7068U, 0x7168U, 0x7268U, 0x7368U, 0x7468U, 0x7568U, 0x7668U, + 0x7768U, 0x7868U, 0x7968U, 0x7A68U, 0x3068U, 0x3168U, 0x3268U, 0x3368U, + 0x3468U, 0x3568U, 0x3668U, 0x3768U, 0x3868U, 0x3968U, 0x2B68U, 0x2F68U, + 0x4169U, 0x4269U, 0x4369U, 0x4469U, 0x4569U, 0x4669U, 0x4769U, 0x4869U, + 0x4969U, 0x4A69U, 0x4B69U, 0x4C69U, 0x4D69U, 0x4E69U, 0x4F69U, 0x5069U, + 0x5169U, 0x5269U, 0x5369U, 0x5469U, 0x5569U, 0x5669U, 0x5769U, 0x5869U, + 0x5969U, 0x5A69U, 0x6169U, 0x6269U, 0x6369U, 0x6469U, 0x6569U, 0x6669U, + 0x6769U, 0x6869U, 0x6969U, 0x6A69U, 0x6B69U, 0x6C69U, 0x6D69U, 0x6E69U, + 0x6F69U, 0x7069U, 0x7169U, 0x7269U, 0x7369U, 0x7469U, 0x7569U, 0x7669U, + 0x7769U, 0x7869U, 0x7969U, 0x7A69U, 0x3069U, 0x3169U, 0x3269U, 0x3369U, + 0x3469U, 0x3569U, 0x3669U, 0x3769U, 0x3869U, 0x3969U, 0x2B69U, 0x2F69U, + 0x416AU, 0x426AU, 0x436AU, 0x446AU, 0x456AU, 0x466AU, 0x476AU, 0x486AU, + 0x496AU, 0x4A6AU, 0x4B6AU, 0x4C6AU, 0x4D6AU, 0x4E6AU, 0x4F6AU, 0x506AU, + 0x516AU, 0x526AU, 0x536AU, 0x546AU, 0x556AU, 0x566AU, 0x576AU, 0x586AU, + 0x596AU, 0x5A6AU, 0x616AU, 0x626AU, 0x636AU, 0x646AU, 0x656AU, 0x666AU, + 0x676AU, 0x686AU, 0x696AU, 0x6A6AU, 0x6B6AU, 0x6C6AU, 0x6D6AU, 0x6E6AU, + 0x6F6AU, 0x706AU, 0x716AU, 0x726AU, 0x736AU, 0x746AU, 0x756AU, 0x766AU, + 0x776AU, 0x786AU, 0x796AU, 0x7A6AU, 0x306AU, 0x316AU, 0x326AU, 0x336AU, + 0x346AU, 0x356AU, 0x366AU, 0x376AU, 0x386AU, 0x396AU, 0x2B6AU, 0x2F6AU, + 0x416BU, 0x426BU, 0x436BU, 0x446BU, 0x456BU, 0x466BU, 0x476BU, 0x486BU, + 0x496BU, 0x4A6BU, 0x4B6BU, 0x4C6BU, 0x4D6BU, 0x4E6BU, 0x4F6BU, 0x506BU, + 0x516BU, 0x526BU, 0x536BU, 0x546BU, 0x556BU, 0x566BU, 0x576BU, 0x586BU, + 0x596BU, 0x5A6BU, 0x616BU, 0x626BU, 0x636BU, 0x646BU, 0x656BU, 0x666BU, + 0x676BU, 0x686BU, 0x696BU, 0x6A6BU, 0x6B6BU, 0x6C6BU, 0x6D6BU, 0x6E6BU, + 0x6F6BU, 0x706BU, 0x716BU, 0x726BU, 0x736BU, 0x746BU, 0x756BU, 0x766BU, + 0x776BU, 0x786BU, 0x796BU, 0x7A6BU, 0x306BU, 0x316BU, 0x326BU, 0x336BU, + 0x346BU, 0x356BU, 0x366BU, 0x376BU, 0x386BU, 0x396BU, 0x2B6BU, 0x2F6BU, + 0x416CU, 0x426CU, 0x436CU, 0x446CU, 0x456CU, 0x466CU, 0x476CU, 0x486CU, + 0x496CU, 0x4A6CU, 0x4B6CU, 0x4C6CU, 0x4D6CU, 0x4E6CU, 0x4F6CU, 0x506CU, + 0x516CU, 0x526CU, 0x536CU, 0x546CU, 0x556CU, 0x566CU, 0x576CU, 0x586CU, + 0x596CU, 0x5A6CU, 0x616CU, 0x626CU, 0x636CU, 0x646CU, 0x656CU, 0x666CU, + 0x676CU, 0x686CU, 0x696CU, 0x6A6CU, 0x6B6CU, 0x6C6CU, 0x6D6CU, 0x6E6CU, + 0x6F6CU, 0x706CU, 0x716CU, 0x726CU, 0x736CU, 0x746CU, 0x756CU, 0x766CU, + 0x776CU, 0x786CU, 0x796CU, 0x7A6CU, 0x306CU, 0x316CU, 0x326CU, 0x336CU, + 0x346CU, 0x356CU, 0x366CU, 0x376CU, 0x386CU, 0x396CU, 0x2B6CU, 0x2F6CU, + 0x416DU, 0x426DU, 0x436DU, 0x446DU, 0x456DU, 0x466DU, 0x476DU, 0x486DU, + 0x496DU, 0x4A6DU, 0x4B6DU, 0x4C6DU, 0x4D6DU, 0x4E6DU, 0x4F6DU, 0x506DU, + 0x516DU, 0x526DU, 0x536DU, 0x546DU, 0x556DU, 0x566DU, 0x576DU, 0x586DU, + 0x596DU, 0x5A6DU, 0x616DU, 0x626DU, 0x636DU, 0x646DU, 0x656DU, 0x666DU, + 0x676DU, 0x686DU, 0x696DU, 0x6A6DU, 0x6B6DU, 0x6C6DU, 0x6D6DU, 0x6E6DU, + 0x6F6DU, 0x706DU, 0x716DU, 0x726DU, 0x736DU, 0x746DU, 0x756DU, 0x766DU, + 0x776DU, 0x786DU, 0x796DU, 0x7A6DU, 0x306DU, 0x316DU, 0x326DU, 0x336DU, + 0x346DU, 0x356DU, 0x366DU, 0x376DU, 0x386DU, 0x396DU, 0x2B6DU, 0x2F6DU, + 0x416EU, 0x426EU, 0x436EU, 0x446EU, 0x456EU, 0x466EU, 0x476EU, 0x486EU, + 0x496EU, 0x4A6EU, 0x4B6EU, 0x4C6EU, 0x4D6EU, 0x4E6EU, 0x4F6EU, 0x506EU, + 0x516EU, 0x526EU, 0x536EU, 0x546EU, 0x556EU, 0x566EU, 0x576EU, 0x586EU, + 0x596EU, 0x5A6EU, 0x616EU, 0x626EU, 0x636EU, 0x646EU, 0x656EU, 0x666EU, + 0x676EU, 0x686EU, 0x696EU, 0x6A6EU, 0x6B6EU, 0x6C6EU, 0x6D6EU, 0x6E6EU, + 0x6F6EU, 0x706EU, 0x716EU, 0x726EU, 0x736EU, 0x746EU, 0x756EU, 0x766EU, + 0x776EU, 0x786EU, 0x796EU, 0x7A6EU, 0x306EU, 0x316EU, 0x326EU, 0x336EU, + 0x346EU, 0x356EU, 0x366EU, 0x376EU, 0x386EU, 0x396EU, 0x2B6EU, 0x2F6EU, + 0x416FU, 0x426FU, 0x436FU, 0x446FU, 0x456FU, 0x466FU, 0x476FU, 0x486FU, + 0x496FU, 0x4A6FU, 0x4B6FU, 0x4C6FU, 0x4D6FU, 0x4E6FU, 0x4F6FU, 0x506FU, + 0x516FU, 0x526FU, 0x536FU, 0x546FU, 0x556FU, 0x566FU, 0x576FU, 0x586FU, + 0x596FU, 0x5A6FU, 0x616FU, 0x626FU, 0x636FU, 0x646FU, 0x656FU, 0x666FU, + 0x676FU, 0x686FU, 0x696FU, 0x6A6FU, 0x6B6FU, 0x6C6FU, 0x6D6FU, 0x6E6FU, + 0x6F6FU, 0x706FU, 0x716FU, 0x726FU, 0x736FU, 0x746FU, 0x756FU, 0x766FU, + 0x776FU, 0x786FU, 0x796FU, 0x7A6FU, 0x306FU, 0x316FU, 0x326FU, 0x336FU, + 0x346FU, 0x356FU, 0x366FU, 0x376FU, 0x386FU, 0x396FU, 0x2B6FU, 0x2F6FU, + 0x4170U, 0x4270U, 0x4370U, 0x4470U, 0x4570U, 0x4670U, 0x4770U, 0x4870U, + 0x4970U, 0x4A70U, 0x4B70U, 0x4C70U, 0x4D70U, 0x4E70U, 0x4F70U, 0x5070U, + 0x5170U, 0x5270U, 0x5370U, 0x5470U, 0x5570U, 0x5670U, 0x5770U, 0x5870U, + 0x5970U, 0x5A70U, 0x6170U, 0x6270U, 0x6370U, 0x6470U, 0x6570U, 0x6670U, + 0x6770U, 0x6870U, 0x6970U, 0x6A70U, 0x6B70U, 0x6C70U, 0x6D70U, 0x6E70U, + 0x6F70U, 0x7070U, 0x7170U, 0x7270U, 0x7370U, 0x7470U, 0x7570U, 0x7670U, + 0x7770U, 0x7870U, 0x7970U, 0x7A70U, 0x3070U, 0x3170U, 0x3270U, 0x3370U, + 0x3470U, 0x3570U, 0x3670U, 0x3770U, 0x3870U, 0x3970U, 0x2B70U, 0x2F70U, + 0x4171U, 0x4271U, 0x4371U, 0x4471U, 0x4571U, 0x4671U, 0x4771U, 0x4871U, + 0x4971U, 0x4A71U, 0x4B71U, 0x4C71U, 0x4D71U, 0x4E71U, 0x4F71U, 0x5071U, + 0x5171U, 0x5271U, 0x5371U, 0x5471U, 0x5571U, 0x5671U, 0x5771U, 0x5871U, + 0x5971U, 0x5A71U, 0x6171U, 0x6271U, 0x6371U, 0x6471U, 0x6571U, 0x6671U, + 0x6771U, 0x6871U, 0x6971U, 0x6A71U, 0x6B71U, 0x6C71U, 0x6D71U, 0x6E71U, + 0x6F71U, 0x7071U, 0x7171U, 0x7271U, 0x7371U, 0x7471U, 0x7571U, 0x7671U, + 0x7771U, 0x7871U, 0x7971U, 0x7A71U, 0x3071U, 0x3171U, 0x3271U, 0x3371U, + 0x3471U, 0x3571U, 0x3671U, 0x3771U, 0x3871U, 0x3971U, 0x2B71U, 0x2F71U, + 0x4172U, 0x4272U, 0x4372U, 0x4472U, 0x4572U, 0x4672U, 0x4772U, 0x4872U, + 0x4972U, 0x4A72U, 0x4B72U, 0x4C72U, 0x4D72U, 0x4E72U, 0x4F72U, 0x5072U, + 0x5172U, 0x5272U, 0x5372U, 0x5472U, 0x5572U, 0x5672U, 0x5772U, 0x5872U, + 0x5972U, 0x5A72U, 0x6172U, 0x6272U, 0x6372U, 0x6472U, 0x6572U, 0x6672U, + 0x6772U, 0x6872U, 0x6972U, 0x6A72U, 0x6B72U, 0x6C72U, 0x6D72U, 0x6E72U, + 0x6F72U, 0x7072U, 0x7172U, 0x7272U, 0x7372U, 0x7472U, 0x7572U, 0x7672U, + 0x7772U, 0x7872U, 0x7972U, 0x7A72U, 0x3072U, 0x3172U, 0x3272U, 0x3372U, + 0x3472U, 0x3572U, 0x3672U, 0x3772U, 0x3872U, 0x3972U, 0x2B72U, 0x2F72U, + 0x4173U, 0x4273U, 0x4373U, 0x4473U, 0x4573U, 0x4673U, 0x4773U, 0x4873U, + 0x4973U, 0x4A73U, 0x4B73U, 0x4C73U, 0x4D73U, 0x4E73U, 0x4F73U, 0x5073U, + 0x5173U, 0x5273U, 0x5373U, 0x5473U, 0x5573U, 0x5673U, 0x5773U, 0x5873U, + 0x5973U, 0x5A73U, 0x6173U, 0x6273U, 0x6373U, 0x6473U, 0x6573U, 0x6673U, + 0x6773U, 0x6873U, 0x6973U, 0x6A73U, 0x6B73U, 0x6C73U, 0x6D73U, 0x6E73U, + 0x6F73U, 0x7073U, 0x7173U, 0x7273U, 0x7373U, 0x7473U, 0x7573U, 0x7673U, + 0x7773U, 0x7873U, 0x7973U, 0x7A73U, 0x3073U, 0x3173U, 0x3273U, 0x3373U, + 0x3473U, 0x3573U, 0x3673U, 0x3773U, 0x3873U, 0x3973U, 0x2B73U, 0x2F73U, + 0x4174U, 0x4274U, 0x4374U, 0x4474U, 0x4574U, 0x4674U, 0x4774U, 0x4874U, + 0x4974U, 0x4A74U, 0x4B74U, 0x4C74U, 0x4D74U, 0x4E74U, 0x4F74U, 0x5074U, + 0x5174U, 0x5274U, 0x5374U, 0x5474U, 0x5574U, 0x5674U, 0x5774U, 0x5874U, + 0x5974U, 0x5A74U, 0x6174U, 0x6274U, 0x6374U, 0x6474U, 0x6574U, 0x6674U, + 0x6774U, 0x6874U, 0x6974U, 0x6A74U, 0x6B74U, 0x6C74U, 0x6D74U, 0x6E74U, + 0x6F74U, 0x7074U, 0x7174U, 0x7274U, 0x7374U, 0x7474U, 0x7574U, 0x7674U, + 0x7774U, 0x7874U, 0x7974U, 0x7A74U, 0x3074U, 0x3174U, 0x3274U, 0x3374U, + 0x3474U, 0x3574U, 0x3674U, 0x3774U, 0x3874U, 0x3974U, 0x2B74U, 0x2F74U, + 0x4175U, 0x4275U, 0x4375U, 0x4475U, 0x4575U, 0x4675U, 0x4775U, 0x4875U, + 0x4975U, 0x4A75U, 0x4B75U, 0x4C75U, 0x4D75U, 0x4E75U, 0x4F75U, 0x5075U, + 0x5175U, 0x5275U, 0x5375U, 0x5475U, 0x5575U, 0x5675U, 0x5775U, 0x5875U, + 0x5975U, 0x5A75U, 0x6175U, 0x6275U, 0x6375U, 0x6475U, 0x6575U, 0x6675U, + 0x6775U, 0x6875U, 0x6975U, 0x6A75U, 0x6B75U, 0x6C75U, 0x6D75U, 0x6E75U, + 0x6F75U, 0x7075U, 0x7175U, 0x7275U, 0x7375U, 0x7475U, 0x7575U, 0x7675U, + 0x7775U, 0x7875U, 0x7975U, 0x7A75U, 0x3075U, 0x3175U, 0x3275U, 0x3375U, + 0x3475U, 0x3575U, 0x3675U, 0x3775U, 0x3875U, 0x3975U, 0x2B75U, 0x2F75U, + 0x4176U, 0x4276U, 0x4376U, 0x4476U, 0x4576U, 0x4676U, 0x4776U, 0x4876U, + 0x4976U, 0x4A76U, 0x4B76U, 0x4C76U, 0x4D76U, 0x4E76U, 0x4F76U, 0x5076U, + 0x5176U, 0x5276U, 0x5376U, 0x5476U, 0x5576U, 0x5676U, 0x5776U, 0x5876U, + 0x5976U, 0x5A76U, 0x6176U, 0x6276U, 0x6376U, 0x6476U, 0x6576U, 0x6676U, + 0x6776U, 0x6876U, 0x6976U, 0x6A76U, 0x6B76U, 0x6C76U, 0x6D76U, 0x6E76U, + 0x6F76U, 0x7076U, 0x7176U, 0x7276U, 0x7376U, 0x7476U, 0x7576U, 0x7676U, + 0x7776U, 0x7876U, 0x7976U, 0x7A76U, 0x3076U, 0x3176U, 0x3276U, 0x3376U, + 0x3476U, 0x3576U, 0x3676U, 0x3776U, 0x3876U, 0x3976U, 0x2B76U, 0x2F76U, + 0x4177U, 0x4277U, 0x4377U, 0x4477U, 0x4577U, 0x4677U, 0x4777U, 0x4877U, + 0x4977U, 0x4A77U, 0x4B77U, 0x4C77U, 0x4D77U, 0x4E77U, 0x4F77U, 0x5077U, + 0x5177U, 0x5277U, 0x5377U, 0x5477U, 0x5577U, 0x5677U, 0x5777U, 0x5877U, + 0x5977U, 0x5A77U, 0x6177U, 0x6277U, 0x6377U, 0x6477U, 0x6577U, 0x6677U, + 0x6777U, 0x6877U, 0x6977U, 0x6A77U, 0x6B77U, 0x6C77U, 0x6D77U, 0x6E77U, + 0x6F77U, 0x7077U, 0x7177U, 0x7277U, 0x7377U, 0x7477U, 0x7577U, 0x7677U, + 0x7777U, 0x7877U, 0x7977U, 0x7A77U, 0x3077U, 0x3177U, 0x3277U, 0x3377U, + 0x3477U, 0x3577U, 0x3677U, 0x3777U, 0x3877U, 0x3977U, 0x2B77U, 0x2F77U, + 0x4178U, 0x4278U, 0x4378U, 0x4478U, 0x4578U, 0x4678U, 0x4778U, 0x4878U, + 0x4978U, 0x4A78U, 0x4B78U, 0x4C78U, 0x4D78U, 0x4E78U, 0x4F78U, 0x5078U, + 0x5178U, 0x5278U, 0x5378U, 0x5478U, 0x5578U, 0x5678U, 0x5778U, 0x5878U, + 0x5978U, 0x5A78U, 0x6178U, 0x6278U, 0x6378U, 0x6478U, 0x6578U, 0x6678U, + 0x6778U, 0x6878U, 0x6978U, 0x6A78U, 0x6B78U, 0x6C78U, 0x6D78U, 0x6E78U, + 0x6F78U, 0x7078U, 0x7178U, 0x7278U, 0x7378U, 0x7478U, 0x7578U, 0x7678U, + 0x7778U, 0x7878U, 0x7978U, 0x7A78U, 0x3078U, 0x3178U, 0x3278U, 0x3378U, + 0x3478U, 0x3578U, 0x3678U, 0x3778U, 0x3878U, 0x3978U, 0x2B78U, 0x2F78U, + 0x4179U, 0x4279U, 0x4379U, 0x4479U, 0x4579U, 0x4679U, 0x4779U, 0x4879U, + 0x4979U, 0x4A79U, 0x4B79U, 0x4C79U, 0x4D79U, 0x4E79U, 0x4F79U, 0x5079U, + 0x5179U, 0x5279U, 0x5379U, 0x5479U, 0x5579U, 0x5679U, 0x5779U, 0x5879U, + 0x5979U, 0x5A79U, 0x6179U, 0x6279U, 0x6379U, 0x6479U, 0x6579U, 0x6679U, + 0x6779U, 0x6879U, 0x6979U, 0x6A79U, 0x6B79U, 0x6C79U, 0x6D79U, 0x6E79U, + 0x6F79U, 0x7079U, 0x7179U, 0x7279U, 0x7379U, 0x7479U, 0x7579U, 0x7679U, + 0x7779U, 0x7879U, 0x7979U, 0x7A79U, 0x3079U, 0x3179U, 0x3279U, 0x3379U, + 0x3479U, 0x3579U, 0x3679U, 0x3779U, 0x3879U, 0x3979U, 0x2B79U, 0x2F79U, + 0x417AU, 0x427AU, 0x437AU, 0x447AU, 0x457AU, 0x467AU, 0x477AU, 0x487AU, + 0x497AU, 0x4A7AU, 0x4B7AU, 0x4C7AU, 0x4D7AU, 0x4E7AU, 0x4F7AU, 0x507AU, + 0x517AU, 0x527AU, 0x537AU, 0x547AU, 0x557AU, 0x567AU, 0x577AU, 0x587AU, + 0x597AU, 0x5A7AU, 0x617AU, 0x627AU, 0x637AU, 0x647AU, 0x657AU, 0x667AU, + 0x677AU, 0x687AU, 0x697AU, 0x6A7AU, 0x6B7AU, 0x6C7AU, 0x6D7AU, 0x6E7AU, + 0x6F7AU, 0x707AU, 0x717AU, 0x727AU, 0x737AU, 0x747AU, 0x757AU, 0x767AU, + 0x777AU, 0x787AU, 0x797AU, 0x7A7AU, 0x307AU, 0x317AU, 0x327AU, 0x337AU, + 0x347AU, 0x357AU, 0x367AU, 0x377AU, 0x387AU, 0x397AU, 0x2B7AU, 0x2F7AU, + 0x4130U, 0x4230U, 0x4330U, 0x4430U, 0x4530U, 0x4630U, 0x4730U, 0x4830U, + 0x4930U, 0x4A30U, 0x4B30U, 0x4C30U, 0x4D30U, 0x4E30U, 0x4F30U, 0x5030U, + 0x5130U, 0x5230U, 0x5330U, 0x5430U, 0x5530U, 0x5630U, 0x5730U, 0x5830U, + 0x5930U, 0x5A30U, 0x6130U, 0x6230U, 0x6330U, 0x6430U, 0x6530U, 0x6630U, + 0x6730U, 0x6830U, 0x6930U, 0x6A30U, 0x6B30U, 0x6C30U, 0x6D30U, 0x6E30U, + 0x6F30U, 0x7030U, 0x7130U, 0x7230U, 0x7330U, 0x7430U, 0x7530U, 0x7630U, + 0x7730U, 0x7830U, 0x7930U, 0x7A30U, 0x3030U, 0x3130U, 0x3230U, 0x3330U, + 0x3430U, 0x3530U, 0x3630U, 0x3730U, 0x3830U, 0x3930U, 0x2B30U, 0x2F30U, + 0x4131U, 0x4231U, 0x4331U, 0x4431U, 0x4531U, 0x4631U, 0x4731U, 0x4831U, + 0x4931U, 0x4A31U, 0x4B31U, 0x4C31U, 0x4D31U, 0x4E31U, 0x4F31U, 0x5031U, + 0x5131U, 0x5231U, 0x5331U, 0x5431U, 0x5531U, 0x5631U, 0x5731U, 0x5831U, + 0x5931U, 0x5A31U, 0x6131U, 0x6231U, 0x6331U, 0x6431U, 0x6531U, 0x6631U, + 0x6731U, 0x6831U, 0x6931U, 0x6A31U, 0x6B31U, 0x6C31U, 0x6D31U, 0x6E31U, + 0x6F31U, 0x7031U, 0x7131U, 0x7231U, 0x7331U, 0x7431U, 0x7531U, 0x7631U, + 0x7731U, 0x7831U, 0x7931U, 0x7A31U, 0x3031U, 0x3131U, 0x3231U, 0x3331U, + 0x3431U, 0x3531U, 0x3631U, 0x3731U, 0x3831U, 0x3931U, 0x2B31U, 0x2F31U, + 0x4132U, 0x4232U, 0x4332U, 0x4432U, 0x4532U, 0x4632U, 0x4732U, 0x4832U, + 0x4932U, 0x4A32U, 0x4B32U, 0x4C32U, 0x4D32U, 0x4E32U, 0x4F32U, 0x5032U, + 0x5132U, 0x5232U, 0x5332U, 0x5432U, 0x5532U, 0x5632U, 0x5732U, 0x5832U, + 0x5932U, 0x5A32U, 0x6132U, 0x6232U, 0x6332U, 0x6432U, 0x6532U, 0x6632U, + 0x6732U, 0x6832U, 0x6932U, 0x6A32U, 0x6B32U, 0x6C32U, 0x6D32U, 0x6E32U, + 0x6F32U, 0x7032U, 0x7132U, 0x7232U, 0x7332U, 0x7432U, 0x7532U, 0x7632U, + 0x7732U, 0x7832U, 0x7932U, 0x7A32U, 0x3032U, 0x3132U, 0x3232U, 0x3332U, + 0x3432U, 0x3532U, 0x3632U, 0x3732U, 0x3832U, 0x3932U, 0x2B32U, 0x2F32U, + 0x4133U, 0x4233U, 0x4333U, 0x4433U, 0x4533U, 0x4633U, 0x4733U, 0x4833U, + 0x4933U, 0x4A33U, 0x4B33U, 0x4C33U, 0x4D33U, 0x4E33U, 0x4F33U, 0x5033U, + 0x5133U, 0x5233U, 0x5333U, 0x5433U, 0x5533U, 0x5633U, 0x5733U, 0x5833U, + 0x5933U, 0x5A33U, 0x6133U, 0x6233U, 0x6333U, 0x6433U, 0x6533U, 0x6633U, + 0x6733U, 0x6833U, 0x6933U, 0x6A33U, 0x6B33U, 0x6C33U, 0x6D33U, 0x6E33U, + 0x6F33U, 0x7033U, 0x7133U, 0x7233U, 0x7333U, 0x7433U, 0x7533U, 0x7633U, + 0x7733U, 0x7833U, 0x7933U, 0x7A33U, 0x3033U, 0x3133U, 0x3233U, 0x3333U, + 0x3433U, 0x3533U, 0x3633U, 0x3733U, 0x3833U, 0x3933U, 0x2B33U, 0x2F33U, + 0x4134U, 0x4234U, 0x4334U, 0x4434U, 0x4534U, 0x4634U, 0x4734U, 0x4834U, + 0x4934U, 0x4A34U, 0x4B34U, 0x4C34U, 0x4D34U, 0x4E34U, 0x4F34U, 0x5034U, + 0x5134U, 0x5234U, 0x5334U, 0x5434U, 0x5534U, 0x5634U, 0x5734U, 0x5834U, + 0x5934U, 0x5A34U, 0x6134U, 0x6234U, 0x6334U, 0x6434U, 0x6534U, 0x6634U, + 0x6734U, 0x6834U, 0x6934U, 0x6A34U, 0x6B34U, 0x6C34U, 0x6D34U, 0x6E34U, + 0x6F34U, 0x7034U, 0x7134U, 0x7234U, 0x7334U, 0x7434U, 0x7534U, 0x7634U, + 0x7734U, 0x7834U, 0x7934U, 0x7A34U, 0x3034U, 0x3134U, 0x3234U, 0x3334U, + 0x3434U, 0x3534U, 0x3634U, 0x3734U, 0x3834U, 0x3934U, 0x2B34U, 0x2F34U, + 0x4135U, 0x4235U, 0x4335U, 0x4435U, 0x4535U, 0x4635U, 0x4735U, 0x4835U, + 0x4935U, 0x4A35U, 0x4B35U, 0x4C35U, 0x4D35U, 0x4E35U, 0x4F35U, 0x5035U, + 0x5135U, 0x5235U, 0x5335U, 0x5435U, 0x5535U, 0x5635U, 0x5735U, 0x5835U, + 0x5935U, 0x5A35U, 0x6135U, 0x6235U, 0x6335U, 0x6435U, 0x6535U, 0x6635U, + 0x6735U, 0x6835U, 0x6935U, 0x6A35U, 0x6B35U, 0x6C35U, 0x6D35U, 0x6E35U, + 0x6F35U, 0x7035U, 0x7135U, 0x7235U, 0x7335U, 0x7435U, 0x7535U, 0x7635U, + 0x7735U, 0x7835U, 0x7935U, 0x7A35U, 0x3035U, 0x3135U, 0x3235U, 0x3335U, + 0x3435U, 0x3535U, 0x3635U, 0x3735U, 0x3835U, 0x3935U, 0x2B35U, 0x2F35U, + 0x4136U, 0x4236U, 0x4336U, 0x4436U, 0x4536U, 0x4636U, 0x4736U, 0x4836U, + 0x4936U, 0x4A36U, 0x4B36U, 0x4C36U, 0x4D36U, 0x4E36U, 0x4F36U, 0x5036U, + 0x5136U, 0x5236U, 0x5336U, 0x5436U, 0x5536U, 0x5636U, 0x5736U, 0x5836U, + 0x5936U, 0x5A36U, 0x6136U, 0x6236U, 0x6336U, 0x6436U, 0x6536U, 0x6636U, + 0x6736U, 0x6836U, 0x6936U, 0x6A36U, 0x6B36U, 0x6C36U, 0x6D36U, 0x6E36U, + 0x6F36U, 0x7036U, 0x7136U, 0x7236U, 0x7336U, 0x7436U, 0x7536U, 0x7636U, + 0x7736U, 0x7836U, 0x7936U, 0x7A36U, 0x3036U, 0x3136U, 0x3236U, 0x3336U, + 0x3436U, 0x3536U, 0x3636U, 0x3736U, 0x3836U, 0x3936U, 0x2B36U, 0x2F36U, + 0x4137U, 0x4237U, 0x4337U, 0x4437U, 0x4537U, 0x4637U, 0x4737U, 0x4837U, + 0x4937U, 0x4A37U, 0x4B37U, 0x4C37U, 0x4D37U, 0x4E37U, 0x4F37U, 0x5037U, + 0x5137U, 0x5237U, 0x5337U, 0x5437U, 0x5537U, 0x5637U, 0x5737U, 0x5837U, + 0x5937U, 0x5A37U, 0x6137U, 0x6237U, 0x6337U, 0x6437U, 0x6537U, 0x6637U, + 0x6737U, 0x6837U, 0x6937U, 0x6A37U, 0x6B37U, 0x6C37U, 0x6D37U, 0x6E37U, + 0x6F37U, 0x7037U, 0x7137U, 0x7237U, 0x7337U, 0x7437U, 0x7537U, 0x7637U, + 0x7737U, 0x7837U, 0x7937U, 0x7A37U, 0x3037U, 0x3137U, 0x3237U, 0x3337U, + 0x3437U, 0x3537U, 0x3637U, 0x3737U, 0x3837U, 0x3937U, 0x2B37U, 0x2F37U, + 0x4138U, 0x4238U, 0x4338U, 0x4438U, 0x4538U, 0x4638U, 0x4738U, 0x4838U, + 0x4938U, 0x4A38U, 0x4B38U, 0x4C38U, 0x4D38U, 0x4E38U, 0x4F38U, 0x5038U, + 0x5138U, 0x5238U, 0x5338U, 0x5438U, 0x5538U, 0x5638U, 0x5738U, 0x5838U, + 0x5938U, 0x5A38U, 0x6138U, 0x6238U, 0x6338U, 0x6438U, 0x6538U, 0x6638U, + 0x6738U, 0x6838U, 0x6938U, 0x6A38U, 0x6B38U, 0x6C38U, 0x6D38U, 0x6E38U, + 0x6F38U, 0x7038U, 0x7138U, 0x7238U, 0x7338U, 0x7438U, 0x7538U, 0x7638U, + 0x7738U, 0x7838U, 0x7938U, 0x7A38U, 0x3038U, 0x3138U, 0x3238U, 0x3338U, + 0x3438U, 0x3538U, 0x3638U, 0x3738U, 0x3838U, 0x3938U, 0x2B38U, 0x2F38U, + 0x4139U, 0x4239U, 0x4339U, 0x4439U, 0x4539U, 0x4639U, 0x4739U, 0x4839U, + 0x4939U, 0x4A39U, 0x4B39U, 0x4C39U, 0x4D39U, 0x4E39U, 0x4F39U, 0x5039U, + 0x5139U, 0x5239U, 0x5339U, 0x5439U, 0x5539U, 0x5639U, 0x5739U, 0x5839U, + 0x5939U, 0x5A39U, 0x6139U, 0x6239U, 0x6339U, 0x6439U, 0x6539U, 0x6639U, + 0x6739U, 0x6839U, 0x6939U, 0x6A39U, 0x6B39U, 0x6C39U, 0x6D39U, 0x6E39U, + 0x6F39U, 0x7039U, 0x7139U, 0x7239U, 0x7339U, 0x7439U, 0x7539U, 0x7639U, + 0x7739U, 0x7839U, 0x7939U, 0x7A39U, 0x3039U, 0x3139U, 0x3239U, 0x3339U, + 0x3439U, 0x3539U, 0x3639U, 0x3739U, 0x3839U, 0x3939U, 0x2B39U, 0x2F39U, + 0x412BU, 0x422BU, 0x432BU, 0x442BU, 0x452BU, 0x462BU, 0x472BU, 0x482BU, + 0x492BU, 0x4A2BU, 0x4B2BU, 0x4C2BU, 0x4D2BU, 0x4E2BU, 0x4F2BU, 0x502BU, + 0x512BU, 0x522BU, 0x532BU, 0x542BU, 0x552BU, 0x562BU, 0x572BU, 0x582BU, + 0x592BU, 0x5A2BU, 0x612BU, 0x622BU, 0x632BU, 0x642BU, 0x652BU, 0x662BU, + 0x672BU, 0x682BU, 0x692BU, 0x6A2BU, 0x6B2BU, 0x6C2BU, 0x6D2BU, 0x6E2BU, + 0x6F2BU, 0x702BU, 0x712BU, 0x722BU, 0x732BU, 0x742BU, 0x752BU, 0x762BU, + 0x772BU, 0x782BU, 0x792BU, 0x7A2BU, 0x302BU, 0x312BU, 0x322BU, 0x332BU, + 0x342BU, 0x352BU, 0x362BU, 0x372BU, 0x382BU, 0x392BU, 0x2B2BU, 0x2F2BU, + 0x412FU, 0x422FU, 0x432FU, 0x442FU, 0x452FU, 0x462FU, 0x472FU, 0x482FU, + 0x492FU, 0x4A2FU, 0x4B2FU, 0x4C2FU, 0x4D2FU, 0x4E2FU, 0x4F2FU, 0x502FU, + 0x512FU, 0x522FU, 0x532FU, 0x542FU, 0x552FU, 0x562FU, 0x572FU, 0x582FU, + 0x592FU, 0x5A2FU, 0x612FU, 0x622FU, 0x632FU, 0x642FU, 0x652FU, 0x662FU, + 0x672FU, 0x682FU, 0x692FU, 0x6A2FU, 0x6B2FU, 0x6C2FU, 0x6D2FU, 0x6E2FU, + 0x6F2FU, 0x702FU, 0x712FU, 0x722FU, 0x732FU, 0x742FU, 0x752FU, 0x762FU, + 0x772FU, 0x782FU, 0x792FU, 0x7A2FU, 0x302FU, 0x312FU, 0x322FU, 0x332FU, + 0x342FU, 0x352FU, 0x362FU, 0x372FU, 0x382FU, 0x392FU, 0x2B2FU, 0x2F2FU, +#else + 0x4141U, 0x4142U, 0x4143U, 0x4144U, 0x4145U, 0x4146U, 0x4147U, 0x4148U, + 0x4149U, 0x414AU, 0x414BU, 0x414CU, 0x414DU, 0x414EU, 0x414FU, 0x4150U, + 0x4151U, 0x4152U, 0x4153U, 0x4154U, 0x4155U, 0x4156U, 0x4157U, 0x4158U, + 0x4159U, 0x415AU, 0x4161U, 0x4162U, 0x4163U, 0x4164U, 0x4165U, 0x4166U, + 0x4167U, 0x4168U, 0x4169U, 0x416AU, 0x416BU, 0x416CU, 0x416DU, 0x416EU, + 0x416FU, 0x4170U, 0x4171U, 0x4172U, 0x4173U, 0x4174U, 0x4175U, 0x4176U, + 0x4177U, 0x4178U, 0x4179U, 0x417AU, 0x4130U, 0x4131U, 0x4132U, 0x4133U, + 0x4134U, 0x4135U, 0x4136U, 0x4137U, 0x4138U, 0x4139U, 0x412BU, 0x412FU, + 0x4241U, 0x4242U, 0x4243U, 0x4244U, 0x4245U, 0x4246U, 0x4247U, 0x4248U, + 0x4249U, 0x424AU, 0x424BU, 0x424CU, 0x424DU, 0x424EU, 0x424FU, 0x4250U, + 0x4251U, 0x4252U, 0x4253U, 0x4254U, 0x4255U, 0x4256U, 0x4257U, 0x4258U, + 0x4259U, 0x425AU, 0x4261U, 0x4262U, 0x4263U, 0x4264U, 0x4265U, 0x4266U, + 0x4267U, 0x4268U, 0x4269U, 0x426AU, 0x426BU, 0x426CU, 0x426DU, 0x426EU, + 0x426FU, 0x4270U, 0x4271U, 0x4272U, 0x4273U, 0x4274U, 0x4275U, 0x4276U, + 0x4277U, 0x4278U, 0x4279U, 0x427AU, 0x4230U, 0x4231U, 0x4232U, 0x4233U, + 0x4234U, 0x4235U, 0x4236U, 0x4237U, 0x4238U, 0x4239U, 0x422BU, 0x422FU, + 0x4341U, 0x4342U, 0x4343U, 0x4344U, 0x4345U, 0x4346U, 0x4347U, 0x4348U, + 0x4349U, 0x434AU, 0x434BU, 0x434CU, 0x434DU, 0x434EU, 0x434FU, 0x4350U, + 0x4351U, 0x4352U, 0x4353U, 0x4354U, 0x4355U, 0x4356U, 0x4357U, 0x4358U, + 0x4359U, 0x435AU, 0x4361U, 0x4362U, 0x4363U, 0x4364U, 0x4365U, 0x4366U, + 0x4367U, 0x4368U, 0x4369U, 0x436AU, 0x436BU, 0x436CU, 0x436DU, 0x436EU, + 0x436FU, 0x4370U, 0x4371U, 0x4372U, 0x4373U, 0x4374U, 0x4375U, 0x4376U, + 0x4377U, 0x4378U, 0x4379U, 0x437AU, 0x4330U, 0x4331U, 0x4332U, 0x4333U, + 0x4334U, 0x4335U, 0x4336U, 0x4337U, 0x4338U, 0x4339U, 0x432BU, 0x432FU, + 0x4441U, 0x4442U, 0x4443U, 0x4444U, 0x4445U, 0x4446U, 0x4447U, 0x4448U, + 0x4449U, 0x444AU, 0x444BU, 0x444CU, 0x444DU, 0x444EU, 0x444FU, 0x4450U, + 0x4451U, 0x4452U, 0x4453U, 0x4454U, 0x4455U, 0x4456U, 0x4457U, 0x4458U, + 0x4459U, 0x445AU, 0x4461U, 0x4462U, 0x4463U, 0x4464U, 0x4465U, 0x4466U, + 0x4467U, 0x4468U, 0x4469U, 0x446AU, 0x446BU, 0x446CU, 0x446DU, 0x446EU, + 0x446FU, 0x4470U, 0x4471U, 0x4472U, 0x4473U, 0x4474U, 0x4475U, 0x4476U, + 0x4477U, 0x4478U, 0x4479U, 0x447AU, 0x4430U, 0x4431U, 0x4432U, 0x4433U, + 0x4434U, 0x4435U, 0x4436U, 0x4437U, 0x4438U, 0x4439U, 0x442BU, 0x442FU, + 0x4541U, 0x4542U, 0x4543U, 0x4544U, 0x4545U, 0x4546U, 0x4547U, 0x4548U, + 0x4549U, 0x454AU, 0x454BU, 0x454CU, 0x454DU, 0x454EU, 0x454FU, 0x4550U, + 0x4551U, 0x4552U, 0x4553U, 0x4554U, 0x4555U, 0x4556U, 0x4557U, 0x4558U, + 0x4559U, 0x455AU, 0x4561U, 0x4562U, 0x4563U, 0x4564U, 0x4565U, 0x4566U, + 0x4567U, 0x4568U, 0x4569U, 0x456AU, 0x456BU, 0x456CU, 0x456DU, 0x456EU, + 0x456FU, 0x4570U, 0x4571U, 0x4572U, 0x4573U, 0x4574U, 0x4575U, 0x4576U, + 0x4577U, 0x4578U, 0x4579U, 0x457AU, 0x4530U, 0x4531U, 0x4532U, 0x4533U, + 0x4534U, 0x4535U, 0x4536U, 0x4537U, 0x4538U, 0x4539U, 0x452BU, 0x452FU, + 0x4641U, 0x4642U, 0x4643U, 0x4644U, 0x4645U, 0x4646U, 0x4647U, 0x4648U, + 0x4649U, 0x464AU, 0x464BU, 0x464CU, 0x464DU, 0x464EU, 0x464FU, 0x4650U, + 0x4651U, 0x4652U, 0x4653U, 0x4654U, 0x4655U, 0x4656U, 0x4657U, 0x4658U, + 0x4659U, 0x465AU, 0x4661U, 0x4662U, 0x4663U, 0x4664U, 0x4665U, 0x4666U, + 0x4667U, 0x4668U, 0x4669U, 0x466AU, 0x466BU, 0x466CU, 0x466DU, 0x466EU, + 0x466FU, 0x4670U, 0x4671U, 0x4672U, 0x4673U, 0x4674U, 0x4675U, 0x4676U, + 0x4677U, 0x4678U, 0x4679U, 0x467AU, 0x4630U, 0x4631U, 0x4632U, 0x4633U, + 0x4634U, 0x4635U, 0x4636U, 0x4637U, 0x4638U, 0x4639U, 0x462BU, 0x462FU, + 0x4741U, 0x4742U, 0x4743U, 0x4744U, 0x4745U, 0x4746U, 0x4747U, 0x4748U, + 0x4749U, 0x474AU, 0x474BU, 0x474CU, 0x474DU, 0x474EU, 0x474FU, 0x4750U, + 0x4751U, 0x4752U, 0x4753U, 0x4754U, 0x4755U, 0x4756U, 0x4757U, 0x4758U, + 0x4759U, 0x475AU, 0x4761U, 0x4762U, 0x4763U, 0x4764U, 0x4765U, 0x4766U, + 0x4767U, 0x4768U, 0x4769U, 0x476AU, 0x476BU, 0x476CU, 0x476DU, 0x476EU, + 0x476FU, 0x4770U, 0x4771U, 0x4772U, 0x4773U, 0x4774U, 0x4775U, 0x4776U, + 0x4777U, 0x4778U, 0x4779U, 0x477AU, 0x4730U, 0x4731U, 0x4732U, 0x4733U, + 0x4734U, 0x4735U, 0x4736U, 0x4737U, 0x4738U, 0x4739U, 0x472BU, 0x472FU, + 0x4841U, 0x4842U, 0x4843U, 0x4844U, 0x4845U, 0x4846U, 0x4847U, 0x4848U, + 0x4849U, 0x484AU, 0x484BU, 0x484CU, 0x484DU, 0x484EU, 0x484FU, 0x4850U, + 0x4851U, 0x4852U, 0x4853U, 0x4854U, 0x4855U, 0x4856U, 0x4857U, 0x4858U, + 0x4859U, 0x485AU, 0x4861U, 0x4862U, 0x4863U, 0x4864U, 0x4865U, 0x4866U, + 0x4867U, 0x4868U, 0x4869U, 0x486AU, 0x486BU, 0x486CU, 0x486DU, 0x486EU, + 0x486FU, 0x4870U, 0x4871U, 0x4872U, 0x4873U, 0x4874U, 0x4875U, 0x4876U, + 0x4877U, 0x4878U, 0x4879U, 0x487AU, 0x4830U, 0x4831U, 0x4832U, 0x4833U, + 0x4834U, 0x4835U, 0x4836U, 0x4837U, 0x4838U, 0x4839U, 0x482BU, 0x482FU, + 0x4941U, 0x4942U, 0x4943U, 0x4944U, 0x4945U, 0x4946U, 0x4947U, 0x4948U, + 0x4949U, 0x494AU, 0x494BU, 0x494CU, 0x494DU, 0x494EU, 0x494FU, 0x4950U, + 0x4951U, 0x4952U, 0x4953U, 0x4954U, 0x4955U, 0x4956U, 0x4957U, 0x4958U, + 0x4959U, 0x495AU, 0x4961U, 0x4962U, 0x4963U, 0x4964U, 0x4965U, 0x4966U, + 0x4967U, 0x4968U, 0x4969U, 0x496AU, 0x496BU, 0x496CU, 0x496DU, 0x496EU, + 0x496FU, 0x4970U, 0x4971U, 0x4972U, 0x4973U, 0x4974U, 0x4975U, 0x4976U, + 0x4977U, 0x4978U, 0x4979U, 0x497AU, 0x4930U, 0x4931U, 0x4932U, 0x4933U, + 0x4934U, 0x4935U, 0x4936U, 0x4937U, 0x4938U, 0x4939U, 0x492BU, 0x492FU, + 0x4A41U, 0x4A42U, 0x4A43U, 0x4A44U, 0x4A45U, 0x4A46U, 0x4A47U, 0x4A48U, + 0x4A49U, 0x4A4AU, 0x4A4BU, 0x4A4CU, 0x4A4DU, 0x4A4EU, 0x4A4FU, 0x4A50U, + 0x4A51U, 0x4A52U, 0x4A53U, 0x4A54U, 0x4A55U, 0x4A56U, 0x4A57U, 0x4A58U, + 0x4A59U, 0x4A5AU, 0x4A61U, 0x4A62U, 0x4A63U, 0x4A64U, 0x4A65U, 0x4A66U, + 0x4A67U, 0x4A68U, 0x4A69U, 0x4A6AU, 0x4A6BU, 0x4A6CU, 0x4A6DU, 0x4A6EU, + 0x4A6FU, 0x4A70U, 0x4A71U, 0x4A72U, 0x4A73U, 0x4A74U, 0x4A75U, 0x4A76U, + 0x4A77U, 0x4A78U, 0x4A79U, 0x4A7AU, 0x4A30U, 0x4A31U, 0x4A32U, 0x4A33U, + 0x4A34U, 0x4A35U, 0x4A36U, 0x4A37U, 0x4A38U, 0x4A39U, 0x4A2BU, 0x4A2FU, + 0x4B41U, 0x4B42U, 0x4B43U, 0x4B44U, 0x4B45U, 0x4B46U, 0x4B47U, 0x4B48U, + 0x4B49U, 0x4B4AU, 0x4B4BU, 0x4B4CU, 0x4B4DU, 0x4B4EU, 0x4B4FU, 0x4B50U, + 0x4B51U, 0x4B52U, 0x4B53U, 0x4B54U, 0x4B55U, 0x4B56U, 0x4B57U, 0x4B58U, + 0x4B59U, 0x4B5AU, 0x4B61U, 0x4B62U, 0x4B63U, 0x4B64U, 0x4B65U, 0x4B66U, + 0x4B67U, 0x4B68U, 0x4B69U, 0x4B6AU, 0x4B6BU, 0x4B6CU, 0x4B6DU, 0x4B6EU, + 0x4B6FU, 0x4B70U, 0x4B71U, 0x4B72U, 0x4B73U, 0x4B74U, 0x4B75U, 0x4B76U, + 0x4B77U, 0x4B78U, 0x4B79U, 0x4B7AU, 0x4B30U, 0x4B31U, 0x4B32U, 0x4B33U, + 0x4B34U, 0x4B35U, 0x4B36U, 0x4B37U, 0x4B38U, 0x4B39U, 0x4B2BU, 0x4B2FU, + 0x4C41U, 0x4C42U, 0x4C43U, 0x4C44U, 0x4C45U, 0x4C46U, 0x4C47U, 0x4C48U, + 0x4C49U, 0x4C4AU, 0x4C4BU, 0x4C4CU, 0x4C4DU, 0x4C4EU, 0x4C4FU, 0x4C50U, + 0x4C51U, 0x4C52U, 0x4C53U, 0x4C54U, 0x4C55U, 0x4C56U, 0x4C57U, 0x4C58U, + 0x4C59U, 0x4C5AU, 0x4C61U, 0x4C62U, 0x4C63U, 0x4C64U, 0x4C65U, 0x4C66U, + 0x4C67U, 0x4C68U, 0x4C69U, 0x4C6AU, 0x4C6BU, 0x4C6CU, 0x4C6DU, 0x4C6EU, + 0x4C6FU, 0x4C70U, 0x4C71U, 0x4C72U, 0x4C73U, 0x4C74U, 0x4C75U, 0x4C76U, + 0x4C77U, 0x4C78U, 0x4C79U, 0x4C7AU, 0x4C30U, 0x4C31U, 0x4C32U, 0x4C33U, + 0x4C34U, 0x4C35U, 0x4C36U, 0x4C37U, 0x4C38U, 0x4C39U, 0x4C2BU, 0x4C2FU, + 0x4D41U, 0x4D42U, 0x4D43U, 0x4D44U, 0x4D45U, 0x4D46U, 0x4D47U, 0x4D48U, + 0x4D49U, 0x4D4AU, 0x4D4BU, 0x4D4CU, 0x4D4DU, 0x4D4EU, 0x4D4FU, 0x4D50U, + 0x4D51U, 0x4D52U, 0x4D53U, 0x4D54U, 0x4D55U, 0x4D56U, 0x4D57U, 0x4D58U, + 0x4D59U, 0x4D5AU, 0x4D61U, 0x4D62U, 0x4D63U, 0x4D64U, 0x4D65U, 0x4D66U, + 0x4D67U, 0x4D68U, 0x4D69U, 0x4D6AU, 0x4D6BU, 0x4D6CU, 0x4D6DU, 0x4D6EU, + 0x4D6FU, 0x4D70U, 0x4D71U, 0x4D72U, 0x4D73U, 0x4D74U, 0x4D75U, 0x4D76U, + 0x4D77U, 0x4D78U, 0x4D79U, 0x4D7AU, 0x4D30U, 0x4D31U, 0x4D32U, 0x4D33U, + 0x4D34U, 0x4D35U, 0x4D36U, 0x4D37U, 0x4D38U, 0x4D39U, 0x4D2BU, 0x4D2FU, + 0x4E41U, 0x4E42U, 0x4E43U, 0x4E44U, 0x4E45U, 0x4E46U, 0x4E47U, 0x4E48U, + 0x4E49U, 0x4E4AU, 0x4E4BU, 0x4E4CU, 0x4E4DU, 0x4E4EU, 0x4E4FU, 0x4E50U, + 0x4E51U, 0x4E52U, 0x4E53U, 0x4E54U, 0x4E55U, 0x4E56U, 0x4E57U, 0x4E58U, + 0x4E59U, 0x4E5AU, 0x4E61U, 0x4E62U, 0x4E63U, 0x4E64U, 0x4E65U, 0x4E66U, + 0x4E67U, 0x4E68U, 0x4E69U, 0x4E6AU, 0x4E6BU, 0x4E6CU, 0x4E6DU, 0x4E6EU, + 0x4E6FU, 0x4E70U, 0x4E71U, 0x4E72U, 0x4E73U, 0x4E74U, 0x4E75U, 0x4E76U, + 0x4E77U, 0x4E78U, 0x4E79U, 0x4E7AU, 0x4E30U, 0x4E31U, 0x4E32U, 0x4E33U, + 0x4E34U, 0x4E35U, 0x4E36U, 0x4E37U, 0x4E38U, 0x4E39U, 0x4E2BU, 0x4E2FU, + 0x4F41U, 0x4F42U, 0x4F43U, 0x4F44U, 0x4F45U, 0x4F46U, 0x4F47U, 0x4F48U, + 0x4F49U, 0x4F4AU, 0x4F4BU, 0x4F4CU, 0x4F4DU, 0x4F4EU, 0x4F4FU, 0x4F50U, + 0x4F51U, 0x4F52U, 0x4F53U, 0x4F54U, 0x4F55U, 0x4F56U, 0x4F57U, 0x4F58U, + 0x4F59U, 0x4F5AU, 0x4F61U, 0x4F62U, 0x4F63U, 0x4F64U, 0x4F65U, 0x4F66U, + 0x4F67U, 0x4F68U, 0x4F69U, 0x4F6AU, 0x4F6BU, 0x4F6CU, 0x4F6DU, 0x4F6EU, + 0x4F6FU, 0x4F70U, 0x4F71U, 0x4F72U, 0x4F73U, 0x4F74U, 0x4F75U, 0x4F76U, + 0x4F77U, 0x4F78U, 0x4F79U, 0x4F7AU, 0x4F30U, 0x4F31U, 0x4F32U, 0x4F33U, + 0x4F34U, 0x4F35U, 0x4F36U, 0x4F37U, 0x4F38U, 0x4F39U, 0x4F2BU, 0x4F2FU, + 0x5041U, 0x5042U, 0x5043U, 0x5044U, 0x5045U, 0x5046U, 0x5047U, 0x5048U, + 0x5049U, 0x504AU, 0x504BU, 0x504CU, 0x504DU, 0x504EU, 0x504FU, 0x5050U, + 0x5051U, 0x5052U, 0x5053U, 0x5054U, 0x5055U, 0x5056U, 0x5057U, 0x5058U, + 0x5059U, 0x505AU, 0x5061U, 0x5062U, 0x5063U, 0x5064U, 0x5065U, 0x5066U, + 0x5067U, 0x5068U, 0x5069U, 0x506AU, 0x506BU, 0x506CU, 0x506DU, 0x506EU, + 0x506FU, 0x5070U, 0x5071U, 0x5072U, 0x5073U, 0x5074U, 0x5075U, 0x5076U, + 0x5077U, 0x5078U, 0x5079U, 0x507AU, 0x5030U, 0x5031U, 0x5032U, 0x5033U, + 0x5034U, 0x5035U, 0x5036U, 0x5037U, 0x5038U, 0x5039U, 0x502BU, 0x502FU, + 0x5141U, 0x5142U, 0x5143U, 0x5144U, 0x5145U, 0x5146U, 0x5147U, 0x5148U, + 0x5149U, 0x514AU, 0x514BU, 0x514CU, 0x514DU, 0x514EU, 0x514FU, 0x5150U, + 0x5151U, 0x5152U, 0x5153U, 0x5154U, 0x5155U, 0x5156U, 0x5157U, 0x5158U, + 0x5159U, 0x515AU, 0x5161U, 0x5162U, 0x5163U, 0x5164U, 0x5165U, 0x5166U, + 0x5167U, 0x5168U, 0x5169U, 0x516AU, 0x516BU, 0x516CU, 0x516DU, 0x516EU, + 0x516FU, 0x5170U, 0x5171U, 0x5172U, 0x5173U, 0x5174U, 0x5175U, 0x5176U, + 0x5177U, 0x5178U, 0x5179U, 0x517AU, 0x5130U, 0x5131U, 0x5132U, 0x5133U, + 0x5134U, 0x5135U, 0x5136U, 0x5137U, 0x5138U, 0x5139U, 0x512BU, 0x512FU, + 0x5241U, 0x5242U, 0x5243U, 0x5244U, 0x5245U, 0x5246U, 0x5247U, 0x5248U, + 0x5249U, 0x524AU, 0x524BU, 0x524CU, 0x524DU, 0x524EU, 0x524FU, 0x5250U, + 0x5251U, 0x5252U, 0x5253U, 0x5254U, 0x5255U, 0x5256U, 0x5257U, 0x5258U, + 0x5259U, 0x525AU, 0x5261U, 0x5262U, 0x5263U, 0x5264U, 0x5265U, 0x5266U, + 0x5267U, 0x5268U, 0x5269U, 0x526AU, 0x526BU, 0x526CU, 0x526DU, 0x526EU, + 0x526FU, 0x5270U, 0x5271U, 0x5272U, 0x5273U, 0x5274U, 0x5275U, 0x5276U, + 0x5277U, 0x5278U, 0x5279U, 0x527AU, 0x5230U, 0x5231U, 0x5232U, 0x5233U, + 0x5234U, 0x5235U, 0x5236U, 0x5237U, 0x5238U, 0x5239U, 0x522BU, 0x522FU, + 0x5341U, 0x5342U, 0x5343U, 0x5344U, 0x5345U, 0x5346U, 0x5347U, 0x5348U, + 0x5349U, 0x534AU, 0x534BU, 0x534CU, 0x534DU, 0x534EU, 0x534FU, 0x5350U, + 0x5351U, 0x5352U, 0x5353U, 0x5354U, 0x5355U, 0x5356U, 0x5357U, 0x5358U, + 0x5359U, 0x535AU, 0x5361U, 0x5362U, 0x5363U, 0x5364U, 0x5365U, 0x5366U, + 0x5367U, 0x5368U, 0x5369U, 0x536AU, 0x536BU, 0x536CU, 0x536DU, 0x536EU, + 0x536FU, 0x5370U, 0x5371U, 0x5372U, 0x5373U, 0x5374U, 0x5375U, 0x5376U, + 0x5377U, 0x5378U, 0x5379U, 0x537AU, 0x5330U, 0x5331U, 0x5332U, 0x5333U, + 0x5334U, 0x5335U, 0x5336U, 0x5337U, 0x5338U, 0x5339U, 0x532BU, 0x532FU, + 0x5441U, 0x5442U, 0x5443U, 0x5444U, 0x5445U, 0x5446U, 0x5447U, 0x5448U, + 0x5449U, 0x544AU, 0x544BU, 0x544CU, 0x544DU, 0x544EU, 0x544FU, 0x5450U, + 0x5451U, 0x5452U, 0x5453U, 0x5454U, 0x5455U, 0x5456U, 0x5457U, 0x5458U, + 0x5459U, 0x545AU, 0x5461U, 0x5462U, 0x5463U, 0x5464U, 0x5465U, 0x5466U, + 0x5467U, 0x5468U, 0x5469U, 0x546AU, 0x546BU, 0x546CU, 0x546DU, 0x546EU, + 0x546FU, 0x5470U, 0x5471U, 0x5472U, 0x5473U, 0x5474U, 0x5475U, 0x5476U, + 0x5477U, 0x5478U, 0x5479U, 0x547AU, 0x5430U, 0x5431U, 0x5432U, 0x5433U, + 0x5434U, 0x5435U, 0x5436U, 0x5437U, 0x5438U, 0x5439U, 0x542BU, 0x542FU, + 0x5541U, 0x5542U, 0x5543U, 0x5544U, 0x5545U, 0x5546U, 0x5547U, 0x5548U, + 0x5549U, 0x554AU, 0x554BU, 0x554CU, 0x554DU, 0x554EU, 0x554FU, 0x5550U, + 0x5551U, 0x5552U, 0x5553U, 0x5554U, 0x5555U, 0x5556U, 0x5557U, 0x5558U, + 0x5559U, 0x555AU, 0x5561U, 0x5562U, 0x5563U, 0x5564U, 0x5565U, 0x5566U, + 0x5567U, 0x5568U, 0x5569U, 0x556AU, 0x556BU, 0x556CU, 0x556DU, 0x556EU, + 0x556FU, 0x5570U, 0x5571U, 0x5572U, 0x5573U, 0x5574U, 0x5575U, 0x5576U, + 0x5577U, 0x5578U, 0x5579U, 0x557AU, 0x5530U, 0x5531U, 0x5532U, 0x5533U, + 0x5534U, 0x5535U, 0x5536U, 0x5537U, 0x5538U, 0x5539U, 0x552BU, 0x552FU, + 0x5641U, 0x5642U, 0x5643U, 0x5644U, 0x5645U, 0x5646U, 0x5647U, 0x5648U, + 0x5649U, 0x564AU, 0x564BU, 0x564CU, 0x564DU, 0x564EU, 0x564FU, 0x5650U, + 0x5651U, 0x5652U, 0x5653U, 0x5654U, 0x5655U, 0x5656U, 0x5657U, 0x5658U, + 0x5659U, 0x565AU, 0x5661U, 0x5662U, 0x5663U, 0x5664U, 0x5665U, 0x5666U, + 0x5667U, 0x5668U, 0x5669U, 0x566AU, 0x566BU, 0x566CU, 0x566DU, 0x566EU, + 0x566FU, 0x5670U, 0x5671U, 0x5672U, 0x5673U, 0x5674U, 0x5675U, 0x5676U, + 0x5677U, 0x5678U, 0x5679U, 0x567AU, 0x5630U, 0x5631U, 0x5632U, 0x5633U, + 0x5634U, 0x5635U, 0x5636U, 0x5637U, 0x5638U, 0x5639U, 0x562BU, 0x562FU, + 0x5741U, 0x5742U, 0x5743U, 0x5744U, 0x5745U, 0x5746U, 0x5747U, 0x5748U, + 0x5749U, 0x574AU, 0x574BU, 0x574CU, 0x574DU, 0x574EU, 0x574FU, 0x5750U, + 0x5751U, 0x5752U, 0x5753U, 0x5754U, 0x5755U, 0x5756U, 0x5757U, 0x5758U, + 0x5759U, 0x575AU, 0x5761U, 0x5762U, 0x5763U, 0x5764U, 0x5765U, 0x5766U, + 0x5767U, 0x5768U, 0x5769U, 0x576AU, 0x576BU, 0x576CU, 0x576DU, 0x576EU, + 0x576FU, 0x5770U, 0x5771U, 0x5772U, 0x5773U, 0x5774U, 0x5775U, 0x5776U, + 0x5777U, 0x5778U, 0x5779U, 0x577AU, 0x5730U, 0x5731U, 0x5732U, 0x5733U, + 0x5734U, 0x5735U, 0x5736U, 0x5737U, 0x5738U, 0x5739U, 0x572BU, 0x572FU, + 0x5841U, 0x5842U, 0x5843U, 0x5844U, 0x5845U, 0x5846U, 0x5847U, 0x5848U, + 0x5849U, 0x584AU, 0x584BU, 0x584CU, 0x584DU, 0x584EU, 0x584FU, 0x5850U, + 0x5851U, 0x5852U, 0x5853U, 0x5854U, 0x5855U, 0x5856U, 0x5857U, 0x5858U, + 0x5859U, 0x585AU, 0x5861U, 0x5862U, 0x5863U, 0x5864U, 0x5865U, 0x5866U, + 0x5867U, 0x5868U, 0x5869U, 0x586AU, 0x586BU, 0x586CU, 0x586DU, 0x586EU, + 0x586FU, 0x5870U, 0x5871U, 0x5872U, 0x5873U, 0x5874U, 0x5875U, 0x5876U, + 0x5877U, 0x5878U, 0x5879U, 0x587AU, 0x5830U, 0x5831U, 0x5832U, 0x5833U, + 0x5834U, 0x5835U, 0x5836U, 0x5837U, 0x5838U, 0x5839U, 0x582BU, 0x582FU, + 0x5941U, 0x5942U, 0x5943U, 0x5944U, 0x5945U, 0x5946U, 0x5947U, 0x5948U, + 0x5949U, 0x594AU, 0x594BU, 0x594CU, 0x594DU, 0x594EU, 0x594FU, 0x5950U, + 0x5951U, 0x5952U, 0x5953U, 0x5954U, 0x5955U, 0x5956U, 0x5957U, 0x5958U, + 0x5959U, 0x595AU, 0x5961U, 0x5962U, 0x5963U, 0x5964U, 0x5965U, 0x5966U, + 0x5967U, 0x5968U, 0x5969U, 0x596AU, 0x596BU, 0x596CU, 0x596DU, 0x596EU, + 0x596FU, 0x5970U, 0x5971U, 0x5972U, 0x5973U, 0x5974U, 0x5975U, 0x5976U, + 0x5977U, 0x5978U, 0x5979U, 0x597AU, 0x5930U, 0x5931U, 0x5932U, 0x5933U, + 0x5934U, 0x5935U, 0x5936U, 0x5937U, 0x5938U, 0x5939U, 0x592BU, 0x592FU, + 0x5A41U, 0x5A42U, 0x5A43U, 0x5A44U, 0x5A45U, 0x5A46U, 0x5A47U, 0x5A48U, + 0x5A49U, 0x5A4AU, 0x5A4BU, 0x5A4CU, 0x5A4DU, 0x5A4EU, 0x5A4FU, 0x5A50U, + 0x5A51U, 0x5A52U, 0x5A53U, 0x5A54U, 0x5A55U, 0x5A56U, 0x5A57U, 0x5A58U, + 0x5A59U, 0x5A5AU, 0x5A61U, 0x5A62U, 0x5A63U, 0x5A64U, 0x5A65U, 0x5A66U, + 0x5A67U, 0x5A68U, 0x5A69U, 0x5A6AU, 0x5A6BU, 0x5A6CU, 0x5A6DU, 0x5A6EU, + 0x5A6FU, 0x5A70U, 0x5A71U, 0x5A72U, 0x5A73U, 0x5A74U, 0x5A75U, 0x5A76U, + 0x5A77U, 0x5A78U, 0x5A79U, 0x5A7AU, 0x5A30U, 0x5A31U, 0x5A32U, 0x5A33U, + 0x5A34U, 0x5A35U, 0x5A36U, 0x5A37U, 0x5A38U, 0x5A39U, 0x5A2BU, 0x5A2FU, + 0x6141U, 0x6142U, 0x6143U, 0x6144U, 0x6145U, 0x6146U, 0x6147U, 0x6148U, + 0x6149U, 0x614AU, 0x614BU, 0x614CU, 0x614DU, 0x614EU, 0x614FU, 0x6150U, + 0x6151U, 0x6152U, 0x6153U, 0x6154U, 0x6155U, 0x6156U, 0x6157U, 0x6158U, + 0x6159U, 0x615AU, 0x6161U, 0x6162U, 0x6163U, 0x6164U, 0x6165U, 0x6166U, + 0x6167U, 0x6168U, 0x6169U, 0x616AU, 0x616BU, 0x616CU, 0x616DU, 0x616EU, + 0x616FU, 0x6170U, 0x6171U, 0x6172U, 0x6173U, 0x6174U, 0x6175U, 0x6176U, + 0x6177U, 0x6178U, 0x6179U, 0x617AU, 0x6130U, 0x6131U, 0x6132U, 0x6133U, + 0x6134U, 0x6135U, 0x6136U, 0x6137U, 0x6138U, 0x6139U, 0x612BU, 0x612FU, + 0x6241U, 0x6242U, 0x6243U, 0x6244U, 0x6245U, 0x6246U, 0x6247U, 0x6248U, + 0x6249U, 0x624AU, 0x624BU, 0x624CU, 0x624DU, 0x624EU, 0x624FU, 0x6250U, + 0x6251U, 0x6252U, 0x6253U, 0x6254U, 0x6255U, 0x6256U, 0x6257U, 0x6258U, + 0x6259U, 0x625AU, 0x6261U, 0x6262U, 0x6263U, 0x6264U, 0x6265U, 0x6266U, + 0x6267U, 0x6268U, 0x6269U, 0x626AU, 0x626BU, 0x626CU, 0x626DU, 0x626EU, + 0x626FU, 0x6270U, 0x6271U, 0x6272U, 0x6273U, 0x6274U, 0x6275U, 0x6276U, + 0x6277U, 0x6278U, 0x6279U, 0x627AU, 0x6230U, 0x6231U, 0x6232U, 0x6233U, + 0x6234U, 0x6235U, 0x6236U, 0x6237U, 0x6238U, 0x6239U, 0x622BU, 0x622FU, + 0x6341U, 0x6342U, 0x6343U, 0x6344U, 0x6345U, 0x6346U, 0x6347U, 0x6348U, + 0x6349U, 0x634AU, 0x634BU, 0x634CU, 0x634DU, 0x634EU, 0x634FU, 0x6350U, + 0x6351U, 0x6352U, 0x6353U, 0x6354U, 0x6355U, 0x6356U, 0x6357U, 0x6358U, + 0x6359U, 0x635AU, 0x6361U, 0x6362U, 0x6363U, 0x6364U, 0x6365U, 0x6366U, + 0x6367U, 0x6368U, 0x6369U, 0x636AU, 0x636BU, 0x636CU, 0x636DU, 0x636EU, + 0x636FU, 0x6370U, 0x6371U, 0x6372U, 0x6373U, 0x6374U, 0x6375U, 0x6376U, + 0x6377U, 0x6378U, 0x6379U, 0x637AU, 0x6330U, 0x6331U, 0x6332U, 0x6333U, + 0x6334U, 0x6335U, 0x6336U, 0x6337U, 0x6338U, 0x6339U, 0x632BU, 0x632FU, + 0x6441U, 0x6442U, 0x6443U, 0x6444U, 0x6445U, 0x6446U, 0x6447U, 0x6448U, + 0x6449U, 0x644AU, 0x644BU, 0x644CU, 0x644DU, 0x644EU, 0x644FU, 0x6450U, + 0x6451U, 0x6452U, 0x6453U, 0x6454U, 0x6455U, 0x6456U, 0x6457U, 0x6458U, + 0x6459U, 0x645AU, 0x6461U, 0x6462U, 0x6463U, 0x6464U, 0x6465U, 0x6466U, + 0x6467U, 0x6468U, 0x6469U, 0x646AU, 0x646BU, 0x646CU, 0x646DU, 0x646EU, + 0x646FU, 0x6470U, 0x6471U, 0x6472U, 0x6473U, 0x6474U, 0x6475U, 0x6476U, + 0x6477U, 0x6478U, 0x6479U, 0x647AU, 0x6430U, 0x6431U, 0x6432U, 0x6433U, + 0x6434U, 0x6435U, 0x6436U, 0x6437U, 0x6438U, 0x6439U, 0x642BU, 0x642FU, + 0x6541U, 0x6542U, 0x6543U, 0x6544U, 0x6545U, 0x6546U, 0x6547U, 0x6548U, + 0x6549U, 0x654AU, 0x654BU, 0x654CU, 0x654DU, 0x654EU, 0x654FU, 0x6550U, + 0x6551U, 0x6552U, 0x6553U, 0x6554U, 0x6555U, 0x6556U, 0x6557U, 0x6558U, + 0x6559U, 0x655AU, 0x6561U, 0x6562U, 0x6563U, 0x6564U, 0x6565U, 0x6566U, + 0x6567U, 0x6568U, 0x6569U, 0x656AU, 0x656BU, 0x656CU, 0x656DU, 0x656EU, + 0x656FU, 0x6570U, 0x6571U, 0x6572U, 0x6573U, 0x6574U, 0x6575U, 0x6576U, + 0x6577U, 0x6578U, 0x6579U, 0x657AU, 0x6530U, 0x6531U, 0x6532U, 0x6533U, + 0x6534U, 0x6535U, 0x6536U, 0x6537U, 0x6538U, 0x6539U, 0x652BU, 0x652FU, + 0x6641U, 0x6642U, 0x6643U, 0x6644U, 0x6645U, 0x6646U, 0x6647U, 0x6648U, + 0x6649U, 0x664AU, 0x664BU, 0x664CU, 0x664DU, 0x664EU, 0x664FU, 0x6650U, + 0x6651U, 0x6652U, 0x6653U, 0x6654U, 0x6655U, 0x6656U, 0x6657U, 0x6658U, + 0x6659U, 0x665AU, 0x6661U, 0x6662U, 0x6663U, 0x6664U, 0x6665U, 0x6666U, + 0x6667U, 0x6668U, 0x6669U, 0x666AU, 0x666BU, 0x666CU, 0x666DU, 0x666EU, + 0x666FU, 0x6670U, 0x6671U, 0x6672U, 0x6673U, 0x6674U, 0x6675U, 0x6676U, + 0x6677U, 0x6678U, 0x6679U, 0x667AU, 0x6630U, 0x6631U, 0x6632U, 0x6633U, + 0x6634U, 0x6635U, 0x6636U, 0x6637U, 0x6638U, 0x6639U, 0x662BU, 0x662FU, + 0x6741U, 0x6742U, 0x6743U, 0x6744U, 0x6745U, 0x6746U, 0x6747U, 0x6748U, + 0x6749U, 0x674AU, 0x674BU, 0x674CU, 0x674DU, 0x674EU, 0x674FU, 0x6750U, + 0x6751U, 0x6752U, 0x6753U, 0x6754U, 0x6755U, 0x6756U, 0x6757U, 0x6758U, + 0x6759U, 0x675AU, 0x6761U, 0x6762U, 0x6763U, 0x6764U, 0x6765U, 0x6766U, + 0x6767U, 0x6768U, 0x6769U, 0x676AU, 0x676BU, 0x676CU, 0x676DU, 0x676EU, + 0x676FU, 0x6770U, 0x6771U, 0x6772U, 0x6773U, 0x6774U, 0x6775U, 0x6776U, + 0x6777U, 0x6778U, 0x6779U, 0x677AU, 0x6730U, 0x6731U, 0x6732U, 0x6733U, + 0x6734U, 0x6735U, 0x6736U, 0x6737U, 0x6738U, 0x6739U, 0x672BU, 0x672FU, + 0x6841U, 0x6842U, 0x6843U, 0x6844U, 0x6845U, 0x6846U, 0x6847U, 0x6848U, + 0x6849U, 0x684AU, 0x684BU, 0x684CU, 0x684DU, 0x684EU, 0x684FU, 0x6850U, + 0x6851U, 0x6852U, 0x6853U, 0x6854U, 0x6855U, 0x6856U, 0x6857U, 0x6858U, + 0x6859U, 0x685AU, 0x6861U, 0x6862U, 0x6863U, 0x6864U, 0x6865U, 0x6866U, + 0x6867U, 0x6868U, 0x6869U, 0x686AU, 0x686BU, 0x686CU, 0x686DU, 0x686EU, + 0x686FU, 0x6870U, 0x6871U, 0x6872U, 0x6873U, 0x6874U, 0x6875U, 0x6876U, + 0x6877U, 0x6878U, 0x6879U, 0x687AU, 0x6830U, 0x6831U, 0x6832U, 0x6833U, + 0x6834U, 0x6835U, 0x6836U, 0x6837U, 0x6838U, 0x6839U, 0x682BU, 0x682FU, + 0x6941U, 0x6942U, 0x6943U, 0x6944U, 0x6945U, 0x6946U, 0x6947U, 0x6948U, + 0x6949U, 0x694AU, 0x694BU, 0x694CU, 0x694DU, 0x694EU, 0x694FU, 0x6950U, + 0x6951U, 0x6952U, 0x6953U, 0x6954U, 0x6955U, 0x6956U, 0x6957U, 0x6958U, + 0x6959U, 0x695AU, 0x6961U, 0x6962U, 0x6963U, 0x6964U, 0x6965U, 0x6966U, + 0x6967U, 0x6968U, 0x6969U, 0x696AU, 0x696BU, 0x696CU, 0x696DU, 0x696EU, + 0x696FU, 0x6970U, 0x6971U, 0x6972U, 0x6973U, 0x6974U, 0x6975U, 0x6976U, + 0x6977U, 0x6978U, 0x6979U, 0x697AU, 0x6930U, 0x6931U, 0x6932U, 0x6933U, + 0x6934U, 0x6935U, 0x6936U, 0x6937U, 0x6938U, 0x6939U, 0x692BU, 0x692FU, + 0x6A41U, 0x6A42U, 0x6A43U, 0x6A44U, 0x6A45U, 0x6A46U, 0x6A47U, 0x6A48U, + 0x6A49U, 0x6A4AU, 0x6A4BU, 0x6A4CU, 0x6A4DU, 0x6A4EU, 0x6A4FU, 0x6A50U, + 0x6A51U, 0x6A52U, 0x6A53U, 0x6A54U, 0x6A55U, 0x6A56U, 0x6A57U, 0x6A58U, + 0x6A59U, 0x6A5AU, 0x6A61U, 0x6A62U, 0x6A63U, 0x6A64U, 0x6A65U, 0x6A66U, + 0x6A67U, 0x6A68U, 0x6A69U, 0x6A6AU, 0x6A6BU, 0x6A6CU, 0x6A6DU, 0x6A6EU, + 0x6A6FU, 0x6A70U, 0x6A71U, 0x6A72U, 0x6A73U, 0x6A74U, 0x6A75U, 0x6A76U, + 0x6A77U, 0x6A78U, 0x6A79U, 0x6A7AU, 0x6A30U, 0x6A31U, 0x6A32U, 0x6A33U, + 0x6A34U, 0x6A35U, 0x6A36U, 0x6A37U, 0x6A38U, 0x6A39U, 0x6A2BU, 0x6A2FU, + 0x6B41U, 0x6B42U, 0x6B43U, 0x6B44U, 0x6B45U, 0x6B46U, 0x6B47U, 0x6B48U, + 0x6B49U, 0x6B4AU, 0x6B4BU, 0x6B4CU, 0x6B4DU, 0x6B4EU, 0x6B4FU, 0x6B50U, + 0x6B51U, 0x6B52U, 0x6B53U, 0x6B54U, 0x6B55U, 0x6B56U, 0x6B57U, 0x6B58U, + 0x6B59U, 0x6B5AU, 0x6B61U, 0x6B62U, 0x6B63U, 0x6B64U, 0x6B65U, 0x6B66U, + 0x6B67U, 0x6B68U, 0x6B69U, 0x6B6AU, 0x6B6BU, 0x6B6CU, 0x6B6DU, 0x6B6EU, + 0x6B6FU, 0x6B70U, 0x6B71U, 0x6B72U, 0x6B73U, 0x6B74U, 0x6B75U, 0x6B76U, + 0x6B77U, 0x6B78U, 0x6B79U, 0x6B7AU, 0x6B30U, 0x6B31U, 0x6B32U, 0x6B33U, + 0x6B34U, 0x6B35U, 0x6B36U, 0x6B37U, 0x6B38U, 0x6B39U, 0x6B2BU, 0x6B2FU, + 0x6C41U, 0x6C42U, 0x6C43U, 0x6C44U, 0x6C45U, 0x6C46U, 0x6C47U, 0x6C48U, + 0x6C49U, 0x6C4AU, 0x6C4BU, 0x6C4CU, 0x6C4DU, 0x6C4EU, 0x6C4FU, 0x6C50U, + 0x6C51U, 0x6C52U, 0x6C53U, 0x6C54U, 0x6C55U, 0x6C56U, 0x6C57U, 0x6C58U, + 0x6C59U, 0x6C5AU, 0x6C61U, 0x6C62U, 0x6C63U, 0x6C64U, 0x6C65U, 0x6C66U, + 0x6C67U, 0x6C68U, 0x6C69U, 0x6C6AU, 0x6C6BU, 0x6C6CU, 0x6C6DU, 0x6C6EU, + 0x6C6FU, 0x6C70U, 0x6C71U, 0x6C72U, 0x6C73U, 0x6C74U, 0x6C75U, 0x6C76U, + 0x6C77U, 0x6C78U, 0x6C79U, 0x6C7AU, 0x6C30U, 0x6C31U, 0x6C32U, 0x6C33U, + 0x6C34U, 0x6C35U, 0x6C36U, 0x6C37U, 0x6C38U, 0x6C39U, 0x6C2BU, 0x6C2FU, + 0x6D41U, 0x6D42U, 0x6D43U, 0x6D44U, 0x6D45U, 0x6D46U, 0x6D47U, 0x6D48U, + 0x6D49U, 0x6D4AU, 0x6D4BU, 0x6D4CU, 0x6D4DU, 0x6D4EU, 0x6D4FU, 0x6D50U, + 0x6D51U, 0x6D52U, 0x6D53U, 0x6D54U, 0x6D55U, 0x6D56U, 0x6D57U, 0x6D58U, + 0x6D59U, 0x6D5AU, 0x6D61U, 0x6D62U, 0x6D63U, 0x6D64U, 0x6D65U, 0x6D66U, + 0x6D67U, 0x6D68U, 0x6D69U, 0x6D6AU, 0x6D6BU, 0x6D6CU, 0x6D6DU, 0x6D6EU, + 0x6D6FU, 0x6D70U, 0x6D71U, 0x6D72U, 0x6D73U, 0x6D74U, 0x6D75U, 0x6D76U, + 0x6D77U, 0x6D78U, 0x6D79U, 0x6D7AU, 0x6D30U, 0x6D31U, 0x6D32U, 0x6D33U, + 0x6D34U, 0x6D35U, 0x6D36U, 0x6D37U, 0x6D38U, 0x6D39U, 0x6D2BU, 0x6D2FU, + 0x6E41U, 0x6E42U, 0x6E43U, 0x6E44U, 0x6E45U, 0x6E46U, 0x6E47U, 0x6E48U, + 0x6E49U, 0x6E4AU, 0x6E4BU, 0x6E4CU, 0x6E4DU, 0x6E4EU, 0x6E4FU, 0x6E50U, + 0x6E51U, 0x6E52U, 0x6E53U, 0x6E54U, 0x6E55U, 0x6E56U, 0x6E57U, 0x6E58U, + 0x6E59U, 0x6E5AU, 0x6E61U, 0x6E62U, 0x6E63U, 0x6E64U, 0x6E65U, 0x6E66U, + 0x6E67U, 0x6E68U, 0x6E69U, 0x6E6AU, 0x6E6BU, 0x6E6CU, 0x6E6DU, 0x6E6EU, + 0x6E6FU, 0x6E70U, 0x6E71U, 0x6E72U, 0x6E73U, 0x6E74U, 0x6E75U, 0x6E76U, + 0x6E77U, 0x6E78U, 0x6E79U, 0x6E7AU, 0x6E30U, 0x6E31U, 0x6E32U, 0x6E33U, + 0x6E34U, 0x6E35U, 0x6E36U, 0x6E37U, 0x6E38U, 0x6E39U, 0x6E2BU, 0x6E2FU, + 0x6F41U, 0x6F42U, 0x6F43U, 0x6F44U, 0x6F45U, 0x6F46U, 0x6F47U, 0x6F48U, + 0x6F49U, 0x6F4AU, 0x6F4BU, 0x6F4CU, 0x6F4DU, 0x6F4EU, 0x6F4FU, 0x6F50U, + 0x6F51U, 0x6F52U, 0x6F53U, 0x6F54U, 0x6F55U, 0x6F56U, 0x6F57U, 0x6F58U, + 0x6F59U, 0x6F5AU, 0x6F61U, 0x6F62U, 0x6F63U, 0x6F64U, 0x6F65U, 0x6F66U, + 0x6F67U, 0x6F68U, 0x6F69U, 0x6F6AU, 0x6F6BU, 0x6F6CU, 0x6F6DU, 0x6F6EU, + 0x6F6FU, 0x6F70U, 0x6F71U, 0x6F72U, 0x6F73U, 0x6F74U, 0x6F75U, 0x6F76U, + 0x6F77U, 0x6F78U, 0x6F79U, 0x6F7AU, 0x6F30U, 0x6F31U, 0x6F32U, 0x6F33U, + 0x6F34U, 0x6F35U, 0x6F36U, 0x6F37U, 0x6F38U, 0x6F39U, 0x6F2BU, 0x6F2FU, + 0x7041U, 0x7042U, 0x7043U, 0x7044U, 0x7045U, 0x7046U, 0x7047U, 0x7048U, + 0x7049U, 0x704AU, 0x704BU, 0x704CU, 0x704DU, 0x704EU, 0x704FU, 0x7050U, + 0x7051U, 0x7052U, 0x7053U, 0x7054U, 0x7055U, 0x7056U, 0x7057U, 0x7058U, + 0x7059U, 0x705AU, 0x7061U, 0x7062U, 0x7063U, 0x7064U, 0x7065U, 0x7066U, + 0x7067U, 0x7068U, 0x7069U, 0x706AU, 0x706BU, 0x706CU, 0x706DU, 0x706EU, + 0x706FU, 0x7070U, 0x7071U, 0x7072U, 0x7073U, 0x7074U, 0x7075U, 0x7076U, + 0x7077U, 0x7078U, 0x7079U, 0x707AU, 0x7030U, 0x7031U, 0x7032U, 0x7033U, + 0x7034U, 0x7035U, 0x7036U, 0x7037U, 0x7038U, 0x7039U, 0x702BU, 0x702FU, + 0x7141U, 0x7142U, 0x7143U, 0x7144U, 0x7145U, 0x7146U, 0x7147U, 0x7148U, + 0x7149U, 0x714AU, 0x714BU, 0x714CU, 0x714DU, 0x714EU, 0x714FU, 0x7150U, + 0x7151U, 0x7152U, 0x7153U, 0x7154U, 0x7155U, 0x7156U, 0x7157U, 0x7158U, + 0x7159U, 0x715AU, 0x7161U, 0x7162U, 0x7163U, 0x7164U, 0x7165U, 0x7166U, + 0x7167U, 0x7168U, 0x7169U, 0x716AU, 0x716BU, 0x716CU, 0x716DU, 0x716EU, + 0x716FU, 0x7170U, 0x7171U, 0x7172U, 0x7173U, 0x7174U, 0x7175U, 0x7176U, + 0x7177U, 0x7178U, 0x7179U, 0x717AU, 0x7130U, 0x7131U, 0x7132U, 0x7133U, + 0x7134U, 0x7135U, 0x7136U, 0x7137U, 0x7138U, 0x7139U, 0x712BU, 0x712FU, + 0x7241U, 0x7242U, 0x7243U, 0x7244U, 0x7245U, 0x7246U, 0x7247U, 0x7248U, + 0x7249U, 0x724AU, 0x724BU, 0x724CU, 0x724DU, 0x724EU, 0x724FU, 0x7250U, + 0x7251U, 0x7252U, 0x7253U, 0x7254U, 0x7255U, 0x7256U, 0x7257U, 0x7258U, + 0x7259U, 0x725AU, 0x7261U, 0x7262U, 0x7263U, 0x7264U, 0x7265U, 0x7266U, + 0x7267U, 0x7268U, 0x7269U, 0x726AU, 0x726BU, 0x726CU, 0x726DU, 0x726EU, + 0x726FU, 0x7270U, 0x7271U, 0x7272U, 0x7273U, 0x7274U, 0x7275U, 0x7276U, + 0x7277U, 0x7278U, 0x7279U, 0x727AU, 0x7230U, 0x7231U, 0x7232U, 0x7233U, + 0x7234U, 0x7235U, 0x7236U, 0x7237U, 0x7238U, 0x7239U, 0x722BU, 0x722FU, + 0x7341U, 0x7342U, 0x7343U, 0x7344U, 0x7345U, 0x7346U, 0x7347U, 0x7348U, + 0x7349U, 0x734AU, 0x734BU, 0x734CU, 0x734DU, 0x734EU, 0x734FU, 0x7350U, + 0x7351U, 0x7352U, 0x7353U, 0x7354U, 0x7355U, 0x7356U, 0x7357U, 0x7358U, + 0x7359U, 0x735AU, 0x7361U, 0x7362U, 0x7363U, 0x7364U, 0x7365U, 0x7366U, + 0x7367U, 0x7368U, 0x7369U, 0x736AU, 0x736BU, 0x736CU, 0x736DU, 0x736EU, + 0x736FU, 0x7370U, 0x7371U, 0x7372U, 0x7373U, 0x7374U, 0x7375U, 0x7376U, + 0x7377U, 0x7378U, 0x7379U, 0x737AU, 0x7330U, 0x7331U, 0x7332U, 0x7333U, + 0x7334U, 0x7335U, 0x7336U, 0x7337U, 0x7338U, 0x7339U, 0x732BU, 0x732FU, + 0x7441U, 0x7442U, 0x7443U, 0x7444U, 0x7445U, 0x7446U, 0x7447U, 0x7448U, + 0x7449U, 0x744AU, 0x744BU, 0x744CU, 0x744DU, 0x744EU, 0x744FU, 0x7450U, + 0x7451U, 0x7452U, 0x7453U, 0x7454U, 0x7455U, 0x7456U, 0x7457U, 0x7458U, + 0x7459U, 0x745AU, 0x7461U, 0x7462U, 0x7463U, 0x7464U, 0x7465U, 0x7466U, + 0x7467U, 0x7468U, 0x7469U, 0x746AU, 0x746BU, 0x746CU, 0x746DU, 0x746EU, + 0x746FU, 0x7470U, 0x7471U, 0x7472U, 0x7473U, 0x7474U, 0x7475U, 0x7476U, + 0x7477U, 0x7478U, 0x7479U, 0x747AU, 0x7430U, 0x7431U, 0x7432U, 0x7433U, + 0x7434U, 0x7435U, 0x7436U, 0x7437U, 0x7438U, 0x7439U, 0x742BU, 0x742FU, + 0x7541U, 0x7542U, 0x7543U, 0x7544U, 0x7545U, 0x7546U, 0x7547U, 0x7548U, + 0x7549U, 0x754AU, 0x754BU, 0x754CU, 0x754DU, 0x754EU, 0x754FU, 0x7550U, + 0x7551U, 0x7552U, 0x7553U, 0x7554U, 0x7555U, 0x7556U, 0x7557U, 0x7558U, + 0x7559U, 0x755AU, 0x7561U, 0x7562U, 0x7563U, 0x7564U, 0x7565U, 0x7566U, + 0x7567U, 0x7568U, 0x7569U, 0x756AU, 0x756BU, 0x756CU, 0x756DU, 0x756EU, + 0x756FU, 0x7570U, 0x7571U, 0x7572U, 0x7573U, 0x7574U, 0x7575U, 0x7576U, + 0x7577U, 0x7578U, 0x7579U, 0x757AU, 0x7530U, 0x7531U, 0x7532U, 0x7533U, + 0x7534U, 0x7535U, 0x7536U, 0x7537U, 0x7538U, 0x7539U, 0x752BU, 0x752FU, + 0x7641U, 0x7642U, 0x7643U, 0x7644U, 0x7645U, 0x7646U, 0x7647U, 0x7648U, + 0x7649U, 0x764AU, 0x764BU, 0x764CU, 0x764DU, 0x764EU, 0x764FU, 0x7650U, + 0x7651U, 0x7652U, 0x7653U, 0x7654U, 0x7655U, 0x7656U, 0x7657U, 0x7658U, + 0x7659U, 0x765AU, 0x7661U, 0x7662U, 0x7663U, 0x7664U, 0x7665U, 0x7666U, + 0x7667U, 0x7668U, 0x7669U, 0x766AU, 0x766BU, 0x766CU, 0x766DU, 0x766EU, + 0x766FU, 0x7670U, 0x7671U, 0x7672U, 0x7673U, 0x7674U, 0x7675U, 0x7676U, + 0x7677U, 0x7678U, 0x7679U, 0x767AU, 0x7630U, 0x7631U, 0x7632U, 0x7633U, + 0x7634U, 0x7635U, 0x7636U, 0x7637U, 0x7638U, 0x7639U, 0x762BU, 0x762FU, + 0x7741U, 0x7742U, 0x7743U, 0x7744U, 0x7745U, 0x7746U, 0x7747U, 0x7748U, + 0x7749U, 0x774AU, 0x774BU, 0x774CU, 0x774DU, 0x774EU, 0x774FU, 0x7750U, + 0x7751U, 0x7752U, 0x7753U, 0x7754U, 0x7755U, 0x7756U, 0x7757U, 0x7758U, + 0x7759U, 0x775AU, 0x7761U, 0x7762U, 0x7763U, 0x7764U, 0x7765U, 0x7766U, + 0x7767U, 0x7768U, 0x7769U, 0x776AU, 0x776BU, 0x776CU, 0x776DU, 0x776EU, + 0x776FU, 0x7770U, 0x7771U, 0x7772U, 0x7773U, 0x7774U, 0x7775U, 0x7776U, + 0x7777U, 0x7778U, 0x7779U, 0x777AU, 0x7730U, 0x7731U, 0x7732U, 0x7733U, + 0x7734U, 0x7735U, 0x7736U, 0x7737U, 0x7738U, 0x7739U, 0x772BU, 0x772FU, + 0x7841U, 0x7842U, 0x7843U, 0x7844U, 0x7845U, 0x7846U, 0x7847U, 0x7848U, + 0x7849U, 0x784AU, 0x784BU, 0x784CU, 0x784DU, 0x784EU, 0x784FU, 0x7850U, + 0x7851U, 0x7852U, 0x7853U, 0x7854U, 0x7855U, 0x7856U, 0x7857U, 0x7858U, + 0x7859U, 0x785AU, 0x7861U, 0x7862U, 0x7863U, 0x7864U, 0x7865U, 0x7866U, + 0x7867U, 0x7868U, 0x7869U, 0x786AU, 0x786BU, 0x786CU, 0x786DU, 0x786EU, + 0x786FU, 0x7870U, 0x7871U, 0x7872U, 0x7873U, 0x7874U, 0x7875U, 0x7876U, + 0x7877U, 0x7878U, 0x7879U, 0x787AU, 0x7830U, 0x7831U, 0x7832U, 0x7833U, + 0x7834U, 0x7835U, 0x7836U, 0x7837U, 0x7838U, 0x7839U, 0x782BU, 0x782FU, + 0x7941U, 0x7942U, 0x7943U, 0x7944U, 0x7945U, 0x7946U, 0x7947U, 0x7948U, + 0x7949U, 0x794AU, 0x794BU, 0x794CU, 0x794DU, 0x794EU, 0x794FU, 0x7950U, + 0x7951U, 0x7952U, 0x7953U, 0x7954U, 0x7955U, 0x7956U, 0x7957U, 0x7958U, + 0x7959U, 0x795AU, 0x7961U, 0x7962U, 0x7963U, 0x7964U, 0x7965U, 0x7966U, + 0x7967U, 0x7968U, 0x7969U, 0x796AU, 0x796BU, 0x796CU, 0x796DU, 0x796EU, + 0x796FU, 0x7970U, 0x7971U, 0x7972U, 0x7973U, 0x7974U, 0x7975U, 0x7976U, + 0x7977U, 0x7978U, 0x7979U, 0x797AU, 0x7930U, 0x7931U, 0x7932U, 0x7933U, + 0x7934U, 0x7935U, 0x7936U, 0x7937U, 0x7938U, 0x7939U, 0x792BU, 0x792FU, + 0x7A41U, 0x7A42U, 0x7A43U, 0x7A44U, 0x7A45U, 0x7A46U, 0x7A47U, 0x7A48U, + 0x7A49U, 0x7A4AU, 0x7A4BU, 0x7A4CU, 0x7A4DU, 0x7A4EU, 0x7A4FU, 0x7A50U, + 0x7A51U, 0x7A52U, 0x7A53U, 0x7A54U, 0x7A55U, 0x7A56U, 0x7A57U, 0x7A58U, + 0x7A59U, 0x7A5AU, 0x7A61U, 0x7A62U, 0x7A63U, 0x7A64U, 0x7A65U, 0x7A66U, + 0x7A67U, 0x7A68U, 0x7A69U, 0x7A6AU, 0x7A6BU, 0x7A6CU, 0x7A6DU, 0x7A6EU, + 0x7A6FU, 0x7A70U, 0x7A71U, 0x7A72U, 0x7A73U, 0x7A74U, 0x7A75U, 0x7A76U, + 0x7A77U, 0x7A78U, 0x7A79U, 0x7A7AU, 0x7A30U, 0x7A31U, 0x7A32U, 0x7A33U, + 0x7A34U, 0x7A35U, 0x7A36U, 0x7A37U, 0x7A38U, 0x7A39U, 0x7A2BU, 0x7A2FU, + 0x3041U, 0x3042U, 0x3043U, 0x3044U, 0x3045U, 0x3046U, 0x3047U, 0x3048U, + 0x3049U, 0x304AU, 0x304BU, 0x304CU, 0x304DU, 0x304EU, 0x304FU, 0x3050U, + 0x3051U, 0x3052U, 0x3053U, 0x3054U, 0x3055U, 0x3056U, 0x3057U, 0x3058U, + 0x3059U, 0x305AU, 0x3061U, 0x3062U, 0x3063U, 0x3064U, 0x3065U, 0x3066U, + 0x3067U, 0x3068U, 0x3069U, 0x306AU, 0x306BU, 0x306CU, 0x306DU, 0x306EU, + 0x306FU, 0x3070U, 0x3071U, 0x3072U, 0x3073U, 0x3074U, 0x3075U, 0x3076U, + 0x3077U, 0x3078U, 0x3079U, 0x307AU, 0x3030U, 0x3031U, 0x3032U, 0x3033U, + 0x3034U, 0x3035U, 0x3036U, 0x3037U, 0x3038U, 0x3039U, 0x302BU, 0x302FU, + 0x3141U, 0x3142U, 0x3143U, 0x3144U, 0x3145U, 0x3146U, 0x3147U, 0x3148U, + 0x3149U, 0x314AU, 0x314BU, 0x314CU, 0x314DU, 0x314EU, 0x314FU, 0x3150U, + 0x3151U, 0x3152U, 0x3153U, 0x3154U, 0x3155U, 0x3156U, 0x3157U, 0x3158U, + 0x3159U, 0x315AU, 0x3161U, 0x3162U, 0x3163U, 0x3164U, 0x3165U, 0x3166U, + 0x3167U, 0x3168U, 0x3169U, 0x316AU, 0x316BU, 0x316CU, 0x316DU, 0x316EU, + 0x316FU, 0x3170U, 0x3171U, 0x3172U, 0x3173U, 0x3174U, 0x3175U, 0x3176U, + 0x3177U, 0x3178U, 0x3179U, 0x317AU, 0x3130U, 0x3131U, 0x3132U, 0x3133U, + 0x3134U, 0x3135U, 0x3136U, 0x3137U, 0x3138U, 0x3139U, 0x312BU, 0x312FU, + 0x3241U, 0x3242U, 0x3243U, 0x3244U, 0x3245U, 0x3246U, 0x3247U, 0x3248U, + 0x3249U, 0x324AU, 0x324BU, 0x324CU, 0x324DU, 0x324EU, 0x324FU, 0x3250U, + 0x3251U, 0x3252U, 0x3253U, 0x3254U, 0x3255U, 0x3256U, 0x3257U, 0x3258U, + 0x3259U, 0x325AU, 0x3261U, 0x3262U, 0x3263U, 0x3264U, 0x3265U, 0x3266U, + 0x3267U, 0x3268U, 0x3269U, 0x326AU, 0x326BU, 0x326CU, 0x326DU, 0x326EU, + 0x326FU, 0x3270U, 0x3271U, 0x3272U, 0x3273U, 0x3274U, 0x3275U, 0x3276U, + 0x3277U, 0x3278U, 0x3279U, 0x327AU, 0x3230U, 0x3231U, 0x3232U, 0x3233U, + 0x3234U, 0x3235U, 0x3236U, 0x3237U, 0x3238U, 0x3239U, 0x322BU, 0x322FU, + 0x3341U, 0x3342U, 0x3343U, 0x3344U, 0x3345U, 0x3346U, 0x3347U, 0x3348U, + 0x3349U, 0x334AU, 0x334BU, 0x334CU, 0x334DU, 0x334EU, 0x334FU, 0x3350U, + 0x3351U, 0x3352U, 0x3353U, 0x3354U, 0x3355U, 0x3356U, 0x3357U, 0x3358U, + 0x3359U, 0x335AU, 0x3361U, 0x3362U, 0x3363U, 0x3364U, 0x3365U, 0x3366U, + 0x3367U, 0x3368U, 0x3369U, 0x336AU, 0x336BU, 0x336CU, 0x336DU, 0x336EU, + 0x336FU, 0x3370U, 0x3371U, 0x3372U, 0x3373U, 0x3374U, 0x3375U, 0x3376U, + 0x3377U, 0x3378U, 0x3379U, 0x337AU, 0x3330U, 0x3331U, 0x3332U, 0x3333U, + 0x3334U, 0x3335U, 0x3336U, 0x3337U, 0x3338U, 0x3339U, 0x332BU, 0x332FU, + 0x3441U, 0x3442U, 0x3443U, 0x3444U, 0x3445U, 0x3446U, 0x3447U, 0x3448U, + 0x3449U, 0x344AU, 0x344BU, 0x344CU, 0x344DU, 0x344EU, 0x344FU, 0x3450U, + 0x3451U, 0x3452U, 0x3453U, 0x3454U, 0x3455U, 0x3456U, 0x3457U, 0x3458U, + 0x3459U, 0x345AU, 0x3461U, 0x3462U, 0x3463U, 0x3464U, 0x3465U, 0x3466U, + 0x3467U, 0x3468U, 0x3469U, 0x346AU, 0x346BU, 0x346CU, 0x346DU, 0x346EU, + 0x346FU, 0x3470U, 0x3471U, 0x3472U, 0x3473U, 0x3474U, 0x3475U, 0x3476U, + 0x3477U, 0x3478U, 0x3479U, 0x347AU, 0x3430U, 0x3431U, 0x3432U, 0x3433U, + 0x3434U, 0x3435U, 0x3436U, 0x3437U, 0x3438U, 0x3439U, 0x342BU, 0x342FU, + 0x3541U, 0x3542U, 0x3543U, 0x3544U, 0x3545U, 0x3546U, 0x3547U, 0x3548U, + 0x3549U, 0x354AU, 0x354BU, 0x354CU, 0x354DU, 0x354EU, 0x354FU, 0x3550U, + 0x3551U, 0x3552U, 0x3553U, 0x3554U, 0x3555U, 0x3556U, 0x3557U, 0x3558U, + 0x3559U, 0x355AU, 0x3561U, 0x3562U, 0x3563U, 0x3564U, 0x3565U, 0x3566U, + 0x3567U, 0x3568U, 0x3569U, 0x356AU, 0x356BU, 0x356CU, 0x356DU, 0x356EU, + 0x356FU, 0x3570U, 0x3571U, 0x3572U, 0x3573U, 0x3574U, 0x3575U, 0x3576U, + 0x3577U, 0x3578U, 0x3579U, 0x357AU, 0x3530U, 0x3531U, 0x3532U, 0x3533U, + 0x3534U, 0x3535U, 0x3536U, 0x3537U, 0x3538U, 0x3539U, 0x352BU, 0x352FU, + 0x3641U, 0x3642U, 0x3643U, 0x3644U, 0x3645U, 0x3646U, 0x3647U, 0x3648U, + 0x3649U, 0x364AU, 0x364BU, 0x364CU, 0x364DU, 0x364EU, 0x364FU, 0x3650U, + 0x3651U, 0x3652U, 0x3653U, 0x3654U, 0x3655U, 0x3656U, 0x3657U, 0x3658U, + 0x3659U, 0x365AU, 0x3661U, 0x3662U, 0x3663U, 0x3664U, 0x3665U, 0x3666U, + 0x3667U, 0x3668U, 0x3669U, 0x366AU, 0x366BU, 0x366CU, 0x366DU, 0x366EU, + 0x366FU, 0x3670U, 0x3671U, 0x3672U, 0x3673U, 0x3674U, 0x3675U, 0x3676U, + 0x3677U, 0x3678U, 0x3679U, 0x367AU, 0x3630U, 0x3631U, 0x3632U, 0x3633U, + 0x3634U, 0x3635U, 0x3636U, 0x3637U, 0x3638U, 0x3639U, 0x362BU, 0x362FU, + 0x3741U, 0x3742U, 0x3743U, 0x3744U, 0x3745U, 0x3746U, 0x3747U, 0x3748U, + 0x3749U, 0x374AU, 0x374BU, 0x374CU, 0x374DU, 0x374EU, 0x374FU, 0x3750U, + 0x3751U, 0x3752U, 0x3753U, 0x3754U, 0x3755U, 0x3756U, 0x3757U, 0x3758U, + 0x3759U, 0x375AU, 0x3761U, 0x3762U, 0x3763U, 0x3764U, 0x3765U, 0x3766U, + 0x3767U, 0x3768U, 0x3769U, 0x376AU, 0x376BU, 0x376CU, 0x376DU, 0x376EU, + 0x376FU, 0x3770U, 0x3771U, 0x3772U, 0x3773U, 0x3774U, 0x3775U, 0x3776U, + 0x3777U, 0x3778U, 0x3779U, 0x377AU, 0x3730U, 0x3731U, 0x3732U, 0x3733U, + 0x3734U, 0x3735U, 0x3736U, 0x3737U, 0x3738U, 0x3739U, 0x372BU, 0x372FU, + 0x3841U, 0x3842U, 0x3843U, 0x3844U, 0x3845U, 0x3846U, 0x3847U, 0x3848U, + 0x3849U, 0x384AU, 0x384BU, 0x384CU, 0x384DU, 0x384EU, 0x384FU, 0x3850U, + 0x3851U, 0x3852U, 0x3853U, 0x3854U, 0x3855U, 0x3856U, 0x3857U, 0x3858U, + 0x3859U, 0x385AU, 0x3861U, 0x3862U, 0x3863U, 0x3864U, 0x3865U, 0x3866U, + 0x3867U, 0x3868U, 0x3869U, 0x386AU, 0x386BU, 0x386CU, 0x386DU, 0x386EU, + 0x386FU, 0x3870U, 0x3871U, 0x3872U, 0x3873U, 0x3874U, 0x3875U, 0x3876U, + 0x3877U, 0x3878U, 0x3879U, 0x387AU, 0x3830U, 0x3831U, 0x3832U, 0x3833U, + 0x3834U, 0x3835U, 0x3836U, 0x3837U, 0x3838U, 0x3839U, 0x382BU, 0x382FU, + 0x3941U, 0x3942U, 0x3943U, 0x3944U, 0x3945U, 0x3946U, 0x3947U, 0x3948U, + 0x3949U, 0x394AU, 0x394BU, 0x394CU, 0x394DU, 0x394EU, 0x394FU, 0x3950U, + 0x3951U, 0x3952U, 0x3953U, 0x3954U, 0x3955U, 0x3956U, 0x3957U, 0x3958U, + 0x3959U, 0x395AU, 0x3961U, 0x3962U, 0x3963U, 0x3964U, 0x3965U, 0x3966U, + 0x3967U, 0x3968U, 0x3969U, 0x396AU, 0x396BU, 0x396CU, 0x396DU, 0x396EU, + 0x396FU, 0x3970U, 0x3971U, 0x3972U, 0x3973U, 0x3974U, 0x3975U, 0x3976U, + 0x3977U, 0x3978U, 0x3979U, 0x397AU, 0x3930U, 0x3931U, 0x3932U, 0x3933U, + 0x3934U, 0x3935U, 0x3936U, 0x3937U, 0x3938U, 0x3939U, 0x392BU, 0x392FU, + 0x2B41U, 0x2B42U, 0x2B43U, 0x2B44U, 0x2B45U, 0x2B46U, 0x2B47U, 0x2B48U, + 0x2B49U, 0x2B4AU, 0x2B4BU, 0x2B4CU, 0x2B4DU, 0x2B4EU, 0x2B4FU, 0x2B50U, + 0x2B51U, 0x2B52U, 0x2B53U, 0x2B54U, 0x2B55U, 0x2B56U, 0x2B57U, 0x2B58U, + 0x2B59U, 0x2B5AU, 0x2B61U, 0x2B62U, 0x2B63U, 0x2B64U, 0x2B65U, 0x2B66U, + 0x2B67U, 0x2B68U, 0x2B69U, 0x2B6AU, 0x2B6BU, 0x2B6CU, 0x2B6DU, 0x2B6EU, + 0x2B6FU, 0x2B70U, 0x2B71U, 0x2B72U, 0x2B73U, 0x2B74U, 0x2B75U, 0x2B76U, + 0x2B77U, 0x2B78U, 0x2B79U, 0x2B7AU, 0x2B30U, 0x2B31U, 0x2B32U, 0x2B33U, + 0x2B34U, 0x2B35U, 0x2B36U, 0x2B37U, 0x2B38U, 0x2B39U, 0x2B2BU, 0x2B2FU, + 0x2F41U, 0x2F42U, 0x2F43U, 0x2F44U, 0x2F45U, 0x2F46U, 0x2F47U, 0x2F48U, + 0x2F49U, 0x2F4AU, 0x2F4BU, 0x2F4CU, 0x2F4DU, 0x2F4EU, 0x2F4FU, 0x2F50U, + 0x2F51U, 0x2F52U, 0x2F53U, 0x2F54U, 0x2F55U, 0x2F56U, 0x2F57U, 0x2F58U, + 0x2F59U, 0x2F5AU, 0x2F61U, 0x2F62U, 0x2F63U, 0x2F64U, 0x2F65U, 0x2F66U, + 0x2F67U, 0x2F68U, 0x2F69U, 0x2F6AU, 0x2F6BU, 0x2F6CU, 0x2F6DU, 0x2F6EU, + 0x2F6FU, 0x2F70U, 0x2F71U, 0x2F72U, 0x2F73U, 0x2F74U, 0x2F75U, 0x2F76U, + 0x2F77U, 0x2F78U, 0x2F79U, 0x2F7AU, 0x2F30U, 0x2F31U, 0x2F32U, 0x2F33U, + 0x2F34U, 0x2F35U, 0x2F36U, 0x2F37U, 0x2F38U, 0x2F39U, 0x2F2BU, 0x2F2FU, +#endif +}; diff --git a/mypyc/lib-rt/base64/tables/tables.c b/mypyc/lib-rt/base64/tables/tables.c new file mode 100644 index 000000000000..45778b6befdd --- /dev/null +++ b/mypyc/lib-rt/base64/tables/tables.c @@ -0,0 +1,40 @@ +#include "tables.h" + +const uint8_t +base64_table_enc_6bit[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789" + "+/"; + +// In the lookup table below, note that the value for '=' (character 61) is +// 254, not 255. This character is used for in-band signaling of the end of +// the datastream, and we will use that later. The characters A-Z, a-z, 0-9 +// and + / are mapped to their "decoded" values. The other bytes all map to +// the value 255, which flags them as "invalid input". + +const uint8_t +base64_table_dec_8bit[] = +{ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0..15 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 16..31 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, // 32..47 + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 254, 255, 255, // 48..63 + 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, // 64..79 + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, // 80..95 + 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, // 96..111 + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255, // 112..127 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 128..143 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +}; + +#if BASE64_WORDSIZE >= 32 +# include "table_dec_32bit.h" +# include "table_enc_12bit.h" +#endif diff --git a/mypyc/lib-rt/base64/tables/tables.h b/mypyc/lib-rt/base64/tables/tables.h new file mode 100644 index 000000000000..cb74268a4bf1 --- /dev/null +++ b/mypyc/lib-rt/base64/tables/tables.h @@ -0,0 +1,23 @@ +#ifndef BASE64_TABLES_H +#define BASE64_TABLES_H + +#include + +#include "../env.h" + +// These tables are used by all codecs for fallback plain encoding/decoding: +extern const uint8_t base64_table_enc_6bit[]; +extern const uint8_t base64_table_dec_8bit[]; + +// These tables are used for the 32-bit and 64-bit generic decoders: +#if BASE64_WORDSIZE >= 32 +extern const uint32_t base64_table_dec_32bit_d0[]; +extern const uint32_t base64_table_dec_32bit_d1[]; +extern const uint32_t base64_table_dec_32bit_d2[]; +extern const uint32_t base64_table_dec_32bit_d3[]; + +// This table is used by the 32 and 64-bit generic encoders: +extern const uint16_t base64_table_enc_12bit[]; +#endif + +#endif // BASE64_TABLES_H diff --git a/mypyc/lib-rt/librt_base64.c b/mypyc/lib-rt/librt_base64.c index 1c3a6f8d01a5..020a56e412f4 100644 --- a/mypyc/lib-rt/librt_base64.c +++ b/mypyc/lib-rt/librt_base64.c @@ -1,24 +1,19 @@ #define PY_SSIZE_T_CLEAN #include #include "librt_base64.h" +#include "libbase64.h" #include "pythoncapi_compat.h" #ifdef MYPYC_EXPERIMENTAL -// b64encode_internal below is adapted from the CPython 3.14.0 binascii module - -static const unsigned char table_b2a_base64[] = -"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -#define BASE64_PAD '=' - -/* Max binary chunk size; limited only by available memory */ #define BASE64_MAXBIN ((PY_SSIZE_T_MAX - 3) / 2) +#define STACK_BUFFER_SIZE 1024 + static PyObject * b64encode_internal(PyObject *obj) { unsigned char *ascii_data; - const unsigned char *bin_data; + char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -31,51 +26,32 @@ b64encode_internal(PyObject *obj) { return NULL; } - bin_data = (const unsigned char *)PyBytes_AS_STRING(obj); + bin_data = PyBytes_AS_STRING(obj); bin_len = PyBytes_GET_SIZE(obj); - assert(bin_len >= 0); - if ( bin_len > BASE64_MAXBIN ) { + if (bin_len > BASE64_MAXBIN) { PyErr_SetString(PyExc_ValueError, "Too much data for base64 line"); return NULL; } - /* We're lazy and allocate too much (fixed up later). - "+2" leaves room for up to two pad characters. - Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ - out_len = bin_len*2 + 2; - if (newline) - out_len++; - writer = PyBytesWriter_Create(out_len); - ascii_data = PyBytesWriter_GetData(writer); - if (writer == NULL) - return NULL; - - for( ; bin_len > 0 ; bin_len--, bin_data++ ) { - /* Shift the data into our buffer */ - leftchar = (leftchar << 8) | *bin_data; - leftbits += 8; - - /* See if there are 6-bit groups ready */ - while ( leftbits >= 6 ) { - this_ch = (leftchar >> (leftbits-6)) & 0x3f; - leftbits -= 6; - *ascii_data++ = table_b2a_base64[this_ch]; + Py_ssize_t buflen = 4 * bin_len / 3 + 4; + char *buf; + char stack_buf[STACK_BUFFER_SIZE]; + if (buflen <= STACK_BUFFER_SIZE) { + buf = stack_buf; + } else { + buf = PyMem_Malloc(buflen); + if (buf == NULL) { + return PyErr_NoMemory(); } } - if ( leftbits == 2 ) { - *ascii_data++ = table_b2a_base64[(leftchar&3) << 4]; - *ascii_data++ = BASE64_PAD; - *ascii_data++ = BASE64_PAD; - } else if ( leftbits == 4 ) { - *ascii_data++ = table_b2a_base64[(leftchar&0xf) << 2]; - *ascii_data++ = BASE64_PAD; - } - if (newline) - *ascii_data++ = '\n'; /* Append a courtesy newline */ - - return PyBytesWriter_FinishWithSize(writer, ascii_data - (unsigned char *)PyBytesWriter_GetData(writer)); + size_t actual_len; + base64_encode(bin_data, bin_len, buf, &actual_len, 0); + PyObject *res = PyBytes_FromStringAndSize(buf, actual_len); + if (buflen > STACK_BUFFER_SIZE) + PyMem_Free(buf); + return res; } static PyObject* diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 3c08c7dbb5ee..acd61458e516 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -98,7 +98,24 @@ def run(self) -> None: extra_compile_args=cflags, ), Extension( - "librt.base64", ["librt_base64.c"], include_dirs=["."], extra_compile_args=cflags + "librt.base64", + [ + "librt_base64.c", + "base64/lib.c", + "base64/codec_choose.c", + "base64/tables/tables.c", + "base64/arch/generic/codec.c", + "base64/arch/ssse3/codec.c", + "base64/arch/sse41/codec.c", + "base64/arch/sse42/codec.c", + "base64/arch/avx/codec.c", + "base64/arch/avx2/codec.c", + "base64/arch/avx512/codec.c", + "base64/arch/neon32/codec.c", + "base64/arch/neon64/codec.c", + ], + include_dirs=[".", "base64"], + extra_compile_args=cflags, ), ] ) From 1b6ebb17b7fe64488a7b3c3b4b0187bb14fe331b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 17 Nov 2025 15:39:31 +0000 Subject: [PATCH 170/183] [mypyc] Enable SIMD for librt.base64 on x86-64 (#20244) Also generally enable SSE4.2 instructions when targeting x86-64. These have been supported by hardware since ~2010, so it seems fine to require them now. This speeds up `b64encode` by up to 100% on Linux running on a recent AMD CPU. Some fairly recent hardware doesn't support AVX2, so it's not enabled. We'd probably need to rely on hardware capability checking for AVX2 support, and we'd need compile different files with different architecture flags probably, and I didn't want to go there (at least not yet). --- mypyc/build.py | 15 ++++++++++++++- mypyc/common.py | 3 +++ mypyc/lib-rt/base64/config.h | 5 +++++ mypyc/lib-rt/setup.py | 7 +++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/mypyc/build.py b/mypyc/build.py index 8505a2d95701..02f427c83426 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -36,7 +36,7 @@ from mypy.util import write_junit_xml from mypyc.annotate import generate_annotated_html from mypyc.codegen import emitmodule -from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, shared_lib_name +from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, X86_64, shared_lib_name from mypyc.errors import Errors from mypyc.ir.pprint import format_modules from mypyc.namegen import exported_name @@ -77,6 +77,12 @@ class ModDesc(NamedTuple): "base64/arch/generic/enc_tail.c", "base64/arch/generic/dec_head.c", "base64/arch/generic/dec_tail.c", + "base64/arch/ssse3/dec_reshuffle.c", + "base64/arch/ssse3/dec_loop.c", + "base64/arch/ssse3/enc_loop_asm.c", + "base64/arch/ssse3/enc_translate.c", + "base64/arch/ssse3/enc_reshuffle.c", + "base64/arch/ssse3/enc_loop.c", "base64/arch/neon64/dec_loop.c", "base64/arch/neon64/enc_loop_asm.c", "base64/codecs.h", @@ -655,6 +661,9 @@ def mypycify( # See https://github.com/mypyc/mypyc/issues/956 "-Wno-cpp", ] + if X86_64: + # Enable SIMD extensions. All CPUs released since ~2010 support SSE4.2. + cflags.append("-msse4.2") if log_trace: cflags.append("-DMYPYC_LOG_TRACE") if experimental_features: @@ -683,6 +692,10 @@ def mypycify( # that we actually get the compilation speed and memory # use wins that multi-file mode is intended for. cflags += ["/GL-", "/wd9025"] # warning about overriding /GL + if X86_64: + # Enable SIMD extensions. All CPUs released since ~2010 support SSE4.2. + # Also Windows 11 requires SSE4.2 since 24H2. + cflags.append("/arch:SSE4.2") if log_trace: cflags.append("/DMYPYC_LOG_TRACE") if experimental_features: diff --git a/mypyc/common.py b/mypyc/common.py index 2de63c09bb2c..98f8a89f6fcb 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -1,5 +1,6 @@ from __future__ import annotations +import platform import sys import sysconfig from typing import Any, Final @@ -44,6 +45,8 @@ IS_32_BIT_PLATFORM: Final = int(SIZEOF_SIZE_T) == 4 +X86_64: Final = platform.machine() in ("x86_64", "AMD64", "amd64") + PLATFORM_SIZE = 4 if IS_32_BIT_PLATFORM else 8 # Maximum value for a short tagged integer. diff --git a/mypyc/lib-rt/base64/config.h b/mypyc/lib-rt/base64/config.h index fd516c4be2d6..b5e47fb04e75 100644 --- a/mypyc/lib-rt/base64/config.h +++ b/mypyc/lib-rt/base64/config.h @@ -7,7 +7,12 @@ #define BASE64_WITH_SSE41 0 #define HAVE_SSE41 BASE64_WITH_SSE41 +#if defined(__x86_64__) || defined(_M_X64) +#define BASE64_WITH_SSE42 1 +#else #define BASE64_WITH_SSE42 0 +#endif + #define HAVE_SSE42 BASE64_WITH_SSE42 #define BASE64_WITH_AVX 0 diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index acd61458e516..6a56c65306ae 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -6,6 +6,7 @@ from __future__ import annotations import os +import platform import subprocess import sys from distutils import ccompiler, sysconfig @@ -24,6 +25,8 @@ "pythonsupport.c", ] +X86_64 = platform.machine() in ("x86_64", "AMD64", "amd64") + class BuildExtGtest(build_ext): def get_library_names(self) -> list[str]: @@ -79,8 +82,12 @@ def run(self) -> None: cflags: list[str] = [] if compiler.compiler_type == "unix": cflags += ["-O3"] + if X86_64: + cflags.append("-msse4.2") # Enable SIMD (see also mypyc/build.py) elif compiler.compiler_type == "msvc": cflags += ["/O2"] + if X86_64: + cflags.append("/arch:SSE4.2") # Enable SIMD (see also mypyc/build.py) setup( ext_modules=[ From 0c6593b117ae3be29d9549c7baa81f591953836c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 18 Nov 2025 10:47:34 +0000 Subject: [PATCH 171/183] [mypyc] Fix async or generator methods in traits (#20246) Disable optimization that doesn't work for trait methods. We could probably make the optimization work properly, but it would take significant work, so focus on fixing a regression. Fixes mypyc/mypyc#1141. #20044 was a previous fix attempt that fixes some other use cases, but it didn't address traits. --- mypyc/irbuild/prepare.py | 4 ++++ mypyc/test-data/run-async.test | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 0f7cc7e3b3c5..9f3c7fc6f270 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -842,6 +842,10 @@ def adjust_generator_classes_of_methods(mapper: Mapper) -> None: if subcls is None: # Override could be of a different type, so we can't make assumptions. precise_ret_type = False + elif class_ir.is_trait: + # Give up on traits. We could possibly have an abstract base class + # for generator return types to make this use precise types. + precise_ret_type = False else: for s in subcls: if name in s.method_decls: diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index cf063310fd89..718325b8b7b6 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1297,6 +1297,8 @@ from typing import final, Coroutine, Any, TypeVar import asyncio +from mypy_extensions import trait + class Base1: async def foo(self) -> int: return 1 @@ -1363,5 +1365,22 @@ def test_override_non_async() -> None: assert asyncio.run(base3_foo(Base3())) == 7 assert asyncio.run(base3_foo(Derived3())) == 8 +class Base4: pass + +@trait +class TraitBase: + async def foo(self, value: int) -> int: + raise NotImplementedError() + +class DerivedFromTrait(Base4, TraitBase): + async def foo(self, value: int) -> int: + return value + 3 + +async def trait_foo(o: TraitBase, x: int) -> int: + return await o.foo(x) + +def test_override_trait() -> None: + assert asyncio.run(trait_foo(DerivedFromTrait(), 7)) == 10 + [file asyncio/__init__.pyi] def run(x: object) -> object: ... From 66797fcdac4ac63b1c7f90ef31cfd104afb0a99b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 18 Nov 2025 13:55:22 +0000 Subject: [PATCH 172/183] [mypyc] Fix calling base class async method using super() (#20254) Fixes mypyc/mypyc#1154. The next label integer value was incorrectly passed as the `self` argument. --- mypyc/irbuild/expression.py | 4 ++-- mypyc/test-data/run-async.test | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index f6636a0e7b62..86cc2c7fb145 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -504,12 +504,12 @@ def translate_super_method_call(builder: IRBuilder, expr: CallExpr, callee: Supe if decl.kind == FUNC_CLASSMETHOD: vself = builder.primitive_op(type_op, [vself], expr.line) elif builder.fn_info.is_generator: - # For generator classes, the self target is the 6th value + # For generator classes, the self target is the 7th value # in the symbol table (which is an ordered dict). This is sort # of ugly, but we can't search by name since the 'self' parameter # could be named anything, and it doesn't get added to the # environment indexes. - self_targ = list(builder.symtables[-1].values())[6] + self_targ = list(builder.symtables[-1].values())[7] vself = builder.read(self_targ, builder.fn_info.fitem.line) arg_values.insert(0, vself) arg_kinds.insert(0, ARG_POS) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 718325b8b7b6..39410e00a024 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -1382,5 +1382,20 @@ async def trait_foo(o: TraitBase, x: int) -> int: def test_override_trait() -> None: assert asyncio.run(trait_foo(DerivedFromTrait(), 7)) == 10 +class Base5: + def __init__(self) -> None: + self._name = "test" + + async def foo(self, x: int) -> int: + assert self._name == "test" + return x + 11 + +class Derived5(Base5): + async def foo(self, x: int) -> int: + return await super().foo(x) + 22 + +def test_call_using_super() -> None: + assert asyncio.run(Derived5().foo(5)) == 38 + [file asyncio/__init__.pyi] def run(x: object) -> object: ... From 7e3fd2479b31109f89b260aae8b124a5422aa2ca Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Nov 2025 17:28:54 +0000 Subject: [PATCH 173/183] Bump librt version (#20256) This pulls the version that contains stubs inside the wheel. I checked that third-party tools can find the stubs, but our local version in typeshed takes precedence as discussed in https://github.com/python/mypy/issues/20245 --- mypy-requirements.txt | 2 +- pyproject.toml | 4 ++-- test-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 06e0a9bffb1c..b0c632dddac5 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -4,4 +4,4 @@ typing_extensions>=4.6.0 mypy_extensions>=1.0.0 pathspec>=0.9.0 tomli>=1.1.0; python_version<'3.11' -librt>=0.6.0 +librt>=0.6.2 diff --git a/pyproject.toml b/pyproject.toml index 336a16c48979..bb41c82b1a3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.6.0", + "librt>=0.6.2", # the following is from build-requirements.txt "types-psutil", "types-setuptools", @@ -54,7 +54,7 @@ dependencies = [ "mypy_extensions>=1.0.0", "pathspec>=0.9.0", "tomli>=1.1.0; python_version<'3.11'", - "librt>=0.6.0", + "librt>=0.6.2", ] dynamic = ["version"] diff --git a/test-requirements.txt b/test-requirements.txt index 4d3f644d6bca..953e7a750c75 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ identify==2.6.15 # via pre-commit iniconfig==2.1.0 # via pytest -librt==0.6.0 +librt==0.6.2 # via -r mypy-requirements.txt lxml==6.0.2 ; python_version < "3.15" # via -r test-requirements.in From 0738db3f9d336622923c7ee143e1c3adf7600a31 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 18 Nov 2025 18:33:48 +0100 Subject: [PATCH 174/183] Do not push partial types to the binder (#20202) Fixes #19996. This was unearthed by #19400 where `is_subtype(Partial, Partial)` started returning True - before that `binder.assign_type(expr, Partial, Partial)` just returned early. If I understand correctly, that is how Partial should be handled here: we do not want to push partials to the binder. I do not think we should add a special case for that (both False and True make some sense for a partial type, I am not convinced that either one is marginally better), so I just add an explicit guard to skip adding partial types here. --- mypy/checker.py | 6 +++++- test-data/unit/check-redefine2.test | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 07f5c520de95..ad7eb3d35568 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3424,7 +3424,11 @@ def check_assignment( and lvalue_type is not None ): lvalue.node.type = remove_instance_last_known_values(lvalue_type) - elif self.options.allow_redefinition_new and lvalue_type is not None: + elif ( + self.options.allow_redefinition_new + and lvalue_type is not None + and not isinstance(lvalue_type, PartialType) + ): # TODO: Can we use put() here? self.binder.assign_type(lvalue, lvalue_type, lvalue_type) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 1abe957240b5..4d99aa17f804 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -369,6 +369,16 @@ class C4: reveal_type(self.x) # N: Revealed type is "builtins.list[builtins.str]" reveal_type(C4().x) # N: Revealed type is "builtins.list[builtins.str]" + +class C5: + def __init__(self) -> None: + if int(): + self.x = None + return + self.x = [""] + reveal_type(self.x) # N: Revealed type is "builtins.list[builtins.str]" + +reveal_type(C5().x) # N: Revealed type is "Union[builtins.list[builtins.str], None]" [builtins fixtures/list.pyi] [case testNewRedefinePartialGenericTypes] From 094f66dc742cec2d69add9296fb21cdef50624d0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 18 Nov 2025 18:44:14 +0000 Subject: [PATCH 175/183] [mypyc] Add __repr__ to AssignmentTarget subclasses (#20258) This makes debugging a little easier, since you can see register names etc. --- mypyc/irbuild/targets.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mypyc/irbuild/targets.py b/mypyc/irbuild/targets.py index 270c2896bc06..8bc9da074f07 100644 --- a/mypyc/irbuild/targets.py +++ b/mypyc/irbuild/targets.py @@ -20,6 +20,9 @@ def __init__(self, register: Register) -> None: self.register = register self.type = register.type + def __repr__(self) -> str: + return f"AssignmentTargetRegister({self.register.name})" + class AssignmentTargetIndex(AssignmentTarget): """base[index] as assignment target""" @@ -31,6 +34,9 @@ def __init__(self, base: Value, index: Value) -> None: # lvalue type in mypy and use a better type to avoid unneeded boxing. self.type = object_rprimitive + def __repr__(self) -> str: + return f"AssignmentTargetIndex({self.base!r}, {self.index!r})" + class AssignmentTargetAttr(AssignmentTarget): """obj.attr as assignment target""" @@ -48,6 +54,10 @@ def __init__(self, obj: Value, attr: str, can_borrow: bool = False) -> None: self.obj_type = object_rprimitive self.type = object_rprimitive + def __repr__(self) -> str: + can_borrow_str = ", can_borrow=True" if self.can_borrow else "" + return f"AssignmentTargetAttr({self.obj!r}.{self.attr}{can_borrow_str})" + class AssignmentTargetTuple(AssignmentTarget): """x, ..., y as assignment target""" @@ -55,3 +65,6 @@ class AssignmentTargetTuple(AssignmentTarget): def __init__(self, items: list[AssignmentTarget], star_idx: int | None = None) -> None: self.items = items self.star_idx = star_idx + + def __repr__(self) -> str: + return f"AssignmentTargetTuple({self.items}, {self.star_idx})" From 35e843cc38cedc1bdf87d9937c06d51189ad0e45 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 19 Nov 2025 17:18:38 +0000 Subject: [PATCH 176/183] [mypyc] Add efficient librt.base64.b64decode (#20263) The performance can be 10x faster than stdlib if input is valid base64, or if input has extra non-base64 characters only at the end of input. Similar to the base64 encode implementation I added recently, this uses SIMD instructions when available. The implementation first tries to decode the input optimistically assuming valid base64. If this fails, we'll perform a slow path with a preprocessing step that removes extra characters, and we'll perform a strict base64 decode on the cleaned up input. The semantics aren't 100% compatible with stdlib. First, we raise ValueError on invalid padding instead of `binascii.Error`, since I don't want a runtime dependency on the unrelated a`binascii` module. This needs to be documented, but stdlib can already raise ValueError on other conditions, so the deviation is not huge. Also, some invalid inputs are checked more strictly for padding violations. The stdlib implementation has some mysterious behaviors with invalid inputs that didn't seem worth replicating. The function only accepts a single ASCII str or bytes argument for now, since that seems to be by the far the most common use case. The stdlib function also accepts buffer objects and a `validate` argument. The slow path is still somewhat faster than stdlib (on the order of 1.3x to 2x for longer inputs), at least if the input is much smaller than L1 cache size. Got the initial fast path implementation from ChatGPT, but did a bunch of manual edits afterwards and reviewed carefully. --- mypy/typeshed/stubs/librt/librt/base64.pyi | 1 + mypyc/lib-rt/librt_base64.c | 191 ++++++++++++++++++++- mypyc/test-data/run-base64.test | 108 +++++++++++- 3 files changed, 297 insertions(+), 3 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/base64.pyi b/mypy/typeshed/stubs/librt/librt/base64.pyi index 36366f5754ce..1cea838505d6 100644 --- a/mypy/typeshed/stubs/librt/librt/base64.pyi +++ b/mypy/typeshed/stubs/librt/librt/base64.pyi @@ -1 +1,2 @@ def b64encode(s: bytes) -> bytes: ... +def b64decode(s: bytes | str) -> bytes: ... diff --git a/mypyc/lib-rt/librt_base64.c b/mypyc/lib-rt/librt_base64.c index 020a56e412f4..1720359ef9a6 100644 --- a/mypyc/lib-rt/librt_base64.c +++ b/mypyc/lib-rt/librt_base64.c @@ -1,11 +1,16 @@ #define PY_SSIZE_T_CLEAN #include +#include #include "librt_base64.h" #include "libbase64.h" #include "pythoncapi_compat.h" #ifdef MYPYC_EXPERIMENTAL +static PyObject * +b64decode_handle_invalid_input( + PyObject *out_bytes, char *outbuf, size_t max_out, const char *src, size_t srclen); + #define BASE64_MAXBIN ((PY_SSIZE_T_MAX - 3) / 2) #define STACK_BUFFER_SIZE 1024 @@ -63,11 +68,193 @@ b64encode(PyObject *self, PyObject *const *args, size_t nargs) { return b64encode_internal(args[0]); } +static inline int +is_valid_base64_char(char c, bool allow_padding) { + return ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || (c == '+') || (c == '/') || (allow_padding && c == '=')); +} + +static PyObject * +b64decode_internal(PyObject *arg) { + const char *src; + Py_ssize_t srclen_ssz; + + // Get input pointer and length + if (PyBytes_Check(arg)) { + src = PyBytes_AS_STRING(arg); + srclen_ssz = PyBytes_GET_SIZE(arg); + } else if (PyUnicode_Check(arg)) { + if (!PyUnicode_IS_ASCII(arg)) { + PyErr_SetString(PyExc_ValueError, + "string argument should contain only ASCII characters"); + return NULL; + } + src = (const char *)PyUnicode_1BYTE_DATA(arg); + srclen_ssz = PyUnicode_GET_LENGTH(arg); + } else { + PyErr_SetString(PyExc_TypeError, + "argument should be a bytes-like object or ASCII string"); + return NULL; + } + + // Fast-path: empty input + if (srclen_ssz == 0) { + return PyBytes_FromStringAndSize(NULL, 0); + } + + // Quickly ignore invalid characters at the end. Other invalid characters + // are also accepted, but they need a slow path. + while (srclen_ssz > 0 && !is_valid_base64_char(src[srclen_ssz - 1], true)) { + srclen_ssz--; + } + + // Compute an output capacity that's at least 3/4 of input, without overflow: + // ceil(3/4 * N) == N - floor(N/4) + size_t srclen = (size_t)srclen_ssz; + size_t max_out = srclen - (srclen / 4); + if (max_out == 0) { + max_out = 1; // defensive (srclen > 0 implies >= 1 anyway) + } + if (max_out > (size_t)PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, "input too large"); + return NULL; + } + + // Allocate output bytes (uninitialized) of the max capacity + PyObject *out_bytes = PyBytes_FromStringAndSize(NULL, (Py_ssize_t)max_out); + if (out_bytes == NULL) { + return NULL; // Propagate memory error + } + + char *outbuf = PyBytes_AS_STRING(out_bytes); + size_t outlen = max_out; + + int ret = base64_decode(src, srclen, outbuf, &outlen, 0); + + if (ret != 1) { + if (ret == 0) { + // Slow path: handle non-base64 input + return b64decode_handle_invalid_input(out_bytes, outbuf, max_out, src, srclen); + } + Py_DECREF(out_bytes); + if (ret == -1) { + PyErr_SetString(PyExc_NotImplementedError, "base64 codec not available in this build"); + } else { + PyErr_SetString(PyExc_RuntimeError, "base64_decode failed"); + } + return NULL; + } + + // Sanity-check contract (decoder must not overflow our buffer) + if (outlen > max_out) { + Py_DECREF(out_bytes); + PyErr_SetString(PyExc_RuntimeError, "decoder wrote past output buffer"); + return NULL; + } + + // Shrink in place to the actual decoded length + if (_PyBytes_Resize(&out_bytes, (Py_ssize_t)outlen) < 0) { + // _PyBytes_Resize sets an exception and may free the old object + return NULL; + } + return out_bytes; +} + +// Process non-base64 input by ignoring non-base64 characters, for compatibility +// with stdlib b64decode. +static PyObject * +b64decode_handle_invalid_input( + PyObject *out_bytes, char *outbuf, size_t max_out, const char *src, size_t srclen) +{ + // Copy input to a temporary buffer, with non-base64 characters and extra suffix + // characters removed + size_t newbuf_len = 0; + char *newbuf = PyMem_Malloc(srclen); + if (newbuf == NULL) { + Py_DECREF(out_bytes); + return PyErr_NoMemory(); + } + + // Copy base64 characters and some padding to the new buffer + for (size_t i = 0; i < srclen; i++) { + char c = src[i]; + if (is_valid_base64_char(c, false)) { + newbuf[newbuf_len++] = c; + } else if (c == '=') { + // Copy a necessary amount of padding + int remainder = newbuf_len % 4; + if (remainder == 0) { + // No padding needed + break; + } + int numpad = 4 - remainder; + // Check that there is at least the required amount padding (CPython ignores + // extra padding) + while (numpad > 0) { + if (i == srclen || src[i] != '=') { + break; + } + newbuf[newbuf_len++] = '='; + i++; + numpad--; + // Skip non-base64 alphabet characters within padding + while (i < srclen && !is_valid_base64_char(src[i], true)) { + i++; + } + } + break; + } + } + + // Stdlib always performs a non-strict padding check + if (newbuf_len % 4 != 0) { + Py_DECREF(out_bytes); + PyMem_Free(newbuf); + PyErr_SetString(PyExc_ValueError, "Incorrect padding"); + return NULL; + } + + size_t outlen = max_out; + int ret = base64_decode(newbuf, newbuf_len, outbuf, &outlen, 0); + PyMem_Free(newbuf); + + if (ret != 1) { + Py_DECREF(out_bytes); + if (ret == 0) { + PyErr_SetString(PyExc_ValueError, "Only base64 data is allowed"); + } + if (ret == -1) { + PyErr_SetString(PyExc_NotImplementedError, "base64 codec not available in this build"); + } else { + PyErr_SetString(PyExc_RuntimeError, "base64_decode failed"); + } + return NULL; + } + + // Shrink in place to the actual decoded length + if (_PyBytes_Resize(&out_bytes, (Py_ssize_t)outlen) < 0) { + // _PyBytes_Resize sets an exception and may free the old object + return NULL; + } + return out_bytes; +} + + +static PyObject* +b64decode(PyObject *self, PyObject *const *args, size_t nargs) { + if (nargs != 1) { + PyErr_SetString(PyExc_TypeError, "b64decode() takes exactly one argument"); + return 0; + } + return b64decode_internal(args[0]); +} + #endif static PyMethodDef librt_base64_module_methods[] = { #ifdef MYPYC_EXPERIMENTAL - {"b64encode", (PyCFunction)b64encode, METH_FASTCALL, PyDoc_STR("Encode bytes-like object using Base64.")}, + {"b64encode", (PyCFunction)b64encode, METH_FASTCALL, PyDoc_STR("Encode bytes object using Base64.")}, + {"b64decode", (PyCFunction)b64decode, METH_FASTCALL, PyDoc_STR("Decode a Base64 encoded bytes object or ASCII string.")}, #endif {NULL, NULL, 0, NULL} }; @@ -111,7 +298,7 @@ static PyModuleDef_Slot librt_base64_module_slots[] = { static PyModuleDef librt_base64_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "base64", - .m_doc = "base64 encoding and decoding optimized for mypyc", + .m_doc = "Fast base64 encoding and decoding optimized for mypyc", .m_size = 0, .m_methods = librt_base64_module_methods, .m_slots = librt_base64_module_slots, diff --git a/mypyc/test-data/run-base64.test b/mypyc/test-data/run-base64.test index 0f9151c2b00b..8d7eb7c13482 100644 --- a/mypyc/test-data/run-base64.test +++ b/mypyc/test-data/run-base64.test @@ -1,8 +1,9 @@ [case testAllBase64Features_librt_experimental] from typing import Any import base64 +import binascii -from librt.base64 import b64encode +from librt.base64 import b64encode, b64decode from testutil import assertRaises @@ -44,6 +45,111 @@ def test_encode_wrapper() -> None: with assertRaises(TypeError): enc(b"x", b"y") +def test_decode_basic() -> None: + assert b64decode(b"eA==") == b"x" + + with assertRaises(TypeError): + b64decode(bytearray(b"eA==")) + + for non_ascii in "\x80", "foo\u100bar", "foo\ua1234bar": + with assertRaises(ValueError): + b64decode(non_ascii) + +def check_decode(b: bytes, encoded: bool = False) -> None: + if encoded: + enc = b + else: + enc = b64encode(b) + assert b64decode(enc) == getattr(base64, "b64decode")(enc) + if getattr(enc, "isascii")(): # Test stub has no "isascii" + enc_str = enc.decode("ascii") + assert b64decode(enc_str) == getattr(base64, "b64decode")(enc_str) + +def test_decode_different_strings() -> None: + for i in range(256): + check_decode(bytes([i])) + check_decode(bytes([i]) + b"x") + check_decode(bytes([i]) + b"xy") + check_decode(bytes([i]) + b"xyz") + check_decode(bytes([i]) + b"xyza") + check_decode(b"x" + bytes([i])) + check_decode(b"xy" + bytes([i])) + check_decode(b"xyz" + bytes([i])) + check_decode(b"xyza" + bytes([i])) + + b = b"a\x00\xb7" * 1000 + for i in range(1000): + check_decode(b[:i]) + + for b in b"", b"ab", b"bac", b"1234", b"xyz88", b"abc" * 200: + check_decode(b) + +def is_base64_char(x: int) -> bool: + c = chr(x) + return ('a' <= c <= 'z') or ('A' <= c <= 'Z') or ('0' <= c <= '9') or c in '+/=' + +def test_decode_with_non_base64_chars() -> None: + # For stdlib compatibility, non-base64 characters should be ignored. + + # Invalid characters as a suffix use a fast path. + check_decode(b"eA== ", encoded=True) + check_decode(b"eA==\n", encoded=True) + check_decode(b"eA== \t\n", encoded=True) + check_decode(b"\n", encoded=True) + + check_decode(b" e A = = ", encoded=True) + + # Special case: Two different encodings of the same data + check_decode(b"eAa=", encoded=True) + check_decode(b"eAY=", encoded=True) + + for x in range(256): + if not is_base64_char(x): + b = bytes([x]) + check_decode(b, encoded=True) + check_decode(b"eA==" + b, encoded=True) + check_decode(b"e" + b + b"A==", encoded=True) + check_decode(b"eA=" + b + b"=", encoded=True) + +def check_decode_error(b: bytes, ignore_stdlib: bool = False) -> None: + if not ignore_stdlib: + with assertRaises(binascii.Error): + getattr(base64, "b64decode")(b) + + # The raised error is different, since librt shouldn't depend on binascii + with assertRaises(ValueError): + b64decode(b) + +def test_decode_with_invalid_padding() -> None: + check_decode_error(b"eA") + check_decode_error(b"eA=") + check_decode_error(b"eHk") + check_decode_error(b"eA = ") + + # Here stdlib behavior seems nonsensical, so we don't try to duplicate it + check_decode_error(b"eA=a=", ignore_stdlib=True) + +def test_decode_with_extra_data_after_padding() -> None: + check_decode(b"=", encoded=True) + check_decode(b"==", encoded=True) + check_decode(b"===", encoded=True) + check_decode(b"====", encoded=True) + check_decode(b"eA===", encoded=True) + check_decode(b"eHk==", encoded=True) + check_decode(b"eA==x", encoded=True) + check_decode(b"eHk=x", encoded=True) + check_decode(b"eA==abc=======efg", encoded=True) + +def test_decode_wrapper() -> None: + dec: Any = b64decode + assert dec(b"eA==") == b"x" + + with assertRaises(TypeError): + dec() + + with assertRaises(TypeError): + dec(b"x", b"y") + [case testBase64FeaturesNotAvailableInNonExperimentalBuild_librt_base64] # This also ensures librt.base64 can be built without experimental features import librt.base64 From a087a5894935cfdbc2eba27a6d04ebca38fd6659 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Nov 2025 11:58:01 +0000 Subject: [PATCH 177/183] Update import map when new modules added (#20271) Fixes https://github.com/python/mypy/issues/20209 --- mypy/server/update.py | 2 ++ test-data/unit/fine-grained-modules.test | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/mypy/server/update.py b/mypy/server/update.py index 839090ca45ac..86ccb57c2a3f 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -631,6 +631,8 @@ def restore(ids: list[str]) -> None: # Find any other modules brought in by imports. changed_modules = [(st.id, st.xpath) for st in new_modules] + for m in new_modules: + manager.import_map[m.id] = set(m.dependencies + m.suppressed) # If there are multiple modules to process, only process one of them and return # the remaining ones to the caller. diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test index f28dbaa1113b..3ee07a03792f 100644 --- a/test-data/unit/fine-grained-modules.test +++ b/test-data/unit/fine-grained-modules.test @@ -2244,3 +2244,19 @@ undefined a.py:1: error: Cannot find implementation or library stub for module named "foobar" a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports == + +[case testDaemonImportMapRefresh] +# cmd: mypy main.py +[file main.py] +[file main.py.2] +import a.b +reveal_type(a.b.foo()) +[file a/__init__.pyi] +[file a/b.pyi] +import a.c +def foo() -> a.c.C: ... +[file a/c.pyi] +class C: ... +[out] +== +main.py:2: note: Revealed type is "a.c.C" From 13369cb25fe450f755f63e59156b86df84c08b3d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 23 Nov 2025 07:10:10 +0000 Subject: [PATCH 178/183] [mypyc] Fix crash on super in generator (#20291) This is another problem caused by https://github.com/python/mypy/pull/19398 and a missing part of https://github.com/python/mypy/pull/20254 cc @JukkaL @hauntsaninja there is a small chance this may be related to the problems with black. --- mypyc/irbuild/expression.py | 4 ++-- mypyc/test-data/run-classes.test | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 86cc2c7fb145..2ed347ca1e79 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -296,8 +296,8 @@ def transform_super_expr(builder: IRBuilder, o: SuperExpr) -> Value: # Grab first argument vself: Value = next(iter_env) if builder.fn_info.is_generator: - # grab sixth argument (see comment in translate_super_method_call) - self_targ = list(builder.symtables[-1].values())[6] + # grab seventh argument (see comment in translate_super_method_call) + self_targ = list(builder.symtables[-1].values())[7] vself = builder.read(self_targ, builder.fn_info.fitem.line) elif not ir.is_ext_class: vself = next(iter_env) # second argument is self if non_extension class diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 2c2eac505797..da72fe59f456 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -322,6 +322,17 @@ if sys.version_info[:2] > (3, 5): assert TestEnum.b.name == 'b' assert TestEnum.b.value == 2 +[case testRunSuperYieldFromDict] +from typing import Any, Iterator + +class DictSubclass(dict): + def items(self) -> Iterator[Any]: + yield 1 + yield from super().items() + +def test_sub_dict() -> None: + assert list(DictSubclass().items()) == [1] + [case testGetAttribute] class C: x: int From 1b94fbb9fbc581de7e057d71e9892e3acbf9a7d3 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Thu, 27 Nov 2025 19:21:10 +0100 Subject: [PATCH 179/183] [mypyc] Fix vtable pointer with inherited dunder new (#20302) Fixes an issue where a subclass would have its vtable pointer set to the base class' vtable when there is a `__new__` method defined in the base class. This resulted in the subclass constructor calling the setup function of the base class because mypyc transforms `object.__new__` into the setup function. The fix is to store the pointers to the setup functions in `tp_methods` of type objects and look them up dynamically when instantiating new objects. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypyc/codegen/emitclass.py | 13 ++++++- mypyc/irbuild/specialize.py | 11 +++++- mypyc/lib-rt/CPy.h | 2 + mypyc/lib-rt/generic_ops.c | 20 ++++++++++ mypyc/primitives/generic_ops.py | 7 ++++ mypyc/test-data/irbuild-classes.test | 28 ++++++++++++++ mypyc/test-data/run-classes.test | 55 ++++++++++++++++++++++++++++ 7 files changed, 132 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index d64940084f12..e190d45a2e93 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -359,7 +359,7 @@ def emit_line() -> None: if cl.is_trait: generate_new_for_trait(cl, new_name, emitter) - generate_methods_table(cl, methods_name, emitter) + generate_methods_table(cl, methods_name, setup_name if generate_full else None, emitter) emit_line() flags = ["Py_TPFLAGS_DEFAULT", "Py_TPFLAGS_HEAPTYPE", "Py_TPFLAGS_BASETYPE"] @@ -960,8 +960,17 @@ def generate_finalize_for_class( emitter.emit_line("}") -def generate_methods_table(cl: ClassIR, name: str, emitter: Emitter) -> None: +def generate_methods_table( + cl: ClassIR, name: str, setup_name: str | None, emitter: Emitter +) -> None: emitter.emit_line(f"static PyMethodDef {name}[] = {{") + if setup_name: + # Store pointer to the setup function so it can be resolved dynamically + # in case of instance creation in __new__. + # CPy_SetupObject expects this method to be the first one in tp_methods. + emitter.emit_line( + f'{{"__internal_mypyc_setup", (PyCFunction){setup_name}, METH_O, NULL}},' + ) for fn in cl.methods.values(): if fn.decl.is_prop_setter or fn.decl.is_prop_getter or fn.internal: continue diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index e810f11bd079..b64f51043b13 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -99,7 +99,7 @@ isinstance_dict, ) from mypyc.primitives.float_ops import isinstance_float -from mypyc.primitives.generic_ops import generic_setattr +from mypyc.primitives.generic_ops import generic_setattr, setup_object from mypyc.primitives.int_ops import isinstance_int from mypyc.primitives.list_ops import isinstance_list, new_list_set_item_op from mypyc.primitives.misc_ops import isinstance_bool @@ -1103,7 +1103,14 @@ def translate_object_new(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> method_args = fn.fitem.arg_names if isinstance(typ_arg, NameExpr) and len(method_args) > 0 and method_args[0] == typ_arg.name: subtype = builder.accept(expr.args[0]) - return builder.add(Call(ir.setup, [subtype], expr.line)) + subs = ir.subclasses() + if subs is not None and len(subs) == 0: + return builder.add(Call(ir.setup, [subtype], expr.line)) + # Call a function that dynamically resolves the setup function of extension classes from the type object. + # This is necessary because the setup involves default attribute initialization and setting up + # the vtable which are specific to a given type and will not work if a subtype is created using + # the setup function of its base. + return builder.call_c(setup_object, [subtype], expr.line) return None diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index c79923f69e69..6d1e7502a7e7 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -958,6 +958,8 @@ static inline int CPyObject_GenericSetAttr(PyObject *self, PyObject *name, PyObj return _PyObject_GenericSetAttrWithDict(self, name, value, NULL); } +PyObject *CPy_SetupObject(PyObject *type); + #if CPY_3_11_FEATURES PyObject *CPy_GetName(PyObject *obj); #endif diff --git a/mypyc/lib-rt/generic_ops.c b/mypyc/lib-rt/generic_ops.c index 260cfec5b360..1e1e184bf290 100644 --- a/mypyc/lib-rt/generic_ops.c +++ b/mypyc/lib-rt/generic_ops.c @@ -62,3 +62,23 @@ PyObject *CPyObject_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) { Py_DECREF(slice); return result; } + +typedef PyObject *(*SetupFunction)(PyObject *); + +PyObject *CPy_SetupObject(PyObject *type) { + PyTypeObject *tp = (PyTypeObject *)type; + PyMethodDef *def = NULL; + for(; tp; tp = tp->tp_base) { + def = tp->tp_methods; + if (!def || !def->ml_name) { + continue; + } + + if (!strcmp(def->ml_name, "__internal_mypyc_setup")) { + return ((SetupFunction)(void(*)(void))def->ml_meth)(type); + } + } + + PyErr_SetString(PyExc_RuntimeError, "Internal mypyc error: Unable to find object setup function"); + return NULL; +} diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index 16bd074396d2..1003fda8d9ae 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -417,3 +417,10 @@ c_function_name="CPyObject_GenericSetAttr", error_kind=ERR_NEG_INT, ) + +setup_object = custom_op( + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPy_SetupObject", + error_kind=ERR_MAGIC, +) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 0f8ec2b094f0..504e6234de24 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1685,6 +1685,13 @@ class Test: obj.val = val return obj +class Test2: + def __new__(cls) -> Test2: + return super().__new__(cls) + +class Sub(Test2): + pass + def fn() -> Test: return Test.__new__(Test, 42) @@ -1719,6 +1726,13 @@ L0: obj = r0 obj.val = val; r1 = is_error return obj +def Test2.__new__(cls): + cls, r0 :: object + r1 :: __main__.Test2 +L0: + r0 = CPy_SetupObject(cls) + r1 = cast(__main__.Test2, r0) + return r1 def fn(): r0 :: object r1 :: __main__.Test @@ -1822,6 +1836,13 @@ class Test: obj.val = val return obj +class Test2: + def __new__(cls) -> Test2: + return object.__new__(cls) + +class Sub(Test2): + pass + def fn() -> Test: return Test.__new__(Test, 42) @@ -1874,6 +1895,13 @@ L0: obj = r0 obj.val = val; r1 = is_error return obj +def Test2.__new__(cls): + cls, r0 :: object + r1 :: __main__.Test2 +L0: + r0 = CPy_SetupObject(cls) + r1 = cast(__main__.Test2, r0) + return r1 def fn(): r0 :: object r1 :: __main__.Test diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index da72fe59f456..02a9934bac71 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3859,6 +3859,7 @@ Add(1, 0)=1 [case testInheritedDunderNew] from __future__ import annotations from mypy_extensions import mypyc_attr +from testutil import assertRaises from typing_extensions import Self from m import interpreted_subclass @@ -3875,7 +3876,11 @@ class Base: def __init__(self, val: int) -> None: self.init_val = val + def method(self) -> int: + raise NotImplementedError + class Sub(Base): + def __new__(cls, val: int) -> Self: return super().__new__(cls, val + 1) @@ -3883,11 +3888,20 @@ class Sub(Base): super().__init__(val) self.init_val = self.init_val * 2 + def method(self) -> int: + return 0 + class SubWithoutNew(Base): + sub_only_str = "" + sub_only_int: int + def __init__(self, val: int) -> None: super().__init__(val) self.init_val = self.init_val * 2 + def method(self) -> int: + return 1 + class BaseWithoutInterpretedSubclasses: val: int @@ -3899,6 +3913,9 @@ class BaseWithoutInterpretedSubclasses: def __init__(self, val: int) -> None: self.init_val = val + def method(self) -> int: + raise NotImplementedError + class SubNoInterpreted(BaseWithoutInterpretedSubclasses): def __new__(cls, val: int) -> Self: return super().__new__(cls, val + 1) @@ -3907,48 +3924,68 @@ class SubNoInterpreted(BaseWithoutInterpretedSubclasses): super().__init__(val) self.init_val = self.init_val * 2 + def method(self) -> int: + return 0 + class SubNoInterpretedWithoutNew(BaseWithoutInterpretedSubclasses): def __init__(self, val: int) -> None: super().__init__(val) self.init_val = self.init_val * 2 + def method(self) -> int: + return 1 + def test_inherited_dunder_new() -> None: b = Base(42) assert type(b) == Base assert b.val == 43 assert b.init_val == 42 + with assertRaises(NotImplementedError): + b.method() s = Sub(42) assert type(s) == Sub assert s.val == 44 assert s.init_val == 84 + assert s.method() == 0 s2 = SubWithoutNew(42) assert type(s2) == SubWithoutNew assert s2.val == 43 assert s2.init_val == 84 + assert s2.method() == 1 + assert s2.sub_only_str == "" + with assertRaises(AttributeError): + s2.sub_only_int + s2.sub_only_int = 11 + assert s2.sub_only_int == 11 def test_inherited_dunder_new_without_interpreted_subclasses() -> None: b = BaseWithoutInterpretedSubclasses(42) assert type(b) == BaseWithoutInterpretedSubclasses assert b.val == 43 assert b.init_val == 42 + with assertRaises(NotImplementedError): + b.method() s = SubNoInterpreted(42) assert type(s) == SubNoInterpreted assert s.val == 44 assert s.init_val == 84 + assert s.method() == 0 s2 = SubNoInterpretedWithoutNew(42) assert type(s2) == SubNoInterpretedWithoutNew assert s2.val == 43 assert s2.init_val == 84 + assert s2.method() == 1 def test_interpreted_subclass() -> None: interpreted_subclass(Base) [file m.py] from __future__ import annotations +from testutil import assertRaises from typing_extensions import Self def interpreted_subclass(base) -> None: @@ -3956,6 +3993,8 @@ def interpreted_subclass(base) -> None: assert type(b) == base assert b.val == 43 assert b.init_val == 42 + with assertRaises(NotImplementedError): + b.method() class InterpretedSub(base): def __new__(cls, val: int) -> Self: @@ -3965,20 +4004,36 @@ def interpreted_subclass(base) -> None: super().__init__(val) self.init_val : int = self.init_val * 2 + def method(self) -> int: + return 3 + s = InterpretedSub(42) assert type(s) == InterpretedSub assert s.val == 44 assert s.init_val == 84 + assert s.method() == 3 class InterpretedSubWithoutNew(base): + sub_only_str = "" + sub_only_int: int + def __init__(self, val: int) -> None: super().__init__(val) self.init_val : int = self.init_val * 2 + def method(self) -> int: + return 4 + s2 = InterpretedSubWithoutNew(42) assert type(s2) == InterpretedSubWithoutNew assert s2.val == 43 assert s2.init_val == 84 + assert s2.method() == 4 + assert s2.sub_only_str == "" + with assertRaises(AttributeError): + s2.sub_only_int + s2.sub_only_int = 11 + assert s2.sub_only_int == 11 [typing fixtures/typing-full.pyi] From 1999a20e9898f673fa2f4c9a91790c075141ba71 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 28 Nov 2025 10:58:29 +0100 Subject: [PATCH 180/183] [mypyc] librt base64: use existing SIMD CPU dispatch by customizing build flags (#20253) Fixes the current SSE4.2 requirement added in https://github.com/python/mypy/commit/1b6ebb17b7fe64488a7b3c3b4b0187bb14fe331b / https://github.com/python/mypy/pull/20244 This PR fully enables the existing x86-64 CPU detection and dispatch code for SSSE3, SSE4.1, SSE4.2, AVX, and AVX2 in the base64 module. To use the existing CPU dispatch from the [upstream base64 code](https://github.com/aklomp/base64), one needs to compile the sources in each of the CPU specific codec directories with a specific compiler flag; alas this is difficult to do with setuptools, but I found a solution inspired by https://stackoverflow.com/a/68508804 Note that I did not enable the AVX512 path with this PR, as many intel CPUs that support AVX512 can come with a performance hit if AVX512 is sporadically used; the performance of the AVX512 (encoding) path need to be evaluated in the context of how mypyc uses base64 in various realistic scenarios. (There is no AVX512 accelerated decoding path in the upstream base64 codebase, it falls back to the avx2 decoder). If there are additional performance concerns, then I suggest benchmarking with the openmp feature of base64 turned on, for multi-core processing. --- mypy_self_check.ini | 2 +- mypyc/build.py | 17 ++++---- mypyc/build_setup.py | 62 +++++++++++++++++++++++++++ mypyc/common.py | 3 -- mypyc/lib-rt/base64/arch/avx/codec.c | 2 +- mypyc/lib-rt/base64/arch/avx2/codec.c | 12 +++--- mypyc/lib-rt/base64/config.h | 28 +++--------- mypyc/lib-rt/setup.py | 54 ++++++++++++++++++++--- setup.py | 1 + 9 files changed, 135 insertions(+), 46 deletions(-) create mode 100644 mypyc/build_setup.py diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 0b49b3de862b..f4f8d2d0e08b 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -8,7 +8,7 @@ pretty = True always_false = MYPYC plugins = mypy.plugins.proper_plugin python_version = 3.9 -exclude = mypy/typeshed/|mypyc/test-data/|mypyc/lib-rt/ +exclude = mypy/typeshed/|mypyc/test-data/ enable_error_code = ignore-without-code,redundant-expr enable_incomplete_feature = PreciseTupleTypes show_error_code_links = True diff --git a/mypyc/build.py b/mypyc/build.py index 02f427c83426..69ef6c3bc435 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -28,6 +28,7 @@ from collections.abc import Iterable from typing import TYPE_CHECKING, Any, NamedTuple, NoReturn, Union, cast +import mypyc.build_setup # noqa: F401 from mypy.build import BuildSource from mypy.errors import CompileError from mypy.fscache import FileSystemCache @@ -36,7 +37,7 @@ from mypy.util import write_junit_xml from mypyc.annotate import generate_annotated_html from mypyc.codegen import emitmodule -from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, X86_64, shared_lib_name +from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, shared_lib_name from mypyc.errors import Errors from mypyc.ir.pprint import format_modules from mypyc.namegen import exported_name @@ -70,6 +71,13 @@ class ModDesc(NamedTuple): "base64/arch/neon64/codec.c", ], [ + "base64/arch/avx/enc_loop_asm.c", + "base64/arch/avx2/enc_loop.c", + "base64/arch/avx2/enc_loop_asm.c", + "base64/arch/avx2/enc_reshuffle.c", + "base64/arch/avx2/enc_translate.c", + "base64/arch/avx2/dec_loop.c", + "base64/arch/avx2/dec_reshuffle.c", "base64/arch/generic/32/enc_loop.c", "base64/arch/generic/64/enc_loop.c", "base64/arch/generic/32/dec_loop.c", @@ -661,9 +669,6 @@ def mypycify( # See https://github.com/mypyc/mypyc/issues/956 "-Wno-cpp", ] - if X86_64: - # Enable SIMD extensions. All CPUs released since ~2010 support SSE4.2. - cflags.append("-msse4.2") if log_trace: cflags.append("-DMYPYC_LOG_TRACE") if experimental_features: @@ -692,10 +697,6 @@ def mypycify( # that we actually get the compilation speed and memory # use wins that multi-file mode is intended for. cflags += ["/GL-", "/wd9025"] # warning about overriding /GL - if X86_64: - # Enable SIMD extensions. All CPUs released since ~2010 support SSE4.2. - # Also Windows 11 requires SSE4.2 since 24H2. - cflags.append("/arch:SSE4.2") if log_trace: cflags.append("/DMYPYC_LOG_TRACE") if experimental_features: diff --git a/mypyc/build_setup.py b/mypyc/build_setup.py new file mode 100644 index 000000000000..a3e7a669abee --- /dev/null +++ b/mypyc/build_setup.py @@ -0,0 +1,62 @@ +import platform +import sys + +try: + # Import setuptools so that it monkey-patch overrides distutils + import setuptools # noqa: F401 +except ImportError: + pass + +if sys.version_info >= (3, 12): + # From setuptools' monkeypatch + from distutils import ccompiler # type: ignore[import-not-found] +else: + from distutils import ccompiler + +EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT = { + "unix": { + "base64/arch/ssse3": ["-mssse3"], + "base64/arch/sse41": ["-msse4.1"], + "base64/arch/sse42": ["-msse4.2"], + "base64/arch/avx2": ["-mavx2"], + "base64/arch/avx": ["-mavx"], + }, + "msvc": { + "base64/arch/sse42": ["/arch:SSE4.2"], + "base64/arch/avx2": ["/arch:AVX2"], + "base64/arch/avx": ["/arch:AVX"], + }, +} + +ccompiler.CCompiler.__spawn = ccompiler.CCompiler.spawn # type: ignore[attr-defined] +X86_64 = platform.machine() in ("x86_64", "AMD64", "amd64") + + +def spawn(self, cmd, **kwargs) -> None: # type: ignore[no-untyped-def] + compiler_type: str = self.compiler_type + extra_options = EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT[compiler_type] + new_cmd = list(cmd) + if X86_64 and extra_options is not None: + # filenames are closer to the end of command line + for argument in reversed(new_cmd): + # Check if the matching argument contains a source filename. + if not str(argument).endswith(".c"): + continue + + for path in extra_options.keys(): + if path in str(argument): + if compiler_type == "bcpp": + compiler = new_cmd.pop() + # Borland accepts a source file name at the end, + # insert the options before it + new_cmd.extend(extra_options[path]) + new_cmd.append(compiler) + else: + new_cmd.extend(extra_options[path]) + + # path component is found, no need to search any further + break + self.__spawn(new_cmd, **kwargs) + + +ccompiler.CCompiler.spawn = spawn # type: ignore[method-assign] diff --git a/mypyc/common.py b/mypyc/common.py index 98f8a89f6fcb..2de63c09bb2c 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -1,6 +1,5 @@ from __future__ import annotations -import platform import sys import sysconfig from typing import Any, Final @@ -45,8 +44,6 @@ IS_32_BIT_PLATFORM: Final = int(SIZEOF_SIZE_T) == 4 -X86_64: Final = platform.machine() in ("x86_64", "AMD64", "amd64") - PLATFORM_SIZE = 4 if IS_32_BIT_PLATFORM else 8 # Maximum value for a short tagged integer. diff --git a/mypyc/lib-rt/base64/arch/avx/codec.c b/mypyc/lib-rt/base64/arch/avx/codec.c index 8e2ef5c2e724..7a64a94be2af 100644 --- a/mypyc/lib-rt/base64/arch/avx/codec.c +++ b/mypyc/lib-rt/base64/arch/avx/codec.c @@ -24,7 +24,7 @@ #include "../ssse3/dec_loop.c" #if BASE64_AVX_USE_ASM -# include "enc_loop_asm.c" +# include "./enc_loop_asm.c" #else # include "../ssse3/enc_translate.c" # include "../ssse3/enc_reshuffle.c" diff --git a/mypyc/lib-rt/base64/arch/avx2/codec.c b/mypyc/lib-rt/base64/arch/avx2/codec.c index fe9200296914..a54385bf89be 100644 --- a/mypyc/lib-rt/base64/arch/avx2/codec.c +++ b/mypyc/lib-rt/base64/arch/avx2/codec.c @@ -20,15 +20,15 @@ # endif #endif -#include "dec_reshuffle.c" -#include "dec_loop.c" +#include "./dec_reshuffle.c" +#include "./dec_loop.c" #if BASE64_AVX2_USE_ASM -# include "enc_loop_asm.c" +# include "./enc_loop_asm.c" #else -# include "enc_translate.c" -# include "enc_reshuffle.c" -# include "enc_loop.c" +# include "./enc_translate.c" +# include "./enc_reshuffle.c" +# include "./enc_loop.c" #endif #endif // HAVE_AVX2 diff --git a/mypyc/lib-rt/base64/config.h b/mypyc/lib-rt/base64/config.h index b5e47fb04e75..467a722c2f11 100644 --- a/mypyc/lib-rt/base64/config.h +++ b/mypyc/lib-rt/base64/config.h @@ -1,29 +1,15 @@ #ifndef BASE64_CONFIG_H #define BASE64_CONFIG_H -#define BASE64_WITH_SSSE3 0 -#define HAVE_SSSE3 BASE64_WITH_SSSE3 - -#define BASE64_WITH_SSE41 0 -#define HAVE_SSE41 BASE64_WITH_SSE41 - -#if defined(__x86_64__) || defined(_M_X64) -#define BASE64_WITH_SSE42 1 -#else -#define BASE64_WITH_SSE42 0 +#if !defined(__APPLE__) && ((defined(__x86_64__) && defined(__LP64__)) || defined(_M_X64)) + #define HAVE_SSSE3 1 + #define HAVE_SSE41 1 + #define HAVE_SSE42 1 + #define HAVE_AVX 1 + #define HAVE_AVX2 1 + #define HAVE_AVX512 0 #endif -#define HAVE_SSE42 BASE64_WITH_SSE42 - -#define BASE64_WITH_AVX 0 -#define HAVE_AVX BASE64_WITH_AVX - -#define BASE64_WITH_AVX2 0 -#define HAVE_AVX2 BASE64_WITH_AVX2 - -#define BASE64_WITH_AVX512 0 -#define HAVE_AVX512 BASE64_WITH_AVX512 - #define BASE64_WITH_NEON32 0 #define HAVE_NEON32 BASE64_WITH_NEON32 diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 6a56c65306ae..72dfc15d8588 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -25,9 +25,55 @@ "pythonsupport.c", ] +EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT = { + "unix": { + "base64/arch/ssse3": ["-mssse3"], + "base64/arch/sse41": ["-msse4.1"], + "base64/arch/sse42": ["-msse4.2"], + "base64/arch/avx2": ["-mavx2"], + "base64/arch/avx": ["-mavx"], + }, + "msvc": { + "base64/arch/sse42": ["/arch:SSE4.2"], + "base64/arch/avx2": ["/arch:AVX2"], + "base64/arch/avx": ["/arch:AVX"], + }, +} + +ccompiler.CCompiler.__spawn = ccompiler.CCompiler.spawn # type: ignore[attr-defined] X86_64 = platform.machine() in ("x86_64", "AMD64", "amd64") +def spawn(self, cmd, **kwargs) -> None: # type: ignore[no-untyped-def] + compiler_type: str = self.compiler_type + extra_options = EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT[compiler_type] + new_cmd = list(cmd) + if X86_64 and extra_options is not None: + # filenames are closer to the end of command line + for argument in reversed(new_cmd): + # Check if the matching argument contains a source filename. + if not str(argument).endswith(".c"): + continue + + for path in extra_options.keys(): + if path in str(argument): + if compiler_type == "bcpp": + compiler = new_cmd.pop() + # Borland accepts a source file name at the end, + # insert the options before it + new_cmd.extend(extra_options[path]) + new_cmd.append(compiler) + else: + new_cmd.extend(extra_options[path]) + + # path component is found, no need to search any further + break + self.__spawn(new_cmd, **kwargs) + + +ccompiler.CCompiler.spawn = spawn # type: ignore[method-assign] + + class BuildExtGtest(build_ext): def get_library_names(self) -> list[str]: return ["gtest"] @@ -80,14 +126,10 @@ def run(self) -> None: compiler = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) cflags: list[str] = [] - if compiler.compiler_type == "unix": + if compiler.compiler_type == "unix": # type: ignore[attr-defined] cflags += ["-O3"] - if X86_64: - cflags.append("-msse4.2") # Enable SIMD (see also mypyc/build.py) - elif compiler.compiler_type == "msvc": + elif compiler.compiler_type == "msvc": # type: ignore[attr-defined] cflags += ["/O2"] - if X86_64: - cflags.append("/arch:SSE4.2") # Enable SIMD (see also mypyc/build.py) setup( ext_modules=[ diff --git a/setup.py b/setup.py index 0037624f9bbc..f20c1db5d045 100644 --- a/setup.py +++ b/setup.py @@ -99,6 +99,7 @@ def run(self) -> None: os.path.join("mypyc", "lib-rt", "setup.py"), # Uses __file__ at top level https://github.com/mypyc/mypyc/issues/700 os.path.join("mypyc", "__main__.py"), + os.path.join("mypyc", "build_setup.py"), # for monkeypatching ) everything = [os.path.join("mypy", x) for x in find_package_data("mypy", ["*.py"])] + [ From 3c813083b27c87cf3a32e7422191b02bf59fab6e Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Thu, 27 Nov 2025 11:50:14 +0100 Subject: [PATCH 181/183] Add draft version of 1.19 release notes (#20296) Added a draft version with the commits filtered to remove internal changes and grouped into sections. --------- Co-authored-by: Jacopo Abramo --- CHANGELOG.md | 150 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 134d251d90b1..ec3f0cbb59bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,156 @@ ## Next Release +## Mypy 1.19 (Unreleased) + +We’ve just uploaded mypy 1.19.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. This release includes new features, performance +improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Performance improvements +- Switch to a more dynamic SCC processing logic (Ivan Levkivskyi, PR [20053](https://github.com/python/mypy/pull/20053)) +- Try some aliases speed-up (Ivan Levkivskyi, PR [19810](https://github.com/python/mypy/pull/19810)) + +### Fixed‑Format Cache +- Force-discard cache if cache format changed (Ivan Levkivskyi, PR [20152](https://github.com/python/mypy/pull/20152)) +- Use more efficient serialization format for long integers in cache files (Jukka Lehtosalo, PR [20151](https://github.com/python/mypy/pull/20151)) +- More robust packing of flats in FF cache (Ivan Levkivskyi, PR [20150](https://github.com/python/mypy/pull/20150)) +- Use self-descriptive cache with type tags (Ivan Levkivskyi, PR [20137](https://github.com/python/mypy/pull/20137)) +- Use fixed format for cache metas (Ivan Levkivskyi, PR [20088](https://github.com/python/mypy/pull/20088)) +- Make metas more compact; fix indirect suppression (Ivan Levkivskyi, PR [20075](https://github.com/python/mypy/pull/20075)) +- Add tool to convert binary cache files to JSON (Jukka Lehtosalo, PR [20071](https://github.com/python/mypy/pull/20071)) +- Use dedicated tags for most common instances (Ivan Levkivskyi, PR [19762](https://github.com/python/mypy/pull/19762)) + +### PEP 747 - Annotating Type Forms +- [PEP 747] Recognize `TypeForm[T]` type and values (#9773) (David Foster, PR [19596](https://github.com/python/mypy/pull/19596)) + +### Fixes to crashes +- Do not push partial types to the binder (Stanislav Terliakov, PR [20202](https://github.com/python/mypy/pull/20202)) +- Fix crash on recursive tuple with Hashable (Ivan Levkivskyi, PR [20232](https://github.com/python/mypy/pull/20232)) +- Do not assume that args of decorated functions can be cleanly mapped to their nodes (Stanislav Terliakov, PR [20203](https://github.com/python/mypy/pull/20203)) +- Do not abort constructing TypeAlias if only type parameters hold us back (Stanislav Terliakov, PR [20162](https://github.com/python/mypy/pull/20162)) +- Use the fallback for `ModuleSpec` early if it can never be resolved (Stanislav Terliakov, PR [20167](https://github.com/python/mypy/pull/20167)) +- Do not store deferred NamedTuple fields as redefinitions (Stanislav Terliakov, PR [20147](https://github.com/python/mypy/pull/20147)) +- Discard partials remaining after inference failure (Stanislav Terliakov, PR [20126](https://github.com/python/mypy/pull/20126)) +- Remember the pair in `is_overlapping_types` if at least one of them is an alias (Stanislav Terliakov, PR [20127](https://github.com/python/mypy/pull/20127)) +- Fix IsADirectoryError for namespace packages when using --linecoverage-report (wyattscarpenter, PR [20109](https://github.com/python/mypy/pull/20109)) +- Fix an INTERNAL ERROR when creating cobertura output for namespace package (wyattscarpenter, PR [20112](https://github.com/python/mypy/pull/20112)) +- Allow type parameters reusing the name missing from current module (Stanislav Terliakov, PR [20081](https://github.com/python/mypy/pull/20081)) +- Prevent TypeGuardedType leak from `narrow_declared_type` as part of typevar bound (Stanislav Terliakov, PR [20046](https://github.com/python/mypy/pull/20046)) +- Fix crash on invalid unpack in base class (Ivan Levkivskyi, PR [19962](https://github.com/python/mypy/pull/19962)) +- Traverse ParamSpec prefix where we should (Ivan Levkivskyi, PR [19800](https://github.com/python/mypy/pull/19800)) + +### Mypyc: Support for `__getattr__`, `__setattr__`, and `__delattr__` +- Support deleting attributes in `__setattr__` wrapper (Piotr Sawicki, PR [19997](https://github.com/python/mypy/pull/19997)) +- Generate `__setattr__` wrapper (Piotr Sawicki, PR [19937](https://github.com/python/mypy/pull/19937)) +- Generate `__getattr__` wrapper (Piotr Sawicki, PR [19909](https://github.com/python/mypy/pull/19909)) + +### Miscellaneous Mypyc Improvements +- Fix crash on `super` in generator (Ivan Levkivskyi, PR [20291](https://github.com/python/mypy/pull/20291)) +- Fix calling base class async method using `super()` (Jukka Lehtosalo, PR [20254](https://github.com/python/mypy/pull/20254)) +- Fix async or generator methods in traits (Jukka Lehtosalo, PR [20246](https://github.com/python/mypy/pull/20246)) +- Optimize equality check with string literals [1/1] (BobTheBuidler, PR [19883](https://github.com/python/mypy/pull/19883)) +- Fix inheritance of async defs (Jukka Lehtosalo, PR [20044](https://github.com/python/mypy/pull/20044)) +- Reject invalid `mypyc_attr` args [1/1] (BobTheBuidler, PR [19963](https://github.com/python/mypy/pull/19963)) +- Optimize `isinstance` with tuple of primitive types (BobTheBuidler, PR [19949](https://github.com/python/mypy/pull/19949)) +- Optimize away first index check in for loops if length > 1 (BobTheBuidler, PR [19933](https://github.com/python/mypy/pull/19933)) +- Fix broken exception/cancellation handling in async def (Jukka Lehtosalo, PR [19951](https://github.com/python/mypy/pull/19951)) +- Transform `object.__new__` inside `__new__` (Piotr Sawicki, PR [19866](https://github.com/python/mypy/pull/19866)) +- Fix crash with NewType and other non-class types in incremental builds (Jukka Lehtosalo, PR [19837](https://github.com/python/mypy/pull/19837)) +- Optimize container creation from expressions with length known at compile time (BobTheBuidler, PR [19503](https://github.com/python/mypy/pull/19503)) +- Allow per-class free list to be used with inheritance (Jukka Lehtosalo, PR [19790](https://github.com/python/mypy/pull/19790)) +- Fix object finalization (Marc Mueller, PR [19749](https://github.com/python/mypy/pull/19749)) +- Allow defining a single-item free "list" for a native class (Jukka Lehtosalo, PR [19785](https://github.com/python/mypy/pull/19785)) +- Speed up unary "not" (Jukka Lehtosalo, PR [19774](https://github.com/python/mypy/pull/19774)) + +### Stubtest Improvements +- Check `_value_` for ellipsis-valued stub enum members (Stanislav Terliakov, PR [19760](https://github.com/python/mypy/pull/19760)) +- Include function name in overload assertion messages (Joren Hammudoglu, PR [20063](https://github.com/python/mypy/pull/20063)) +- Small fix in get_default_function_sig (iap, PR [19822](https://github.com/python/mypy/pull/19822)) +- Adjust stubtest test stubs for PEP 728 (Python 3.15) (Marc Mueller, PR [20009](https://github.com/python/mypy/pull/20009)) +- Improve `allowlist` docs with better example (sobolevn, PR [20007](https://github.com/python/mypy/pull/20007)) + +### Documentation Updates +- Update duck_type_compatibility.rst: mention strict-bytes & mypy 2.0 (wyattscarpenter, PR [20121](https://github.com/python/mypy/pull/20121)) +- document --enable-incomplete-feature TypeForm, minimally but sufficiently (wyattscarpenter, PR [20173](https://github.com/python/mypy/pull/20173)) +- Change the InlineTypedDict example (wyattscarpenter, PR [20172](https://github.com/python/mypy/pull/20172)) +- Update kinds_of_types.rst: keep old anchor for #no-strict-optional (wyattscarpenter, PR [19828](https://github.com/python/mypy/pull/19828)) +- Replace `List` with built‑in `list` (PEP 585) (Thiago J. Barbalho, PR [20000](https://github.com/python/mypy/pull/20000)) +- main.py: junit documentation elaboration (wyattscarpenter, PR [19867](https://github.com/python/mypy/pull/19867)) + +### Other Notable Fixes and Improvements +- Update import map when new modules added (Ivan Levkivskyi, PR [20271](https://github.com/python/mypy/pull/20271)) +- Fix annotated with function as type keyword list parameter (KarelKenens, PR [20094](https://github.com/python/mypy/pull/20094)) +- Fix errors for raise NotImplemented (Shantanu, PR [20168](https://github.com/python/mypy/pull/20168)) +- Don't let help formatter line-wrap URLs (Frank Dana, PR [19825](https://github.com/python/mypy/pull/19825)) +- Do not cache fast container types inside lambdas (Stanislav Terliakov, PR [20166](https://github.com/python/mypy/pull/20166)) +- Respect force-union-syntax flag in error hint (Marc Mueller, PR [20165](https://github.com/python/mypy/pull/20165)) +- Fix type checking of dict type aliases (Shantanu, PR [20170](https://github.com/python/mypy/pull/20170)) +- Use pretty_callable more often for callable expressions (Theodore Ando, PR [20128](https://github.com/python/mypy/pull/20128)) +- Use dummy concrete type instead of `Any` when checking protocol variance (bzoracler, PR [20110](https://github.com/python/mypy/pull/20110)) +- [PEP 696] Fix swapping TypeVars with defaults (Randolf Scholz, PR [19449](https://github.com/python/mypy/pull/19449)) +- Fix narrowing of class pattern with union-argument (Randolf Scholz, PR [19517](https://github.com/python/mypy/pull/19517)) +- Do not emit unreachable warnings for lines that return `NotImplemented` (Christoph Tyralla, PR [20083](https://github.com/python/mypy/pull/20083)) +- fix matching against `typing.Callable` and `Protocol` types (Randolf Scholz, PR [19471](https://github.com/python/mypy/pull/19471)) +- Make --pretty work better on multi-line issues (A5rocks, PR [20056](https://github.com/python/mypy/pull/20056)) +- More precise return types for `TypedDict.get` (Randolf Scholz, PR [19897](https://github.com/python/mypy/pull/19897)) +- prevent false unreachable warnings for @final instances that occur when strict optional checking is disabled (Christoph Tyralla, PR [20045](https://github.com/python/mypy/pull/20045)) +- Check class references to catch non-existent classes in match cases (A5rocks, PR [20042](https://github.com/python/mypy/pull/20042)) +- Do not sort unused error codes in unused error codes warning (wyattscarpenter, PR [20036](https://github.com/python/mypy/pull/20036)) +- Fix `[name-defined]` false-positive in `class A[X, Y=X]:` case (sobolevn, PR [20021](https://github.com/python/mypy/pull/20021)) +- Filter SyntaxWarnings during AST parsing (Marc Mueller, PR [20023](https://github.com/python/mypy/pull/20023)) +- Make untyped decorator its own code (wyattscarpenter, PR [19911](https://github.com/python/mypy/pull/19911)) +- Support error codes from plugins in options (Sigve Sebastian Farstad, PR [19719](https://github.com/python/mypy/pull/19719)) +- Allow returning Literals in `__new__` (James Hilton-Balfe, PR [15687](https://github.com/python/mypy/pull/15687)) +- Inverse interface freshness logic (Ivan Levkivskyi, PR [19809](https://github.com/python/mypy/pull/19809)) +- Do not report exhaustive-match after deferral (Stanislav Terliakov, PR [19804](https://github.com/python/mypy/pull/19804)) +- Make untyped_calls_exclude invalidate cache (Ivan Levkivskyi, PR [19801](https://github.com/python/mypy/pull/19801)) +- Add await to empty context hack (Stanislav Terliakov, PR [19777](https://github.com/python/mypy/pull/19777)) +- Consider non-empty enums assignable to Self (Stanislav Terliakov, PR [19779](https://github.com/python/mypy/pull/19779)) + +### Typeshed updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=ebce8d766b41fbf4d83cf47c1297563a9508ff60+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements + +Thanks to all mypy contributors who contributed to this release: +- A5rocks +- BobTheBuidler +- bzoracler +- Chainfire +- Christoph Tyralla +- David Foster +- Frank Dana +- Guo Ci +- iap +- Ivan Levkivskyi +- James Hilton-Balfe +- jhance +- Joren Hammudoglu +- Jukka Lehtosalo +- KarelKenens +- Kevin Kannammalil +- Marc Mueller +- Michael Carlstrom +- Michael J. Sullivan +- Piotr Sawicki +- Randolf Scholz +- Shantanu +- Sigve Sebastian Farstad +- sobolevn +- Stanislav Terliakov +- Stephen Morton +- Theodore Ando +- Thiago J. Barbalho +- wyattscarpenter + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 1.18.1 We’ve just uploaded mypy 1.18.1 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). From 6d5cf52e67da306b62455cdce4ce9a9ccec35d02 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 28 Nov 2025 11:53:21 +0000 Subject: [PATCH 182/183] Various updates to 1.19 changelog (#20304) The first draft was added in #20296. --- CHANGELOG.md | 155 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 120 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec3f0cbb59bf..0be81310c6e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Next Release -## Mypy 1.19 (Unreleased) +## Mypy 1.19 We’ve just uploaded mypy 1.19.0 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance @@ -12,51 +12,139 @@ improvements and bug fixes. You can install it as follows: You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -### Performance improvements +### Performance Improvements - Switch to a more dynamic SCC processing logic (Ivan Levkivskyi, PR [20053](https://github.com/python/mypy/pull/20053)) -- Try some aliases speed-up (Ivan Levkivskyi, PR [19810](https://github.com/python/mypy/pull/19810)) +- Speed up type aliases (Ivan Levkivskyi, PR [19810](https://github.com/python/mypy/pull/19810)) + +### Fixed‑Format Cache Improvements + +Mypy uses a cache by default to speed up incremental runs by reusing partial results +from earlier runs. Mypy 1.18 added a new binary fixed-format cache representation as +an experimental feature. The feature is no longer experimental, and we are planning +to enable it by default in a future mypy release (possibly 1.20), since it's faster +and uses less space than the original, JSON-based cache format. Use +`--fixed-format-cache` to enable the fixed-format cache. + +Mypy now has an extra dependency on the `librt` PyPI package, as it's needed for +cache serialization and deserialization. + +Mypy ships with a tool to convert fixed-format cache files to the old JSON format. +Example of how to use this: +``` +$ python -m mypy.exportjson .mypy_cache/.../my_module.data.ff +``` + +This way existing use cases that parse JSON cache files can be supported when using +the new format, though an extra conversion step is needed. + +This release includes these improvements: -### Fixed‑Format Cache - Force-discard cache if cache format changed (Ivan Levkivskyi, PR [20152](https://github.com/python/mypy/pull/20152)) +- Add tool to convert binary cache files to JSON (Jukka Lehtosalo, PR [20071](https://github.com/python/mypy/pull/20071)) - Use more efficient serialization format for long integers in cache files (Jukka Lehtosalo, PR [20151](https://github.com/python/mypy/pull/20151)) -- More robust packing of flats in FF cache (Ivan Levkivskyi, PR [20150](https://github.com/python/mypy/pull/20150)) +- More robust packing of floats in fixed-format cache (Ivan Levkivskyi, PR [20150](https://github.com/python/mypy/pull/20150)) - Use self-descriptive cache with type tags (Ivan Levkivskyi, PR [20137](https://github.com/python/mypy/pull/20137)) - Use fixed format for cache metas (Ivan Levkivskyi, PR [20088](https://github.com/python/mypy/pull/20088)) - Make metas more compact; fix indirect suppression (Ivan Levkivskyi, PR [20075](https://github.com/python/mypy/pull/20075)) -- Add tool to convert binary cache files to JSON (Jukka Lehtosalo, PR [20071](https://github.com/python/mypy/pull/20071)) -- Use dedicated tags for most common instances (Ivan Levkivskyi, PR [19762](https://github.com/python/mypy/pull/19762)) +- Use dedicated tags for most common cached instances (Ivan Levkivskyi, PR [19762](https://github.com/python/mypy/pull/19762)) + +### PEP 747: Annotating Type Forms + +Mypy now recognizes `TypeForm[T]` as a type and implements +[PEP 747](https://peps.python.org/pep-0747/). The feature is still experimental, +and it's disabled by default. Use `--enable-incomplete-feature=TypeForm` to +enable type forms. A type form object captures the type information provided by a +runtime type expression. Example: + +```python +from typing_extensions import TypeForm -### PEP 747 - Annotating Type Forms -- [PEP 747] Recognize `TypeForm[T]` type and values (#9773) (David Foster, PR [19596](https://github.com/python/mypy/pull/19596)) +def trycast[T](typx: TypeForm[T], value: object) -> T | None: ... -### Fixes to crashes +def example(o: object) -> None: + # 'int | str' below is an expression that represents a type. + # Unlike type[T], TypeForm[T] can be used with all kinds of types, + # including union types. + x = trycast(int | str, o) + if x is not None: + # Type of 'x' is 'int | str' here + ... +``` + +This feature was contributed by David Foster (PR [19596](https://github.com/python/mypy/pull/19596)). + +### Fixes to Crashes - Do not push partial types to the binder (Stanislav Terliakov, PR [20202](https://github.com/python/mypy/pull/20202)) - Fix crash on recursive tuple with Hashable (Ivan Levkivskyi, PR [20232](https://github.com/python/mypy/pull/20232)) -- Do not assume that args of decorated functions can be cleanly mapped to their nodes (Stanislav Terliakov, PR [20203](https://github.com/python/mypy/pull/20203)) +- Fix crash related to decorated functions (Stanislav Terliakov, PR [20203](https://github.com/python/mypy/pull/20203)) - Do not abort constructing TypeAlias if only type parameters hold us back (Stanislav Terliakov, PR [20162](https://github.com/python/mypy/pull/20162)) - Use the fallback for `ModuleSpec` early if it can never be resolved (Stanislav Terliakov, PR [20167](https://github.com/python/mypy/pull/20167)) - Do not store deferred NamedTuple fields as redefinitions (Stanislav Terliakov, PR [20147](https://github.com/python/mypy/pull/20147)) -- Discard partials remaining after inference failure (Stanislav Terliakov, PR [20126](https://github.com/python/mypy/pull/20126)) -- Remember the pair in `is_overlapping_types` if at least one of them is an alias (Stanislav Terliakov, PR [20127](https://github.com/python/mypy/pull/20127)) +- Discard partial types remaining after inference failure (Stanislav Terliakov, PR [20126](https://github.com/python/mypy/pull/20126)) +- Fix an infinite recursion bug (Stanislav Terliakov, PR [20127](https://github.com/python/mypy/pull/20127)) - Fix IsADirectoryError for namespace packages when using --linecoverage-report (wyattscarpenter, PR [20109](https://github.com/python/mypy/pull/20109)) -- Fix an INTERNAL ERROR when creating cobertura output for namespace package (wyattscarpenter, PR [20112](https://github.com/python/mypy/pull/20112)) +- Fix an internal error when creating cobertura output for namespace package (wyattscarpenter, PR [20112](https://github.com/python/mypy/pull/20112)) - Allow type parameters reusing the name missing from current module (Stanislav Terliakov, PR [20081](https://github.com/python/mypy/pull/20081)) -- Prevent TypeGuardedType leak from `narrow_declared_type` as part of typevar bound (Stanislav Terliakov, PR [20046](https://github.com/python/mypy/pull/20046)) +- Prevent TypeGuardedType leak from narrowing declared type as part of type variable bound (Stanislav Terliakov, PR [20046](https://github.com/python/mypy/pull/20046)) - Fix crash on invalid unpack in base class (Ivan Levkivskyi, PR [19962](https://github.com/python/mypy/pull/19962)) - Traverse ParamSpec prefix where we should (Ivan Levkivskyi, PR [19800](https://github.com/python/mypy/pull/19800)) +- Fix daemon crash related to imports (Ivan Levkivskyi, PR [20271](https://github.com/python/mypy/pull/20271)) ### Mypyc: Support for `__getattr__`, `__setattr__`, and `__delattr__` -- Support deleting attributes in `__setattr__` wrapper (Piotr Sawicki, PR [19997](https://github.com/python/mypy/pull/19997)) + +Mypyc now has partial support for `__getattr__`, `__setattr__` and +`__delattr__` methods in native classes. + +Note that native attributes are not stored using `__dict__`. Setting attributes +directly while bypassing `__setattr__` is possible by using +`super().__setattr__(...)` or `object.__setattr__(...)`, but not via `__dict__`. + +Example: +```python +class Demo: + _data: dict[str, str] + + def __init__(self) -> None: + # Initialize data dict without calling our __setattr__ + super().__setattr__("_data", {}) + + def __setattr__(self, name: str, value: str) -> None: + print(f"Setting {name} = {value!r}") + + if name == "_data": + raise AttributeError("'_data' cannot be set") + + self._data[name] = value + + def __getattr__(self, name: str) -> str: + print(f"Getting {name}") + + try: + return self._data[name] + except KeyError: + raise AttributeError(name) + +d = Demo() +d.x = "hello" +d.y = "world" + +print(d.x) +print(d.y) +``` + +Related PRs: - Generate `__setattr__` wrapper (Piotr Sawicki, PR [19937](https://github.com/python/mypy/pull/19937)) - Generate `__getattr__` wrapper (Piotr Sawicki, PR [19909](https://github.com/python/mypy/pull/19909)) +- Support deleting attributes in `__setattr__` wrapper (Piotr Sawicki, PR [19997](https://github.com/python/mypy/pull/19997)) ### Miscellaneous Mypyc Improvements +- Fix `__new__` in native classes with inheritance (Piotr Sawicki, PR [20302](https://github.com/python/mypy/pull/20302)) - Fix crash on `super` in generator (Ivan Levkivskyi, PR [20291](https://github.com/python/mypy/pull/20291)) - Fix calling base class async method using `super()` (Jukka Lehtosalo, PR [20254](https://github.com/python/mypy/pull/20254)) - Fix async or generator methods in traits (Jukka Lehtosalo, PR [20246](https://github.com/python/mypy/pull/20246)) -- Optimize equality check with string literals [1/1] (BobTheBuidler, PR [19883](https://github.com/python/mypy/pull/19883)) +- Optimize equality check with string literals (BobTheBuidler, PR [19883](https://github.com/python/mypy/pull/19883)) - Fix inheritance of async defs (Jukka Lehtosalo, PR [20044](https://github.com/python/mypy/pull/20044)) -- Reject invalid `mypyc_attr` args [1/1] (BobTheBuidler, PR [19963](https://github.com/python/mypy/pull/19963)) +- Reject invalid `mypyc_attr` args (BobTheBuidler, PR [19963](https://github.com/python/mypy/pull/19963)) - Optimize `isinstance` with tuple of primitive types (BobTheBuidler, PR [19949](https://github.com/python/mypy/pull/19949)) - Optimize away first index check in for loops if length > 1 (BobTheBuidler, PR [19933](https://github.com/python/mypy/pull/19933)) - Fix broken exception/cancellation handling in async def (Jukka Lehtosalo, PR [19951](https://github.com/python/mypy/pull/19951)) @@ -71,45 +159,42 @@ You can read the full documentation for this release on [Read the Docs](http://m ### Stubtest Improvements - Check `_value_` for ellipsis-valued stub enum members (Stanislav Terliakov, PR [19760](https://github.com/python/mypy/pull/19760)) - Include function name in overload assertion messages (Joren Hammudoglu, PR [20063](https://github.com/python/mypy/pull/20063)) -- Small fix in get_default_function_sig (iap, PR [19822](https://github.com/python/mypy/pull/19822)) -- Adjust stubtest test stubs for PEP 728 (Python 3.15) (Marc Mueller, PR [20009](https://github.com/python/mypy/pull/20009)) +- Fix special case in analyzing function signature (iap, PR [19822](https://github.com/python/mypy/pull/19822)) - Improve `allowlist` docs with better example (sobolevn, PR [20007](https://github.com/python/mypy/pull/20007)) ### Documentation Updates -- Update duck_type_compatibility.rst: mention strict-bytes & mypy 2.0 (wyattscarpenter, PR [20121](https://github.com/python/mypy/pull/20121)) -- document --enable-incomplete-feature TypeForm, minimally but sufficiently (wyattscarpenter, PR [20173](https://github.com/python/mypy/pull/20173)) -- Change the InlineTypedDict example (wyattscarpenter, PR [20172](https://github.com/python/mypy/pull/20172)) -- Update kinds_of_types.rst: keep old anchor for #no-strict-optional (wyattscarpenter, PR [19828](https://github.com/python/mypy/pull/19828)) +- Update duck type compatibility: mention strict-bytes and mypy 2.0 (wyattscarpenter, PR [20121](https://github.com/python/mypy/pull/20121)) +- Document `--enable-incomplete-feature TypeForm` (wyattscarpenter, PR [20173](https://github.com/python/mypy/pull/20173)) +- Change the inline TypedDict example (wyattscarpenter, PR [20172](https://github.com/python/mypy/pull/20172)) - Replace `List` with built‑in `list` (PEP 585) (Thiago J. Barbalho, PR [20000](https://github.com/python/mypy/pull/20000)) -- main.py: junit documentation elaboration (wyattscarpenter, PR [19867](https://github.com/python/mypy/pull/19867)) +- Improve junit documentation (wyattscarpenter, PR [19867](https://github.com/python/mypy/pull/19867)) ### Other Notable Fixes and Improvements -- Update import map when new modules added (Ivan Levkivskyi, PR [20271](https://github.com/python/mypy/pull/20271)) - Fix annotated with function as type keyword list parameter (KarelKenens, PR [20094](https://github.com/python/mypy/pull/20094)) - Fix errors for raise NotImplemented (Shantanu, PR [20168](https://github.com/python/mypy/pull/20168)) - Don't let help formatter line-wrap URLs (Frank Dana, PR [19825](https://github.com/python/mypy/pull/19825)) - Do not cache fast container types inside lambdas (Stanislav Terliakov, PR [20166](https://github.com/python/mypy/pull/20166)) - Respect force-union-syntax flag in error hint (Marc Mueller, PR [20165](https://github.com/python/mypy/pull/20165)) - Fix type checking of dict type aliases (Shantanu, PR [20170](https://github.com/python/mypy/pull/20170)) -- Use pretty_callable more often for callable expressions (Theodore Ando, PR [20128](https://github.com/python/mypy/pull/20128)) +- Use pretty callable formatting more often for callable expressions (Theodore Ando, PR [20128](https://github.com/python/mypy/pull/20128)) - Use dummy concrete type instead of `Any` when checking protocol variance (bzoracler, PR [20110](https://github.com/python/mypy/pull/20110)) -- [PEP 696] Fix swapping TypeVars with defaults (Randolf Scholz, PR [19449](https://github.com/python/mypy/pull/19449)) -- Fix narrowing of class pattern with union-argument (Randolf Scholz, PR [19517](https://github.com/python/mypy/pull/19517)) +- PEP 696: Fix swapping TypeVars with defaults (Randolf Scholz, PR [19449](https://github.com/python/mypy/pull/19449)) +- Fix narrowing of class pattern with union type (Randolf Scholz, PR [19517](https://github.com/python/mypy/pull/19517)) - Do not emit unreachable warnings for lines that return `NotImplemented` (Christoph Tyralla, PR [20083](https://github.com/python/mypy/pull/20083)) -- fix matching against `typing.Callable` and `Protocol` types (Randolf Scholz, PR [19471](https://github.com/python/mypy/pull/19471)) -- Make --pretty work better on multi-line issues (A5rocks, PR [20056](https://github.com/python/mypy/pull/20056)) +- Fix matching against `typing.Callable` and `Protocol` types (Randolf Scholz, PR [19471](https://github.com/python/mypy/pull/19471)) +- Make `--pretty` work better on multi-line issues (A5rocks, PR [20056](https://github.com/python/mypy/pull/20056)) - More precise return types for `TypedDict.get` (Randolf Scholz, PR [19897](https://github.com/python/mypy/pull/19897)) -- prevent false unreachable warnings for @final instances that occur when strict optional checking is disabled (Christoph Tyralla, PR [20045](https://github.com/python/mypy/pull/20045)) +- Prevent false unreachable warnings for `@final` instances that occur when strict optional checking is disabled (Christoph Tyralla, PR [20045](https://github.com/python/mypy/pull/20045)) - Check class references to catch non-existent classes in match cases (A5rocks, PR [20042](https://github.com/python/mypy/pull/20042)) - Do not sort unused error codes in unused error codes warning (wyattscarpenter, PR [20036](https://github.com/python/mypy/pull/20036)) -- Fix `[name-defined]` false-positive in `class A[X, Y=X]:` case (sobolevn, PR [20021](https://github.com/python/mypy/pull/20021)) +- Fix `[name-defined]` false positive in `class A[X, Y=X]:` case (sobolevn, PR [20021](https://github.com/python/mypy/pull/20021)) - Filter SyntaxWarnings during AST parsing (Marc Mueller, PR [20023](https://github.com/python/mypy/pull/20023)) -- Make untyped decorator its own code (wyattscarpenter, PR [19911](https://github.com/python/mypy/pull/19911)) +- Make untyped decorator its own error code (wyattscarpenter, PR [19911](https://github.com/python/mypy/pull/19911)) - Support error codes from plugins in options (Sigve Sebastian Farstad, PR [19719](https://github.com/python/mypy/pull/19719)) - Allow returning Literals in `__new__` (James Hilton-Balfe, PR [15687](https://github.com/python/mypy/pull/15687)) - Inverse interface freshness logic (Ivan Levkivskyi, PR [19809](https://github.com/python/mypy/pull/19809)) - Do not report exhaustive-match after deferral (Stanislav Terliakov, PR [19804](https://github.com/python/mypy/pull/19804)) -- Make untyped_calls_exclude invalidate cache (Ivan Levkivskyi, PR [19801](https://github.com/python/mypy/pull/19801)) +- Make `untyped_calls_exclude` invalidate cache (Ivan Levkivskyi, PR [19801](https://github.com/python/mypy/pull/19801)) - Add await to empty context hack (Stanislav Terliakov, PR [19777](https://github.com/python/mypy/pull/19777)) - Consider non-empty enums assignable to Self (Stanislav Terliakov, PR [19779](https://github.com/python/mypy/pull/19779)) From 0f068c9ec604daa09e69c92545b059f4b44f566e Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Fri, 28 Nov 2025 12:57:08 +0100 Subject: [PATCH 183/183] Remove +dev --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index af216bddded1..f550cd929eb5 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.19.0+dev" +__version__ = "1.19.0" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))