From d82da0f0e9151bc9afae381f803948fe5836c084 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 28 Jan 2025 20:52:03 -0800 Subject: [PATCH 01/69] Fix hatch build (#4565) --- pyproject.toml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 30d2962248c..8f9e3ff350c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,8 @@ dependencies = [ "tomli>=1.1.0; python_version < '3.11'", "typing_extensions>=4.0.1; python_version < '3.11'", ] -dynamic = ["readme", "version"] +version = "25.1.0" +dynamic = ["readme"] [project.optional-dependencies] colorama = ["colorama>=0.4.3"] @@ -186,16 +187,6 @@ MYPYC_DEBUG_LEVEL = "0" # Black needs Clang to compile successfully on Linux. CC = "clang" -[tool.cibuildwheel.macos] -build-frontend = { name = "build", args = ["--no-isolation"] } -# Unfortunately, hatch doesn't respect MACOSX_DEPLOYMENT_TARGET -# Note we don't have a good test for this sed horror, so if you futz with it -# make sure to test manually -before-build = [ - "python -m pip install 'hatchling==1.20.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy>=1.12' 'click>=8.1.7'", - """sed -i '' -e "600,700s/'10_16'/os.environ['MACOSX_DEPLOYMENT_TARGET'].replace('.', '_')/" $(python -c 'import hatchling.builders.wheel as h; print(h.__file__)') """, -] - [tool.isort] atomic = true profile = "black" From b844c8a1360b2c00e0204d079e3a81e7342a0509 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 28 Jan 2025 21:54:46 -0800 Subject: [PATCH 02/69] unhack pyproject.toml (#4566) --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8f9e3ff350c..8c28d254be1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,8 +72,7 @@ dependencies = [ "tomli>=1.1.0; python_version < '3.11'", "typing_extensions>=4.0.1; python_version < '3.11'", ] -version = "25.1.0" -dynamic = ["readme"] +dynamic = ["readme", "version"] [project.optional-dependencies] colorama = ["colorama>=0.4.3"] From edaf085a189a529c9af3f0c284c97679bbd43fed Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 28 Jan 2025 21:55:17 -0800 Subject: [PATCH 03/69] new changelog template --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index d7acf6b7b9d..25a4b23375f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 25.1.0 ### Highlights From c02ca47daaad8063539aed6ab2755b04bb63a038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Wed, 29 Jan 2025 21:25:00 +0100 Subject: [PATCH 04/69] Fix mis-synced version check in black.vim (#4567) The message has been updated to indicate Python 3.9+, but the check still compares to 3.8 --- CHANGES.md | 2 +- autoload/black.vim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 25a4b23375f..7a2d1430f35 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,7 +20,7 @@ ### Packaging - +- Fix the version check in the vim file to reject Python 3.8 (#4567) ### Parser diff --git a/autoload/black.vim b/autoload/black.vim index 1cae6adab49..cb2233c465a 100644 --- a/autoload/black.vim +++ b/autoload/black.vim @@ -75,7 +75,7 @@ def _initialize_black_env(upgrade=False): return True pyver = sys.version_info[:3] - if pyver < (3, 8): + if pyver < (3, 9): print("Sorry, Black requires Python 3.9+ to run.") return False From 9c129567e75728e2b9382f4c79bf72ef6beac37e Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Wed, 29 Jan 2025 14:29:55 -0800 Subject: [PATCH 05/69] Re-add packaging CHANGES.md comment (#4568) --- CHANGES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7a2d1430f35..d8348c98f1a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,7 +20,7 @@ ### Packaging -- Fix the version check in the vim file to reject Python 3.8 (#4567) + ### Parser @@ -42,6 +42,8 @@ +- Fix the version check in the vim file to reject Python 3.8 (#4567) + ### Documentation +- Fix bug where certain unusual expressions (e.g., lambdas) were not accepted + in type parameter bounds and defaults. (#4602) + ### Performance diff --git a/src/blib2to3/Grammar.txt b/src/blib2to3/Grammar.txt index c8800e21931..b779b49eefb 100644 --- a/src/blib2to3/Grammar.txt +++ b/src/blib2to3/Grammar.txt @@ -12,9 +12,9 @@ file_input: (NEWLINE | stmt)* ENDMARKER single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE eval_input: testlist NEWLINE* ENDMARKER -typevar: NAME [':' expr] ['=' expr] -paramspec: '**' NAME ['=' expr] -typevartuple: '*' NAME ['=' (expr|star_expr)] +typevar: NAME [':' test] ['=' test] +paramspec: '**' NAME ['=' test] +typevartuple: '*' NAME ['=' (test|star_expr)] typeparam: typevar | paramspec | typevartuple typeparams: '[' typeparam (',' typeparam)* [','] ']' diff --git a/tests/data/cases/type_param_defaults.py b/tests/data/cases/type_param_defaults.py index feba64a2c72..f8c24eb7cff 100644 --- a/tests/data/cases/type_param_defaults.py +++ b/tests/data/cases/type_param_defaults.py @@ -20,6 +20,8 @@ def trailing_comma1[T=int,](a: str): def trailing_comma2[T=int](a: str,): pass +def weird_syntax[T=lambda: 42, **P=lambda: 43, *Ts=lambda: 44](): pass + # output type A[T = int] = float @@ -61,3 +63,7 @@ def trailing_comma2[T = int]( a: str, ): pass + + +def weird_syntax[T = lambda: 42, **P = lambda: 43, *Ts = lambda: 44](): + pass diff --git a/tests/data/cases/type_params.py b/tests/data/cases/type_params.py index 720a775ef31..f8fc3855741 100644 --- a/tests/data/cases/type_params.py +++ b/tests/data/cases/type_params.py @@ -13,6 +13,8 @@ def it_gets_worse[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplit def magic[Trailing, Comma,](): pass +def weird_syntax[T: lambda: 42, U: a or b](): pass + # output @@ -56,3 +58,7 @@ def magic[ Comma, ](): pass + + +def weird_syntax[T: lambda: 42, U: a or b](): + pass From bb802cf19a2346820fe8f192de77837c5ad6da6a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 06:24:03 -0800 Subject: [PATCH 17/69] Bump sphinx from 8.2.1 to 8.2.3 in /docs (#4603) Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 8.2.1 to 8.2.3. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v8.2.1...v8.2.3) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f86cb77818c..528b2972200 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ # Used by ReadTheDocs; pinned requirements for stability. myst-parser==4.0.1 -Sphinx==8.2.1 +Sphinx==8.2.3 # Older versions break Sphinx even though they're declared to be supported. docutils==0.21.2 sphinxcontrib-programoutput==0.18 From 3e9dd25dadb2a3fb02b1936d874173808020c481 Mon Sep 17 00:00:00 2001 From: Pedro Mezacasa Muller <114496585+Pedro-Muller29@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:11:21 -0300 Subject: [PATCH 18/69] Fix bug where # fmt: skip is not being respected with one-liner functions (#4552) --- CHANGES.md | 3 ++ docs/the_black_code_style/future_style.md | 3 ++ src/black/comments.py | 61 +++++++++++++++++++---- src/black/mode.py | 1 + src/black/resources/black.schema.json | 3 +- tests/data/cases/fmtskip10.py | 9 ++++ 6 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 tests/data/cases/fmtskip10.py diff --git a/CHANGES.md b/CHANGES.md index a81e407a943..8d8808c2573 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,9 @@ +- Fix a bug where one-liner functions/conditionals marked with `# fmt: skip` + would still be formatted (#4552) + ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index da2e8b93433..e801874a4f0 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -26,6 +26,9 @@ Currently, the following features are included in the preview style: statements, except when the line after the import is a comment or an import statement - `wrap_long_dict_values_in_parens`: Add parentheses around long values in dictionaries ([see below](labels/wrap-long-dict-values)) +- `fix_fmt_skip_in_one_liners`: Fix `# fmt: skip` behaviour on one-liner declarations, + such as `def foo(): return "mock" # fmt: skip`, where previously the declaration + would have been incorrectly collapsed. (labels/unstable-features)= diff --git a/src/black/comments.py b/src/black/comments.py index f42a51033db..1054e7ae8a2 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -4,7 +4,7 @@ from functools import lru_cache from typing import Final, Optional, Union -from black.mode import Mode +from black.mode import Mode, Preview from black.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, @@ -270,7 +270,7 @@ def generate_ignored_nodes( Stops at the end of the block. """ if _contains_fmt_skip_comment(comment.value, mode): - yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment) + yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment, mode) return container: Optional[LN] = container_of(leaf) while container is not None and container.type != token.ENDMARKER: @@ -309,11 +309,12 @@ def generate_ignored_nodes( def _generate_ignored_nodes_from_fmt_skip( - leaf: Leaf, comment: ProtoComment + leaf: Leaf, comment: ProtoComment, mode: Mode ) -> Iterator[LN]: """Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`.""" prev_sibling = leaf.prev_sibling parent = leaf.parent + ignored_nodes: list[LN] = [] # Need to properly format the leaf prefix to compare it to comment.value, # which is also formatted comments = list_comments(leaf.prefix, is_endmarker=False) @@ -321,11 +322,54 @@ def _generate_ignored_nodes_from_fmt_skip( return if prev_sibling is not None: leaf.prefix = "" - siblings = [prev_sibling] - while "\n" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None: - prev_sibling = prev_sibling.prev_sibling - siblings.insert(0, prev_sibling) - yield from siblings + + if Preview.fix_fmt_skip_in_one_liners not in mode: + siblings = [prev_sibling] + while ( + "\n" not in prev_sibling.prefix + and prev_sibling.prev_sibling is not None + ): + prev_sibling = prev_sibling.prev_sibling + siblings.insert(0, prev_sibling) + yield from siblings + return + + # Generates the nodes to be ignored by `fmt: skip`. + + # Nodes to ignore are the ones on the same line as the + # `# fmt: skip` comment, excluding the `# fmt: skip` + # node itself. + + # Traversal process (starting at the `# fmt: skip` node): + # 1. Move to the `prev_sibling` of the current node. + # 2. If `prev_sibling` has children, go to its rightmost leaf. + # 3. If there’s no `prev_sibling`, move up to the parent + # node and repeat. + # 4. Continue until: + # a. You encounter an `INDENT` or `NEWLINE` node (indicates + # start of the line). + # b. You reach the root node. + + # Include all visited LEAVES in the ignored list, except INDENT + # or NEWLINE leaves. + + current_node = prev_sibling + ignored_nodes = [current_node] + if current_node.prev_sibling is None and current_node.parent is not None: + current_node = current_node.parent + while "\n" not in current_node.prefix and current_node.prev_sibling is not None: + leaf_nodes = list(current_node.prev_sibling.leaves()) + current_node = leaf_nodes[-1] if leaf_nodes else current_node + + if current_node.type in (token.NEWLINE, token.INDENT): + current_node.prefix = "" + break + + ignored_nodes.insert(0, current_node) + + if current_node.prev_sibling is None and current_node.parent is not None: + current_node = current_node.parent + yield from ignored_nodes elif ( parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE ): @@ -333,7 +377,6 @@ def _generate_ignored_nodes_from_fmt_skip( # statements. The ignored nodes should be previous siblings of the # parent suite node. leaf.prefix = "" - ignored_nodes: list[LN] = [] parent_sibling = parent.prev_sibling while parent_sibling is not None and parent_sibling.type != syms.suite: ignored_nodes.insert(0, parent_sibling) diff --git a/src/black/mode.py b/src/black/mode.py index 7335bd12078..362607efc86 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -203,6 +203,7 @@ class Preview(Enum): wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() always_one_newline_after_import = auto() + fix_fmt_skip_in_one_liners = auto() UNSTABLE_FEATURES: set[Preview] = { diff --git a/src/black/resources/black.schema.json b/src/black/resources/black.schema.json index b9b61489136..572e5bbfa1e 100644 --- a/src/black/resources/black.schema.json +++ b/src/black/resources/black.schema.json @@ -83,7 +83,8 @@ "hug_parens_with_braces_and_square_brackets", "wrap_long_dict_values_in_parens", "multiline_string_handling", - "always_one_newline_after_import" + "always_one_newline_after_import", + "fix_fmt_skip_in_one_liners" ] }, "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." diff --git a/tests/data/cases/fmtskip10.py b/tests/data/cases/fmtskip10.py new file mode 100644 index 00000000000..0c017719b2d --- /dev/null +++ b/tests/data/cases/fmtskip10.py @@ -0,0 +1,9 @@ +# flags: --preview +def foo(): return "mock" # fmt: skip +if True: print("yay") # fmt: skip +for i in range(10): print(i) # fmt: skip + +j = 1 # fmt: skip +while j < 10: j += 1 # fmt: skip + +b = [c for c in "A very long string that would normally generate some kind of collapse, since it is this long"] # fmt: skip From 9f38928414e6a39044f9b148692e90f3e1fd3433 Mon Sep 17 00:00:00 2001 From: Glyph Date: Wed, 5 Mar 2025 18:26:00 -0800 Subject: [PATCH 19/69] github is deprecating the ubuntu 20.04 actions runner image (#4607) see https://github.com/actions/runner-images/issues/11101 --- .github/workflows/upload_binary.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upload_binary.yml b/.github/workflows/upload_binary.yml index 1bde446442a..41231016cde 100644 --- a/.github/workflows/upload_binary.yml +++ b/.github/workflows/upload_binary.yml @@ -13,13 +13,13 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-2019, ubuntu-20.04, macos-latest] + os: [windows-2019, ubuntu-22.04, macos-latest] include: - os: windows-2019 pathsep: ";" asset_name: black_windows.exe executable_mime: "application/vnd.microsoft.portable-executable" - - os: ubuntu-20.04 + - os: ubuntu-22.04 pathsep: ":" asset_name: black_linux executable_mime: "application/x-executable" From 5342d2eeda6935a96d2bd0c74072b3c441d76bde Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sun, 16 Mar 2025 06:11:19 +0530 Subject: [PATCH 20/69] Replace the blib2to3 tokenizer with pytokens (#4536) --- .pre-commit-config.yaml | 1 + CHANGES.md | 1 + pyproject.toml | 1 + src/blib2to3/pgen2/driver.py | 20 +- src/blib2to3/pgen2/pgen.py | 7 +- src/blib2to3/pgen2/tokenize.py | 1176 +++----------------- tests/data/miscellaneous/debug_visitor.out | 20 - tests/test_black.py | 13 +- tests/test_tokenize.py | 32 +- 9 files changed, 183 insertions(+), 1088 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37ad51d097a..b683ae8cf0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,6 +51,7 @@ repos: - click >= 8.1.0, != 8.1.4, != 8.1.5 - packaging >= 22.0 - platformdirs >= 2.1.0 + - pytokens >= 0.1.10 - pytest - hypothesis - aiohttp >= 3.7.4 diff --git a/CHANGES.md b/CHANGES.md index 8d8808c2573..cc42cde14ef 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,6 +29,7 @@ +- Rewrite tokenizer to improve performance and compliance (#4536) - Fix bug where certain unusual expressions (e.g., lambdas) were not accepted in type parameter bounds and defaults. (#4602) diff --git a/pyproject.toml b/pyproject.toml index eb46241c398..bcd3ebceab8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ dependencies = [ "packaging>=22.0", "pathspec>=0.9.0", "platformdirs>=2", + "pytokens>=0.1.10", "tomli>=1.1.0; python_version < '3.11'", "typing_extensions>=4.0.1; python_version < '3.11'", ] diff --git a/src/blib2to3/pgen2/driver.py b/src/blib2to3/pgen2/driver.py index d17fd1d7bfb..056fab2127b 100644 --- a/src/blib2to3/pgen2/driver.py +++ b/src/blib2to3/pgen2/driver.py @@ -28,7 +28,7 @@ from typing import IO, Any, Optional, Union, cast from blib2to3.pgen2.grammar import Grammar -from blib2to3.pgen2.tokenize import GoodTokenInfo +from blib2to3.pgen2.tokenize import TokenInfo from blib2to3.pytree import NL # Pgen imports @@ -112,7 +112,7 @@ def __init__(self, grammar: Grammar, logger: Optional[Logger] = None) -> None: logger = logging.getLogger(__name__) self.logger = logger - def parse_tokens(self, tokens: Iterable[GoodTokenInfo], debug: bool = False) -> NL: + def parse_tokens(self, tokens: Iterable[TokenInfo], debug: bool = False) -> NL: """Parse a series of tokens and return the syntax tree.""" # XXX Move the prefix computation into a wrapper around tokenize. proxy = TokenProxy(tokens) @@ -180,27 +180,17 @@ def parse_tokens(self, tokens: Iterable[GoodTokenInfo], debug: bool = False) -> assert p.rootnode is not None return p.rootnode - def parse_stream_raw(self, stream: IO[str], debug: bool = False) -> NL: - """Parse a stream and return the syntax tree.""" - tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar) - return self.parse_tokens(tokens, debug) - - def parse_stream(self, stream: IO[str], debug: bool = False) -> NL: - """Parse a stream and return the syntax tree.""" - return self.parse_stream_raw(stream, debug) - def parse_file( self, filename: Path, encoding: Optional[str] = None, debug: bool = False ) -> NL: """Parse a file and return the syntax tree.""" with open(filename, encoding=encoding) as stream: - return self.parse_stream(stream, debug) + text = stream.read() + return self.parse_string(text, debug) def parse_string(self, text: str, debug: bool = False) -> NL: """Parse a string and return the syntax tree.""" - tokens = tokenize.generate_tokens( - io.StringIO(text).readline, grammar=self.grammar - ) + tokens = tokenize.tokenize(text, grammar=self.grammar) return self.parse_tokens(tokens, debug) def _partially_consume_prefix(self, prefix: str, column: int) -> tuple[str, str]: diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py index ea6d8cc19a5..6599c1f226c 100644 --- a/src/blib2to3/pgen2/pgen.py +++ b/src/blib2to3/pgen2/pgen.py @@ -6,7 +6,7 @@ from typing import IO, Any, NoReturn, Optional, Union from blib2to3.pgen2 import grammar, token, tokenize -from blib2to3.pgen2.tokenize import GoodTokenInfo +from blib2to3.pgen2.tokenize import TokenInfo Path = Union[str, "os.PathLike[str]"] @@ -18,7 +18,7 @@ class PgenGrammar(grammar.Grammar): class ParserGenerator: filename: Path stream: IO[str] - generator: Iterator[GoodTokenInfo] + generator: Iterator[TokenInfo] first: dict[str, Optional[dict[str, int]]] def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None: @@ -27,8 +27,7 @@ def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None: stream = open(filename, encoding="utf-8") close_stream = stream.close self.filename = filename - self.stream = stream - self.generator = tokenize.generate_tokens(stream.readline) + self.generator = tokenize.tokenize(stream.read()) self.gettoken() # Initialize lookahead self.dfas, self.startsymbol = self.parse() if close_stream is not None: diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 407c184dd74..5cbfd5148d8 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -27,11 +27,9 @@ function to which the 5 fields described above are passed as 5 arguments, each time a new token is found.""" -import builtins import sys -from collections.abc import Callable, Iterable, Iterator -from re import Pattern -from typing import Final, Optional, Union +from collections.abc import Iterator +from typing import Optional from blib2to3.pgen2.grammar import Grammar from blib2to3.pgen2.token import ( @@ -45,13 +43,11 @@ FSTRING_MIDDLE, FSTRING_START, INDENT, - LBRACE, NAME, NEWLINE, NL, NUMBER, OP, - RBRACE, STRING, tok_name, ) @@ -59,1056 +55,206 @@ __author__ = "Ka-Ping Yee " __credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro" -import re -from codecs import BOM_UTF8, lookup +import pytokens +from pytokens import TokenType -from . import token +from . import token as _token -__all__ = [x for x in dir(token) if x[0] != "_"] + [ +__all__ = [x for x in dir(_token) if x[0] != "_"] + [ "tokenize", "generate_tokens", "untokenize", ] -del token - - -def group(*choices: str) -> str: - return "(" + "|".join(choices) + ")" - - -def any(*choices: str) -> str: - return group(*choices) + "*" - - -def maybe(*choices: str) -> str: - return group(*choices) + "?" - - -def _combinations(*l: str) -> set[str]: - return {x + y for x in l for y in l + ("",) if x.casefold() != y.casefold()} - - -Whitespace = r"[ \f\t]*" -Comment = r"#[^\r\n]*" -Ignore = Whitespace + any(r"\\\r?\n" + Whitespace) + maybe(Comment) -Name = ( # this is invalid but it's fine because Name comes after Number in all groups - r"[^\s#\(\)\[\]\{\}+\-*/!@$%^&=|;:'\",\.<>/?`~\\]+" -) - -Binnumber = r"0[bB]_?[01]+(?:_[01]+)*" -Hexnumber = r"0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?" -Octnumber = r"0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?" -Decnumber = group(r"[1-9]\d*(?:_\d+)*[lL]?", "0[lL]?") -Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r"[eE][-+]?\d+(?:_\d+)*" -Pointfloat = group(r"\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?", r"\.\d+(?:_\d+)*") + maybe( - Exponent -) -Expfloat = r"\d+(?:_\d+)*" + Exponent -Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r"\d+(?:_\d+)*[jJ]", Floatnumber + r"[jJ]") -Number = group(Imagnumber, Floatnumber, Intnumber) - -# Tail end of ' string. -Single = r"(?:\\.|[^'\\])*'" -# Tail end of " string. -Double = r'(?:\\.|[^"\\])*"' -# Tail end of ''' string. -Single3 = r"(?:\\.|'(?!'')|[^'\\])*'''" -# Tail end of """ string. -Double3 = r'(?:\\.|"(?!"")|[^"\\])*"""' -_litprefix = r"(?:[uUrRbB]|[rR][bB]|[bBuU][rR])?" -_fstringlitprefix = r"(?:rF|FR|Fr|fr|RF|F|rf|f|Rf|fR)" -Triple = group( - _litprefix + "'''", - _litprefix + '"""', - _fstringlitprefix + '"""', - _fstringlitprefix + "'''", -) - -# beginning of a single quoted f-string. must not end with `{{` or `\N{` -SingleLbrace = r"(?:\\N{|{{|\\'|[^\n'{])*(?>=?", - r"<<=?", - r"<>", - r"!=", - r"//=?", - r"->", - r"[+\-*/%&@|^=<>:]=?", - r"~", -) - -Bracket = "[][(){}]" -Special = group(r"\r?\n", r"[:;.,`@]") -Funny = group(Operator, Bracket, Special) - -_string_middle_single = r"(?:[^\n'\\]|\\.)*" -_string_middle_double = r'(?:[^\n"\\]|\\.)*' - -# FSTRING_MIDDLE and LBRACE, must not end with a `{{` or `\N{` -_fstring_middle_single = SingleLbrace -_fstring_middle_double = DoubleLbrace - -# First (or only) line of ' or " string. -ContStr = group( - _litprefix + "'" + _string_middle_single + group("'", r"\\\r?\n"), - _litprefix + '"' + _string_middle_double + group('"', r"\\\r?\n"), - group(_fstringlitprefix + "'") + _fstring_middle_single, - group(_fstringlitprefix + '"') + _fstring_middle_double, - group(_fstringlitprefix + "'") + _string_middle_single + group("'", r"\\\r?\n"), - group(_fstringlitprefix + '"') + _string_middle_double + group('"', r"\\\r?\n"), -) -PseudoExtras = group(r"\\\r?\n", Comment, Triple) -PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) - -pseudoprog: Final = re.compile(PseudoToken, re.UNICODE) - -singleprog = re.compile(Single) -singleprog_plus_lbrace = re.compile(group(SingleLbrace, Single)) -doubleprog = re.compile(Double) -doubleprog_plus_lbrace = re.compile(group(DoubleLbrace, Double)) - -single3prog = re.compile(Single3) -single3prog_plus_lbrace = re.compile(group(Single3Lbrace, Single3)) -double3prog = re.compile(Double3) -double3prog_plus_lbrace = re.compile(group(Double3Lbrace, Double3)) - -_strprefixes = _combinations("r", "R", "b", "B") | {"u", "U", "ur", "uR", "Ur", "UR"} -_fstring_prefixes = _combinations("r", "R", "f", "F") - {"r", "R"} - -endprogs: Final = { - "'": singleprog, - '"': doubleprog, - "'''": single3prog, - '"""': double3prog, - **{f"{prefix}'": singleprog for prefix in _strprefixes}, - **{f'{prefix}"': doubleprog for prefix in _strprefixes}, - **{f"{prefix}'": singleprog_plus_lbrace for prefix in _fstring_prefixes}, - **{f'{prefix}"': doubleprog_plus_lbrace for prefix in _fstring_prefixes}, - **{f"{prefix}'''": single3prog for prefix in _strprefixes}, - **{f'{prefix}"""': double3prog for prefix in _strprefixes}, - **{f"{prefix}'''": single3prog_plus_lbrace for prefix in _fstring_prefixes}, - **{f'{prefix}"""': double3prog_plus_lbrace for prefix in _fstring_prefixes}, -} - -triple_quoted: Final = ( - {"'''", '"""'} - | {f"{prefix}'''" for prefix in _strprefixes | _fstring_prefixes} - | {f'{prefix}"""' for prefix in _strprefixes | _fstring_prefixes} -) -single_quoted: Final = ( - {"'", '"'} - | {f"{prefix}'" for prefix in _strprefixes | _fstring_prefixes} - | {f'{prefix}"' for prefix in _strprefixes | _fstring_prefixes} -) -fstring_prefix: Final = tuple( - {f"{prefix}'" for prefix in _fstring_prefixes} - | {f'{prefix}"' for prefix in _fstring_prefixes} - | {f"{prefix}'''" for prefix in _fstring_prefixes} - | {f'{prefix}"""' for prefix in _fstring_prefixes} -) - -tabsize = 8 - - -class TokenError(Exception): - pass - - -class StopTokenizing(Exception): - pass - +del _token Coord = tuple[int, int] +TokenInfo = tuple[int, str, Coord, Coord, str] + +TOKEN_TYPE_MAP = { + TokenType.indent: INDENT, + TokenType.dedent: DEDENT, + TokenType.newline: NEWLINE, + TokenType.nl: NL, + TokenType.comment: COMMENT, + TokenType.semicolon: OP, + TokenType.lparen: OP, + TokenType.rparen: OP, + TokenType.lbracket: OP, + TokenType.rbracket: OP, + TokenType.lbrace: OP, + TokenType.rbrace: OP, + TokenType.colon: OP, + TokenType.op: OP, + TokenType.identifier: NAME, + TokenType.number: NUMBER, + TokenType.string: STRING, + TokenType.fstring_start: FSTRING_START, + TokenType.fstring_middle: FSTRING_MIDDLE, + TokenType.fstring_end: FSTRING_END, + TokenType.endmarker: ENDMARKER, +} -def printtoken( - type: int, token: str, srow_col: Coord, erow_col: Coord, line: str -) -> None: # for testing - (srow, scol) = srow_col - (erow, ecol) = erow_col - print( - "%d,%d-%d,%d:\t%s\t%s" % (srow, scol, erow, ecol, tok_name[type], repr(token)) - ) - - -TokenEater = Callable[[int, str, Coord, Coord, str], None] - - -def tokenize(readline: Callable[[], str], tokeneater: TokenEater = printtoken) -> None: - """ - The tokenize() function accepts two parameters: one representing the - input stream, and one providing an output mechanism for tokenize(). +class TokenError(Exception): ... - The first parameter, readline, must be a callable object which provides - the same interface as the readline() method of built-in file objects. - Each call to the function should return one line of input as a string. - The second parameter, tokeneater, must also be a callable object. It is - called once for each token, with five arguments, corresponding to the - tuples generated by generate_tokens(). +def transform_whitespace( + token: pytokens.Token, source: str, prev_token: Optional[pytokens.Token] +) -> pytokens.Token: + r""" + Black treats `\\\n` at the end of a line as a 'NL' token, while it + is ignored as whitespace in the regular Python parser. + But, only the first one. If there's a `\\\n` following it + (as in, a \ just by itself on a line), that is not made into NL. """ - try: - tokenize_loop(readline, tokeneater) - except StopTokenizing: - pass - - -# backwards compatible interface -def tokenize_loop(readline: Callable[[], str], tokeneater: TokenEater) -> None: - for token_info in generate_tokens(readline): - tokeneater(*token_info) - - -GoodTokenInfo = tuple[int, str, Coord, Coord, str] -TokenInfo = Union[tuple[int, str], GoodTokenInfo] - - -class Untokenizer: - tokens: list[str] - prev_row: int - prev_col: int - - def __init__(self) -> None: - self.tokens = [] - self.prev_row = 1 - self.prev_col = 0 - - def add_whitespace(self, start: Coord) -> None: - row, col = start - assert row <= self.prev_row - col_offset = col - self.prev_col - if col_offset: - self.tokens.append(" " * col_offset) - - def untokenize(self, iterable: Iterable[TokenInfo]) -> str: - for t in iterable: - if len(t) == 2: - self.compat(t, iterable) - break - tok_type, token, start, end, line = t - self.add_whitespace(start) - self.tokens.append(token) - self.prev_row, self.prev_col = end - if tok_type in (NEWLINE, NL): - self.prev_row += 1 - self.prev_col = 0 - return "".join(self.tokens) - - def compat(self, token: tuple[int, str], iterable: Iterable[TokenInfo]) -> None: - startline = False - indents = [] - toks_append = self.tokens.append - toknum, tokval = token - if toknum in (NAME, NUMBER): - tokval += " " - if toknum in (NEWLINE, NL): - startline = True - for tok in iterable: - toknum, tokval = tok[:2] - - if toknum in (NAME, NUMBER, ASYNC, AWAIT): - tokval += " " - - if toknum == INDENT: - indents.append(tokval) - continue - elif toknum == DEDENT: - indents.pop() - continue - elif toknum in (NEWLINE, NL): - startline = True - elif startline and indents: - toks_append(indents[-1]) - startline = False - toks_append(tokval) - - -cookie_re = re.compile(r"^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)", re.ASCII) -blank_re = re.compile(rb"^[ \t\f]*(?:[#\r\n]|$)", re.ASCII) - - -def _get_normal_name(orig_enc: str) -> str: - """Imitates get_normal_name in tokenizer.c.""" - # Only care about the first 12 characters. - enc = orig_enc[:12].lower().replace("_", "-") - if enc == "utf-8" or enc.startswith("utf-8-"): - return "utf-8" - if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or enc.startswith( - ("latin-1-", "iso-8859-1-", "iso-latin-1-") + if ( + token.type == TokenType.whitespace + and prev_token is not None + and prev_token.type not in (TokenType.nl, TokenType.newline) ): - return "iso-8859-1" - return orig_enc - - -def detect_encoding(readline: Callable[[], bytes]) -> tuple[str, list[bytes]]: - """ - The detect_encoding() function is used to detect the encoding that should - be used to decode a Python source file. It requires one argument, readline, - in the same way as the tokenize() generator. - - It will call readline a maximum of twice, and return the encoding used - (as a string) and a list of any lines (left as bytes) it has read - in. - - It detects the encoding from the presence of a utf-8 bom or an encoding - cookie as specified in pep-0263. If both a bom and a cookie are present, but - disagree, a SyntaxError will be raised. If the encoding cookie is an invalid - charset, raise a SyntaxError. Note that if a utf-8 bom is found, - 'utf-8-sig' is returned. - - If no encoding is specified, then the default of 'utf-8' will be returned. - """ - bom_found = False - encoding = None - default = "utf-8" - - def read_or_stop() -> bytes: - try: - return readline() - except StopIteration: - return b"" - - def find_cookie(line: bytes) -> Optional[str]: - try: - line_string = line.decode("ascii") - except UnicodeDecodeError: - return None - match = cookie_re.match(line_string) - if not match: - return None - encoding = _get_normal_name(match.group(1)) - try: - codec = lookup(encoding) - except LookupError: - # This behaviour mimics the Python interpreter - raise SyntaxError("unknown encoding: " + encoding) - - if bom_found: - if codec.name != "utf-8": - # This behaviour mimics the Python interpreter - raise SyntaxError("encoding problem: utf-8") - encoding += "-sig" - return encoding - - first = read_or_stop() - if first.startswith(BOM_UTF8): - bom_found = True - first = first[3:] - default = "utf-8-sig" - if not first: - return default, [] - - encoding = find_cookie(first) - if encoding: - return encoding, [first] - if not blank_re.match(first): - return default, [first] - - second = read_or_stop() - if not second: - return default, [first] - - encoding = find_cookie(second) - if encoding: - return encoding, [first, second] - - return default, [first, second] - - -def untokenize(iterable: Iterable[TokenInfo]) -> str: - """Transform tokens back into Python source code. - - Each element returned by the iterable must be a token sequence - with at least two elements, a token number and token value. If - only two tokens are passed, the resulting output is poor. - - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited input: - # Output text will tokenize the back to the input - t1 = [tok[:2] for tok in generate_tokens(f.readline)] - newcode = untokenize(t1) - readline = iter(newcode.splitlines(1)).next - t2 = [tok[:2] for tokin generate_tokens(readline)] - assert t1 == t2 - """ - ut = Untokenizer() - return ut.untokenize(iterable) - - -def is_fstring_start(token: str) -> bool: - return token.startswith(fstring_prefix) - - -def _split_fstring_start_and_middle(token: str) -> tuple[str, str]: - for prefix in fstring_prefix: - _, prefix, rest = token.partition(prefix) - if prefix != "": - return prefix, rest - - raise ValueError(f"Token {token!r} is not a valid f-string start") - - -STATE_NOT_FSTRING: Final = 0 # not in an f-string -STATE_MIDDLE: Final = 1 # in the string portion of an f-string (outside braces) -STATE_IN_BRACES: Final = 2 # between braces in an f-string -# in the format specifier (between the colon and the closing brace) -STATE_IN_COLON: Final = 3 - - -class FStringState: - """Keeps track of state around f-strings. - - The tokenizer should call the appropriate method on this class when - it transitions to a different part of an f-string. This is needed - because the tokenization depends on knowing where exactly we are in - the f-string. - - For example, consider the following f-string: - - f"a{1:b{2}c}d" - - The following is the tokenization of this string and the states - tracked by this class: - - 1,0-1,2: FSTRING_START 'f"' # [STATE_NOT_FSTRING, STATE_MIDDLE] - 1,2-1,3: FSTRING_MIDDLE 'a' - 1,3-1,4: LBRACE '{' # [STATE_NOT_FSTRING, STATE_IN_BRACES] - 1,4-1,5: NUMBER '1' - 1,5-1,6: OP ':' # [STATE_NOT_FSTRING, STATE_IN_COLON] - 1,6-1,7: FSTRING_MIDDLE 'b' - 1,7-1,8: LBRACE '{' # [STATE_NOT_FSTRING, STATE_IN_COLON, STATE_IN_BRACES] - 1,8-1,9: NUMBER '2' - 1,9-1,10: RBRACE '}' # [STATE_NOT_FSTRING, STATE_IN_COLON] - 1,10-1,11: FSTRING_MIDDLE 'c' - 1,11-1,12: RBRACE '}' # [STATE_NOT_FSTRING, STATE_MIDDLE] - 1,12-1,13: FSTRING_MIDDLE 'd' - 1,13-1,14: FSTRING_END '"' # [STATE_NOT_FSTRING] - 1,14-1,15: NEWLINE '\n' - 2,0-2,0: ENDMARKER '' - - Notice that the nested braces in the format specifier are represented - by adding a STATE_IN_BRACES entry to the state stack. The stack is - also used if there are nested f-strings. - - """ - - def __init__(self) -> None: - self.stack: list[int] = [STATE_NOT_FSTRING] - - def is_in_fstring_expression(self) -> bool: - return self.stack[-1] not in (STATE_MIDDLE, STATE_NOT_FSTRING) - - def current(self) -> int: - return self.stack[-1] - - def enter_fstring(self) -> None: - self.stack.append(STATE_MIDDLE) - - def leave_fstring(self) -> None: - state = self.stack.pop() - assert state == STATE_MIDDLE - - def consume_lbrace(self) -> None: - current_state = self.stack[-1] - if current_state == STATE_MIDDLE: - self.stack[-1] = STATE_IN_BRACES - elif current_state == STATE_IN_COLON: - self.stack.append(STATE_IN_BRACES) - else: - assert False, current_state - - def consume_rbrace(self) -> None: - current_state = self.stack[-1] - assert current_state in (STATE_IN_BRACES, STATE_IN_COLON) - if len(self.stack) > 1 and self.stack[-2] == STATE_IN_COLON: - self.stack.pop() - else: - self.stack[-1] = STATE_MIDDLE - - def consume_colon(self) -> None: - assert self.stack[-1] == STATE_IN_BRACES, self.stack - self.stack[-1] = STATE_IN_COLON - - -def generate_tokens( - readline: Callable[[], str], grammar: Optional[Grammar] = None -) -> Iterator[GoodTokenInfo]: - """ - The generate_tokens() generator requires one argument, readline, which - must be a callable object which provides the same interface as the - readline() method of built-in file objects. Each call to the function - should return one line of input as a string. Alternately, readline - can be a callable function terminating with StopIteration: - readline = open(myfile).next # Example of alternate readline - - The generator produces 5-tuples with these members: the token type; the - token string; a 2-tuple (srow, scol) of ints specifying the row and - column where the token begins in the source; a 2-tuple (erow, ecol) of - ints specifying the row and column where the token ends in the source; - and the line on which the token was found. The line passed is the - logical line; continuation lines are included. - """ - lnum = parenlev = continued = 0 - parenlev_stack: list[int] = [] - fstring_state = FStringState() - formatspec = "" - numchars: Final[str] = "0123456789" - contstr, needcont = "", 0 - contline: Optional[str] = None - indents = [0] - - # If we know we're parsing 3.7+, we can unconditionally parse `async` and - # `await` as keywords. + token_str = source[token.start_index : token.end_index] + if token_str.startswith("\\\n"): + return pytokens.Token( + TokenType.nl, + token.start_index, + token.start_index + 2, + token.start_line, + token.start_col, + token.start_line, + token.start_col + 2, + ) + + return token + + +def tokenize(source: str, grammar: Optional[Grammar] = None) -> Iterator[TokenInfo]: async_keywords = False if grammar is None else grammar.async_keywords - # 'stashed' and 'async_*' are used for async/await parsing - stashed: Optional[GoodTokenInfo] = None - async_def = False - async_def_indent = 0 - async_def_nl = False - strstart: tuple[int, int] - endprog_stack: list[Pattern[str]] = [] - formatspec_start: tuple[int, int] + lines = source.split("\n") + lines += [""] # For newline tokens in files that don't end in a newline + line, column = 1, 0 - while 1: # loop over lines in stream - try: - line = readline() - except StopIteration: - line = "" - lnum += 1 + token_iterator = pytokens.tokenize(source) + is_async = False + current_indent = 0 + async_indent = 0 - # skip lines that are just indent characters ending with a slash - # to avoid storing that line's indent information. - if not contstr and line.rstrip("\n").strip(" \t\f") == "\\": - continue + prev_token: Optional[pytokens.Token] = None + try: + for token in token_iterator: + token = transform_whitespace(token, source, prev_token) - pos, max = 0, len(line) + line, column = token.start_line, token.start_col + if token.type == TokenType.whitespace: + continue - if contstr: # continued string - assert contline is not None - if not line: - raise TokenError("EOF in multi-line string", strstart) - endprog = endprog_stack[-1] - endmatch = endprog.match(line) - if endmatch: - end = endmatch.end(0) - token = contstr + line[:end] - spos = strstart - epos = (lnum, end) - tokenline = contline + line - if fstring_state.current() in ( - STATE_NOT_FSTRING, - STATE_IN_BRACES, - ) and not is_fstring_start(token): - yield (STRING, token, spos, epos, tokenline) - endprog_stack.pop() - parenlev = parenlev_stack.pop() - else: - if is_fstring_start(token): - fstring_start, token = _split_fstring_start_and_middle(token) - fstring_start_epos = (spos[0], spos[1] + len(fstring_start)) - yield ( - FSTRING_START, - fstring_start, - spos, - fstring_start_epos, - tokenline, - ) - fstring_state.enter_fstring() - # increase spos to the end of the fstring start - spos = fstring_start_epos + token_str = source[token.start_index : token.end_index] - if token.endswith("{"): - fstring_middle, lbrace = token[:-1], token[-1] - fstring_middle_epos = lbrace_spos = (lnum, end - 1) - yield ( - FSTRING_MIDDLE, - fstring_middle, - spos, - fstring_middle_epos, - line, - ) - yield (LBRACE, lbrace, lbrace_spos, epos, line) - fstring_state.consume_lbrace() - else: - if token.endswith(('"""', "'''")): - fstring_middle, fstring_end = token[:-3], token[-3:] - fstring_middle_epos = end_spos = (lnum, end - 3) - else: - fstring_middle, fstring_end = token[:-1], token[-1] - fstring_middle_epos = end_spos = (lnum, end - 1) - yield ( - FSTRING_MIDDLE, - fstring_middle, - spos, - fstring_middle_epos, - line, - ) - yield ( - FSTRING_END, - fstring_end, - end_spos, - epos, - line, - ) - fstring_state.leave_fstring() - endprog_stack.pop() - parenlev = parenlev_stack.pop() - pos = end - contstr, needcont = "", 0 - contline = None - elif needcont and line[-2:] != "\\\n" and line[-3:] != "\\\r\n": - yield ( - ERRORTOKEN, - contstr + line, - strstart, - (lnum, len(line)), - contline, - ) - contstr = "" - contline = None - continue - else: - contstr = contstr + line - contline = contline + line + if token.type == TokenType.newline and token_str == "": + # Black doesn't yield empty newline tokens at the end of a file + # if there's no newline at the end of a file. + prev_token = token continue - # new statement - elif ( - parenlev == 0 - and not continued - and not fstring_state.is_in_fstring_expression() - ): - if not line: - break - column = 0 - while pos < max: # measure leading whitespace - if line[pos] == " ": - column += 1 - elif line[pos] == "\t": - column = (column // tabsize + 1) * tabsize - elif line[pos] == "\f": - column = 0 - else: + if token.type == TokenType.indent: + current_indent += 1 + if token.type == TokenType.dedent: + current_indent -= 1 + if is_async and current_indent < async_indent: + is_async = False + + source_line = lines[token.start_line - 1] + + if token.type == TokenType.identifier and token_str in ("async", "await"): + # Black uses `async` and `await` token types just for those two keywords + while True: + next_token = next(token_iterator) + next_str = source[next_token.start_index : next_token.end_index] + next_token = transform_whitespace(next_token, next_str, token) + if next_token.type == TokenType.whitespace: + continue break - pos += 1 - if pos == max: - break - if stashed: - yield stashed - stashed = None + next_token_type = TOKEN_TYPE_MAP[next_token.type] + next_line = lines[next_token.start_line - 1] - if line[pos] in "\r\n": # skip blank lines - yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line) - continue + if token_str == "async" and ( + async_keywords + or (next_token_type == NAME and next_str in ("def", "for")) + ): + is_async = True + async_indent = current_indent + 1 + current_token_type = ASYNC + elif token_str == "await" and (async_keywords or is_async): + current_token_type = AWAIT + else: + current_token_type = TOKEN_TYPE_MAP[token.type] - if line[pos] == "#": # skip comments - comment_token = line[pos:].rstrip("\r\n") - nl_pos = pos + len(comment_token) yield ( - COMMENT, - comment_token, - (lnum, pos), - (lnum, nl_pos), - line, + current_token_type, + token_str, + (token.start_line, token.start_col), + (token.end_line, token.end_col), + source_line, ) - yield (NL, line[nl_pos:], (lnum, nl_pos), (lnum, len(line)), line) + yield ( + next_token_type, + next_str, + (next_token.start_line, next_token.start_col), + (next_token.end_line, next_token.end_col), + next_line, + ) + prev_token = token continue - if column > indents[-1]: # count indents - indents.append(column) - yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) - - while column < indents[-1]: # count dedents - if column not in indents: - raise IndentationError( - "unindent does not match any outer indentation level", - ("", lnum, pos, line), - ) - indents = indents[:-1] - - if async_def and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - - yield (DEDENT, "", (lnum, pos), (lnum, pos), line) + if token.type == TokenType.op and token_str == "...": + # Black doesn't have an ellipsis token yet, yield 3 DOTs instead + assert token.start_line == token.end_line + assert token.end_col == token.start_col + 3 - if async_def and async_def_nl and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - - else: # continued statement - if not line: - raise TokenError("EOF in multi-line statement", (lnum, 0)) - continued = 0 - - while pos < max: - if fstring_state.current() == STATE_MIDDLE: - endprog = endprog_stack[-1] - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - start, end = endmatch.span(0) - token = line[start:end] - if token.endswith(('"""', "'''")): - middle_token, end_token = token[:-3], token[-3:] - middle_epos = end_spos = (lnum, end - 3) - else: - middle_token, end_token = token[:-1], token[-1] - middle_epos = end_spos = (lnum, end - 1) - # TODO: unsure if this can be safely removed - if stashed: - yield stashed - stashed = None + token_str = "." + for start_col in range(token.start_col, token.start_col + 3): + end_col = start_col + 1 yield ( - FSTRING_MIDDLE, - middle_token, - (lnum, pos), - middle_epos, - line, + TOKEN_TYPE_MAP[token.type], + token_str, + (token.start_line, start_col), + (token.end_line, end_col), + source_line, ) - if not token.endswith("{"): - yield ( - FSTRING_END, - end_token, - end_spos, - (lnum, end), - line, - ) - fstring_state.leave_fstring() - endprog_stack.pop() - parenlev = parenlev_stack.pop() - else: - yield (LBRACE, "{", (lnum, end - 1), (lnum, end), line) - fstring_state.consume_lbrace() - pos = end - continue - else: # multiple lines - strstart = (lnum, end) - contstr = line[end:] - contline = line - break - - if fstring_state.current() == STATE_IN_COLON: - match = fstring_middle_after_colon.match(line, pos) - if match is None: - formatspec += line[pos:] - pos = max - continue - - start, end = match.span(1) - token = line[start:end] - formatspec += token - - brace_start, brace_end = match.span(2) - brace_or_nl = line[brace_start:brace_end] - if brace_or_nl == "\n": - pos = brace_end - - yield (FSTRING_MIDDLE, formatspec, formatspec_start, (lnum, end), line) - formatspec = "" - - if brace_or_nl == "{": - yield (LBRACE, "{", (lnum, brace_start), (lnum, brace_end), line) - fstring_state.consume_lbrace() - end = brace_end - elif brace_or_nl == "}": - yield (RBRACE, "}", (lnum, brace_start), (lnum, brace_end), line) - fstring_state.consume_rbrace() - end = brace_end - formatspec_start = (lnum, brace_end) - - pos = end + prev_token = token continue - if fstring_state.current() == STATE_IN_BRACES and parenlev == 0: - match = bang.match(line, pos) - if match: - start, end = match.span(1) - yield (OP, "!", (lnum, start), (lnum, end), line) - pos = end - continue + yield ( + TOKEN_TYPE_MAP[token.type], + token_str, + (token.start_line, token.start_col), + (token.end_line, token.end_col), + source_line, + ) + prev_token = token - match = colon.match(line, pos) - if match: - start, end = match.span(1) - yield (OP, ":", (lnum, start), (lnum, end), line) - fstring_state.consume_colon() - formatspec_start = (lnum, end) - pos = end - continue + except pytokens.UnexpectedEOF: + raise TokenError("Unexpected EOF in multi-line statement", (line, column)) + except pytokens.TokenizeError as exc: + raise TokenError(f"Failed to parse: {type(exc).__name__}", (line, column)) - pseudomatch = pseudoprog.match(line, pos) - if pseudomatch: # scan for tokens - start, end = pseudomatch.span(1) - spos, epos, pos = (lnum, start), (lnum, end), end - token, initial = line[start:end], line[start] - - if initial in numchars or ( - initial == "." and token != "." - ): # ordinary number - yield (NUMBER, token, spos, epos, line) - elif initial in "\r\n": - newline = NEWLINE - if parenlev > 0 or fstring_state.is_in_fstring_expression(): - newline = NL - elif async_def: - async_def_nl = True - if stashed: - yield stashed - stashed = None - yield (newline, token, spos, epos, line) - - elif initial == "#": - assert not token.endswith("\n") - if stashed: - yield stashed - stashed = None - yield (COMMENT, token, spos, epos, line) - elif token in triple_quoted: - endprog = endprogs[token] - endprog_stack.append(endprog) - parenlev_stack.append(parenlev) - parenlev = 0 - if is_fstring_start(token): - yield (FSTRING_START, token, spos, epos, line) - fstring_state.enter_fstring() - - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - if stashed: - yield stashed - stashed = None - if not is_fstring_start(token): - pos = endmatch.end(0) - token = line[start:pos] - epos = (lnum, pos) - yield (STRING, token, spos, epos, line) - endprog_stack.pop() - parenlev = parenlev_stack.pop() - else: - end = endmatch.end(0) - token = line[pos:end] - spos, epos = (lnum, pos), (lnum, end) - if not token.endswith("{"): - fstring_middle, fstring_end = token[:-3], token[-3:] - fstring_middle_epos = fstring_end_spos = (lnum, end - 3) - yield ( - FSTRING_MIDDLE, - fstring_middle, - spos, - fstring_middle_epos, - line, - ) - yield ( - FSTRING_END, - fstring_end, - fstring_end_spos, - epos, - line, - ) - fstring_state.leave_fstring() - endprog_stack.pop() - parenlev = parenlev_stack.pop() - else: - fstring_middle, lbrace = token[:-1], token[-1] - fstring_middle_epos = lbrace_spos = (lnum, end - 1) - yield ( - FSTRING_MIDDLE, - fstring_middle, - spos, - fstring_middle_epos, - line, - ) - yield (LBRACE, lbrace, lbrace_spos, epos, line) - fstring_state.consume_lbrace() - pos = end - else: - # multiple lines - if is_fstring_start(token): - strstart = (lnum, pos) - contstr = line[pos:] - else: - strstart = (lnum, start) - contstr = line[start:] - contline = line - break - elif ( - initial in single_quoted - or token[:2] in single_quoted - or token[:3] in single_quoted - ): - maybe_endprog = ( - endprogs.get(initial) - or endprogs.get(token[:2]) - or endprogs.get(token[:3]) - ) - assert maybe_endprog is not None, f"endprog not found for {token}" - endprog = maybe_endprog - if token[-1] == "\n": # continued string - endprog_stack.append(endprog) - parenlev_stack.append(parenlev) - parenlev = 0 - strstart = (lnum, start) - contstr, needcont = line[start:], 1 - contline = line - break - else: # ordinary string - if stashed: - yield stashed - stashed = None - if not is_fstring_start(token): - yield (STRING, token, spos, epos, line) - else: - if pseudomatch[20] is not None: - fstring_start = pseudomatch[20] - offset = pseudomatch.end(20) - pseudomatch.start(1) - elif pseudomatch[22] is not None: - fstring_start = pseudomatch[22] - offset = pseudomatch.end(22) - pseudomatch.start(1) - elif pseudomatch[24] is not None: - fstring_start = pseudomatch[24] - offset = pseudomatch.end(24) - pseudomatch.start(1) - else: - fstring_start = pseudomatch[26] - offset = pseudomatch.end(26) - pseudomatch.start(1) - - start_epos = (lnum, start + offset) - yield (FSTRING_START, fstring_start, spos, start_epos, line) - fstring_state.enter_fstring() - endprog = endprogs[fstring_start] - endprog_stack.append(endprog) - parenlev_stack.append(parenlev) - parenlev = 0 - - end_offset = pseudomatch.end(1) - 1 - fstring_middle = line[start + offset : end_offset] - middle_spos = (lnum, start + offset) - middle_epos = (lnum, end_offset) - yield ( - FSTRING_MIDDLE, - fstring_middle, - middle_spos, - middle_epos, - line, - ) - if not token.endswith("{"): - end_spos = (lnum, end_offset) - end_epos = (lnum, end_offset + 1) - yield (FSTRING_END, token[-1], end_spos, end_epos, line) - fstring_state.leave_fstring() - endprog_stack.pop() - parenlev = parenlev_stack.pop() - else: - end_spos = (lnum, end_offset) - end_epos = (lnum, end_offset + 1) - yield (LBRACE, "{", end_spos, end_epos, line) - fstring_state.consume_lbrace() - - elif initial.isidentifier(): # ordinary name - if token in ("async", "await"): - if async_keywords or async_def: - yield ( - ASYNC if token == "async" else AWAIT, - token, - spos, - epos, - line, - ) - continue - - tok = (NAME, token, spos, epos, line) - if token == "async" and not stashed: - stashed = tok - continue - - if token in ("def", "for"): - if stashed and stashed[0] == NAME and stashed[1] == "async": - if token == "def": - async_def = True - async_def_indent = indents[-1] - - yield ( - ASYNC, - stashed[1], - stashed[2], - stashed[3], - stashed[4], - ) - stashed = None - - if stashed: - yield stashed - stashed = None - - yield tok - elif initial == "\\": # continued stmt - # This yield is new; needed for better idempotency: - if stashed: - yield stashed - stashed = None - yield (NL, token, spos, (lnum, pos), line) - continued = 1 - elif ( - initial == "}" - and parenlev == 0 - and fstring_state.is_in_fstring_expression() - ): - yield (RBRACE, token, spos, epos, line) - fstring_state.consume_rbrace() - formatspec_start = epos - else: - if initial in "([{": - parenlev += 1 - elif initial in ")]}": - parenlev -= 1 - if stashed: - yield stashed - stashed = None - yield (OP, token, spos, epos, line) - else: - yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos + 1), line) - pos += 1 - - if stashed: - yield stashed - stashed = None - - for _indent in indents[1:]: # pop remaining indent levels - yield (DEDENT, "", (lnum, 0), (lnum, 0), "") - yield (ENDMARKER, "", (lnum, 0), (lnum, 0), "") - assert len(endprog_stack) == 0 - assert len(parenlev_stack) == 0 +def printtoken( + type: int, token: str, srow_col: Coord, erow_col: Coord, line: str +) -> None: # for testing + (srow, scol) = srow_col + (erow, ecol) = erow_col + print( + "%d,%d-%d,%d:\t%s\t%s" % (srow, scol, erow, ecol, tok_name[type], repr(token)) + ) if __name__ == "__main__": # testing if len(sys.argv) > 1: - tokenize(open(sys.argv[1]).readline) + token_iterator = tokenize(open(sys.argv[1]).read()) else: - tokenize(sys.stdin.readline) + token_iterator = tokenize(sys.stdin.read()) + + for tok in token_iterator: + printtoken(*tok) diff --git a/tests/data/miscellaneous/debug_visitor.out b/tests/data/miscellaneous/debug_visitor.out index 24d7ed82472..a243ab72734 100644 --- a/tests/data/miscellaneous/debug_visitor.out +++ b/tests/data/miscellaneous/debug_visitor.out @@ -232,8 +232,6 @@ file_input fstring FSTRING_START "f'" - FSTRING_MIDDLE - '' fstring_replacement_field LBRACE '{' @@ -242,8 +240,6 @@ file_input RBRACE '}' /fstring_replacement_field - FSTRING_MIDDLE - '' fstring_replacement_field LBRACE '{' @@ -252,8 +248,6 @@ file_input RBRACE '}' /fstring_replacement_field - FSTRING_MIDDLE - '' FSTRING_END "'" /fstring @@ -399,8 +393,6 @@ file_input fstring FSTRING_START "f'" - FSTRING_MIDDLE - '' fstring_replacement_field LBRACE '{' @@ -419,8 +411,6 @@ file_input RBRACE '}' /fstring_replacement_field - FSTRING_MIDDLE - '' FSTRING_END "'" /fstring @@ -549,8 +539,6 @@ file_input fstring FSTRING_START "f'" - FSTRING_MIDDLE - '' fstring_replacement_field LBRACE '{' @@ -559,8 +547,6 @@ file_input RBRACE '}' /fstring_replacement_field - FSTRING_MIDDLE - '' fstring_replacement_field LBRACE '{' @@ -569,8 +555,6 @@ file_input RBRACE '}' /fstring_replacement_field - FSTRING_MIDDLE - '' FSTRING_END "'" /fstring @@ -660,8 +644,6 @@ file_input RBRACE '}' /fstring_replacement_field - FSTRING_MIDDLE - '' FSTRING_END "'" /fstring @@ -744,8 +726,6 @@ file_input RBRACE '}' /fstring_replacement_field - FSTRING_MIDDLE - '' FSTRING_END "'" /fstring diff --git a/tests/test_black.py b/tests/test_black.py index ca19c17678b..acafb521619 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -463,17 +463,6 @@ def test_tab_comment_indentation(self) -> None: self.assertFormatEqual(contents_spc, fs(contents_spc)) self.assertFormatEqual(contents_spc, fs(contents_tab)) - # mixed tabs and spaces (valid Python 2 code) - contents_tab = "if 1:\n if 2:\n\t\tpass\n\t# comment\n pass\n" - contents_spc = "if 1:\n if 2:\n pass\n # comment\n pass\n" - self.assertFormatEqual(contents_spc, fs(contents_spc)) - self.assertFormatEqual(contents_spc, fs(contents_tab)) - - contents_tab = "if 1:\n if 2:\n\t\tpass\n\t\t# comment\n pass\n" - contents_spc = "if 1:\n if 2:\n pass\n # comment\n pass\n" - self.assertFormatEqual(contents_spc, fs(contents_spc)) - self.assertFormatEqual(contents_spc, fs(contents_tab)) - def test_false_positive_symlink_output_issue_3384(self) -> None: # Emulate the behavior when using the CLI (`black ./child --verbose`), which # involves patching some `pathlib.Path` methods. In particular, `is_dir` is @@ -1980,7 +1969,7 @@ def test_for_handled_unexpected_eof_error(self) -> None: with pytest.raises(black.parsing.InvalidInput) as exc_info: black.lib2to3_parse("print(", {}) - exc_info.match("Cannot parse: 2:0: EOF in multi-line statement") + exc_info.match("Cannot parse: 1:6: Unexpected EOF in multi-line statement") def test_line_ranges_with_code_option(self) -> None: code = textwrap.dedent("""\ diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index 71773069546..efa7ad5e80d 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -1,6 +1,5 @@ """Tests for the blib2to3 tokenizer.""" -import io import sys import textwrap from dataclasses import dataclass @@ -19,16 +18,10 @@ class Token: def get_tokens(text: str) -> list[Token]: """Return the tokens produced by the tokenizer.""" - readline = io.StringIO(text).readline - tokens: list[Token] = [] - - def tokeneater( - type: int, string: str, start: tokenize.Coord, end: tokenize.Coord, line: str - ) -> None: - tokens.append(Token(token.tok_name[type], string, start, end)) - - tokenize.tokenize(readline, tokeneater) - return tokens + return [ + Token(token.tok_name[tok_type], string, start, end) + for tok_type, string, start, end, _ in tokenize.tokenize(text) + ] def assert_tokenizes(text: str, tokens: list[Token]) -> None: @@ -69,11 +62,9 @@ def test_fstring() -> None: 'f"{x}"', [ Token("FSTRING_START", 'f"', (1, 0), (1, 2)), - Token("FSTRING_MIDDLE", "", (1, 2), (1, 2)), - Token("LBRACE", "{", (1, 2), (1, 3)), + Token("OP", "{", (1, 2), (1, 3)), Token("NAME", "x", (1, 3), (1, 4)), - Token("RBRACE", "}", (1, 4), (1, 5)), - Token("FSTRING_MIDDLE", "", (1, 5), (1, 5)), + Token("OP", "}", (1, 4), (1, 5)), Token("FSTRING_END", '"', (1, 5), (1, 6)), Token("ENDMARKER", "", (2, 0), (2, 0)), ], @@ -82,13 +73,11 @@ def test_fstring() -> None: 'f"{x:y}"\n', [ Token(type="FSTRING_START", string='f"', start=(1, 0), end=(1, 2)), - Token(type="FSTRING_MIDDLE", string="", start=(1, 2), end=(1, 2)), - Token(type="LBRACE", string="{", start=(1, 2), end=(1, 3)), + Token(type="OP", string="{", start=(1, 2), end=(1, 3)), Token(type="NAME", string="x", start=(1, 3), end=(1, 4)), Token(type="OP", string=":", start=(1, 4), end=(1, 5)), Token(type="FSTRING_MIDDLE", string="y", start=(1, 5), end=(1, 6)), - Token(type="RBRACE", string="}", start=(1, 6), end=(1, 7)), - Token(type="FSTRING_MIDDLE", string="", start=(1, 7), end=(1, 7)), + Token(type="OP", string="}", start=(1, 6), end=(1, 7)), Token(type="FSTRING_END", string='"', start=(1, 7), end=(1, 8)), Token(type="NEWLINE", string="\n", start=(1, 8), end=(1, 9)), Token(type="ENDMARKER", string="", start=(2, 0), end=(2, 0)), @@ -99,10 +88,9 @@ def test_fstring() -> None: [ Token(type="FSTRING_START", string='f"', start=(1, 0), end=(1, 2)), Token(type="FSTRING_MIDDLE", string="x\\\n", start=(1, 2), end=(2, 0)), - Token(type="LBRACE", string="{", start=(2, 0), end=(2, 1)), + Token(type="OP", string="{", start=(2, 0), end=(2, 1)), Token(type="NAME", string="a", start=(2, 1), end=(2, 2)), - Token(type="RBRACE", string="}", start=(2, 2), end=(2, 3)), - Token(type="FSTRING_MIDDLE", string="", start=(2, 3), end=(2, 3)), + Token(type="OP", string="}", start=(2, 2), end=(2, 3)), Token(type="FSTRING_END", string='"', start=(2, 3), end=(2, 4)), Token(type="NEWLINE", string="\n", start=(2, 4), end=(2, 5)), Token(type="ENDMARKER", string="", start=(3, 0), end=(3, 0)), From dbb14eac93637cca7f9b88685a56c0b381e42caa Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Thu, 20 Mar 2025 03:32:40 +0530 Subject: [PATCH 21/69] Recursively unwrap tuples in `del` statements (#4628) --- CHANGES.md | 1 + src/black/parsing.py | 10 +++++++++- tests/data/cases/cantfit.py | 9 ++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cc42cde14ef..148f8bcb8f5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,7 @@ ### Stable style +- Fix crash while formatting a long `del` statement containing tuples (#4628) ### Preview style diff --git a/src/black/parsing.py b/src/black/parsing.py index 0019b0c006a..93d017cbbb5 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -213,7 +213,7 @@ def _stringify_ast(node: ast.AST, parent_stack: list[ast.AST]) -> Iterator[str]: and isinstance(node, ast.Delete) and isinstance(item, ast.Tuple) ): - for elt in item.elts: + for elt in _unwrap_tuples(item): yield from _stringify_ast_with_new_parent( elt, parent_stack, node ) @@ -250,3 +250,11 @@ def _stringify_ast(node: ast.AST, parent_stack: list[ast.AST]) -> Iterator[str]: ) yield f"{' ' * len(parent_stack)}) # /{node.__class__.__name__}" + + +def _unwrap_tuples(node: ast.Tuple) -> Iterator[ast.AST]: + for elt in node.elts: + if isinstance(elt, ast.Tuple): + yield from _unwrap_tuples(elt) + else: + yield elt diff --git a/tests/data/cases/cantfit.py b/tests/data/cases/cantfit.py index f002326947d..61911927957 100644 --- a/tests/data/cases/cantfit.py +++ b/tests/data/cases/cantfit.py @@ -31,7 +31,8 @@ raise ValueError(err.format(key)) concatenated_strings = "some strings that are " "concatenated implicitly, so if you put them on separate " "lines it will fit" del concatenated_strings, string_variable_name, normal_function_name, normal_name, need_more_to_make_the_line_long_enough - +del ([], name_1, name_2), [(), [], name_4, name_3], name_1[[name_2 for name_1 in name_0]] +del (), # output @@ -91,3 +92,9 @@ normal_name, need_more_to_make_the_line_long_enough, ) +del ( + ([], name_1, name_2), + [(), [], name_4, name_3], + name_1[[name_2 for name_1 in name_0]], +) +del ((),) From dd278cb316d75868716a0478c35b1fcd600a5249 Mon Sep 17 00:00:00 2001 From: Tsvika Shapira Date: Thu, 20 Mar 2025 17:01:31 +0200 Subject: [PATCH 22/69] update github-action to look for black version in "dependency-groups" (#4606) "dependency-groups" is the mechanism for storing package requirements in `pyproject.toml`, recommended for formatting tools (see https://packaging.python.org/en/latest/specifications/dependency-groups/ ) this change allow the black action to look also in those locations when determining the version of black to install --- CHANGES.md | 2 ++ action/main.py | 1 + docs/integrations/github_actions.md | 8 ++++---- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 148f8bcb8f5..05c0011ad94 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -51,6 +51,8 @@ - Fix the version check in the vim file to reject Python 3.8 (#4567) +- Enhance GitHub Action `psf/black` to read Black version from an additional + section in pyproject.toml: `[project.dependency-groups]` (#4606) ### Documentation diff --git a/action/main.py b/action/main.py index 2b347e149cb..f7fdda7efb6 100644 --- a/action/main.py +++ b/action/main.py @@ -71,6 +71,7 @@ def read_version_specifier_from_pyproject() -> str: return f"=={version}" arrays = [ + *pyproject.get("dependency-groups", {}).values(), pyproject.get("project", {}).get("dependencies"), *pyproject.get("project", {}).get("optional-dependencies", {}).values(), ] diff --git a/docs/integrations/github_actions.md b/docs/integrations/github_actions.md index 61689799731..3270e0f858f 100644 --- a/docs/integrations/github_actions.md +++ b/docs/integrations/github_actions.md @@ -37,10 +37,10 @@ the `pyproject.toml` file. `version` can be any [valid version specifier](https://packaging.python.org/en/latest/glossary/#term-Version-Specifier) or just the version number if you want an exact version. To read the version from the `pyproject.toml` file instead, set `use_pyproject` to `true`. This will first look into -the `tool.black.required-version` field, then the `project.dependencies` array and -finally the `project.optional-dependencies` table. The action defaults to the latest -release available on PyPI. Only versions available from PyPI are supported, so no commit -SHAs or branch names. +the `tool.black.required-version` field, then the `dependency-groups` table, then the +`project.dependencies` array and finally the `project.optional-dependencies` table. +The action defaults to the latest release available on PyPI. Only versions available +from PyPI are supported, so no commit SHAs or branch names. If you want to include Jupyter Notebooks, _Black_ must be installed with the `jupyter` extra. Installing the extra and including Jupyter Notebook files can be configured via From 6144c46c6a6960aeaf62484d3d8cbafedf0092f3 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Fri, 21 Mar 2025 02:30:11 +0530 Subject: [PATCH 23/69] Fix parsing of walrus operator in complex with statements (#4630) --- CHANGES.md | 2 ++ src/black/linegen.py | 1 + tests/data/cases/pep_572_py310.py | 5 +++++ 3 files changed, 8 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 05c0011ad94..4cbc3d8bd95 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,8 @@ - Fix crash while formatting a long `del` statement containing tuples (#4628) +- Fix crash while formatting expressions using the walrus operator in complex + `with` statements (#4630) ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index ee65a7a6e40..1ee9f6a6be6 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1649,6 +1649,7 @@ def maybe_make_parens_invisible_in_atom( syms.except_clause, syms.funcdef, syms.with_stmt, + syms.testlist_gexp, syms.tname, # these ones aren't useful to end users, but they do please fuzzers syms.for_stmt, diff --git a/tests/data/cases/pep_572_py310.py b/tests/data/cases/pep_572_py310.py index ba488d4741c..f79024931f4 100644 --- a/tests/data/cases/pep_572_py310.py +++ b/tests/data/cases/pep_572_py310.py @@ -14,3 +14,8 @@ f((a := b + c for c in range(10)), x) f(y=(a := b + c for c in range(10))) f(x, (a := b + c for c in range(10)), y=z, **q) + + +# Don't remove parens when assignment expr is one of the exprs in a with statement +with x, (a := b): + pass From 2c135edf3732a8efcc89450446fcaa7589e2a1c8 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Sun, 23 Mar 2025 08:00:40 +0530 Subject: [PATCH 24/69] Handle `# fmt: skip` followed by a comment (#4635) --- CHANGES.md | 1 + src/black/comments.py | 2 +- tests/data/cases/fmtskip11.py | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 tests/data/cases/fmtskip11.py diff --git a/CHANGES.md b/CHANGES.md index 4cbc3d8bd95..65bcab1c2cc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,7 @@ - Fix crash while formatting a long `del` statement containing tuples (#4628) - Fix crash while formatting expressions using the walrus operator in complex `with` statements (#4630) +- Handle `# fmt: skip` followed by a comment at the end of file (#4635) ### Preview style diff --git a/src/black/comments.py b/src/black/comments.py index 1054e7ae8a2..81d3cfd4a35 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -321,7 +321,7 @@ def _generate_ignored_nodes_from_fmt_skip( if not comments or comment.value != comments[0].value: return if prev_sibling is not None: - leaf.prefix = "" + leaf.prefix = leaf.prefix[comment.consumed :] if Preview.fix_fmt_skip_in_one_liners not in mode: siblings = [prev_sibling] diff --git a/tests/data/cases/fmtskip11.py b/tests/data/cases/fmtskip11.py new file mode 100644 index 00000000000..5d3f7874e55 --- /dev/null +++ b/tests/data/cases/fmtskip11.py @@ -0,0 +1,6 @@ +def foo(): + pass + + +# comment 1 # fmt: skip +# comment 2 From 950ec38c119916868833cb9896dab324c5d8eadd Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Tue, 1 Apr 2025 20:19:37 +0530 Subject: [PATCH 25/69] Disallow unwrapping tuples in an `as` clause (#4634) --- CHANGES.md | 2 ++ src/black/linegen.py | 2 ++ src/black/nodes.py | 11 +++++++++++ tests/data/cases/context_managers_39.py | 10 ++++++++++ 4 files changed, 25 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 65bcab1c2cc..b7520f3f93a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,8 @@ - Fix crash while formatting expressions using the walrus operator in complex `with` statements (#4630) - Handle `# fmt: skip` followed by a comment at the end of file (#4635) +- Fix crash when a tuple appears in the `as` clause of a `with` statement + (#4634) ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index 1ee9f6a6be6..52ef2cf0131 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -56,6 +56,7 @@ is_rpar_token, is_stub_body, is_stub_suite, + is_tuple, is_tuple_containing_star, is_tuple_containing_walrus, is_type_ignore_comment_string, @@ -1626,6 +1627,7 @@ def maybe_make_parens_invisible_in_atom( node.type not in (syms.atom, syms.expr) or is_empty_tuple(node) or is_one_tuple(node) + or (is_tuple(node) and parent.type == syms.asexpr_test) or (is_yield(node) and parent.type != syms.expr_stmt) or ( # This condition tries to prevent removing non-optional brackets diff --git a/src/black/nodes.py b/src/black/nodes.py index 3b74e2db0be..665cb15d910 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -603,6 +603,17 @@ def is_one_tuple(node: LN) -> bool: ) +def is_tuple(node: LN) -> bool: + """Return True if `node` holds a tuple.""" + if node.type != syms.atom: + return False + gexp = unwrap_singleton_parenthesis(node) + if gexp is None or gexp.type != syms.testlist_gexp: + return False + + return True + + def is_tuple_containing_walrus(node: LN) -> bool: """Return True if `node` holds a tuple that contains a walrus operator.""" if node.type != syms.atom: diff --git a/tests/data/cases/context_managers_39.py b/tests/data/cases/context_managers_39.py index c9fcf9c8ba2..ff4289d3a4b 100644 --- a/tests/data/cases/context_managers_39.py +++ b/tests/data/cases/context_managers_39.py @@ -84,6 +84,11 @@ async def func(): pass + +# don't remove the brackets here, it changes the meaning of the code. +with (x, y) as z: + pass + # output @@ -172,3 +177,8 @@ async def func(): some_other_function(argument1, argument2, argument3="some_value"), ): pass + + +# don't remove the brackets here, it changes the meaning of the code. +with (x, y) as z: + pass From a41dc89f1f7e2b60d5d7dba236e12c6c87cb921d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Apr 2025 14:45:01 -0700 Subject: [PATCH 26/69] [pre-commit.ci] pre-commit autoupdate (#4644) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/isort: 5.13.2 → 6.0.1](https://github.com/pycqa/isort/compare/5.13.2...6.0.1) - [github.com/pycqa/flake8: 7.1.1 → 7.2.0](https://github.com/pycqa/flake8/compare/7.1.1...7.2.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b683ae8cf0d..2cd8c44fdab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,12 +24,12 @@ repos: additional_dependencies: *version_check_dependencies - repo: https://github.com/pycqa/isort - rev: 5.13.2 + rev: 6.0.1 hooks: - id: isort - repo: https://github.com/pycqa/flake8 - rev: 7.1.1 + rev: 7.2.0 hooks: - id: flake8 additional_dependencies: From d0ff3bd6cb82e35b1529155d29fca2c13442e68d Mon Sep 17 00:00:00 2001 From: Pedro Mezacasa Muller <114496585+Pedro-Muller29@users.noreply.github.com> Date: Wed, 9 Apr 2025 01:42:17 -0300 Subject: [PATCH 27/69] Fix crash when a tuple is used as a ContextManager (#4646) --- CHANGES.md | 1 + src/black/linegen.py | 6 ++++ src/black/nodes.py | 18 +++++++++++ tests/data/cases/context_managers_39.py | 40 +++++++++++++++++++++++++ 4 files changed, 65 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index b7520f3f93a..65bd285f452 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,7 @@ - Handle `# fmt: skip` followed by a comment at the end of file (#4635) - Fix crash when a tuple appears in the `as` clause of a `with` statement (#4634) +- Fix crash when tuple is used as a context manager inside a `with` statement (#4646) ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index 52ef2cf0131..fa574ca215e 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -40,6 +40,7 @@ ensure_visible, fstring_to_string, get_annotation_type, + has_sibling_with_type, is_arith_like, is_async_stmt_or_funcdef, is_atom_with_invisible_parens, @@ -1628,6 +1629,11 @@ def maybe_make_parens_invisible_in_atom( or is_empty_tuple(node) or is_one_tuple(node) or (is_tuple(node) and parent.type == syms.asexpr_test) + or ( + is_tuple(node) + and parent.type == syms.with_stmt + and has_sibling_with_type(node, token.COMMA) + ) or (is_yield(node) and parent.type != syms.expr_stmt) or ( # This condition tries to prevent removing non-optional brackets diff --git a/src/black/nodes.py b/src/black/nodes.py index 665cb15d910..ac346f2411b 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -1058,3 +1058,21 @@ def furthest_ancestor_with_last_leaf(leaf: Leaf) -> LN: while node.parent and node.parent.children and node is node.parent.children[-1]: node = node.parent return node + + +def has_sibling_with_type(node: LN, type: int) -> bool: + # Check previous siblings + sibling = node.prev_sibling + while sibling is not None: + if sibling.type == type: + return True + sibling = sibling.prev_sibling + + # Check next siblings + sibling = node.next_sibling + while sibling is not None: + if sibling.type == type: + return True + sibling = sibling.next_sibling + + return False diff --git a/tests/data/cases/context_managers_39.py b/tests/data/cases/context_managers_39.py index ff4289d3a4b..f4934cb07e4 100644 --- a/tests/data/cases/context_managers_39.py +++ b/tests/data/cases/context_managers_39.py @@ -89,6 +89,26 @@ async def func(): with (x, y) as z: pass + +# don't remove the brackets here, it changes the meaning of the code. +# even though the code will always trigger a runtime error +with (name_5, name_4), name_5: + pass + + +def test_tuple_as_contextmanager(): + from contextlib import nullcontext + + try: + with (nullcontext(),nullcontext()),nullcontext(): + pass + except TypeError: + # test passed + pass + else: + # this should be a type error + assert False + # output @@ -182,3 +202,23 @@ async def func(): # don't remove the brackets here, it changes the meaning of the code. with (x, y) as z: pass + + +# don't remove the brackets here, it changes the meaning of the code. +# even though the code will always trigger a runtime error +with (name_5, name_4), name_5: + pass + + +def test_tuple_as_contextmanager(): + from contextlib import nullcontext + + try: + with (nullcontext(), nullcontext()), nullcontext(): + pass + except TypeError: + # test passed + pass + else: + # this should be a type error + assert False From 314f8cf92b285de3d95bb6b86c66cc7ce252b6c1 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+cobaltt7@users.noreply.github.com> Date: Sun, 11 May 2025 19:21:50 -0500 Subject: [PATCH 28/69] Update Prettier pre-commit configuration (#4662) * Update Prettier configuration Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * Update .github/workflows/diff_shades.yml Co-authored-by: Jelle Zijlstra --------- Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- .github/workflows/diff_shades.yml | 12 +++++------ .pre-commit-config.yaml | 6 +++--- CHANGES.md | 31 ++++++++++++++-------------- README.md | 4 ++-- docs/contributing/release_process.md | 9 ++++---- docs/integrations/github_actions.md | 6 +++--- 6 files changed, 35 insertions(+), 33 deletions(-) diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index 159e163dc21..5fc4983be40 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -34,7 +34,8 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: > - python scripts/diff_shades_gha_helper.py config ${{ github.event_name }} ${{ matrix.mode }} + python scripts/diff_shades_gha_helper.py config ${{ github.event_name }} + ${{ matrix.mode }} analysis: name: analysis / ${{ matrix.mode }} @@ -48,7 +49,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.configure.outputs.matrix )}} + include: ${{ fromJson(needs.configure.outputs.matrix) }} steps: - name: Checkout this repository (full clone) @@ -130,10 +131,9 @@ jobs: - name: Generate summary file (PR only) if: github.event_name == 'pull_request' && matrix.mode == 'preview-changes' run: > - python helper.py comment-body - ${{ matrix.baseline-analysis }} ${{ matrix.target-analysis }} - ${{ matrix.baseline-sha }} ${{ matrix.target-sha }} - ${{ github.event.pull_request.number }} + python helper.py comment-body ${{ matrix.baseline-analysis }} + ${{ matrix.target-analysis }} ${{ matrix.baseline-sha }} + ${{ matrix.target-sha }} ${{ github.event.pull_request.number }} - name: Upload summary file (PR only) if: github.event_name == 'pull_request' && matrix.mode == 'preview-changes' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2cd8c44fdab..dda279db36b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,11 +64,11 @@ repos: args: ["--python-version=3.10"] additional_dependencies: *mypy_deps - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.8 + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.5.3 hooks: - id: prettier - types_or: [css, javascript, html, json, yaml] + types_or: [markdown, yaml, json] exclude: \.github/workflows/diff_shades\.yml - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/CHANGES.md b/CHANGES.md index 65bd285f452..cf415f15fc3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,20 +9,20 @@ ### Stable style + - Fix crash while formatting a long `del` statement containing tuples (#4628) -- Fix crash while formatting expressions using the walrus operator in complex - `with` statements (#4630) +- Fix crash while formatting expressions using the walrus operator in complex `with` + statements (#4630) - Handle `# fmt: skip` followed by a comment at the end of file (#4635) -- Fix crash when a tuple appears in the `as` clause of a `with` statement - (#4634) +- Fix crash when a tuple appears in the `as` clause of a `with` statement (#4634) - Fix crash when tuple is used as a context manager inside a `with` statement (#4646) ### Preview style -- Fix a bug where one-liner functions/conditionals marked with `# fmt: skip` - would still be formatted (#4552) +- Fix a bug where one-liner functions/conditionals marked with `# fmt: skip` would still + be formatted (#4552) ### Configuration @@ -37,8 +37,8 @@ - Rewrite tokenizer to improve performance and compliance (#4536) -- Fix bug where certain unusual expressions (e.g., lambdas) were not accepted - in type parameter bounds and defaults. (#4602) +- Fix bug where certain unusual expressions (e.g., lambdas) were not accepted in type + parameter bounds and defaults. (#4602) ### Performance @@ -57,8 +57,8 @@ - Fix the version check in the vim file to reject Python 3.8 (#4567) -- Enhance GitHub Action `psf/black` to read Black version from an additional - section in pyproject.toml: `[project.dependency-groups]` (#4606) +- Enhance GitHub Action `psf/black` to read Black version from an additional section in + pyproject.toml: `[project.dependency-groups]` (#4606) ### Documentation @@ -69,8 +69,8 @@ ### Highlights -This release introduces the new 2025 stable style (#4558), stabilizing -the following changes: +This release introduces the new 2025 stable style (#4558), stabilizing the following +changes: - Normalize casing of Unicode escape characters in strings to lowercase (#2916) - Fix inconsistencies in whether certain strings are detected as docstrings (#4095) @@ -78,15 +78,16 @@ the following changes: - Remove redundant parentheses in if guards for case blocks (#4214) - Add parentheses to if clauses in case blocks when the line is too long (#4269) - Whitespace before `# fmt: skip` comments is no longer normalized (#4146) -- Fix line length computation for certain expressions that involve the power operator (#4154) +- Fix line length computation for certain expressions that involve the power operator + (#4154) - Check if there is a newline before the terminating quotes of a docstring (#4185) - Fix type annotation spacing between `*` and more complex type variable tuple (#4440) The following changes were not in any previous release: - Remove parentheses around sole list items (#4312) -- Generic function definitions are now formatted more elegantly: parameters are - split over multiple lines first instead of type parameter definitions (#4553) +- Generic function definitions are now formatted more elegantly: parameters are split + over multiple lines first instead of type parameter definitions (#4553) ### Stable style diff --git a/README.md b/README.md index cb3cf71f3dc..2450b8b2eba 100644 --- a/README.md +++ b/README.md @@ -137,8 +137,8 @@ SQLAlchemy, Poetry, PyPA applications (Warehouse, Bandersnatch, Pipenv, virtuale pandas, Pillow, Twisted, LocalStack, every Datadog Agent Integration, Home Assistant, Zulip, Kedro, OpenOA, FLORIS, ORBIT, WOMBAT, and many more. -The following organizations use _Black_: Dropbox, KeepTruckin, Lyft, Mozilla, -Quora, Duolingo, QuantumBlack, Tesla, Archer Aviation. +The following organizations use _Black_: Dropbox, KeepTruckin, Lyft, Mozilla, Quora, +Duolingo, QuantumBlack, Tesla, Archer Aviation. Are we missing anyone? Let us know. diff --git a/docs/contributing/release_process.md b/docs/contributing/release_process.md index 2c904fb95c4..c66ffae8ace 100644 --- a/docs/contributing/release_process.md +++ b/docs/contributing/release_process.md @@ -29,8 +29,8 @@ frequently than monthly nets rapidly diminishing returns. **You must have `write` permissions for the _Black_ repository to cut a release.** The 10,000 foot view of the release process is that you prepare a release PR and then -publish a [GitHub Release]. This triggers [release automation](#release-workflows) that builds -all release artifacts and publishes them to the various platforms we publish to. +publish a [GitHub Release]. This triggers [release automation](#release-workflows) that +builds all release artifacts and publishes them to the various platforms we publish to. We now have a `scripts/release.py` script to help with cutting the release PRs. @@ -96,8 +96,9 @@ In the end, use your best judgement and ask other maintainers for their thoughts ## Release workflows -All of _Black_'s release automation uses [GitHub Actions]. All workflows are therefore configured -using YAML files in the `.github/workflows` directory of the _Black_ repository. +All of _Black_'s release automation uses [GitHub Actions]. All workflows are therefore +configured using YAML files in the `.github/workflows` directory of the _Black_ +repository. They are triggered by the publication of a [GitHub Release]. diff --git a/docs/integrations/github_actions.md b/docs/integrations/github_actions.md index 3270e0f858f..af7dee8d2db 100644 --- a/docs/integrations/github_actions.md +++ b/docs/integrations/github_actions.md @@ -38,9 +38,9 @@ the `pyproject.toml` file. `version` can be any or just the version number if you want an exact version. To read the version from the `pyproject.toml` file instead, set `use_pyproject` to `true`. This will first look into the `tool.black.required-version` field, then the `dependency-groups` table, then the -`project.dependencies` array and finally the `project.optional-dependencies` table. -The action defaults to the latest release available on PyPI. Only versions available -from PyPI are supported, so no commit SHAs or branch names. +`project.dependencies` array and finally the `project.optional-dependencies` table. The +action defaults to the latest release available on PyPI. Only versions available from +PyPI are supported, so no commit SHAs or branch names. If you want to include Jupyter Notebooks, _Black_ must be installed with the `jupyter` extra. Installing the extra and including Jupyter Notebook files can be configured via From b0f36f5b4233ef4cf613daca0adc3896d5424159 Mon Sep 17 00:00:00 2001 From: danigm Date: Thu, 15 May 2025 14:04:00 +0200 Subject: [PATCH 29/69] Update test_code_option_safe to work with click 8.2.0 (#4666) --- tests/test_black.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_black.py b/tests/test_black.py index acafb521619..f5c950244ef 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1907,7 +1907,8 @@ def test_code_option_safe(self) -> None: args = ["--safe", "--code", code] result = CliRunner().invoke(black.main, args) - self.compare_results(result, error_msg, 123) + assert error_msg == result.output + assert result.exit_code == 123 def test_code_option_fast(self) -> None: """Test that the code option ignores errors when the sanity checks fail.""" From 2630801f95178dd95ac7f6d5bc9914d50767651e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 May 2025 07:22:11 -0500 Subject: [PATCH 30/69] Bump pypa/cibuildwheel from 2.22.0 to 2.23.3 (#4660) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.22.0 to 2.23.3. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.22.0...v2.23.3) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-version: 2.23.3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index ea13767eeeb..1b075635477 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -92,7 +92,7 @@ jobs: steps: - uses: actions/checkout@v4 # Keep cibuildwheel version in sync with above - - uses: pypa/cibuildwheel@v2.22.0 + - uses: pypa/cibuildwheel@v2.23.3 with: only: ${{ matrix.only }} From 71e380aedf9e319be24b3b6b68a093a5066d158c Mon Sep 17 00:00:00 2001 From: cobalt <61329810+cobaltt7@users.noreply.github.com> Date: Sun, 25 May 2025 18:23:42 -0500 Subject: [PATCH 31/69] CI: Remove now-uneeded workarounds (#4665) --- .pre-commit-config.yaml | 4 +++- scripts/fuzz.py | 21 +-------------------- tests/test_black.py | 2 +- tox.ini | 24 ++++++------------------ 4 files changed, 11 insertions(+), 40 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dda279db36b..79fc8708129 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,9 @@ repos: - types-PyYAML - types-atheris - tomli >= 0.2.6, < 2.0.0 - - click >= 8.1.0, != 8.1.4, != 8.1.5 + - click >= 8.2.0 + # Click is intentionally out-of-sync with pyproject.toml + # v8.2 has breaking changes. We work around them at runtime, but we need the newer stubs. - packaging >= 22.0 - platformdirs >= 2.1.0 - pytokens >= 0.1.10 diff --git a/scripts/fuzz.py b/scripts/fuzz.py index 0c507381d92..915a036b4ae 100644 --- a/scripts/fuzz.py +++ b/scripts/fuzz.py @@ -5,14 +5,11 @@ a coverage-guided fuzzer I'm working on. """ -import re - import hypothesmith from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st import black -from blib2to3.pgen2.tokenize import TokenError # This test uses the Hypothesis and Hypothesmith libraries to generate random @@ -45,23 +42,7 @@ def test_idempotent_any_syntatically_valid_python( compile(src_contents, "", "exec") # else the bug is in hypothesmith # Then format the code... - try: - dst_contents = black.format_str(src_contents, mode=mode) - except black.InvalidInput: - # This is a bug - if it's valid Python code, as above, Black should be - # able to cope with it. See issues #970, #1012 - # TODO: remove this try-except block when issues are resolved. - return - except TokenError as e: - if ( # Special-case logic for backslashes followed by newlines or end-of-input - e.args[0] == "EOF in multi-line statement" - and re.search(r"\\($|\r?\n)", src_contents) is not None - ): - # This is a bug - if it's valid Python code, as above, Black should be - # able to cope with it. See issue #1012. - # TODO: remove this block when the issue is resolved. - return - raise + dst_contents = black.format_str(src_contents, mode=mode) # And check that we got equivalent and stable output. black.assert_equivalent(src_contents, dst_contents) diff --git a/tests/test_black.py b/tests/test_black.py index f5c950244ef..ee026f312de 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -119,7 +119,7 @@ def __init__(self) -> None: if Version(imp_version("click")) >= Version("8.2.0"): super().__init__() else: - super().__init__(mix_stderr=False) + super().__init__(mix_stderr=False) # type: ignore def invokeBlack( diff --git a/tox.ini b/tox.ini index d64fe7f2210..d4450219dc0 100644 --- a/tox.ini +++ b/tox.ini @@ -13,18 +13,16 @@ skip_install = True recreate = True deps = -r{toxinidir}/test_requirements.txt -; parallelization is disabled on CI because pytest-dev/pytest-xdist#620 occurs too frequently -; local runs can stay parallelized since they aren't rolling the dice so many times as like on CI commands = pip install -e .[d] coverage erase pytest tests --run-optional no_jupyter \ - !ci: --numprocesses auto \ + --numprocesses auto \ --cov {posargs} pip install -e .[jupyter] pytest tests --run-optional jupyter \ -m jupyter \ - !ci: --numprocesses auto \ + --numprocesses auto \ --cov --cov-append {posargs} coverage report @@ -34,20 +32,15 @@ skip_install = True recreate = True deps = -r{toxinidir}/test_requirements.txt -; a separate worker is required in ci due to https://foss.heptapod.net/pypy/pypy/-/issues/3317 -; this seems to cause tox to wait forever -; remove this when pypy releases the bugfix commands = pip install -e .[d] pytest tests \ --run-optional no_jupyter \ - !ci: --numprocesses auto \ - ci: --numprocesses 1 + --numprocesses auto pip install -e .[jupyter] pytest tests --run-optional jupyter \ -m jupyter \ - !ci: --numprocesses auto \ - ci: --numprocesses 1 + --numprocesses auto [testenv:{,ci-}311] setenv = @@ -59,22 +52,17 @@ deps = ; We currently need > aiohttp 3.8.1 that is on PyPI for 3.11 git+https://github.com/aio-libs/aiohttp -r{toxinidir}/test_requirements.txt -; a separate worker is required in ci due to https://foss.heptapod.net/pypy/pypy/-/issues/3317 -; this seems to cause tox to wait forever -; remove this when pypy releases the bugfix commands = pip install -e .[d] coverage erase pytest tests \ --run-optional no_jupyter \ - !ci: --numprocesses auto \ - ci: --numprocesses 1 \ + --numprocesses auto \ --cov {posargs} pip install -e .[jupyter] pytest tests --run-optional jupyter \ -m jupyter \ - !ci: --numprocesses auto \ - ci: --numprocesses 1 \ + --numprocesses auto \ --cov --cov-append {posargs} coverage report From e7bf7b4619928da69d486f36fcb456fb201ff53e Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Thu, 29 May 2025 14:10:29 -0700 Subject: [PATCH 32/69] Fix CI mypyc 1.16 failure (#4671) --- src/blib2to3/pgen2/parse.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/blib2to3/pgen2/parse.py b/src/blib2to3/pgen2/parse.py index 10202ab6002..36e865cdb46 100644 --- a/src/blib2to3/pgen2/parse.py +++ b/src/blib2to3/pgen2/parse.py @@ -89,18 +89,12 @@ def backtrack(self) -> Iterator[None]: self.parser.is_backtracking = is_backtracking def add_token(self, tok_type: int, tok_val: str, raw: bool = False) -> None: - func: Callable[..., Any] - if raw: - func = self.parser._addtoken - else: - func = self.parser.addtoken - for ilabel in self.ilabels: with self.switch_to(ilabel): - args = [tok_type, tok_val, self.context] if raw: - args.insert(0, ilabel) - func(*args) + self.parser._addtoken(ilabel, tok_type, tok_val, self.context) + else: + self.parser.addtoken(tok_type, tok_val, self.context) def determine_route( self, value: Optional[str] = None, force: bool = False From 24e4cb20ab03c20af390fec7303207af2a4a09e8 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Thu, 5 Jun 2025 18:49:15 -0700 Subject: [PATCH 33/69] Fix backslash cr nl bug (#4673) * Update tokenize.py * Update CHANGES.md * Update test_black.py * Update test_black.py * Update test_black.py --- CHANGES.md | 1 + src/blib2to3/pgen2/tokenize.py | 12 +++++++++++- tests/test_black.py | 20 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index cf415f15fc3..ae8bb78b6ad 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,7 @@ - Handle `# fmt: skip` followed by a comment at the end of file (#4635) - Fix crash when a tuple appears in the `as` clause of a `with` statement (#4634) - Fix crash when tuple is used as a context manager inside a `with` statement (#4646) +- Fix crash on a `\\r\n` (#4673) ### Preview style diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 5cbfd5148d8..46c43195a6f 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -113,7 +113,17 @@ def transform_whitespace( and prev_token.type not in (TokenType.nl, TokenType.newline) ): token_str = source[token.start_index : token.end_index] - if token_str.startswith("\\\n"): + if token_str.startswith("\\\r\n"): + return pytokens.Token( + TokenType.nl, + token.start_index, + token.start_index + 3, + token.start_line, + token.start_col, + token.start_line, + token.start_col + 3, + ) + elif token_str.startswith("\\\n") or token_str.startswith("\\\r"): return pytokens.Token( TokenType.nl, token.start_index, diff --git a/tests/test_black.py b/tests/test_black.py index ee026f312de..4588addab4b 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2065,6 +2065,26 @@ def test_lines_with_leading_tabs_expanded(self) -> None: assert lines_with_leading_tabs_expanded("\t\tx") == [f"{tab}{tab}x"] assert lines_with_leading_tabs_expanded("\tx\n y") == [f"{tab}x", " y"] + def test_backslash_carriage_return(self) -> None: + # These tests are here instead of in the normal cases because + # of git's newline normalization and because it's hard to + # get `\r` vs `\r\n` vs `\n` to display properly in editors + assert black.format_str("x=\\\r\n1", mode=black.FileMode()) == "x = 1\n" + assert black.format_str("x=\\\n1", mode=black.FileMode()) == "x = 1\n" + assert black.format_str("x=\\\r1", mode=black.FileMode()) == "x = 1\n" + assert ( + black.format_str("class A\\\r\n:...", mode=black.FileMode()) + == "class A: ...\n" + ) + assert ( + black.format_str("class A\\\n:...", mode=black.FileMode()) + == "class A: ...\n" + ) + assert ( + black.format_str("class A\\\r:...", mode=black.FileMode()) + == "class A: ...\n" + ) + class TestCaching: def test_get_cache_dir( From e5e5dad79225b70892dd7a8e731be1f7000689a3 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Thu, 5 Jun 2025 18:50:42 -0700 Subject: [PATCH 34/69] Fix await ellipses and remove `async/await` soft keyword/identifier support (#4676) * Update tokenize.py * Update driver.py * Update test_black.py * Update test_black.py * Update python37.py * Update tokenize.py * Update CHANGES.md * Update CHANGES.md * Update faq.md * Update driver.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.md | 3 ++ docs/faq.md | 2 + src/blib2to3/pgen2/tokenize.py | 71 ++++++---------------------------- tests/data/cases/python37.py | 2 + tests/test_black.py | 17 -------- 5 files changed, 18 insertions(+), 77 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ae8bb78b6ad..f5b123e8a70 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,9 @@ - Fix crash when a tuple appears in the `as` clause of a `with` statement (#4634) - Fix crash when tuple is used as a context manager inside a `with` statement (#4646) - Fix crash on a `\\r\n` (#4673) +- Fix crash on `await ...` (where `...` is a literal `Ellipsis`) (#4676) +- Remove support for pre-python 3.7 `await/async` as soft keywords/variable names + (#4676) ### Preview style diff --git a/docs/faq.md b/docs/faq.md index 51db1c90390..9efcf316852 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -93,6 +93,8 @@ Support for formatting Python 2 code was removed in version 22.0. While we've ma plans to stop supporting older Python 3 minor versions immediately, their support might also be removed some time in the future without a deprecation period. +`await`/`async` as soft keywords/indentifiers are no longer supported as of 25.2.0. + Runtime support for 3.6 was removed in version 22.10.0, for 3.7 in version 23.7.0, and for 3.8 in version 24.10.0. diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 46c43195a6f..1d515632bfb 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -138,20 +138,13 @@ def transform_whitespace( def tokenize(source: str, grammar: Optional[Grammar] = None) -> Iterator[TokenInfo]: - async_keywords = False if grammar is None else grammar.async_keywords - lines = source.split("\n") lines += [""] # For newline tokens in files that don't end in a newline line, column = 1, 0 - token_iterator = pytokens.tokenize(source) - is_async = False - current_indent = 0 - async_indent = 0 - prev_token: Optional[pytokens.Token] = None try: - for token in token_iterator: + for token in pytokens.tokenize(source): token = transform_whitespace(token, source, prev_token) line, column = token.start_line, token.start_col @@ -166,58 +159,18 @@ def tokenize(source: str, grammar: Optional[Grammar] = None) -> Iterator[TokenIn prev_token = token continue - if token.type == TokenType.indent: - current_indent += 1 - if token.type == TokenType.dedent: - current_indent -= 1 - if is_async and current_indent < async_indent: - is_async = False - source_line = lines[token.start_line - 1] if token.type == TokenType.identifier and token_str in ("async", "await"): # Black uses `async` and `await` token types just for those two keywords - while True: - next_token = next(token_iterator) - next_str = source[next_token.start_index : next_token.end_index] - next_token = transform_whitespace(next_token, next_str, token) - if next_token.type == TokenType.whitespace: - continue - break - - next_token_type = TOKEN_TYPE_MAP[next_token.type] - next_line = lines[next_token.start_line - 1] - - if token_str == "async" and ( - async_keywords - or (next_token_type == NAME and next_str in ("def", "for")) - ): - is_async = True - async_indent = current_indent + 1 - current_token_type = ASYNC - elif token_str == "await" and (async_keywords or is_async): - current_token_type = AWAIT - else: - current_token_type = TOKEN_TYPE_MAP[token.type] - yield ( - current_token_type, + ASYNC if token_str == "async" else AWAIT, token_str, (token.start_line, token.start_col), (token.end_line, token.end_col), source_line, ) - yield ( - next_token_type, - next_str, - (next_token.start_line, next_token.start_col), - (next_token.end_line, next_token.end_col), - next_line, - ) - prev_token = token - continue - - if token.type == TokenType.op and token_str == "...": + elif token.type == TokenType.op and token_str == "...": # Black doesn't have an ellipsis token yet, yield 3 DOTs instead assert token.start_line == token.end_line assert token.end_col == token.start_col + 3 @@ -232,16 +185,14 @@ def tokenize(source: str, grammar: Optional[Grammar] = None) -> Iterator[TokenIn (token.end_line, end_col), source_line, ) - prev_token = token - continue - - yield ( - TOKEN_TYPE_MAP[token.type], - token_str, - (token.start_line, token.start_col), - (token.end_line, token.end_col), - source_line, - ) + else: + yield ( + TOKEN_TYPE_MAP[token.type], + token_str, + (token.start_line, token.start_col), + (token.end_line, token.end_col), + source_line, + ) prev_token = token except pytokens.UnexpectedEOF: diff --git a/tests/data/cases/python37.py b/tests/data/cases/python37.py index f69f6b4e58c..a600621e0d1 100644 --- a/tests/data/cases/python37.py +++ b/tests/data/cases/python37.py @@ -10,6 +10,7 @@ def g(): async def func(): + await ... if test: out_batched = [ i @@ -42,6 +43,7 @@ def g(): async def func(): + await ... if test: out_batched = [ i diff --git a/tests/test_black.py b/tests/test_black.py index 4588addab4b..f0a5fc74e1f 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -422,21 +422,6 @@ def test_skip_magic_trailing_comma(self) -> None: ) self.assertEqual(expected, actual, msg) - @patch("black.dump_to_file", dump_to_stderr) - def test_async_as_identifier(self) -> None: - source_path = get_case_path("miscellaneous", "async_as_identifier") - _, source, expected = read_data_from_file(source_path) - actual = fs(source) - self.assertFormatEqual(expected, actual) - major, minor = sys.version_info[:2] - if major < 3 or (major <= 3 and minor < 7): - black.assert_equivalent(source, actual) - black.assert_stable(source, actual, DEFAULT_MODE) - # ensure black can parse this when the target is 3.6 - self.invokeBlack([str(source_path), "--target-version", "py36"]) - # but not on 3.7, because async/await is no longer an identifier - self.invokeBlack([str(source_path), "--target-version", "py37"], exit_code=123) - @patch("black.dump_to_file", dump_to_stderr) def test_python37(self) -> None: source_path = get_case_path("cases", "python37") @@ -449,8 +434,6 @@ def test_python37(self) -> None: black.assert_stable(source, actual, DEFAULT_MODE) # ensure black can parse this when the target is 3.7 self.invokeBlack([str(source_path), "--target-version", "py37"]) - # but not on 3.6, because we use async as a reserved keyword - self.invokeBlack([str(source_path), "--target-version", "py36"], exit_code=123) def test_tab_comment_indentation(self) -> None: contents_tab = "if 1:\n\tif 2:\n\t\tpass\n\t# comment\n\tpass\n" From 7987951e246b4e76cc5225b2ccd89b5519a25ac8 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Thu, 5 Jun 2025 18:51:26 -0700 Subject: [PATCH 35/69] Convert legacy string formatting to f-strings (#4685) * the changes * Update driver.py --- scripts/migrate-black.py | 2 +- src/blib2to3/pgen2/literals.py | 6 +++--- src/blib2to3/pgen2/pgen.py | 26 +++++++++----------------- src/blib2to3/pgen2/tokenize.py | 4 +--- src/blib2to3/pytree.py | 15 +++++---------- 5 files changed, 19 insertions(+), 34 deletions(-) diff --git a/scripts/migrate-black.py b/scripts/migrate-black.py index efd6f3ccf98..f410c96b0e3 100755 --- a/scripts/migrate-black.py +++ b/scripts/migrate-black.py @@ -77,7 +77,7 @@ def blackify(base_branch: str, black_command: str, logger: logging.Logger) -> in git("commit", "--allow-empty", "-aqC", commit) for commit in commits: - git("branch", "-qD", "%s-black" % commit) + git("branch", "-qD", f"{commit}-black") return 0 diff --git a/src/blib2to3/pgen2/literals.py b/src/blib2to3/pgen2/literals.py index a738c10f460..a384c08dcec 100644 --- a/src/blib2to3/pgen2/literals.py +++ b/src/blib2to3/pgen2/literals.py @@ -28,16 +28,16 @@ def escape(m: re.Match[str]) -> str: if tail.startswith("x"): hexes = tail[1:] if len(hexes) < 2: - raise ValueError("invalid hex string escape ('\\%s')" % tail) + raise ValueError(f"invalid hex string escape ('\\{tail}')") try: i = int(hexes, 16) except ValueError: - raise ValueError("invalid hex string escape ('\\%s')" % tail) from None + raise ValueError(f"invalid hex string escape ('\\{tail}')") from None else: try: i = int(tail, 8) except ValueError: - raise ValueError("invalid octal string escape ('\\%s')" % tail) from None + raise ValueError(f"invalid octal string escape ('\\{tail}')") from None return chr(i) diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py index 6599c1f226c..dc76c999e98 100644 --- a/src/blib2to3/pgen2/pgen.py +++ b/src/blib2to3/pgen2/pgen.py @@ -140,7 +140,7 @@ def calcfirst(self, name: str) -> None: if label in self.first: fset = self.first[label] if fset is None: - raise ValueError("recursion for rule %r" % name) + raise ValueError(f"recursion for rule {name!r}") else: self.calcfirst(label) fset = self.first[label] @@ -155,8 +155,8 @@ def calcfirst(self, name: str) -> None: for symbol in itsfirst: if symbol in inverse: raise ValueError( - "rule %s is ambiguous; %s is in the first sets of %s as well" - " as %s" % (name, symbol, label, inverse[symbol]) + f"rule {name} is ambiguous; {symbol} is in the first sets of" + f" {label} as well as {inverse[symbol]}" ) inverse[symbol] = label self.first[name] = totalset @@ -237,16 +237,16 @@ def dump_nfa(self, name: str, start: "NFAState", finish: "NFAState") -> None: j = len(todo) todo.append(next) if label is None: - print(" -> %d" % j) + print(f" -> {j}") else: - print(" %s -> %d" % (label, j)) + print(f" {label} -> {j}") def dump_dfa(self, name: str, dfa: Sequence["DFAState"]) -> None: print("Dump of DFA for", name) for i, state in enumerate(dfa): print(" State", i, state.isfinal and "(final)" or "") for label, next in sorted(state.arcs.items()): - print(" %s -> %d" % (label, dfa.index(next))) + print(f" {label} -> {dfa.index(next)}") def simplify_dfa(self, dfa: list["DFAState"]) -> None: # This is not theoretically optimal, but works well enough. @@ -330,15 +330,12 @@ def parse_atom(self) -> tuple["NFAState", "NFAState"]: return a, z else: self.raise_error( - "expected (...) or NAME or STRING, got %s/%s", self.type, self.value + f"expected (...) or NAME or STRING, got {self.type}/{self.value}" ) - raise AssertionError def expect(self, type: int, value: Optional[Any] = None) -> str: if self.type != type or (value is not None and self.value != value): - self.raise_error( - "expected %s/%s, got %s/%s", type, value, self.type, self.value - ) + self.raise_error(f"expected {type}/{value}, got {self.type}/{self.value}") value = self.value self.gettoken() return value @@ -350,12 +347,7 @@ def gettoken(self) -> None: self.type, self.value, self.begin, self.end, self.line = tup # print token.tok_name[self.type], repr(self.value) - def raise_error(self, msg: str, *args: Any) -> NoReturn: - if args: - try: - msg = msg % args - except Exception: - msg = " ".join([msg] + list(map(str, args))) + def raise_error(self, msg: str) -> NoReturn: raise SyntaxError( msg, (str(self.filename), self.end[0], self.end[1], self.line) ) diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 1d515632bfb..375d1773397 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -206,9 +206,7 @@ def printtoken( ) -> None: # for testing (srow, scol) = srow_col (erow, ecol) = erow_col - print( - "%d,%d-%d,%d:\t%s\t%s" % (srow, scol, erow, ecol, tok_name[type], repr(token)) - ) + print(f"{srow},{scol}-{erow},{ecol}:\t{tok_name[type]}\t{token!r}") if __name__ == "__main__": # testing diff --git a/src/blib2to3/pytree.py b/src/blib2to3/pytree.py index d57584685a2..01229743253 100644 --- a/src/blib2to3/pytree.py +++ b/src/blib2to3/pytree.py @@ -268,11 +268,7 @@ def __init__( def __repr__(self) -> str: """Return a canonical string representation.""" assert self.type is not None - return "{}({}, {!r})".format( - self.__class__.__name__, - type_repr(self.type), - self.children, - ) + return f"{self.__class__.__name__}({type_repr(self.type)}, {self.children!r})" def __str__(self) -> str: """ @@ -421,10 +417,9 @@ def __repr__(self) -> str: from .pgen2.token import tok_name assert self.type is not None - return "{}({}, {!r})".format( - self.__class__.__name__, - tok_name.get(self.type, self.type), - self.value, + return ( + f"{self.__class__.__name__}({tok_name.get(self.type, self.type)}," + f" {self.value!r})" ) def __str__(self) -> str: @@ -527,7 +522,7 @@ def __repr__(self) -> str: args = [type_repr(self.type), self.content, self.name] while args and args[-1] is None: del args[-1] - return "{}({})".format(self.__class__.__name__, ", ".join(map(repr, args))) + return f"{self.__class__.__name__}({', '.join(map(repr, args))})" def _submatch(self, node, results=None) -> bool: raise NotImplementedError From 59775327810403b7324c68cd74eef370ef570381 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Wed, 11 Jun 2025 12:05:27 -0700 Subject: [PATCH 36/69] Fix f-string format spec CI failure (#4690) Update pep_701.py --- tests/data/cases/pep_701.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/data/cases/pep_701.py b/tests/data/cases/pep_701.py index 9acee951e71..6f86988f0fb 100644 --- a/tests/data/cases/pep_701.py +++ b/tests/data/cases/pep_701.py @@ -74,9 +74,9 @@ x = f"a{2+2:=^{foo(x+y**2):something else}one more}b" f'{(abc:=10)}' -f"This is a really long string, but just make sure that you reflow fstrings { +f"""This is a really long string, but just make sure that you reflow fstrings { 2+2:d -}" +}""" f"This is a really long string, but just make sure that you reflow fstrings correctly {2+2:d}" f"{2+2=}" @@ -213,9 +213,9 @@ x = f"a{2+2:=^{foo(x+y**2):something else}one more}b" f"{(abc:=10)}" -f"This is a really long string, but just make sure that you reflow fstrings { +f"""This is a really long string, but just make sure that you reflow fstrings { 2+2:d -}" +}""" f"This is a really long string, but just make sure that you reflow fstrings correctly {2+2:d}" f"{2+2=}" From e2dc6b3e19dfccf2d3bd3621b3f3b37af1525b36 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Thu, 12 Jun 2025 15:22:31 -0700 Subject: [PATCH 37/69] Prerelease improvements (#4691) * Update parsing.py * Update release.py * Update release_process.md * Update release_process.md * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update release.py * Update parsing.py --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/contributing/release_process.md | 2 -- scripts/release.py | 16 ++++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/contributing/release_process.md b/docs/contributing/release_process.md index c66ffae8ace..056f63ac8d4 100644 --- a/docs/contributing/release_process.md +++ b/docs/contributing/release_process.md @@ -46,8 +46,6 @@ To cut a release: - `release.py` will calculate this and log to stderr for you copy paste pleasure 1. File a PR editing `CHANGES.md` and the docs to version the latest changes - Run `python3 scripts/release.py [--debug]` to generate most changes - - Sub headings in the template, if they have no bullet points need manual removal - _PR welcome to improve :D_ 1. If `release.py` fail manually edit; otherwise, yay, skip this step! 1. Replace the `## Unreleased` header with the version number 1. Remove any empty sections for the current release diff --git a/scripts/release.py b/scripts/release.py index 05df2f715c6..41913540b5c 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -8,6 +8,7 @@ import argparse import logging +import re import sys from datetime import datetime from pathlib import Path @@ -142,19 +143,18 @@ def cleanup_changes_template_for_release(self) -> None: changes_string = cfp.read() # Change Unreleased to next version - versioned_changes = changes_string.replace( + changes_string = changes_string.replace( "## Unreleased", f"## {self.next_version}" ) - # Remove all comments (subheadings are harder - Human required still) - no_comments_changes = [] - for line in versioned_changes.splitlines(): - if line.startswith(""): - continue - no_comments_changes.append(line) + # Remove all comments + changes_string = re.sub(r"^)\n\n", "", changes_string) + + # Remove empty subheadings + changes_string = re.sub(r"^###.+\n\n(?=#)", "", changes_string) with self.changes_path.open("w") as cfp: - cfp.write("\n".join(no_comments_changes) + "\n") + cfp.write(changes_string) LOG.debug(f"Finished Cleaning up {self.changes_path}") From 6681d5a56fe032b4cd92202d545cb8650aa010c0 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Thu, 12 Jun 2025 15:23:33 -0700 Subject: [PATCH 38/69] Small cleanup to parsing.py (#4692) Update parsing.py --- src/black/parsing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/black/parsing.py b/src/black/parsing.py index 93d017cbbb5..b6794e00f72 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -70,7 +70,7 @@ def lib2to3_parse( for grammar in grammars: drv = driver.Driver(grammar) try: - result = drv.parse_string(src_txt, True) + result = drv.parse_string(src_txt, False) break except ParseError as pe: @@ -105,7 +105,7 @@ def lib2to3_parse( def matches_grammar(src_txt: str, grammar: Grammar) -> bool: drv = driver.Driver(grammar) try: - drv.parse_string(src_txt, True) + drv.parse_string(src_txt, False) except (ParseError, TokenError, IndentationError): return False else: From 5adbb6c3f18da88a8c9b69388b7c4a8e94524207 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Thu, 12 Jun 2025 18:39:55 -0700 Subject: [PATCH 39/69] Improve line ranges docs note (#4693) Update the_basics.md --- docs/usage_and_configuration/the_basics.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 197f4feed94..a7de4a3ae46 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -250,8 +250,9 @@ This option is mainly for editor integrations, such as "Format Selection". ```{note} Due to [#4052](https://github.com/psf/black/issues/4052), `--line-ranges` might format -extra lines outside of the ranges when ther are unformatted lines with the exact -content. It also disables _Black_'s formatting stability check in `--safe` mode. +extra lines outside of the ranges when there are unformatted lines with the exact +formatted content next to the requested lines. It also disables _Black_'s formatting +stability check in `--safe` mode. ``` #### `--fast` / `--safe` From a34d23659a21260aed3c48b627d1e6c630f6cdb9 Mon Sep 17 00:00:00 2001 From: huisman <23581164+huisman@users.noreply.github.com> Date: Sun, 15 Jun 2025 04:46:18 +0200 Subject: [PATCH 40/69] Update python image version and reduce size for gallery docker image (#4686) * Update python image version and reduce size Update python image to 3.13 Reduce size by adding `--no-install-recommands` and removing `/var/lib/apt/lists/*` * Use python base image without minor version Use python:3-slim instead of python:3.13-slim as base image for the gallery Dockerfile * Update CHANGES.md --------- Co-authored-by: Cooper Lees --- CHANGES.md | 1 + gallery/Dockerfile | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f5b123e8a70..13b57e76959 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -63,6 +63,7 @@ - Fix the version check in the vim file to reject Python 3.8 (#4567) - Enhance GitHub Action `psf/black` to read Black version from an additional section in pyproject.toml: `[project.dependency-groups]` (#4606) +- Build gallery docker image with python3-slim and reduce image size (#4686) ### Documentation diff --git a/gallery/Dockerfile b/gallery/Dockerfile index 7a18b7e9a18..ced85e58e6e 100644 --- a/gallery/Dockerfile +++ b/gallery/Dockerfile @@ -1,11 +1,12 @@ -FROM python:3.8.2-slim +FROM python:3-slim # note: a single RUN to avoid too many image layers being produced RUN apt-get update \ && apt-get upgrade -y \ - && apt-get install git apt-utils -y \ + && apt-get install git apt-utils -y --no-install-recommends\ && git config --global user.email "black@psf.github.com" \ - && git config --global user.name "Gallery/Black" + && git config --global user.name "Gallery/Black" \ + && rm -rf /var/lib/apt/lists/* COPY gallery.py / ENTRYPOINT ["python", "/gallery.py"] From 95bc5691cf1078fb2105ea734d61b2386d9560e8 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 16 Jun 2025 23:17:31 +0300 Subject: [PATCH 41/69] CI: Replace unsupported windows-2019 with window-2025 (#4697) Replace unsupported windows-2019 with window-2025 --- .github/workflows/upload_binary.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upload_binary.yml b/.github/workflows/upload_binary.yml index 41231016cde..918c0ee85fe 100644 --- a/.github/workflows/upload_binary.yml +++ b/.github/workflows/upload_binary.yml @@ -13,9 +13,9 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-2019, ubuntu-22.04, macos-latest] + os: [windows-2025, ubuntu-22.04, macos-latest] include: - - os: windows-2019 + - os: windows-2025 pathsep: ";" asset_name: black_windows.exe executable_mime: "application/vnd.microsoft.portable-executable" From 60d734a84d5c30f1ca52e6364acf3843f2f612dc Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Sun, 22 Jun 2025 12:14:11 -0700 Subject: [PATCH 42/69] Fix backslash carriage return comments (#4663) * Update comments.py * Update CHANGES.md * Update test_black.py * Update tests/test_black.py Co-authored-by: Jelle Zijlstra * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Jelle Zijlstra Co-authored-by: cobalt <61329810+cobaltt7@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.md | 1 + src/black/comments.py | 2 +- tests/test_black.py | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 13b57e76959..6ebf7c3a8f3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,7 @@ - Handle `# fmt: skip` followed by a comment at the end of file (#4635) - Fix crash when a tuple appears in the `as` clause of a `with` statement (#4634) - Fix crash when tuple is used as a context manager inside a `with` statement (#4646) +- Fix crash when formatting a `\` followed by a `\r` followed by a comment (#4663) - Fix crash on a `\\r\n` (#4673) - Fix crash on `await ...` (where `...` is a literal `Ellipsis`) (#4676) - Remove support for pre-python 3.7 `await/async` as soft keywords/variable names diff --git a/src/black/comments.py b/src/black/comments.py index 81d3cfd4a35..2b530f2b910 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -88,7 +88,7 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> list[ProtoComment]: nlines = 0 ignored_lines = 0 form_feed = False - for index, full_line in enumerate(re.split("\r?\n", prefix)): + for index, full_line in enumerate(re.split("\r?\n|\r", prefix)): consumed += len(full_line) + 1 # adding the length of the split '\n' match = re.match(r"^(\s*)(\S.*|)$", full_line) assert match diff --git a/tests/test_black.py b/tests/test_black.py index f0a5fc74e1f..98d03652abe 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2048,10 +2048,20 @@ def test_lines_with_leading_tabs_expanded(self) -> None: assert lines_with_leading_tabs_expanded("\t\tx") == [f"{tab}{tab}x"] assert lines_with_leading_tabs_expanded("\tx\n y") == [f"{tab}x", " y"] - def test_backslash_carriage_return(self) -> None: + def test_carriage_return_edge_cases(self) -> None: # These tests are here instead of in the normal cases because # of git's newline normalization and because it's hard to - # get `\r` vs `\r\n` vs `\n` to display properly in editors + # get `\r` vs `\r\n` vs `\n` to display properly + assert ( + black.format_str( + "try:\\\r# type: ignore\n pass\nfinally:\n pass\n", + mode=black.FileMode(), + ) + == "try: # type: ignore\n pass\nfinally:\n pass\n" + ) + assert black.format_str("{\r}", mode=black.FileMode()) == "{}\n" + assert black.format_str("pass #\r#\n", mode=black.FileMode()) == "pass #\n#\n" + assert black.format_str("x=\\\r\n1", mode=black.FileMode()) == "x = 1\n" assert black.format_str("x=\\\n1", mode=black.FileMode()) == "x = 1\n" assert black.format_str("x=\\\r1", mode=black.FileMode()) == "x = 1\n" From df0ac360607e44d12fa1349ec544a9702899f918 Mon Sep 17 00:00:00 2001 From: Pedro Mezacasa Muller <114496585+Pedro-Muller29@users.noreply.github.com> Date: Fri, 27 Jun 2025 21:39:24 -0300 Subject: [PATCH 43/69] New tests for better covering of tuples inside with statements (#4650) Co-authored-by: cobalt <61329810+cobaltt7@users.noreply.github.com> --- tests/data/cases/context_managers_39.py | 39 ------------------------- tests/data/cases/tuple_with_stmt.py | 30 +++++++++++++++++++ 2 files changed, 30 insertions(+), 39 deletions(-) create mode 100644 tests/data/cases/tuple_with_stmt.py diff --git a/tests/data/cases/context_managers_39.py b/tests/data/cases/context_managers_39.py index f4934cb07e4..7f1ee2d576b 100644 --- a/tests/data/cases/context_managers_39.py +++ b/tests/data/cases/context_managers_39.py @@ -90,25 +90,6 @@ async def func(): pass -# don't remove the brackets here, it changes the meaning of the code. -# even though the code will always trigger a runtime error -with (name_5, name_4), name_5: - pass - - -def test_tuple_as_contextmanager(): - from contextlib import nullcontext - - try: - with (nullcontext(),nullcontext()),nullcontext(): - pass - except TypeError: - # test passed - pass - else: - # this should be a type error - assert False - # output @@ -202,23 +183,3 @@ async def func(): # don't remove the brackets here, it changes the meaning of the code. with (x, y) as z: pass - - -# don't remove the brackets here, it changes the meaning of the code. -# even though the code will always trigger a runtime error -with (name_5, name_4), name_5: - pass - - -def test_tuple_as_contextmanager(): - from contextlib import nullcontext - - try: - with (nullcontext(), nullcontext()), nullcontext(): - pass - except TypeError: - # test passed - pass - else: - # this should be a type error - assert False diff --git a/tests/data/cases/tuple_with_stmt.py b/tests/data/cases/tuple_with_stmt.py new file mode 100644 index 00000000000..885a300f446 --- /dev/null +++ b/tests/data/cases/tuple_with_stmt.py @@ -0,0 +1,30 @@ +# don't remove the brackets here, it changes the meaning of the code. +# even though the code will always trigger a runtime error +with (name_5, name_4), name_5: + pass + + +with c, (a, b): + pass + + +with c, (a, b), d: + pass + + +with c, (a, b, e, f, g), d: + pass + + +def test_tuple_as_contextmanager(): + from contextlib import nullcontext + + try: + with (nullcontext(), nullcontext()), nullcontext(): + pass + except TypeError: + # test passed + pass + else: + # this should be a type error + assert False From 8310a118ba6690fef55aaf8d67c967852b4c997e Mon Sep 17 00:00:00 2001 From: Ashton Taylor Stasko <114166091+AshSta512@users.noreply.github.com> Date: Fri, 27 Jun 2025 20:42:18 -0400 Subject: [PATCH 44/69] Add default exclusions and inclusions to docs (#4432) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/usage_and_configuration/the_basics.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index a7de4a3ae46..e232a2175bd 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -297,6 +297,9 @@ recursive searches. An empty value means no paths are excluded. Use forward slas directories on all platforms (Windows, too). By default, Black also ignores all paths listed in `.gitignore`. Changing this value will override all default exclusions. +Default Exclusions: +`['.direnv', '.eggs', '.git', '.hg', '.ipynb_checkpoints', '.mypy_cache', '.nox', '.pytest_cache', '.ruff_cache', '.tox', '.svn', '.venv', '.vscode', '__pypackages__', '_build', 'buck-out', 'build', 'dist', 'venv'] ` + If the regular expression contains newlines, it is treated as a [verbose regular expression](https://docs.python.org/3/library/re.html#re.VERBOSE). This is typically useful when setting these options in a `pyproject.toml` configuration file; @@ -325,6 +328,8 @@ recursive searches. An empty value means all files are included regardless of th Use forward slashes for directories on all platforms (Windows, too). Overrides all exclusions, including from `.gitignore` and command line options. +Default Inclusions: `['.pyi', '.ipynb']` + #### `-W`, `--workers` When _Black_ formats multiple files, it may use a process pool to speed up formatting. From 262ad62ca9f1d1b1537cc95a91643fb781d7c501 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Sun, 29 Jun 2025 18:55:29 -0700 Subject: [PATCH 45/69] Fix f string expr split after (#4680) --- CHANGES.md | 2 ++ src/black/trans.py | 2 +- tests/data/cases/preview_long_strings.py | 32 ++++++++++++------------ 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6ebf7c3a8f3..ca6fe3f653a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -28,6 +28,8 @@ - Fix a bug where one-liner functions/conditionals marked with `# fmt: skip` would still be formatted (#4552) +- Fix a bug where `string_processing` would not split f-strings directly after + expressions (#4680) ### Configuration diff --git a/src/black/trans.py b/src/black/trans.py index fabc7051108..97dd1ce366a 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -1755,7 +1755,7 @@ def _get_illegal_split_indices(self, string: str) -> set[Index]: ] for it in iterators: for begin, end in it: - illegal_indices.update(range(begin, end + 1)) + illegal_indices.update(range(begin, end)) return illegal_indices def _get_break_idx(self, string: str, max_break_idx: int) -> Optional[int]: diff --git a/tests/data/cases/preview_long_strings.py b/tests/data/cases/preview_long_strings.py index cf1d12b6e3e..a267a509057 100644 --- a/tests/data/cases/preview_long_strings.py +++ b/tests/data/cases/preview_long_strings.py @@ -883,43 +883,43 @@ def foo(): call(body="%s %s" % (",".join(items), suffix)) log.info( - "Skipping:" - f' {desc["db_id"]=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]=} {desc["exposure_max"]=}' + f'Skipping: {desc["db_id"]=} {desc["ms_name"]} {money=} {dte=} {pos_share=}' + f' {desc["status"]=} {desc["exposure_max"]=}' ) log.info( - "Skipping:" - f" {desc['db_id']=} {desc['ms_name']} {money=} {dte=} {pos_share=} {desc['status']=} {desc['exposure_max']=}" + f"Skipping: {desc['db_id']=} {desc['ms_name']} {money=} {dte=} {pos_share=}" + f" {desc['status']=} {desc['exposure_max']=}" ) log.info( - "Skipping:" - f' {desc["db_id"]} {foo("bar",x=123)} {"foo" != "bar"} {(x := "abc=")} {pos_share=} {desc["status"]} {desc["exposure_max"]}' + f'Skipping: {desc["db_id"]} {foo("bar",x=123)} {"foo" != "bar"} {(x := "abc=")}' + f' {pos_share=} {desc["status"]} {desc["exposure_max"]}' ) log.info( - "Skipping:" - f' {desc["db_id"]} {desc["ms_name"]} {money=} {(x := "abc=")=} {pos_share=} {desc["status"]} {desc["exposure_max"]}' + f'Skipping: {desc["db_id"]} {desc["ms_name"]} {money=} {(x := "abc=")=}' + f' {pos_share=} {desc["status"]} {desc["exposure_max"]}' ) log.info( - "Skipping:" - f' {desc["db_id"]} {foo("bar",x=123)=} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}' + f'Skipping: {desc["db_id"]} {foo("bar",x=123)=} {money=} {dte=} {pos_share=}' + f' {desc["status"]} {desc["exposure_max"]}' ) log.info( - "Skipping:" - f' {foo("asdf")=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}' + f'Skipping: {foo("asdf")=} {desc["ms_name"]} {money=} {dte=} {pos_share=}' + f' {desc["status"]} {desc["exposure_max"]}' ) log.info( - "Skipping:" - f' {"a" == "b" == "c" == "d"} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}' + f'Skipping: {"a" == "b" == "c" == "d"} {desc["ms_name"]} {money=} {dte=}' + f' {pos_share=} {desc["status"]} {desc["exposure_max"]}' ) log.info( - "Skipping:" - f' {"a" == "b" == "c" == "d"=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}' + f'Skipping: {"a" == "b" == "c" == "d"=} {desc["ms_name"]} {money=} {dte=}' + f' {pos_share=} {desc["status"]} {desc["exposure_max"]}' ) log.info( From b085473cc65da4fe76bbd6b1723ea76897e5e40b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 10 Jul 2025 19:58:54 -0700 Subject: [PATCH 46/69] [pre-commit.ci] pre-commit autoupdate (#4702) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/pycqa/flake8: 7.2.0 → 7.3.0](https://github.com/pycqa/flake8/compare/7.2.0...7.3.0) - [github.com/pre-commit/mirrors-mypy: v1.15.0 → v1.16.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.15.0...v1.16.1) - [github.com/rbubley/mirrors-prettier: v3.5.3 → v3.6.2](https://github.com/rbubley/mirrors-prettier/compare/v3.5.3...v3.6.2) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Create _black_version.pyi * Update pyproject.toml --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: cobalt <61329810+cobaltt7@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- CHANGES.md | 2 -- docs/integrations/editors.md | 4 ---- pyproject.toml | 2 +- src/_black_version.pyi | 1 + 5 files changed, 5 insertions(+), 10 deletions(-) create mode 100644 src/_black_version.pyi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 79fc8708129..ee8dc225b5b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: isort - repo: https://github.com/pycqa/flake8 - rev: 7.2.0 + rev: 7.3.0 hooks: - id: flake8 additional_dependencies: @@ -39,7 +39,7 @@ repos: exclude: ^src/blib2to3/ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.15.0 + rev: v1.16.1 hooks: - id: mypy exclude: ^(docs/conf.py|scripts/generate_schema.py)$ @@ -67,7 +67,7 @@ repos: additional_dependencies: *mypy_deps - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.5.3 + rev: v3.6.2 hooks: - id: prettier types_or: [markdown, yaml, json] diff --git a/CHANGES.md b/CHANGES.md index ca6fe3f653a..4bf98d87cfe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1642,7 +1642,6 @@ and the first release covered by our new ## 18.9b0 - numeric literals are now formatted by _Black_ (#452, #461, #464, #469): - - numeric literals are normalized to include `_` separators on Python 3.6+ code - added `--skip-numeric-underscore-normalization` to disable the above behavior and @@ -1692,7 +1691,6 @@ and the first release covered by our new - typing stub files (`.pyi`) now have blank lines added after constants (#340) - `# fmt: off` and `# fmt: on` are now much more dependable: - - they now work also within bracket pairs (#329) - they now correctly work across function/class boundaries (#335) diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index d2940f114ba..272cade7398 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -53,7 +53,6 @@ There are several different ways you can use _Black_ from PyCharm: `File -> Settings -> Tools -> BlackConnect` 1. In `Local Instance (shared between projects)` section: - 1. Check `Start local blackd instance when plugin loads`. 1. Press the `Detect` button near `Path` input. The plugin should detect the `blackd` executable. @@ -65,7 +64,6 @@ There are several different ways you can use _Black_ from PyCharm: shortcut. 1. Optionally, to run _Black_ on every file save: - - In `Trigger Settings` section of plugin configuration check `Trigger when saving changed files`. @@ -107,14 +105,12 @@ There are several different ways you can use _Black_ from PyCharm: `File -> Settings -> Tools -> External Tools` 1. Click the + icon to add a new external tool with the following values: - - Name: Black - Description: Black is the uncompromising Python code formatter. - Program: \ - Arguments: `"$FilePath$"` 1. Format the currently opened file by selecting `Tools -> External Tools -> black`. - - Alternatively, you can set a keyboard shortcut by navigating to `Preferences or Settings -> Keymap -> External Tools -> External Tools - Black`. diff --git a/pyproject.toml b/pyproject.toml index bcd3ebceab8..be6c8f9b9d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,7 +234,7 @@ show_error_codes = true show_column_numbers = true [[tool.mypy.overrides]] -module = ["pathspec.*", "IPython.*", "colorama.*", "tokenize_rt.*", "uvloop.*", "_black_version.*"] +module = ["pathspec.*", "IPython.*", "colorama.*", "tokenize_rt.*", "uvloop.*"] ignore_missing_imports = true # CI only checks src/, but in case users are running LSP or similar we explicitly ignore diff --git a/src/_black_version.pyi b/src/_black_version.pyi new file mode 100644 index 00000000000..c2ee2cab489 --- /dev/null +++ b/src/_black_version.pyi @@ -0,0 +1 @@ +version: str From 9a87babc210b702afabcf9325339574ff74f448b Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Fri, 11 Jul 2025 18:40:54 -0700 Subject: [PATCH 47/69] Fix type params tuple error (#4684) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 1 + src/black/linegen.py | 16 +++++++++++++++- tests/data/cases/type_params.py | 8 ++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4bf98d87cfe..4e97c1f2e46 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,7 @@ - Fix crash on `await ...` (where `...` is a literal `Ellipsis`) (#4676) - Remove support for pre-python 3.7 `await/async` as soft keywords/variable names (#4676) +- Fix crash on parenthesized expression inside a type parameter bound (#4684) ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index fa574ca215e..032ab687835 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -787,12 +787,26 @@ def left_hand_split( head_leaves: list[Leaf] = [] current_leaves = head_leaves matching_bracket: Optional[Leaf] = None - for leaf in line.leaves: + depth = 0 + for index, leaf in enumerate(line.leaves): + if index == 2 and leaf.type == token.LSQB: + # A [ at index 2 means this is a type param, so start + # tracking the depth + depth += 1 + elif depth > 0: + if leaf.type == token.LSQB: + depth += 1 + elif leaf.type == token.RSQB: + depth -= 1 if ( current_leaves is body_leaves and leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is matching_bracket and isinstance(matching_bracket, Leaf) + # If the code is still on LPAR and we are inside a type + # param, ignore the match since this is searching + # for the function arguments + and not (leaf_type == token.LPAR and depth > 0) ): ensure_visible(leaf) ensure_visible(matching_bracket) diff --git a/tests/data/cases/type_params.py b/tests/data/cases/type_params.py index f8fc3855741..124292d6d54 100644 --- a/tests/data/cases/type_params.py +++ b/tests/data/cases/type_params.py @@ -15,6 +15,8 @@ def magic[Trailing, Comma,](): pass def weird_syntax[T: lambda: 42, U: a or b](): pass +def name_3[name_0: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa if aaaaaaaaaaa else name_3](): pass + # output @@ -62,3 +64,9 @@ def magic[ def weird_syntax[T: lambda: 42, U: a or b](): pass + + +def name_3[ + name_0: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa if aaaaaaaaaaa else name_3 +](): + pass From a8de14f882ec4c6d08c62c580eb9b742011ecf61 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Fri, 11 Jul 2025 18:43:07 -0700 Subject: [PATCH 48/69] Misc lints (#4694) --- src/black/__init__.py | 8 +++----- src/black/comments.py | 5 ++--- src/black/handle_ipynb_magics.py | 10 +++++----- src/black/lines.py | 2 +- src/black/ranges.py | 4 ++-- src/black/strings.py | 2 +- src/black/trans.py | 6 +++--- 7 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 93a08a8d88a..48d2bb8154b 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -184,9 +184,7 @@ def spellcheck_pyproject_toml_keys( ) -> None: invalid_keys: list[str] = [] available_config_options = {param.name for param in ctx.command.params} - for key in config_keys: - if key not in available_config_options: - invalid_keys.append(key) + invalid_keys = [key for key in config_keys if key not in available_config_options] if invalid_keys: keys_str = ", ".join(map(repr, invalid_keys)) out( @@ -778,7 +776,7 @@ def get_sources( continue if is_stdin: - path = Path(f"{STDIN_PLACEHOLDER}{str(path)}") + path = Path(f"{STDIN_PLACEHOLDER}{path}") if path.suffix == ".ipynb" and not jupyter_dependencies_are_installed( warn=verbose or not quiet @@ -1281,7 +1279,7 @@ def decode_bytes(src: bytes) -> tuple[FileContent, Encoding, NewLine]: if not lines: return "", encoding, "\n" - newline = "\r\n" if b"\r\n" == lines[0][-2:] else "\n" + newline = "\r\n" if lines[0][-2:] == b"\r\n" else "\n" srcbuf.seek(0) with io.TextIOWrapper(srcbuf, encoding) as tiow: return tiow.read(), encoding, newline diff --git a/src/black/comments.py b/src/black/comments.py index 2b530f2b910..8c8866860d2 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -154,10 +154,9 @@ def make_comment(content: str) -> str: if content[0] == "#": content = content[1:] - NON_BREAKING_SPACE = " " if ( content - and content[0] == NON_BREAKING_SPACE + and content[0] == "\N{NO-BREAK SPACE}" and not content.lstrip().startswith("type:") ): content = " " + content[1:] # Replace NBSP by a simple space @@ -343,7 +342,7 @@ def _generate_ignored_nodes_from_fmt_skip( # Traversal process (starting at the `# fmt: skip` node): # 1. Move to the `prev_sibling` of the current node. # 2. If `prev_sibling` has children, go to its rightmost leaf. - # 3. If there’s no `prev_sibling`, move up to the parent + # 3. If there's no `prev_sibling`, move up to the parent # node and repeat. # 4. Continue until: # a. You encounter an `INDENT` or `NEWLINE` node (indicates diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index dd680bffffb..e9ef6ae3d04 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -66,7 +66,7 @@ def jupyter_dependencies_are_installed(*, warn: bool) -> bool: def validate_cell(src: str, mode: Mode) -> None: - """Check that cell does not already contain TransformerManager transformations, + r"""Check that cell does not already contain TransformerManager transformations, or non-Python cell magics, which might cause tokenizer_rt to break because of indentations. @@ -228,14 +228,14 @@ def get_token(src: str, magic: str) -> str: def replace_cell_magics(src: str) -> tuple[str, list[Replacement]]: - """Replace cell magic with token. + r"""Replace cell magic with token. Note that 'src' will already have been processed by IPython's TransformerManager().transform_cell. Example, - get_ipython().run_cell_magic('t', '-n1', 'ls =!ls\\n') + get_ipython().run_cell_magic('t', '-n1', 'ls =!ls\n') becomes @@ -370,7 +370,7 @@ def header(self) -> str: # ast.NodeVisitor + dataclass = breakage under mypyc. class CellMagicFinder(ast.NodeVisitor): - """Find cell magics. + r"""Find cell magics. Note that the source of the abstract syntax tree will already have been processed by IPython's @@ -383,7 +383,7 @@ class CellMagicFinder(ast.NodeVisitor): would have been transformed to - get_ipython().run_cell_magic('time', '', 'foo()\\n') + get_ipython().run_cell_magic('time', '', 'foo()\n') and we look for instances of the latter. """ diff --git a/src/black/lines.py b/src/black/lines.py index 2a719def3c9..9a889152433 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -71,7 +71,7 @@ def append( if not has_value: return - if token.COLON == leaf.type and self.is_class_paren_empty: + if leaf.type == token.COLON and self.is_class_paren_empty: del self.leaves[-2:] if self.leaves and not preformatted: # Note: at this point leaf.prefix should be empty except for diff --git a/src/black/ranges.py b/src/black/ranges.py index 90649137d2e..92e1c93aaf7 100644 --- a/src/black/ranges.py +++ b/src/black/ranges.py @@ -155,7 +155,7 @@ def adjusted_lines( def convert_unchanged_lines(src_node: Node, lines: Collection[tuple[int, int]]) -> None: - """Converts unchanged lines to STANDALONE_COMMENT. + r"""Converts unchanged lines to STANDALONE_COMMENT. The idea is similar to how `# fmt: on/off` is implemented. It also converts the nodes between those markers as a single `STANDALONE_COMMENT` leaf node with @@ -275,7 +275,7 @@ def _convert_unchanged_line_by_line(node: Node, lines_set: set[int]) -> None: # We will check `simple_stmt` and `stmt+` separately against the lines set parent_sibling = leaf.parent.prev_sibling nodes_to_ignore = [] - while parent_sibling and not parent_sibling.type == syms.suite: + while parent_sibling and parent_sibling.type != syms.suite: # NOTE: Multiple suite nodes can exist as siblings in e.g. `if_stmt`. nodes_to_ignore.insert(0, parent_sibling) parent_sibling = parent_sibling.prev_sibling diff --git a/src/black/strings.py b/src/black/strings.py index a3018990ee8..5d60accff04 100644 --- a/src/black/strings.py +++ b/src/black/strings.py @@ -153,7 +153,7 @@ def normalize_string_prefix(s: str) -> str: ) # Python syntax guarantees max 2 prefixes and that one of them is "r" - if len(new_prefix) == 2 and "r" != new_prefix[0].lower(): + if len(new_prefix) == 2 and new_prefix[0].lower() != "r": new_prefix = new_prefix[::-1] return f"{new_prefix}{match.group(2)}" diff --git a/src/black/trans.py b/src/black/trans.py index 97dd1ce366a..0b595630450 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -584,7 +584,7 @@ def _merge_string_group( <= i < previous_merged_string_idx + previous_merged_num_of_strings ): - for comment_leaf in line.comments_after(LL[i]): + for comment_leaf in line.comments_after(leaf): new_line.append(comment_leaf, preformatted=True) continue @@ -1706,10 +1706,10 @@ def more_splits_should_be_made() -> bool: yield Ok(last_line) def _iter_nameescape_slices(self, string: str) -> Iterator[tuple[Index, Index]]: - """ + r""" Yields: All ranges of @string which, if @string were to be split there, - would result in the splitting of an \\N{...} expression (which is NOT + would result in the splitting of an \N{...} expression (which is NOT allowed). """ # True - the previous backslash was unescaped From 89f36108caaa8e3281779a00e3fc14db1cacbf7d Mon Sep 17 00:00:00 2001 From: cobalt <61329810+cobaltt7@users.noreply.github.com> Date: Fri, 11 Jul 2025 21:31:44 -0500 Subject: [PATCH 49/69] Improve `multiline_string_handling` with ternaries and dictionaries (#4657) --- CHANGES.md | 1 + src/black/lines.py | 6 + tests/data/cases/preview_multiline_strings.py | 171 ++++++++++++++++++ 3 files changed, 178 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 4e97c1f2e46..fda755cd3d9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,6 +29,7 @@ - Fix a bug where one-liner functions/conditionals marked with `# fmt: skip` would still be formatted (#4552) +- Improve `multiline_string_handling` with ternaries and dictionaries (#4657) - Fix a bug where `string_processing` would not split f-strings directly after expressions (#4680) diff --git a/src/black/lines.py b/src/black/lines.py index 9a889152433..8e84f46bf95 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -872,6 +872,12 @@ def is_line_short_enough( # noqa: C901 max_level_to_update = min(max_level_to_update, leaf.bracket_depth) if is_multiline_string(leaf): + if leaf.parent and ( + leaf.parent.type == syms.test + or (leaf.parent.parent and leaf.parent.parent.type == syms.dictsetmaker) + ): + # Keep ternary and dictionary values parenthesized + return False if len(multiline_string_contexts) > 0: # >1 multiline string cannot fit on a single line - force split return False diff --git a/tests/data/cases/preview_multiline_strings.py b/tests/data/cases/preview_multiline_strings.py index 1daf6f4d784..e441fdc4ff8 100644 --- a/tests/data/cases/preview_multiline_strings.py +++ b/tests/data/cases/preview_multiline_strings.py @@ -182,7 +182,75 @@ def dastardly_default_value( expected: {expected_result} actual: {some_var}""" + +def foo(): + a = { + xxxx_xxxxxxx.xxxxxx_xxxxxxxxxxxx_xxxxxx_xx_xxx_xxxxxx: { + "xxxxx": """Sxxxxx xxxxxxxxxxxx xxx xxxxx (xxxxxx xxx xxxxxxx)""", + "xxxxxxxx": ( + """Sxxxxxxx xxxxxxxx, xxxxxxx xx xxxxxxxxx + xxxxxxxxxxxxx xxxxxxx xxxxxxxxx xxx-xxxxxxxxxx xxxxxx xx xxx-xxxxxx""" + ), + "xxxxxxxx": """Sxxxxxxx xxxxxxxx, xxxxxxx xx xxxxxxxxx + xxxxxxxxxxxxx xxxxxxx xxxxxxxxx xxx-xxxxxxxxxx xxxxxx xx xxx-xxxxxx""" + }, + } + + +xxxx_xxxxxxx.xxxxxx_xxxxxxxxxxxx_xxxxxx_xx_xxx_xxxxxx = { + "xxxxx": """Sxxxxx xxxxxxxxxxxx xxx xxxxx (xxxxxx xxx xxxxxxx)""", + "xxxxxxxx": ( + """Sxxxxxxx xxxxxxxx, xxxxxxx xx xxxxxxxxx + xxxxxxxxxxxxx xxxxxxx xxxxxxxxx xxx-xxxxxxxxxx xxxxxx xx xxx-xxxxxx""" + ), + "xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + """ +a +a +a +a +a""" + ), + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": """ +a +a +a +a +a""", +} + +a = """ +""" if """ +""" == """ +""" else """ +""" + +a = """ +""" if b else """ +""" + +a = """ +""" if """ +""" == """ +""" else b + +a = b if """ +""" == """ +""" else """ +""" + +a = """ +""" if b else c + +a = c if b else """ +""" + +a = b if """ +""" == """ +""" else c + # output + """cow say""", call( @@ -399,3 +467,106 @@ def dastardly_default_value( assert some_var == expected_result, f""" expected: {expected_result} actual: {some_var}""" + + +def foo(): + a = { + xxxx_xxxxxxx.xxxxxx_xxxxxxxxxxxx_xxxxxx_xx_xxx_xxxxxx: { + "xxxxx": """Sxxxxx xxxxxxxxxxxx xxx xxxxx (xxxxxx xxx xxxxxxx)""", + "xxxxxxxx": ( + """Sxxxxxxx xxxxxxxx, xxxxxxx xx xxxxxxxxx + xxxxxxxxxxxxx xxxxxxx xxxxxxxxx xxx-xxxxxxxxxx xxxxxx xx xxx-xxxxxx""" + ), + "xxxxxxxx": ( + """Sxxxxxxx xxxxxxxx, xxxxxxx xx xxxxxxxxx + xxxxxxxxxxxxx xxxxxxx xxxxxxxxx xxx-xxxxxxxxxx xxxxxx xx xxx-xxxxxx""" + ), + }, + } + + +xxxx_xxxxxxx.xxxxxx_xxxxxxxxxxxx_xxxxxx_xx_xxx_xxxxxx = { + "xxxxx": """Sxxxxx xxxxxxxxxxxx xxx xxxxx (xxxxxx xxx xxxxxxx)""", + "xxxxxxxx": ( + """Sxxxxxxx xxxxxxxx, xxxxxxx xx xxxxxxxxx + xxxxxxxxxxxxx xxxxxxx xxxxxxxxx xxx-xxxxxxxxxx xxxxxx xx xxx-xxxxxx""" + ), + "xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + """ +a +a +a +a +a""" + ), + "xx_xxxxx_xxxxxxxxxx_xxxxxxxxx_xx": ( + """ +a +a +a +a +a""" + ), +} + +a = ( + """ +""" + if """ +""" + == """ +""" + else """ +""" +) + +a = ( + """ +""" + if b + else """ +""" +) + +a = ( + """ +""" + if """ +""" + == """ +""" + else b +) + +a = ( + b + if """ +""" + == """ +""" + else """ +""" +) + +a = ( + """ +""" + if b + else c +) + +a = ( + c + if b + else """ +""" +) + +a = ( + b + if """ +""" + == """ +""" + else c +) From bbc36ea205292e213b0c9a448f630d19726605e4 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 14 Jul 2025 08:49:58 +0200 Subject: [PATCH 50/69] Fix minor typo (#4707) Add missing space Signed-off-by: Philippe Ombredanne --- docs/the_black_code_style/future_style.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index e801874a4f0..b5eb3690f87 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -152,7 +152,7 @@ plain strings. f-strings will not be merged if they contain internal quotes and change their quotation mark style. User-made splits are respected when they do not exceed the line length limit. Line continuation backslashes are converted into parenthesized strings. Unnecessary parentheses are stripped. The stability and status of -this feature istracked in [this issue](https://github.com/psf/black/issues/2188). +this feature is tracked in [this issue](https://github.com/psf/black/issues/2188). (labels/multiline-string-handling)= From 1fb95422c8719055ba5766f5bde7caf5d2e3bf21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:13:30 -0500 Subject: [PATCH 51/69] Bump furo from 2024.8.6 to 2025.7.19 in /docs (#4713) --- updated-dependencies: - dependency-name: furo dependency-version: 2025.7.19 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 528b2972200..10f6bbbd8a8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ Sphinx==8.2.3 docutils==0.21.2 sphinxcontrib-programoutput==0.18 sphinx_copybutton==0.5.2 -furo==2024.8.6 +furo==2025.7.19 From f4926ace179123942d5713a11196e4a4afae1d2b Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Wed, 23 Jul 2025 15:38:48 -0700 Subject: [PATCH 52/69] Add FAQ section on Windows emojis not working (#4714) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.md | 2 ++ docs/faq.md | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index fda755cd3d9..da991a69eee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -75,6 +75,8 @@ +- Add FAQ entry for windows emoji not displaying (#4714) + ## 25.1.0 ### Highlights diff --git a/docs/faq.md b/docs/faq.md index 9efcf316852..a1ba0041079 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -137,3 +137,13 @@ wheels (including the pure Python wheel), so this command will use the [sdist]. [mypyc]: https://mypyc.readthedocs.io/en/latest/ [sdist]: https://packaging.python.org/en/latest/glossary/#term-Source-Distribution-or-sdist + +## Why are emoji not displaying correctly on Windows? + +When using Windows, the emoji in _Black_'s output may not display correctly. This is not +fixable from _Black_'s end. + +Instead, run your chosen command line/shell through [Windows Terminal], which will +properly handle rendering the emoji. + +[Windows Terminal]: https://github.com/microsoft/terminal From 30275ed2c3695e4c8bd9ab862c28c3d53071b17c Mon Sep 17 00:00:00 2001 From: cobalt <61329810+cobaltt7@users.noreply.github.com> Date: Sun, 27 Jul 2025 10:15:18 -0500 Subject: [PATCH 53/69] Update PR Template (#4716) --- .github/ISSUE_TEMPLATE/bug_report.md | 4 +-- .github/PULL_REQUEST_TEMPLATE.md | 39 ++++++++++++++++------------ 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7e6309b5d88..48aa9291b05 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -12,9 +12,7 @@ current development version. To confirm this, you have three options: 1. Update Black's version if a newer release exists: `pip install -U black` 2. Use the online formatter at , which will use - the latest main branch. Note that the online formatter currently runs on - an older version of Python and may not support newer syntax, such as the - extended f-string syntax added in Python 3.12. + the latest main branch. 3. Or run _Black_ on your machine: - create a new virtualenv (make sure it's the same Python version); - clone this repository; diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a039718cd70..1500a84a493 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,5 +1,5 @@ - + ### Description @@ -10,27 +10,34 @@ ### Checklist - did you ... - - + - Please familiarize yourself with Black's stability policy, linked + below. Code style changes are only allowed under the `--preview` flag + until maintainers move them to stable in the next calendar year. + - All user-facing changes should get a changelog entry. If this isn't + user-facing, signal to us that this should get the magical label to + silence the check. + - Tests are required for all bugfixes and new features. + - Documentation changes are necessary for most formatting changes and + other enhancements. --> + +- [ ] Implement any code style changes under the `--preview` style, following the + stability policy? - [ ] Add an entry in `CHANGES.md` if necessary? - [ ] Add / update tests if necessary? - [ ] Add new / update outdated documentation? - + - PSF COC: https://www.python.org/psf/conduct/ + - Contributing docs: https://black.readthedocs.io/en/latest/contributing/index.html + - Chat on Python Discord: https://discord.gg/RtVdv86PrH + - Stability policy: https://black.readthedocs.io/en/latest/the_black_code_style/index.html --> From bdd7fcd9396350b2887e22dec67072db3aeeeaa9 Mon Sep 17 00:00:00 2001 From: Ranjodh Singh Date: Tue, 29 Jul 2025 20:01:26 +0530 Subject: [PATCH 54/69] Fix test failures on Python 3.14 caused by deprecated `asyncio.get_event_loop_policy()` (#4719) Replace deprecated asyncio.get_event_loop_policy() for Python 3.14+ --- tests/test_black.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_black.py b/tests/test_black.py index 98d03652abe..23e1e94c41d 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -84,8 +84,7 @@ def cache_dir(exists: bool = True) -> Iterator[Path]: @contextmanager def event_loop() -> Iterator[None]: - policy = asyncio.get_event_loop_policy() - loop = policy.new_event_loop() + loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: yield From 3a96e06025daf0519ba1db113f779a5779a4a702 Mon Sep 17 00:00:00 2001 From: Ranjodh Singh Date: Mon, 4 Aug 2025 17:54:49 +0530 Subject: [PATCH 55/69] Fix function `supports_feature` to return False when `target_versions` is empty. [skip news] (#4726) * Fix function `supports_feature` to return False when `target_versions` is empty. * supports_feature: raise ValueError if target_versions is empty --- src/black/mode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/black/mode.py b/src/black/mode.py index 362607efc86..1a238e7210c 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -190,6 +190,9 @@ class Feature(Enum): def supports_feature(target_versions: set[TargetVersion], feature: Feature) -> bool: + if not target_versions: + raise ValueError("target_versions must not be empty") + return all(feature in VERSION_TO_FEATURES[version] for version in target_versions) From b3ae57b22a515fa8d9c8a5076ec1296a46aead74 Mon Sep 17 00:00:00 2001 From: Ranjodh Singh Date: Mon, 11 Aug 2025 05:08:22 +0530 Subject: [PATCH 56/69] Fix `--target-version` flag for unit tests. (#4722) --- tests/data/cases/target_version_flag.py | 10 ++++++++++ tests/util.py | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 tests/data/cases/target_version_flag.py diff --git a/tests/data/cases/target_version_flag.py b/tests/data/cases/target_version_flag.py new file mode 100644 index 00000000000..56972812896 --- /dev/null +++ b/tests/data/cases/target_version_flag.py @@ -0,0 +1,10 @@ +# flags: --minimum-version=3.12 --target-version=py312 +# this is invalid in versions below py312 +class ClassA[T: str]: + def method1(self) -> T: + ... + +# output +# this is invalid in versions below py312 +class ClassA[T: str]: + def method1(self) -> T: ... diff --git a/tests/util.py b/tests/util.py index 5384af9b8a5..a1b7f87df54 100644 --- a/tests/util.py +++ b/tests/util.py @@ -236,8 +236,8 @@ def get_flags_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument( "--target-version", - action="append", - type=lambda val: TargetVersion[val.upper()], + action="store", + type=lambda val: (TargetVersion[val.upper()],), default=(), ) parser.add_argument("--line-length", default=DEFAULT_LINE_LENGTH, type=int) @@ -258,7 +258,7 @@ def get_flags_parser() -> argparse.ArgumentParser: default=None, help=( "Minimum version of Python where this test case is parseable. If this is" - " set, the test case will be run twice: once with the specified" + " set, the test case will be run twice: once without the specified" " --target-version, and once with --target-version set to exactly the" " specified version. This ensures that Black's autodetection of the target" " version works correctly." From 9c47b6e20c934c9137a4562bda49a039820488ba Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 17 Aug 2025 17:17:50 -0700 Subject: [PATCH 57/69] Avoid using an extra process when running with only one worker (#4734) --- CHANGES.md | 2 ++ src/black/concurrency.py | 23 +++++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index da991a69eee..e87b61dbaeb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -53,6 +53,8 @@ +- Avoid using an extra process when running with only one worker (#4734) + ### Output diff --git a/src/black/concurrency.py b/src/black/concurrency.py index 4b3cf48d901..f6a2b8a93be 100644 --- a/src/black/concurrency.py +++ b/src/black/concurrency.py @@ -4,6 +4,8 @@ NOTE: this module is only imported if we need to format several files at once. """ +from __future__ import annotations + import asyncio import logging import os @@ -80,20 +82,25 @@ def reformat_many( """Reformat multiple files using a ProcessPoolExecutor.""" maybe_install_uvloop() - executor: Executor if workers is None: workers = int(os.environ.get("BLACK_NUM_WORKERS", 0)) workers = workers or os.cpu_count() or 1 if sys.platform == "win32": # Work around https://bugs.python.org/issue26903 workers = min(workers, 60) - try: - executor = ProcessPoolExecutor(max_workers=workers) - except (ImportError, NotImplementedError, OSError): - # we arrive here if the underlying system does not support multi-processing - # like in AWS Lambda or Termux, in which case we gracefully fallback to - # a ThreadPoolExecutor with just a single worker (more workers would not do us - # any good due to the Global Interpreter Lock) + + executor: Executor | None = None + if workers > 1: + try: + executor = ProcessPoolExecutor(max_workers=workers) + except (ImportError, NotImplementedError, OSError): + # we arrive here if the underlying system does not support multi-processing + # like in AWS Lambda or Termux, in which case we gracefully fallback to + # a ThreadPoolExecutor with just a single worker (more workers would not do + # us any good due to the Global Interpreter Lock) + pass + + if executor is None: executor = ThreadPoolExecutor(max_workers=1) loop = asyncio.new_event_loop() From abd5f08107b09f768b1ec98c7a48e65ec7fdd661 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 08:51:05 -0500 Subject: [PATCH 58/69] Bump actions/checkout from 4 to 5 (#4735) Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/changelog.yml | 2 +- .github/workflows/diff_shades.yml | 4 ++-- .github/workflows/diff_shades_comment.yml | 2 +- .github/workflows/doc.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/fuzz.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pypi_upload.yml | 8 ++++---- .github/workflows/release_tests.yml | 2 +- .github/workflows/test.yml | 6 +++--- .github/workflows/upload_binary.yml | 2 +- 11 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index a1804597d7d..b6af8f355e7 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Grep CHANGES.md for PR number if: contains(github.event.pull_request.labels.*.name, 'skip news') != true diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index 5fc4983be40..d7658aa6067 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -19,7 +19,7 @@ jobs: matrix: ${{ steps.set-config.outputs.matrix }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: actions/setup-python@v5 with: python-version: "3.11" @@ -53,7 +53,7 @@ jobs: steps: - name: Checkout this repository (full clone) - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: # The baseline revision could be rather old so a full clone is ideal. fetch-depth: 0 diff --git a/.github/workflows/diff_shades_comment.yml b/.github/workflows/diff_shades_comment.yml index 9deb3a0373d..3f446687a9a 100644 --- a/.github/workflows/diff_shades_comment.yml +++ b/.github/workflows/diff_shades_comment.yml @@ -12,7 +12,7 @@ jobs: comment: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: actions/setup-python@v5 with: python-version: "*" diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index f34e5041091..1c0dd398a8c 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -21,7 +21,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up latest Python uses: actions/setup-python@v5 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 43d7a2453b7..4862c608d13 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 48f101c206f..b9090b4a1c5 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -25,7 +25,7 @@ jobs: python-version: ["3.9", "3.10", "3.11", "3.12.4", "3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2d14092481a..73889d914ab 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Assert PR target is main if: github.event_name == 'pull_request' && github.repository == 'psf/black' diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 1b075635477..bf3cd385c84 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -18,7 +18,7 @@ jobs: if: github.event_name == 'release' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up latest Python uses: actions/setup-python@v5 @@ -46,7 +46,7 @@ jobs: outputs: include: ${{ steps.set-matrix.outputs.include }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 # Keep cibuildwheel version in sync with below - name: Install cibuildwheel and pypyp run: | @@ -90,7 +90,7 @@ jobs: include: ${{ fromJson(needs.generate_wheels_matrix.outputs.include) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 # Keep cibuildwheel version in sync with above - uses: pypa/cibuildwheel@v2.23.3 with: @@ -118,7 +118,7 @@ jobs: steps: - name: Checkout stable branch - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: stable fetch-depth: 0 diff --git a/.github/workflows/release_tests.yml b/.github/workflows/release_tests.yml index 6d0af004aae..50d47154d99 100644 --- a/.github/workflows/release_tests.yml +++ b/.github/workflows/release_tests.yml @@ -29,7 +29,7 @@ jobs: os: [macOS-latest, ubuntu-latest, windows-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: # Give us all history, branches and tags fetch-depth: 0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 004256e563d..5b5987b7bce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: os: [ubuntu-latest, macOS-latest, windows-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -76,7 +76,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Send finished signal to Coveralls uses: AndreMiras/coveralls-python-action@ac868b9540fad490f7ca82b8ca00480fd751ed19 with: @@ -94,7 +94,7 @@ jobs: os: [ubuntu-latest, macOS-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up latest Python uses: actions/setup-python@v5 diff --git a/.github/workflows/upload_binary.yml b/.github/workflows/upload_binary.yml index 918c0ee85fe..4f0044f482e 100644 --- a/.github/workflows/upload_binary.yml +++ b/.github/workflows/upload_binary.yml @@ -29,7 +29,7 @@ jobs: executable_mime: "application/x-mach-binary" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up latest Python uses: actions/setup-python@v5 From dd937dcc7e6fd260c87868ed044a29042056cb00 Mon Sep 17 00:00:00 2001 From: Ranjodh Singh Date: Tue, 19 Aug 2025 18:01:00 +0530 Subject: [PATCH 59/69] Remove parentheses around multiple exception types in `except` and `except*` without `as`. (#4720) * Remove parentheses around multiple exception types in `except` and `except*` when not using the `as` clause. (#4678) * Add Changelog Entry * Add Unparenthesized Except Tuple Detection. * Add tests for `except*`. * Oops! Wrong Version. * Fix function `supports_feature` to return False when `target_versions` is empty. [skip news] (#4726) * Fix function `supports_feature` to return False when `target_versions` is empty. * supports_feature: raise ValueError if target_versions is empty * Simplify conditional logic for removing except type parens. --- CHANGES.md | 2 + docs/the_black_code_style/future_style.md | 2 + src/black/__init__.py | 42 ++- src/black/linegen.py | 63 ++++- src/black/mode.py | 27 ++ src/black/resources/black.schema.json | 6 +- src/blib2to3/Grammar.txt | 2 +- .../data/cases/remove_except_types_parens.py | 249 ++++++++++++++++++ .../remove_except_types_parens_pre_py314.py | 225 ++++++++++++++++ tests/test_black.py | 6 + 10 files changed, 601 insertions(+), 23 deletions(-) create mode 100644 tests/data/cases/remove_except_types_parens.py create mode 100644 tests/data/cases/remove_except_types_parens_pre_py314.py diff --git a/CHANGES.md b/CHANGES.md index e87b61dbaeb..4361e1e376f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -32,6 +32,8 @@ - Improve `multiline_string_handling` with ternaries and dictionaries (#4657) - Fix a bug where `string_processing` would not split f-strings directly after expressions (#4680) +- Remove parentheses around multiple exception types in `except` and `except*` without + `as`. (#4720) ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index b5eb3690f87..45ec1e67ed8 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -29,6 +29,8 @@ Currently, the following features are included in the preview style: - `fix_fmt_skip_in_one_liners`: Fix `# fmt: skip` behaviour on one-liner declarations, such as `def foo(): return "mock" # fmt: skip`, where previously the declaration would have been incorrectly collapsed. +- `remove_parens_around_except_types`: Remove parentheses around multiple exception + types in `except` and `except*` without `as`. See PEP 758 for details. (labels/unstable-features)= diff --git a/src/black/__init__.py b/src/black/__init__.py index 48d2bb8154b..082f705f196 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1225,9 +1225,12 @@ def _format_str_once( future_imports = get_future_imports(src_node) versions = detect_target_versions(src_node, future_imports=future_imports) - context_manager_features = { + line_generation_features = { feature - for feature in {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + for feature in { + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + Feature.UNPARENTHESIZED_EXCEPT_TYPES, + } if supports_feature(versions, feature) } normalize_fmt_off(src_node, mode, lines) @@ -1235,7 +1238,7 @@ def _format_str_once( # This should be called after normalize_fmt_off. convert_unchanged_lines(src_node, lines) - line_generator = LineGenerator(mode=mode, features=context_manager_features) + line_generator = LineGenerator(mode=mode, features=line_generation_features) elt = EmptyLineTracker(mode=mode) split_line_features = { feature @@ -1395,13 +1398,6 @@ def get_features_used( # noqa: C901 elif n.type == syms.match_stmt: features.add(Feature.PATTERN_MATCHING) - elif ( - n.type == syms.except_clause - and len(n.children) >= 2 - and n.children[1].type == token.STAR - ): - features.add(Feature.EXCEPT_STAR) - elif n.type in {syms.subscriptlist, syms.trailer} and any( child.type == syms.star_expr for child in n.children ): @@ -1423,6 +1419,32 @@ def get_features_used( # noqa: C901 ): features.add(Feature.TYPE_PARAM_DEFAULTS) + elif ( + n.type == syms.except_clause + and len(n.children) >= 2 + and ( + n.children[1].type == token.STAR or n.children[1].type == syms.testlist + ) + ): + is_star_except = n.children[1].type == token.STAR + + if is_star_except: + features.add(Feature.EXCEPT_STAR) + + # Presence of except* pushes as clause 1 index back + has_as_clause = ( + len(n.children) >= is_star_except + 3 + and n.children[is_star_except + 2].type == token.NAME + and n.children[is_star_except + 2].value == "as" # type: ignore + ) + + # If there's no 'as' clause and the except expression is a testlist. + if not has_as_clause and ( + (is_star_except and n.children[2].type == syms.testlist) + or (not is_star_except and n.children[1].type == syms.testlist) + ): + features.add(Feature.UNPARENTHESIZED_EXCEPT_TYPES) + return features diff --git a/src/black/linegen.py b/src/black/linegen.py index 032ab687835..49fab818a6d 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -250,6 +250,8 @@ def visit_dictsetmaker(self, node: Node) -> Iterator[Line]: maybe_make_parens_invisible_in_atom( child, parent=node, + mode=self.mode, + features=self.features, remove_brackets_around_comma=False, ) else: @@ -270,6 +272,8 @@ def visit_funcdef(self, node: Node) -> Iterator[Line]: if maybe_make_parens_invisible_in_atom( child, parent=node, + mode=self.mode, + features=self.features, remove_brackets_around_comma=False, ): wrap_in_parentheses(node, child, visible=False) @@ -363,7 +367,7 @@ def visit_power(self, node: Node) -> Iterator[Line]: ): wrap_in_parentheses(node, leaf) - remove_await_parens(node) + remove_await_parens(node, mode=self.mode, features=self.features) yield from self.visit_default(node) @@ -410,7 +414,9 @@ def foo(a: int, b: float = 7): ... def foo(a: (int), b: (float) = 7): ... """ assert len(node.children) == 3 - if maybe_make_parens_invisible_in_atom(node.children[2], parent=node): + if maybe_make_parens_invisible_in_atom( + node.children[2], parent=node, mode=self.mode, features=self.features + ): wrap_in_parentheses(node, node.children[2], visible=False) yield from self.visit_default(node) @@ -516,7 +522,12 @@ def visit_atom(self, node: Node) -> Iterator[Line]: first.type == token.LBRACE and last.type == token.RBRACE ): # Lists or sets of one item - maybe_make_parens_invisible_in_atom(node.children[1], parent=node) + maybe_make_parens_invisible_in_atom( + node.children[1], + parent=node, + mode=self.mode, + features=self.features, + ) yield from self.visit_default(node) @@ -1448,15 +1459,16 @@ def normalize_invisible_parens( # noqa: C901 if maybe_make_parens_invisible_in_atom( child, parent=node, + mode=mode, + features=features, remove_brackets_around_comma=True, ): wrap_in_parentheses(node, child, visible=False) elif isinstance(child, Node) and node.type == syms.with_stmt: - remove_with_parens(child, node) + remove_with_parens(child, node, mode=mode, features=features) elif child.type == syms.atom: if maybe_make_parens_invisible_in_atom( - child, - parent=node, + child, parent=node, mode=mode, features=features ): wrap_in_parentheses(node, child, visible=False) elif is_one_tuple(child): @@ -1508,7 +1520,7 @@ def _normalize_import_from(parent: Node, child: LN, index: int) -> None: parent.append_child(Leaf(token.RPAR, "")) -def remove_await_parens(node: Node) -> None: +def remove_await_parens(node: Node, mode: Mode, features: Collection[Feature]) -> None: if node.children[0].type == token.AWAIT and len(node.children) > 1: if ( node.children[1].type == syms.atom @@ -1517,6 +1529,8 @@ def remove_await_parens(node: Node) -> None: if maybe_make_parens_invisible_in_atom( node.children[1], parent=node, + mode=mode, + features=features, remove_brackets_around_comma=True, ): wrap_in_parentheses(node, node.children[1], visible=False) @@ -1585,7 +1599,9 @@ def _maybe_wrap_cms_in_parens( node.insert_child(1, new_child) -def remove_with_parens(node: Node, parent: Node) -> None: +def remove_with_parens( + node: Node, parent: Node, mode: Mode, features: Collection[Feature] +) -> None: """Recursively hide optional parens in `with` statements.""" # Removing all unnecessary parentheses in with statements in one pass is a tad # complex as different variations of bracketed statements result in pretty @@ -1607,21 +1623,25 @@ def remove_with_parens(node: Node, parent: Node) -> None: if maybe_make_parens_invisible_in_atom( node, parent=parent, + mode=mode, + features=features, remove_brackets_around_comma=True, ): wrap_in_parentheses(parent, node, visible=False) if isinstance(node.children[1], Node): - remove_with_parens(node.children[1], node) + remove_with_parens(node.children[1], node, mode=mode, features=features) elif node.type == syms.testlist_gexp: for child in node.children: if isinstance(child, Node): - remove_with_parens(child, node) + remove_with_parens(child, node, mode=mode, features=features) elif node.type == syms.asexpr_test and not any( leaf.type == token.COLONEQUAL for leaf in node.leaves() ): if maybe_make_parens_invisible_in_atom( node.children[0], parent=node, + mode=mode, + features=features, remove_brackets_around_comma=True, ): wrap_in_parentheses(node, node.children[0], visible=False) @@ -1630,6 +1650,8 @@ def remove_with_parens(node: Node, parent: Node) -> None: def maybe_make_parens_invisible_in_atom( node: LN, parent: LN, + mode: Mode, + features: Collection[Feature], remove_brackets_around_comma: bool = False, ) -> bool: """If it's safe, make the parens in the atom `node` invisible, recursively. @@ -1655,6 +1677,25 @@ def maybe_make_parens_invisible_in_atom( # and option to skip this check for `for` and `with` statements. not remove_brackets_around_comma and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY + # Skip this check in Preview mode in order to + # Remove parentheses around multiple exception types in except and + # except* without as. See PEP 758 for details. + and not ( + Preview.remove_parens_around_except_types in mode + and Feature.UNPARENTHESIZED_EXCEPT_TYPES in features + # is a tuple + and is_tuple(node) + # has a parent node + and node.parent is not None + # parent is an except clause + and node.parent.type == syms.except_clause + # is not immediately followed by as clause + and not ( + node.next_sibling is not None + and is_name_token(node.next_sibling) + and node.next_sibling.value == "as" + ) + ) ) or is_tuple_containing_walrus(node) or is_tuple_containing_star(node) @@ -1695,6 +1736,8 @@ def maybe_make_parens_invisible_in_atom( maybe_make_parens_invisible_in_atom( middle, parent=parent, + mode=mode, + features=features, remove_brackets_around_comma=remove_brackets_around_comma, ) diff --git a/src/black/mode.py b/src/black/mode.py index 1a238e7210c..86e0bfcb1c2 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -25,6 +25,7 @@ class TargetVersion(Enum): PY311 = 11 PY312 = 12 PY313 = 13 + PY314 = 14 def pretty(self) -> str: assert self.name[:2] == "PY" @@ -53,6 +54,7 @@ class Feature(Enum): TYPE_PARAMS = 18 FSTRING_PARSING = 19 TYPE_PARAM_DEFAULTS = 20 + UNPARENTHESIZED_EXCEPT_TYPES = 21 FORCE_OPTIONAL_PARENTHESES = 50 # __future__ flags @@ -186,6 +188,28 @@ class Feature(Enum): Feature.FSTRING_PARSING, Feature.TYPE_PARAM_DEFAULTS, }, + TargetVersion.PY314: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + Feature.PATTERN_MATCHING, + Feature.EXCEPT_STAR, + Feature.VARIADIC_GENERICS, + Feature.TYPE_PARAMS, + Feature.FSTRING_PARSING, + Feature.TYPE_PARAM_DEFAULTS, + Feature.UNPARENTHESIZED_EXCEPT_TYPES, + }, } @@ -207,6 +231,9 @@ class Preview(Enum): multiline_string_handling = auto() always_one_newline_after_import = auto() fix_fmt_skip_in_one_liners = auto() + # Remove parentheses around multiple exception types in except and + # except* without as. See PEP 758 for details. + remove_parens_around_except_types = auto() UNSTABLE_FEATURES: set[Preview] = { diff --git a/src/black/resources/black.schema.json b/src/black/resources/black.schema.json index 572e5bbfa1e..b342f2d2204 100644 --- a/src/black/resources/black.schema.json +++ b/src/black/resources/black.schema.json @@ -28,7 +28,8 @@ "py310", "py311", "py312", - "py313" + "py313", + "py314" ] }, "description": "Python versions that should be supported by Black's output. You should include all versions that your code supports. By default, Black will infer target versions from the project metadata in pyproject.toml. If this does not yield conclusive results, Black will use per-file auto-detection." @@ -84,7 +85,8 @@ "wrap_long_dict_values_in_parens", "multiline_string_handling", "always_one_newline_after_import", - "fix_fmt_skip_in_one_liners" + "fix_fmt_skip_in_one_liners", + "remove_parens_around_except_types" ] }, "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." diff --git a/src/blib2to3/Grammar.txt b/src/blib2to3/Grammar.txt index b779b49eefb..406a21f764d 100644 --- a/src/blib2to3/Grammar.txt +++ b/src/blib2to3/Grammar.txt @@ -123,7 +123,7 @@ try_stmt: ('try' ':' suite with_stmt: 'with' asexpr_test (',' asexpr_test)* ':' suite # NB compile.c makes sure that the default except clause is last -except_clause: 'except' ['*'] [test [(',' | 'as') test]] +except_clause: 'except' ['*'] [testlist ['as' test]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT # Backward compatibility cruft to support: diff --git a/tests/data/cases/remove_except_types_parens.py b/tests/data/cases/remove_except_types_parens.py new file mode 100644 index 00000000000..71f2d229d3a --- /dev/null +++ b/tests/data/cases/remove_except_types_parens.py @@ -0,0 +1,249 @@ +# flags: --preview --minimum-version=3.14 +# SEE PEP 758 FOR MORE DETAILS +# remains unchanged +try: + pass +except: + pass + +# remains unchanged +try: + pass +except ValueError: + pass + +try: + pass +except* ValueError: + pass + +# parenthesis are removed +try: + pass +except (ValueError): + pass + +try: + pass +except* (ValueError): + pass + +# parenthesis are removed +try: + pass +except (ValueError) as e: + pass + +try: + pass +except* (ValueError) as e: + pass + +# remains unchanged +try: + pass +except (ValueError,): + pass + +try: + pass +except* (ValueError,): + pass + +# remains unchanged +try: + pass +except (ValueError,) as e: + pass + +try: + pass +except* (ValueError,) as e: + pass + +# remains unchanged +try: + pass +except ValueError, TypeError, KeyboardInterrupt: + pass + +try: + pass +except* ValueError, TypeError, KeyboardInterrupt: + pass + +# parenthesis are removed +try: + pass +except (ValueError, TypeError, KeyboardInterrupt): + pass + +try: + pass +except* (ValueError, TypeError, KeyboardInterrupt): + pass + +# parenthesis are not removed +try: + pass +except (ValueError, TypeError, KeyboardInterrupt) as e: + pass + +try: + pass +except* (ValueError, TypeError, KeyboardInterrupt) as e: + pass + +# parenthesis are removed +try: + pass +except (ValueError if True else TypeError): + pass + +try: + pass +except* (ValueError if True else TypeError): + pass + +# inner except: parenthesis are removed +# outer except: parenthsis are not removed +try: + try: + pass + except (TypeError, KeyboardInterrupt): + pass +except (ValueError,): + pass + +try: + try: + pass + except* (TypeError, KeyboardInterrupt): + pass +except* (ValueError,): + pass + +# output +# SEE PEP 758 FOR MORE DETAILS +# remains unchanged +try: + pass +except: + pass + +# remains unchanged +try: + pass +except ValueError: + pass + +try: + pass +except* ValueError: + pass + +# parenthesis are removed +try: + pass +except ValueError: + pass + +try: + pass +except* ValueError: + pass + +# parenthesis are removed +try: + pass +except ValueError as e: + pass + +try: + pass +except* ValueError as e: + pass + +# remains unchanged +try: + pass +except (ValueError,): + pass + +try: + pass +except* (ValueError,): + pass + +# remains unchanged +try: + pass +except (ValueError,) as e: + pass + +try: + pass +except* (ValueError,) as e: + pass + +# remains unchanged +try: + pass +except ValueError, TypeError, KeyboardInterrupt: + pass + +try: + pass +except* ValueError, TypeError, KeyboardInterrupt: + pass + +# parenthesis are removed +try: + pass +except ValueError, TypeError, KeyboardInterrupt: + pass + +try: + pass +except* ValueError, TypeError, KeyboardInterrupt: + pass + +# parenthesis are not removed +try: + pass +except (ValueError, TypeError, KeyboardInterrupt) as e: + pass + +try: + pass +except* (ValueError, TypeError, KeyboardInterrupt) as e: + pass + +# parenthesis are removed +try: + pass +except ValueError if True else TypeError: + pass + +try: + pass +except* ValueError if True else TypeError: + pass + +# inner except: parenthesis are removed +# outer except: parenthsis are not removed +try: + try: + pass + except TypeError, KeyboardInterrupt: + pass +except (ValueError,): + pass + +try: + try: + pass + except* TypeError, KeyboardInterrupt: + pass +except* (ValueError,): + pass diff --git a/tests/data/cases/remove_except_types_parens_pre_py314.py b/tests/data/cases/remove_except_types_parens_pre_py314.py new file mode 100644 index 00000000000..9f3a3b25652 --- /dev/null +++ b/tests/data/cases/remove_except_types_parens_pre_py314.py @@ -0,0 +1,225 @@ +# flags: --preview --minimum-version=3.11 +# SEE PEP 758 FOR MORE DETAILS +# remains unchanged +try: + pass +except: + pass + +# remains unchanged +try: + pass +except ValueError: + pass + +try: + pass +except* ValueError: + pass + +# parenthesis are removed +try: + pass +except (ValueError): + pass + +try: + pass +except* (ValueError): + pass + +# parenthesis are removed +try: + pass +except (ValueError) as e: + pass + +try: + pass +except* (ValueError) as e: + pass + +# remains unchanged +try: + pass +except (ValueError,): + pass + +try: + pass +except* (ValueError,): + pass + +# remains unchanged +try: + pass +except (ValueError,) as e: + pass + +try: + pass +except* (ValueError,) as e: + pass + +# parenthesis are not removed +try: + pass +except (ValueError, TypeError, KeyboardInterrupt): + pass + +try: + pass +except* (ValueError, TypeError, KeyboardInterrupt): + pass + +# parenthesis are not removed +try: + pass +except (ValueError, TypeError, KeyboardInterrupt) as e: + pass + +try: + pass +except* (ValueError, TypeError, KeyboardInterrupt) as e: + pass + +# parenthesis are removed +try: + pass +except (ValueError if True else TypeError): + pass + +try: + pass +except* (ValueError if True else TypeError): + pass + +# parenthesis are not removed +try: + try: + pass + except (TypeError, KeyboardInterrupt): + pass +except (ValueError,): + pass + +try: + try: + pass + except* (TypeError, KeyboardInterrupt): + pass +except* (ValueError,): + pass + +# output +# SEE PEP 758 FOR MORE DETAILS +# remains unchanged +try: + pass +except: + pass + +# remains unchanged +try: + pass +except ValueError: + pass + +try: + pass +except* ValueError: + pass + +# parenthesis are removed +try: + pass +except ValueError: + pass + +try: + pass +except* ValueError: + pass + +# parenthesis are removed +try: + pass +except ValueError as e: + pass + +try: + pass +except* ValueError as e: + pass + +# remains unchanged +try: + pass +except (ValueError,): + pass + +try: + pass +except* (ValueError,): + pass + +# remains unchanged +try: + pass +except (ValueError,) as e: + pass + +try: + pass +except* (ValueError,) as e: + pass + +# parenthesis are not removed +try: + pass +except (ValueError, TypeError, KeyboardInterrupt): + pass + +try: + pass +except* (ValueError, TypeError, KeyboardInterrupt): + pass + +# parenthesis are not removed +try: + pass +except (ValueError, TypeError, KeyboardInterrupt) as e: + pass + +try: + pass +except* (ValueError, TypeError, KeyboardInterrupt) as e: + pass + +# parenthesis are removed +try: + pass +except ValueError if True else TypeError: + pass + +try: + pass +except* ValueError if True else TypeError: + pass + +# parenthesis are not removed +try: + try: + pass + except (TypeError, KeyboardInterrupt): + pass +except (ValueError,): + pass + +try: + try: + pass + except* (TypeError, KeyboardInterrupt): + pass +except* (ValueError,): + pass diff --git a/tests/test_black.py b/tests/test_black.py index 23e1e94c41d..a9ea4b9a765 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1523,6 +1523,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY311, TargetVersion.PY312, TargetVersion.PY313, + TargetVersion.PY314, ], ), ( @@ -1532,6 +1533,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY311, TargetVersion.PY312, TargetVersion.PY313, + TargetVersion.PY314, ], ), ("<3.6", [TargetVersion.PY33, TargetVersion.PY34, TargetVersion.PY35]), @@ -1543,6 +1545,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY311, TargetVersion.PY312, TargetVersion.PY313, + TargetVersion.PY314, ], ), ( @@ -1553,6 +1556,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY311, TargetVersion.PY312, TargetVersion.PY313, + TargetVersion.PY314, ], ), ( @@ -1567,6 +1571,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY311, TargetVersion.PY312, TargetVersion.PY313, + TargetVersion.PY314, ], ), ( @@ -1583,6 +1588,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY311, TargetVersion.PY312, TargetVersion.PY313, + TargetVersion.PY314, ], ), ("==3.8.*", [TargetVersion.PY38]), From 203fd6b5cdad975178b8174394a7f7fb13d14f02 Mon Sep 17 00:00:00 2001 From: Aleksis Vezenkov <89858747+av-runner@users.noreply.github.com> Date: Mon, 1 Sep 2025 01:28:21 +0200 Subject: [PATCH 60/69] Optimize Line string method (#4739) Co-authored-by: cobalt <61329810+cobaltt7@users.noreply.github.com> --- src/black/lines.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/black/lines.py b/src/black/lines.py index 8e84f46bf95..21e6cec571d 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -484,10 +484,10 @@ def __str__(self) -> str: leaves = iter(self.leaves) first = next(leaves) res = f"{first.prefix}{indent}{first.value}" - for leaf in leaves: - res += str(leaf) - for comment in itertools.chain.from_iterable(self.comments.values()): - res += str(comment) + res += "".join(str(leaf) for leaf in leaves) + comments_iter = itertools.chain.from_iterable(self.comments.values()) + comments = [str(comment) for comment in comments_iter] + res += "".join(comments) return res + "\n" From 1f779dec013db37475fa56a5c9939a09eab7e7d6 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Sun, 31 Aug 2025 17:08:05 -0700 Subject: [PATCH 61/69] Fix line ranges decorator edge case (#4670) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.md | 2 ++ src/black/ranges.py | 13 +++++++++++++ tests/data/cases/line_ranges_decorator_edge_case.py | 8 ++++++++ 3 files changed, 23 insertions(+) create mode 100644 tests/data/cases/line_ranges_decorator_edge_case.py diff --git a/CHANGES.md b/CHANGES.md index 4361e1e376f..cc23ec4d9ad 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,8 @@ - Remove support for pre-python 3.7 `await/async` as soft keywords/variable names (#4676) - Fix crash on parenthesized expression inside a type parameter bound (#4684) +- Fix crash when using line ranges excluding indented single line decorated items + (#4670) ### Preview style diff --git a/src/black/ranges.py b/src/black/ranges.py index 92e1c93aaf7..26407cc7cfd 100644 --- a/src/black/ranges.py +++ b/src/black/ranges.py @@ -330,6 +330,19 @@ def _convert_node_to_standalone_comment(node: LN) -> None: first.prefix = "" index = node.remove() if index is not None: + # Because of the special handling of multiple decorators, if the decorated + # item is a single line then there will be a missing newline between the + # decorator and item, so add it back. This doesn't affect any other case + # since a decorated item with a newline would hit the earlier suite case + # in _convert_unchanged_line_by_line that correctly handles the newlines. + if node.type == syms.decorated: + # A leaf of type decorated wouldn't make sense, since it should always + # have at least the decorator + the decorated item, so if this assert + # hits that means there's a problem in the parser. + assert isinstance(node, Node) + # 1 will always be the correct index since before this function is + # called all the decorators are collapsed into a single leaf + node.insert_child(1, Leaf(NEWLINE, "\n")) # Remove the '\n', as STANDALONE_COMMENT will have '\n' appended when # generating the formatted code. value = str(node)[:-1] diff --git a/tests/data/cases/line_ranges_decorator_edge_case.py b/tests/data/cases/line_ranges_decorator_edge_case.py new file mode 100644 index 00000000000..483fbe8c57d --- /dev/null +++ b/tests/data/cases/line_ranges_decorator_edge_case.py @@ -0,0 +1,8 @@ +# flags: --line-ranges=6-7 +class Foo: + + @overload + def foo(): ... + + def fox(self): + print() From 0cf39efdbc3aaea455f95d31e5b42efb6bd61478 Mon Sep 17 00:00:00 2001 From: Aleksis Vezenkov <89858747+av-runner@users.noreply.github.com> Date: Tue, 2 Sep 2025 23:25:34 +0200 Subject: [PATCH 62/69] Improve the performance of get_string_prefix (#4742) --- src/black/strings.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/black/strings.py b/src/black/strings.py index 5d60accff04..7e47f13062a 100644 --- a/src/black/strings.py +++ b/src/black/strings.py @@ -96,13 +96,13 @@ def get_string_prefix(string: str) -> str: """ assert_is_leaf_string(string) - prefix = "" - prefix_idx = 0 - while string[prefix_idx] in STRING_PREFIX_CHARS: - prefix += string[prefix_idx] - prefix_idx += 1 - - return prefix + prefix = [] + for char in string: + if char in STRING_PREFIX_CHARS: + prefix.append(char) + else: + break + return "".join(prefix) def assert_is_leaf_string(string: str) -> None: From 4d55e6017993ee3f0927524e8bc19ead8de9e8ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 07:47:21 -0500 Subject: [PATCH 63/69] Bump actions/setup-python from 5 to 6 (#4744) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/diff_shades.yml | 4 ++-- .github/workflows/diff_shades_comment.yml | 2 +- .github/workflows/doc.yml | 2 +- .github/workflows/fuzz.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pypi_upload.yml | 2 +- .github/workflows/release_tests.yml | 2 +- .github/workflows/test.yml | 4 ++-- .github/workflows/upload_binary.yml | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index d7658aa6067..2cbe86f2166 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.11" @@ -58,7 +58,7 @@ jobs: # The baseline revision could be rather old so a full clone is ideal. fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.11" diff --git a/.github/workflows/diff_shades_comment.yml b/.github/workflows/diff_shades_comment.yml index 3f446687a9a..2b4e0932eb7 100644 --- a/.github/workflows/diff_shades_comment.yml +++ b/.github/workflows/diff_shades_comment.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "*" diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 1c0dd398a8c..5d477caf6c3 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v5 - name: Set up latest Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.13" allow-prereleases: true diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index b9090b4a1c5..506b1d1e7ce 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 73889d914ab..0e559214a6c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,7 +24,7 @@ jobs: fi - name: Set up latest Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.13" allow-prereleases: true diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index bf3cd385c84..35393ce2e39 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v5 - name: Set up latest Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.13" allow-prereleases: true diff --git a/.github/workflows/release_tests.yml b/.github/workflows/release_tests.yml index 50d47154d99..078c7e8f0f7 100644 --- a/.github/workflows/release_tests.yml +++ b/.github/workflows/release_tests.yml @@ -34,7 +34,7 @@ jobs: # Give us all history, branches and tags fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b5987b7bce..e4cbe736dfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true @@ -97,7 +97,7 @@ jobs: - uses: actions/checkout@v5 - name: Set up latest Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12.4" diff --git a/.github/workflows/upload_binary.yml b/.github/workflows/upload_binary.yml index 4f0044f482e..17b591f6320 100644 --- a/.github/workflows/upload_binary.yml +++ b/.github/workflows/upload_binary.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v5 - name: Set up latest Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12.4" From 24f516961720c5578069dee30415b776359b7be5 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+cobaltt7@users.noreply.github.com> Date: Mon, 8 Sep 2025 07:52:08 -0500 Subject: [PATCH 64/69] ci: Run diff-shades on unstable instead of preview (#4741) Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> --- .github/workflows/diff_shades.yml | 4 ++-- docs/contributing/gauging_changes.md | 14 +++++++------- scripts/diff_shades_gha_helper.py | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index 2cbe86f2166..acfe39e5a65 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -129,14 +129,14 @@ jobs: path: ${{ matrix.target-analysis }} - name: Generate summary file (PR only) - if: github.event_name == 'pull_request' && matrix.mode == 'preview-changes' + if: github.event_name == 'pull_request' && matrix.mode == 'preview-new-changes' run: > python helper.py comment-body ${{ matrix.baseline-analysis }} ${{ matrix.target-analysis }} ${{ matrix.baseline-sha }} ${{ matrix.target-sha }} ${{ github.event.pull_request.number }} - name: Upload summary file (PR only) - if: github.event_name == 'pull_request' && matrix.mode == 'preview-changes' + if: github.event_name == 'pull_request' && matrix.mode == 'preview-new-changes' uses: actions/upload-artifact@v4 with: name: .pr-comment.json diff --git a/docs/contributing/gauging_changes.md b/docs/contributing/gauging_changes.md index 8562a83ed0c..8b921368a91 100644 --- a/docs/contributing/gauging_changes.md +++ b/docs/contributing/gauging_changes.md @@ -27,20 +27,20 @@ workflow that analyzes and compares two revisions of _Black_ according to these | On PRs | latest commit on `main` | PR commit with `main` merged | | On pushes (main only) | latest PyPI version | the pushed commit | -For pushes to main, there's only one analysis job named `preview-changes` where the -preview style is used for all projects. +For pushes to main, there's only one analysis job named `preview-new-changes` where the +unstable style is used for all projects. For PRs they get one more analysis job: `assert-no-changes`. It's similar to -`preview-changes` but runs with the stable code style. It will fail if changes were +`preview-new-changes` but runs with the stable code style. It will fail if changes were made. This makes sure code won't be reformatted again and again within the same year in accordance to Black's stability policy. -Additionally for PRs, a PR comment will be posted embedding a summary of the preview -changes and links to further information. If there's a pre-existing diff-shades comment, -it'll be updated instead the next time the workflow is triggered on the same PR. +Additionally for PRs, a PR comment will be posted embedding a summary previewing the +changes in the unstable style and links to further information. The next time the +workflow is triggered on the same PR, it'll update the pre-existing diff-shades comment. ```{note} -The `preview-changes` job will only fail intentionally if while analyzing a file failed to +The `preview-new-changes` job will only fail intentionally if while analyzing a file failed to format. Otherwise a failure indicates a bug in the workflow. ``` diff --git a/scripts/diff_shades_gha_helper.py b/scripts/diff_shades_gha_helper.py index 1782c76fe8d..e085562f50e 100644 --- a/scripts/diff_shades_gha_helper.py +++ b/scripts/diff_shades_gha_helper.py @@ -117,7 +117,7 @@ def config(event: Literal["push", "pull_request"]) -> None: import diff_shades # type: ignore[import-not-found] if event == "push": - jobs = [{"mode": "preview-changes", "force-flag": "--force-preview-style"}] + jobs = [{"mode": "preview-new-changes", "force-flag": "--force-unstable-style"}] # Push on main, let's use PyPI Black as the baseline. baseline_name = str(get_pypi_version()) baseline_cmd = f"git checkout {baseline_name}" @@ -128,7 +128,7 @@ def config(event: Literal["push", "pull_request"]) -> None: elif event == "pull_request": jobs = [ - {"mode": "preview-changes", "force-flag": "--force-preview-style"}, + {"mode": "preview-new-changes", "force-flag": "--force-unstable-style"}, {"mode": "assert-no-changes", "force-flag": "--force-stable-style"}, ] # PR, let's use main as the baseline. @@ -205,7 +205,7 @@ def comment_details(run_id: str) -> None: set_output("needs-comment", "true") jobs = http_get(data["jobs_url"])["jobs"] - job = next(j for j in jobs if j["name"] == "analysis / preview-changes") + job = next(j for j in jobs if j["name"] == "analysis / preview-new-changes") diff_step = next(s for s in job["steps"] if s["name"] == DIFF_STEP_NAME) diff_url = job["html_url"] + f"#step:{diff_step['number']}:1" From 4f6ad7cf8c3092e0fb4d82f54fe77ccde134468a Mon Sep 17 00:00:00 2001 From: cobalt <61329810+cobaltt7@users.noreply.github.com> Date: Mon, 8 Sep 2025 08:19:33 -0500 Subject: [PATCH 65/69] Wrap the `in` clause of comprehensions across lines if necessary (#4699) * CI: Remove now-uneeded workarounds The issues mentioned in the comments have all been fixed Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * Fix Click typing issue in run self * oops, run self * i swear this was failing earlier ig it's not that related to this pr anyway * has click updated itself yet on pre-commit.ci * Bump click Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * Split the `in` clause of comprehensions onto its own line if necessary Fixes #3498 Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * Lint issues Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * Move to preview style Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * run self Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * Alternative approach Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * oops! Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * run test on preview, not unstable Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * Don't remove parens around ternaries Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update src/black/linegen.py * Update docs/the_black_code_style/future_style.md --------- Signed-off-by: cobalt <61329810+cobaltt7@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.md | 1 + docs/the_black_code_style/future_style.md | 6 +- src/black/linegen.py | 18 +- src/black/mode.py | 1 + src/black/resources/black.schema.json | 1 + .../cases/preview_wrap_comprehension_in.py | 161 ++++++++++++++++++ 6 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 tests/data/cases/preview_wrap_comprehension_in.py diff --git a/CHANGES.md b/CHANGES.md index cc23ec4d9ad..7953d0dc626 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,7 @@ - Improve `multiline_string_handling` with ternaries and dictionaries (#4657) - Fix a bug where `string_processing` would not split f-strings directly after expressions (#4680) +- Wrap the `in` clause of comprehensions across lines if necessary (#4699) - Remove parentheses around multiple exception types in `except` and `except*` without `as`. (#4720) diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 45ec1e67ed8..13bcaa94e5d 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -27,8 +27,10 @@ Currently, the following features are included in the preview style: - `wrap_long_dict_values_in_parens`: Add parentheses around long values in dictionaries ([see below](labels/wrap-long-dict-values)) - `fix_fmt_skip_in_one_liners`: Fix `# fmt: skip` behaviour on one-liner declarations, - such as `def foo(): return "mock" # fmt: skip`, where previously the declaration - would have been incorrectly collapsed. + such as `def foo(): return "mock" # fmt: skip`, where previously the declaration would + have been incorrectly collapsed. +- `wrap_comprehension_in`: Wrap the `in` clause of list and dictionary comprehensions + across lines if it would otherwise exceed the maximum line length. - `remove_parens_around_except_types`: Remove parentheses around multiple exception types in `except` and `except*` without `as`. See PEP 758 for details. diff --git a/src/black/linegen.py b/src/black/linegen.py index 49fab818a6d..27c2c92d9b2 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -579,6 +579,16 @@ def visit_fstring(self, node: Node) -> Iterator[Line]: # yield from self.visit_default(node) + def visit_comp_for(self, node: Node) -> Iterator[Line]: + if Preview.wrap_comprehension_in in self.mode: + normalize_invisible_parens( + node, parens_after={"in"}, mode=self.mode, features=self.features + ) + yield from self.visit_default(node) + + def visit_old_comp_for(self, node: Node) -> Iterator[Line]: + yield from self.visit_comp_for(node) + def __post_init__(self) -> None: """You are in a twisty little maze of passages.""" self.current_line = Line(mode=self.mode) @@ -1466,7 +1476,13 @@ def normalize_invisible_parens( # noqa: C901 wrap_in_parentheses(node, child, visible=False) elif isinstance(child, Node) and node.type == syms.with_stmt: remove_with_parens(child, node, mode=mode, features=features) - elif child.type == syms.atom: + elif child.type == syms.atom and not ( + "in" in parens_after + and len(child.children) == 3 + and is_lpar_token(child.children[0]) + and is_rpar_token(child.children[-1]) + and child.children[1].type == syms.test + ): if maybe_make_parens_invisible_in_atom( child, parent=node, mode=mode, features=features ): diff --git a/src/black/mode.py b/src/black/mode.py index 86e0bfcb1c2..4d85358d5c5 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -231,6 +231,7 @@ class Preview(Enum): multiline_string_handling = auto() always_one_newline_after_import = auto() fix_fmt_skip_in_one_liners = auto() + wrap_comprehension_in = auto() # Remove parentheses around multiple exception types in except and # except* without as. See PEP 758 for details. remove_parens_around_except_types = auto() diff --git a/src/black/resources/black.schema.json b/src/black/resources/black.schema.json index b342f2d2204..c3d7d03d4cc 100644 --- a/src/black/resources/black.schema.json +++ b/src/black/resources/black.schema.json @@ -86,6 +86,7 @@ "multiline_string_handling", "always_one_newline_after_import", "fix_fmt_skip_in_one_liners", + "wrap_comprehension_in", "remove_parens_around_except_types" ] }, diff --git a/tests/data/cases/preview_wrap_comprehension_in.py b/tests/data/cases/preview_wrap_comprehension_in.py new file mode 100644 index 00000000000..e457f0e772f --- /dev/null +++ b/tests/data/cases/preview_wrap_comprehension_in.py @@ -0,0 +1,161 @@ +# flags: --preview --line-length=79 + +[a for graph_path_expression in refined_constraint.condition_as_predicate.variables] +[ + a + for graph_path_expression in refined_constraint.condition_as_predicate.variables +] +[ + a + for graph_path_expression + in refined_constraint.condition_as_predicate.variables +] +[ + a + for graph_path_expression in ( + refined_constraint.condition_as_predicate.variables + ) +] + +[ + (foobar_very_long_key, foobar_very_long_value) + for foobar_very_long_key, foobar_very_long_value in foobar_very_long_dictionary.items() +] + +# Don't split the `in` if it's not too long +lcomp3 = [ + element.split("\n", 1)[0] + for element in collection.select_elements() + # right + if element is not None +] + +# Don't remove parens around ternaries +expected = [i for i in (a if b else c)] + +# Nested arrays +# First in will not be split because it would still be too long +[[ + x + for x in bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + for y in xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +]] + +# Multiple comprehensions, only split the second `in` +graph_path_expressions_in_local_constraint_refinements = [ + graph_path_expression + for refined_constraint in self._local_constraint_refinements.values() + if refined_constraint is not None + for graph_path_expression in refined_constraint.condition_as_predicate.variables +] + +# Dictionary comprehensions +dict_with_really_long_names = { + really_really_long_key_name: an_even_longer_really_really_long_key_value + for really_really_long_key_name, an_even_longer_really_really_long_key_value in really_really_really_long_dict_name.items() +} +{ + key_with_super_really_long_name: key_with_super_really_long_name + for key_with_super_really_long_name in dictionary_with_super_really_long_name +} +{ + key_with_super_really_long_name: key_with_super_really_long_name + for key_with_super_really_long_name + in dictionary_with_super_really_long_name +} +{ + key_with_super_really_long_name: key_with_super_really_long_name + for key in ( + dictionary + ) +} + +# output +[ + a + for graph_path_expression in ( + refined_constraint.condition_as_predicate.variables + ) +] +[ + a + for graph_path_expression in ( + refined_constraint.condition_as_predicate.variables + ) +] +[ + a + for graph_path_expression in ( + refined_constraint.condition_as_predicate.variables + ) +] +[ + a + for graph_path_expression in ( + refined_constraint.condition_as_predicate.variables + ) +] + +[ + (foobar_very_long_key, foobar_very_long_value) + for foobar_very_long_key, foobar_very_long_value in ( + foobar_very_long_dictionary.items() + ) +] + +# Don't split the `in` if it's not too long +lcomp3 = [ + element.split("\n", 1)[0] + for element in collection.select_elements() + # right + if element is not None +] + +# Don't remove parens around ternaries +expected = [i for i in (a if b else c)] + +# Nested arrays +# First in will not be split because it would still be too long +[ + [ + x + for x in bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + for y in ( + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + ) + ] +] + +# Multiple comprehensions, only split the second `in` +graph_path_expressions_in_local_constraint_refinements = [ + graph_path_expression + for refined_constraint in self._local_constraint_refinements.values() + if refined_constraint is not None + for graph_path_expression in ( + refined_constraint.condition_as_predicate.variables + ) +] + +# Dictionary comprehensions +dict_with_really_long_names = { + really_really_long_key_name: an_even_longer_really_really_long_key_value + for really_really_long_key_name, an_even_longer_really_really_long_key_value in ( + really_really_really_long_dict_name.items() + ) +} +{ + key_with_super_really_long_name: key_with_super_really_long_name + for key_with_super_really_long_name in ( + dictionary_with_super_really_long_name + ) +} +{ + key_with_super_really_long_name: key_with_super_really_long_name + for key_with_super_really_long_name in ( + dictionary_with_super_really_long_name + ) +} +{ + key_with_super_really_long_name: key_with_super_really_long_name + for key in dictionary +} From 57a461258f324e33bca189b2eb49d7f7a944ffe7 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+cobaltt7@users.noreply.github.com> Date: Thu, 11 Sep 2025 23:18:32 -0500 Subject: [PATCH 66/69] Fix mypy type issue (#4745) --- .github/workflows/pypi_upload.yml | 2 +- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- src/black/trans.py | 10 ++++++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 35393ce2e39..bb9989b0a4d 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -50,7 +50,7 @@ jobs: # Keep cibuildwheel version in sync with below - name: Install cibuildwheel and pypyp run: | - pipx install cibuildwheel==2.22.0 + pipx install cibuildwheel==2.23.3 pipx install pypyp==1.3.0 - name: generate matrix if: github.event_name != 'pull_request' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ee8dc225b5b..d2335a9b64f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: exclude: ^src/blib2to3/ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.16.1 + rev: v1.17.1 hooks: - id: mypy exclude: ^(docs/conf.py|scripts/generate_schema.py)$ diff --git a/pyproject.toml b/pyproject.toml index be6c8f9b9d5..b3e7f2414fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,7 @@ macos-max-compat = true enable-by-default = false dependencies = [ "hatch-mypyc>=0.16.0", - "mypy>=1.12", + "mypy==1.17.1", "click>=8.1.7", ] require-runtime-dependencies = true diff --git a/src/black/trans.py b/src/black/trans.py index 0b595630450..de24d723e1e 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -334,6 +334,9 @@ class CustomSplit: break_idx: int +CustomSplitMapKey = tuple[StringID, str] + + @trait class CustomSplitMapMixin: """ @@ -342,13 +345,12 @@ class CustomSplitMapMixin: the resultant substrings go over the configured max line length. """ - _Key: ClassVar = tuple[StringID, str] - _CUSTOM_SPLIT_MAP: ClassVar[dict[_Key, tuple[CustomSplit, ...]]] = defaultdict( - tuple + _CUSTOM_SPLIT_MAP: ClassVar[dict[CustomSplitMapKey, tuple[CustomSplit, ...]]] = ( + defaultdict(tuple) ) @staticmethod - def _get_key(string: str) -> "CustomSplitMapMixin._Key": + def _get_key(string: str) -> CustomSplitMapKey: """ Returns: A unique identifier that is used internally to map @string to a From 626b32fe2b5387656be6694da9a4b7a3148fb892 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Thu, 18 Sep 2025 11:49:07 -0700 Subject: [PATCH 67/69] Add normalizing for `\r` style newlines (#4710) * Normalize newlines * Fix tired mistakes * Add changelog entry * Update documentation * Move changes to preview * Fix test formatting * Update schema * Fix typo in CHANGES.md Co-authored-by: Jelle Zijlstra * Fix typo in docs/the_black_code_style/future_style.md Co-authored-by: Jelle Zijlstra * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Jelle Zijlstra Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.md | 2 + docs/the_black_code_style/future_style.md | 2 + src/black/__init__.py | 69 ++++++++++++++++++----- src/black/mode.py | 1 + src/black/resources/black.schema.json | 3 +- src/blackd/__init__.py | 16 +++--- tests/test_black.py | 7 +++ 7 files changed, 79 insertions(+), 21 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7953d0dc626..8bbaa13e2db 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -37,6 +37,8 @@ - Wrap the `in` clause of comprehensions across lines if necessary (#4699) - Remove parentheses around multiple exception types in `except` and `except*` without `as`. (#4720) +- Add `\r` style newlines to the potential newlines to normalize file newlines both from + and to (#4710) ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 13bcaa94e5d..837aec457b0 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -33,6 +33,8 @@ Currently, the following features are included in the preview style: across lines if it would otherwise exceed the maximum line length. - `remove_parens_around_except_types`: Remove parentheses around multiple exception types in `except` and `except*` without `as`. See PEP 758 for details. +- `normalize_cr_newlines`: Add `\r` style newlines to the potential newlines to + normalize file newlines both from and to. (labels/unstable-features)= diff --git a/src/black/__init__.py b/src/black/__init__.py index 082f705f196..79541df2149 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -946,7 +946,7 @@ def format_file_in_place( with open(src, "rb") as buf: if mode.skip_source_first_line: header = buf.readline() - src_contents, encoding, newline = decode_bytes(buf.read()) + src_contents, encoding, newline = decode_bytes(buf.read(), mode) try: dst_contents = format_file_contents( src_contents, fast=fast, mode=mode, lines=lines @@ -1008,7 +1008,9 @@ def format_stdin_to_stdout( then = datetime.now(timezone.utc) if content is None: - src, encoding, newline = decode_bytes(sys.stdin.buffer.read()) + src, encoding, newline = decode_bytes(sys.stdin.buffer.read(), mode) + elif Preview.normalize_cr_newlines in mode: + src, encoding, newline = content, "utf-8", "\n" else: src, encoding, newline = content, "utf-8", "" @@ -1026,8 +1028,12 @@ def format_stdin_to_stdout( ) if write_back == WriteBack.YES: # Make sure there's a newline after the content - if dst and dst[-1] != "\n": - dst += "\n" + if Preview.normalize_cr_newlines in mode: + if dst and dst[-1] != "\n" and dst[-1] != "\r": + dst += newline + else: + if dst and dst[-1] != "\n": + dst += "\n" f.write(dst) elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): now = datetime.now(timezone.utc) @@ -1217,7 +1223,17 @@ def f( def _format_str_once( src_contents: str, *, mode: Mode, lines: Collection[tuple[int, int]] = () ) -> str: - src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions) + if Preview.normalize_cr_newlines in mode: + normalized_contents, _, newline_type = decode_bytes( + src_contents.encode("utf-8"), mode + ) + + src_node = lib2to3_parse( + normalized_contents.lstrip(), target_versions=mode.target_versions + ) + else: + src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions) + dst_blocks: list[LinesBlock] = [] if mode.target_versions: versions = mode.target_versions @@ -1262,16 +1278,25 @@ def _format_str_once( for block in dst_blocks: dst_contents.extend(block.all_lines()) if not dst_contents: - # Use decode_bytes to retrieve the correct source newline (CRLF or LF), - # and check if normalized_content has more than one line - normalized_content, _, newline = decode_bytes(src_contents.encode("utf-8")) - if "\n" in normalized_content: - return newline + if Preview.normalize_cr_newlines in mode: + if "\n" in normalized_contents: + return newline_type + else: + # Use decode_bytes to retrieve the correct source newline (CRLF or LF), + # and check if normalized_content has more than one line + normalized_content, _, newline = decode_bytes( + src_contents.encode("utf-8"), mode + ) + if "\n" in normalized_content: + return newline return "" - return "".join(dst_contents) + if Preview.normalize_cr_newlines in mode: + return "".join(dst_contents).replace("\n", newline_type) + else: + return "".join(dst_contents) -def decode_bytes(src: bytes) -> tuple[FileContent, Encoding, NewLine]: +def decode_bytes(src: bytes, mode: Mode) -> tuple[FileContent, Encoding, NewLine]: """Return a tuple of (decoded_contents, encoding, newline). `newline` is either CRLF or LF but `decoded_contents` is decoded with @@ -1282,7 +1307,25 @@ def decode_bytes(src: bytes) -> tuple[FileContent, Encoding, NewLine]: if not lines: return "", encoding, "\n" - newline = "\r\n" if lines[0][-2:] == b"\r\n" else "\n" + if Preview.normalize_cr_newlines in mode: + if lines[0][-2:] == b"\r\n": + if b"\r" in lines[0][:-2]: + newline = "\r" + else: + newline = "\r\n" + elif lines[0][-1:] == b"\n": + if b"\r" in lines[0][:-1]: + newline = "\r" + else: + newline = "\n" + else: + if b"\r" in lines[0]: + newline = "\r" + else: + newline = "\n" + else: + newline = "\r\n" if lines[0][-2:] == b"\r\n" else "\n" + srcbuf.seek(0) with io.TextIOWrapper(srcbuf, encoding) as tiow: return tiow.read(), encoding, newline diff --git a/src/black/mode.py b/src/black/mode.py index 4d85358d5c5..85a205949dc 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -235,6 +235,7 @@ class Preview(Enum): # Remove parentheses around multiple exception types in except and # except* without as. See PEP 758 for details. remove_parens_around_except_types = auto() + normalize_cr_newlines = auto() UNSTABLE_FEATURES: set[Preview] = { diff --git a/src/black/resources/black.schema.json b/src/black/resources/black.schema.json index c3d7d03d4cc..549e0e8049f 100644 --- a/src/black/resources/black.schema.json +++ b/src/black/resources/black.schema.json @@ -87,7 +87,8 @@ "always_one_newline_after_import", "fix_fmt_skip_in_one_liners", "wrap_comprehension_in", - "remove_parens_around_except_types" + "remove_parens_around_except_types", + "normalize_cr_newlines" ] }, "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index 86309da0ef0..2f9a516d6e5 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -22,6 +22,7 @@ import black from _black_version import version as __version__ from black.concurrency import maybe_install_uvloop +from black.mode import Preview # This is used internally by tests to shut down the server prematurely _stop_signal = asyncio.Event() @@ -129,13 +130,14 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: executor, partial(black.format_file_contents, req_str, fast=fast, mode=mode) ) - # Preserve CRLF line endings - nl = req_str.find("\n") - if nl > 0 and req_str[nl - 1] == "\r": - formatted_str = formatted_str.replace("\n", "\r\n") - # If, after swapping line endings, nothing changed, then say so - if formatted_str == req_str: - raise black.NothingChanged + if Preview.normalize_cr_newlines not in mode: + # Preserve CRLF line endings + nl = req_str.find("\n") + if nl > 0 and req_str[nl - 1] == "\r": + formatted_str = formatted_str.replace("\n", "\r\n") + # If, after swapping line endings, nothing changed, then say so + if formatted_str == req_str: + raise black.NothingChanged # Put the source first line back req_str = header + req_str diff --git a/tests/test_black.py b/tests/test_black.py index a9ea4b9a765..36ee7d9e1b9 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -3,6 +3,7 @@ import asyncio import inspect import io +import itertools import logging import multiprocessing import os @@ -2083,6 +2084,12 @@ def test_carriage_return_edge_cases(self) -> None: == "class A: ...\n" ) + def test_preview_newline_type_detection(self) -> None: + mode = Mode(enabled_features={Preview.normalize_cr_newlines}) + newline_types = ["A\n", "A\r\n", "A\r"] + for test_case in itertools.permutations(newline_types): + assert black.format_str("".join(test_case), mode=mode) == test_case[0] * 3 + class TestCaching: def test_get_cache_dir( From ffc01a027580d99401abb3197e83d50dc4f4c746 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Thu, 18 Sep 2025 14:33:22 -0700 Subject: [PATCH 68/69] Fix schema generation error caused by new click version (#4750) * Fix schema generation error caused by new click version Not sure on the policy for updating versions, so to cope with the new click 8.3.0 version I used some code from #4577 for version conditional behavior. When >= 8.3.0, unset defaults are now the special UNSET sentinal instead of None. Hopefully nothing breaks on the click._utils usage since it's an _ module, might have to find a different solution if it does. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add ignore for mypy error * Actually fix mypy errors with ignore * Add explanation for ignores --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- scripts/generate_schema.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/generate_schema.py b/scripts/generate_schema.py index dcdfc74de62..82abfaec13d 100755 --- a/scripts/generate_schema.py +++ b/scripts/generate_schema.py @@ -1,7 +1,9 @@ import json +from importlib.metadata import version as imp_version from typing import IO, Any import click +from packaging.version import Version import black @@ -38,6 +40,11 @@ def generate_schema_from_click( result[name]["description"] = param.help if param.default is not None and not param.multiple: + if Version(imp_version("click")) >= Version("8.3.0"): + # Ignore the attr-defined error from running with click < 8.3.0, and + # also ignore the error for unused-ignore with click >= 8.3.0 + if param.default is click._utils.UNSET: # type: ignore[attr-defined, unused-ignore] + continue result[name]["default"] = param.default return result From af0ba72a73598c76189d6dd1b21d8532255d5942 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Thu, 18 Sep 2025 16:39:02 -0700 Subject: [PATCH 69/69] Prepare docs for release 25.9.0 (#4751) * Prepare docs for release 25.9.0 * Remove unreleased changelog section --- CHANGES.md | 36 ++------------------- docs/integrations/source_version_control.md | 4 +-- docs/usage_and_configuration/the_basics.md | 6 ++-- 3 files changed, 8 insertions(+), 38 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8bbaa13e2db..c528f57b346 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,15 +1,14 @@ # Change Log -## Unreleased +## 25.9.0 ### Highlights - +- Remove support for pre-python 3.7 `await/async` as soft keywords/variable names + (#4676) ### Stable style - - - Fix crash while formatting a long `del` statement containing tuples (#4628) - Fix crash while formatting expressions using the walrus operator in complex `with` statements (#4630) @@ -19,16 +18,12 @@ - Fix crash when formatting a `\` followed by a `\r` followed by a comment (#4663) - Fix crash on a `\\r\n` (#4673) - Fix crash on `await ...` (where `...` is a literal `Ellipsis`) (#4676) -- Remove support for pre-python 3.7 `await/async` as soft keywords/variable names - (#4676) - Fix crash on parenthesized expression inside a type parameter bound (#4684) - Fix crash when using line ranges excluding indented single line decorated items (#4670) ### Preview style - - - Fix a bug where one-liner functions/conditionals marked with `# fmt: skip` would still be formatted (#4552) - Improve `multiline_string_handling` with ternaries and dictionaries (#4657) @@ -40,40 +35,18 @@ - Add `\r` style newlines to the potential newlines to normalize file newlines both from and to (#4710) -### Configuration - - - -### Packaging - - - ### Parser - - - Rewrite tokenizer to improve performance and compliance (#4536) - Fix bug where certain unusual expressions (e.g., lambdas) were not accepted in type parameter bounds and defaults. (#4602) ### Performance - - - Avoid using an extra process when running with only one worker (#4734) -### Output - - - -### _Blackd_ - - - ### Integrations - - - Fix the version check in the vim file to reject Python 3.8 (#4567) - Enhance GitHub Action `psf/black` to read Black version from an additional section in pyproject.toml: `[project.dependency-groups]` (#4606) @@ -81,9 +54,6 @@ ### Documentation - - - Add FAQ entry for windows emoji not displaying (#4714) ## 25.1.0 diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 0e7cf324d63..dd89f897651 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 + rev: 25.9.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 + rev: 25.9.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index e232a2175bd..0c032261796 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -270,8 +270,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 25.1.0 (compiled: yes) -$ black --required-version 25.1.0 -c "format = 'this'" +black, 25.9.0 (compiled: yes) +$ black --required-version 25.9.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -372,7 +372,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 25.1.0 +black, 25.9.0 ``` #### `--config`