From 83ebdfcbf2106c9b0d77252004ded620042632be Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Wed, 10 Jan 2024 06:21:40 +0100 Subject: [PATCH 001/176] Fix `--theme` help message typo. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index eac019a..ac532a2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -584,7 +584,7 @@ def parse_args(): parser.add_argument( "--theme", default="python-docs-theme", - help="Python package to use for python-docs-theme: Usefull to test branches:" + help="Python package to use for python-docs-theme: Useful to test branches:" " --theme git+https://github.com/obulat/python-docs-theme@master", ) args = parser.parse_args() From b25088dac8e17f1686580ba75a1adcad4cc0ca7c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 9 Aug 2024 22:29:26 +0100 Subject: [PATCH 002/176] Use ``--upgrade-strategy=eager`` for pip --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index 212989e..429ee45 100755 --- a/build_docs.py +++ b/build_docs.py @@ -790,6 +790,7 @@ def build_venv(self): run([sys.executable, "-m", "venv", venv_path]) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] + + ["--upgrade-strategy=eager"] + [self.theme] + self.version.requirements, cwd=self.checkout / "Doc", From 108fd45c0ae1f0cd5fe836ac45add045d6d03f5d Mon Sep 17 00:00:00 2001 From: egeakman Date: Sun, 11 Aug 2024 18:37:31 +0300 Subject: [PATCH 003/176] migrate sentry from 1.x to 2.x --- build_docs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 212989e..a86d95e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1124,9 +1124,9 @@ def build_docs(args) -> bool: ) ) if sentry_sdk: - with sentry_sdk.configure_scope() as scope: - scope.set_tag("version", version.name) - scope.set_tag("language", language.tag) + scope = sentry_sdk.get_isolation_scope() + scope.set_tag("version", version.name) + scope.set_tag("language", language.tag) builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) From 1babbddb3ac2f956e31b344d7f4d8001cfdd6060 Mon Sep 17 00:00:00 2001 From: egeakman Date: Sun, 11 Aug 2024 19:52:16 +0300 Subject: [PATCH 004/176] ignore build_docs.lock --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a29939..79fca5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /build_root/ /logs/ /www/ +build_docs.lock # Created by https://www.gitignore.io/api/python From 12f773580f470cb9a38a4ba6aa304557ab925d45 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 11 Aug 2024 21:23:28 +0300 Subject: [PATCH 005/176] Speed up purges by re-using a requests session --- build_docs.py | 123 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 91 insertions(+), 32 deletions(-) diff --git a/build_docs.py b/build_docs.py index 212989e..719ee01 100755 --- a/build_docs.py +++ b/build_docs.py @@ -434,7 +434,8 @@ def build_robots_txt( www_root: Path, group, skip_cache_invalidation, -): + session: requests.Session, +) -> None: """Disallow crawl of EOL versions in robots.txt.""" if not www_root.exists(): logging.info("Skipping robots.txt generation (www root does not even exists).") @@ -449,7 +450,7 @@ def build_robots_txt( robots_file.chmod(0o775) run(["chgrp", group, robots_file]) if not skip_cache_invalidation: - purge("robots.txt") + purge(session, "robots.txt") def build_sitemap( @@ -642,7 +643,7 @@ def full_build(self): """ return not self.quick and not self.language.html_only - def run(self) -> bool: + def run(self, session: requests.Session) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() logging.info("Running.") @@ -653,7 +654,7 @@ def run(self) -> bool: if self.should_rebuild(): self.build_venv() self.build() - self.copy_build_to_webroot() + self.copy_build_to_webroot(session) self.save_state(build_duration=perf_counter() - start_time) except Exception as err: logging.exception("Badly handled exception, human, please help.") @@ -797,7 +798,7 @@ def build_venv(self): run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) self.venv = venv_path - def copy_build_to_webroot(self): + def copy_build_to_webroot(self, session: requests.Session) -> None: """Copy a given build to the appropriate webroot with appropriate rights.""" logging.info("Publishing start.") self.www_root.mkdir(parents=True, exist_ok=True) @@ -909,9 +910,9 @@ def copy_build_to_webroot(self): prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout prefixes = prefixes.replace(targets_dir + "/", "") prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix] - purge(*prefixes) + purge(session, *prefixes) for prefix in prefixes: - purge(*[prefix + p for p in changed]) + purge(session, *[prefix + p for p in changed]) logging.info("Publishing done") def should_rebuild(self): @@ -977,7 +978,15 @@ def save_state(self, build_duration: float): state_file.write_text(tomlkit.dumps(states), encoding="UTF-8") -def symlink(www_root: Path, language: Language, directory: str, name: str, group: str, skip_cache_invalidation: bool): +def symlink( + www_root: Path, + language: Language, + directory: str, + name: str, + group: str, + skip_cache_invalidation: bool, + session: requests.Session, +) -> None: """Used by major_symlinks and dev_symlink to maintain symlinks.""" if language.tag == "en": # english is rooted on /, no /en/ path = www_root @@ -994,12 +1003,17 @@ def symlink(www_root: Path, language: Language, directory: str, name: str, group link.symlink_to(directory) run(["chown", "-h", ":" + group, str(link)]) if not skip_cache_invalidation: - purge_path(www_root, link) + purge_path(session, www_root, link) def major_symlinks( - www_root: Path, group, versions: Iterable[Version], languages: Iterable[Language], skip_cache_invalidation: bool -): + www_root: Path, + group: str, + versions: Iterable[Version], + languages: Iterable[Language], + skip_cache_invalidation: bool, + session: requests.Session, +) -> None: """Maintains the /2/ and /3/ symlinks for each languages. Like: @@ -1009,11 +1023,26 @@ def major_symlinks( """ current_stable = Version.current_stable(versions).name for language in languages: - symlink(www_root, language, current_stable, "3", group, skip_cache_invalidation) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation) + symlink( + www_root, + language, + current_stable, + "3", + group, + skip_cache_invalidation, + session, + ) + symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, session) -def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidation: bool): +def dev_symlink( + www_root: Path, + group, + versions, + languages, + skip_cache_invalidation: bool, + session: requests.Session, +) -> None: """Maintains the /dev/ symlinks for each languages. Like: @@ -1023,10 +1052,18 @@ def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidat """ current_dev = Version.current_dev(versions).name for language in languages: - symlink(www_root, language, current_dev, "dev", group, skip_cache_invalidation) + symlink( + www_root, + language, + current_dev, + "dev", + group, + skip_cache_invalidation, + session, + ) -def purge(*paths): +def purge(session: requests.Session, *paths: Path | str) -> None: """Remove one or many paths from docs.python.org's CDN. To be used when a file change, so the CDN fetch the new one. @@ -1035,20 +1072,22 @@ def purge(*paths): for path in paths: url = urljoin(base, str(path)) logging.debug("Purging %s from CDN", url) - requests.request("PURGE", url, timeout=30) + session.request("PURGE", url, timeout=30) -def purge_path(www_root: Path, path: Path): +def purge_path(session: requests.Session, www_root: Path, path: Path) -> None: """Recursively remove a path from docs.python.org's CDN. To be used when a directory change, so the CDN fetch the new one. """ - purge(*[file.relative_to(www_root) for file in path.glob("**/*")]) - purge(path.relative_to(www_root)) - purge(str(path.relative_to(www_root)) + "/") + purge(session, *[file.relative_to(www_root) for file in path.glob("**/*")]) + purge(session, path.relative_to(www_root)) + purge(session, str(path.relative_to(www_root)) + "/") -def proofread_canonicals(www_root: Path, skip_cache_invalidation: bool) -> None: +def proofread_canonicals( + www_root: Path, skip_cache_invalidation: bool, session: requests.Session +) -> None: """In www_root we check that all canonical links point to existing contents. It can happen that a canonical is "broken": @@ -1070,11 +1109,11 @@ def proofread_canonicals(www_root: Path, skip_cache_invalidation: bool) -> None: html = html.replace(canonical.group(0), "") file.write_text(html, encoding="UTF-8", errors="surrogateescape") if not skip_cache_invalidation: - purge(str(file).replace("/srv/docs.python.org/", "")) + purge(session, str(file).replace("/srv/docs.python.org/", "")) -def parse_versions_from_devguide(): - releases = requests.get( +def parse_versions_from_devguide(session: requests.Session) -> list[Version]: + releases = session.get( "https://raw.githubusercontent.com/" "python/devguide/main/include/release-cycle.json", timeout=30, @@ -1101,8 +1140,9 @@ def parse_languages_from_config(): def build_docs(args) -> bool: - """Build all docs (each languages and each versions).""" - versions = parse_versions_from_devguide() + """Build all docs (each language and each version).""" + session = requests.Session() + versions = parse_versions_from_devguide(session) languages = parse_languages_from_config() todo = [ (version, language) @@ -1130,7 +1170,7 @@ def build_docs(args) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - all_built_successfully &= builder.run() + all_built_successfully &= builder.run(session) logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") ) @@ -1138,11 +1178,30 @@ def build_docs(args) -> bool: build_sitemap(versions, languages, args.www_root, args.group) build_404(args.www_root, args.group) build_robots_txt( - versions, languages, args.www_root, args.group, args.skip_cache_invalidation + versions, + languages, + args.www_root, + args.group, + args.skip_cache_invalidation, + session, + ) + major_symlinks( + args.www_root, + args.group, + versions, + languages, + args.skip_cache_invalidation, + session, + ) + dev_symlink( + args.www_root, + args.group, + versions, + languages, + args.skip_cache_invalidation, + session, ) - major_symlinks(args.www_root, args.group, versions, languages, args.skip_cache_invalidation) - dev_symlink(args.www_root, args.group, versions, languages, args.skip_cache_invalidation) - proofread_canonicals(args.www_root, args.skip_cache_invalidation) + proofread_canonicals(args.www_root, args.skip_cache_invalidation, session) return all_built_successfully From 4d05bb0bfe729078bc2f547352bb46934a916740 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 11 Aug 2024 22:12:20 +0300 Subject: [PATCH 006/176] Replace requests with urllib3 --- build_docs.py | 68 ++++++++++++++++++++++++------------------------ requirements.txt | 2 +- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/build_docs.py b/build_docs.py index 719ee01..9b3d592 100755 --- a/build_docs.py +++ b/build_docs.py @@ -46,11 +46,10 @@ from typing import Iterable from urllib.parse import urljoin -import zc.lockfile import jinja2 -import requests import tomlkit - +import urllib3 +import zc.lockfile try: from os import EX_OK, EX_SOFTWARE as EX_FAILURE @@ -434,7 +433,7 @@ def build_robots_txt( www_root: Path, group, skip_cache_invalidation, - session: requests.Session, + http: urllib3.PoolManager, ) -> None: """Disallow crawl of EOL versions in robots.txt.""" if not www_root.exists(): @@ -450,7 +449,7 @@ def build_robots_txt( robots_file.chmod(0o775) run(["chgrp", group, robots_file]) if not skip_cache_invalidation: - purge(session, "robots.txt") + purge(http, "robots.txt") def build_sitemap( @@ -643,7 +642,7 @@ def full_build(self): """ return not self.quick and not self.language.html_only - def run(self, session: requests.Session) -> bool: + def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() logging.info("Running.") @@ -654,7 +653,7 @@ def run(self, session: requests.Session) -> bool: if self.should_rebuild(): self.build_venv() self.build() - self.copy_build_to_webroot(session) + self.copy_build_to_webroot(http) self.save_state(build_duration=perf_counter() - start_time) except Exception as err: logging.exception("Badly handled exception, human, please help.") @@ -798,7 +797,7 @@ def build_venv(self): run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) self.venv = venv_path - def copy_build_to_webroot(self, session: requests.Session) -> None: + def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: """Copy a given build to the appropriate webroot with appropriate rights.""" logging.info("Publishing start.") self.www_root.mkdir(parents=True, exist_ok=True) @@ -910,9 +909,9 @@ def copy_build_to_webroot(self, session: requests.Session) -> None: prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout prefixes = prefixes.replace(targets_dir + "/", "") prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix] - purge(session, *prefixes) + purge(http, *prefixes) for prefix in prefixes: - purge(session, *[prefix + p for p in changed]) + purge(http, *[prefix + p for p in changed]) logging.info("Publishing done") def should_rebuild(self): @@ -985,7 +984,7 @@ def symlink( name: str, group: str, skip_cache_invalidation: bool, - session: requests.Session, + http: urllib3.PoolManager, ) -> None: """Used by major_symlinks and dev_symlink to maintain symlinks.""" if language.tag == "en": # english is rooted on /, no /en/ @@ -1003,7 +1002,7 @@ def symlink( link.symlink_to(directory) run(["chown", "-h", ":" + group, str(link)]) if not skip_cache_invalidation: - purge_path(session, www_root, link) + purge_path(http, www_root, link) def major_symlinks( @@ -1012,7 +1011,7 @@ def major_symlinks( versions: Iterable[Version], languages: Iterable[Language], skip_cache_invalidation: bool, - session: requests.Session, + http: urllib3.PoolManager, ) -> None: """Maintains the /2/ and /3/ symlinks for each languages. @@ -1030,9 +1029,9 @@ def major_symlinks( "3", group, skip_cache_invalidation, - session, + http, ) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, session) + symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) def dev_symlink( @@ -1041,7 +1040,7 @@ def dev_symlink( versions, languages, skip_cache_invalidation: bool, - session: requests.Session, + http: urllib3.PoolManager, ) -> None: """Maintains the /dev/ symlinks for each languages. @@ -1059,11 +1058,11 @@ def dev_symlink( "dev", group, skip_cache_invalidation, - session, + http, ) -def purge(session: requests.Session, *paths: Path | str) -> None: +def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: """Remove one or many paths from docs.python.org's CDN. To be used when a file change, so the CDN fetch the new one. @@ -1072,21 +1071,21 @@ def purge(session: requests.Session, *paths: Path | str) -> None: for path in paths: url = urljoin(base, str(path)) logging.debug("Purging %s from CDN", url) - session.request("PURGE", url, timeout=30) + http.request("PURGE", url, timeout=30) -def purge_path(session: requests.Session, www_root: Path, path: Path) -> None: +def purge_path(http: urllib3.PoolManager, www_root: Path, path: Path) -> None: """Recursively remove a path from docs.python.org's CDN. To be used when a directory change, so the CDN fetch the new one. """ - purge(session, *[file.relative_to(www_root) for file in path.glob("**/*")]) - purge(session, path.relative_to(www_root)) - purge(session, str(path.relative_to(www_root)) + "/") + purge(http, *[file.relative_to(www_root) for file in path.glob("**/*")]) + purge(http, path.relative_to(www_root)) + purge(http, str(path.relative_to(www_root)) + "/") def proofread_canonicals( - www_root: Path, skip_cache_invalidation: bool, session: requests.Session + www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager ) -> None: """In www_root we check that all canonical links point to existing contents. @@ -1109,11 +1108,12 @@ def proofread_canonicals( html = html.replace(canonical.group(0), "") file.write_text(html, encoding="UTF-8", errors="surrogateescape") if not skip_cache_invalidation: - purge(session, str(file).replace("/srv/docs.python.org/", "")) + purge(http, str(file).replace("/srv/docs.python.org/", "")) -def parse_versions_from_devguide(session: requests.Session) -> list[Version]: - releases = session.get( +def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: + releases = http.request( + "GET", "https://raw.githubusercontent.com/" "python/devguide/main/include/release-cycle.json", timeout=30, @@ -1141,8 +1141,8 @@ def parse_languages_from_config(): def build_docs(args) -> bool: """Build all docs (each language and each version).""" - session = requests.Session() - versions = parse_versions_from_devguide(session) + http = urllib3.PoolManager() + versions = parse_versions_from_devguide(http) languages = parse_languages_from_config() todo = [ (version, language) @@ -1170,7 +1170,7 @@ def build_docs(args) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - all_built_successfully &= builder.run(session) + all_built_successfully &= builder.run(http) logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") ) @@ -1183,7 +1183,7 @@ def build_docs(args) -> bool: args.www_root, args.group, args.skip_cache_invalidation, - session, + http, ) major_symlinks( args.www_root, @@ -1191,7 +1191,7 @@ def build_docs(args) -> bool: versions, languages, args.skip_cache_invalidation, - session, + http, ) dev_symlink( args.www_root, @@ -1199,9 +1199,9 @@ def build_docs(args) -> bool: versions, languages, args.skip_cache_invalidation, - session, + http, ) - proofread_canonicals(args.www_root, args.skip_cache_invalidation, session) + proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) return all_built_successfully diff --git a/requirements.txt b/requirements.txt index f51c7d0..cf12434 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ jinja2 -requests sentry-sdk tomlkit +urllib3 zc.lockfile From 81cce71c9e46add68452f77a5a0fe558e13ffb49 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 12 Aug 2024 00:01:51 +0300 Subject: [PATCH 007/176] Install urllib3 >= v2 Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cf12434..4602e89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ jinja2 sentry-sdk tomlkit -urllib3 +urllib3>=2 zc.lockfile From dd53442012ceaf090519ceb9f83c9b5f39b180eb Mon Sep 17 00:00:00 2001 From: egeakman Date: Mon, 12 Aug 2024 13:10:38 +0300 Subject: [PATCH 008/176] require sentry-sdk>=2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f51c7d0..b7e25bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ jinja2 requests -sentry-sdk +sentry-sdk>=2 tomlkit zc.lockfile From 4e7d3d91daef2288f406e9f6d095b3ed041823e1 Mon Sep 17 00:00:00 2001 From: egeakman Date: Mon, 12 Aug 2024 13:48:04 +0300 Subject: [PATCH 009/176] Fix typos --- build_docs.py | 55 +++++++++++++++++++++++------------------------ check_versions.py | 4 ++-- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/build_docs.py b/build_docs.py index 212989e..7cd3737 100755 --- a/build_docs.py +++ b/build_docs.py @@ -10,12 +10,11 @@ -q selects "quick build", which means to build only HTML. -Translations are fetched from github repositories according to PEP -545. `--languages` allows to select translations, like `--languages -en` to just build the english documents. +Translations are fetched from GitHub repositories according to PEP +545. `--languages` allows selecting translations, like `--languages +en` to just build the English documents. -This script was originally created and by Georg Brandl in March -2010. +This script was originally created by Georg Brandl in March 2010. Modified by Benjamin Peterson to do CDN cache invalidation. Modified by Julien Palard to build translations. @@ -69,7 +68,7 @@ @total_ordering class Version: - """Represents a cpython version and its documentation builds dependencies.""" + """Represents a CPython version and its documentation build dependencies.""" STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"} @@ -147,7 +146,7 @@ def filter(versions, branch=None): If *branch* is given, only *versions* matching *branch* are returned. - Else all live version are returned (this mean no EOL and no + Else all live versions are returned (this means no EOL and no security-fixes branches). """ if branch: @@ -156,12 +155,12 @@ def filter(versions, branch=None): @staticmethod def current_stable(versions): - """Find the current stable cPython version.""" + """Find the current stable CPython version.""" return max((v for v in versions if v.status == "stable"), key=Version.as_tuple) @staticmethod def current_dev(versions): - """Find the current de cPython version.""" + """Find the current CPython version in development.""" return max(versions, key=Version.as_tuple) @property @@ -360,7 +359,7 @@ def locate_nearest_version(available_versions, target_version): def edit(file: Path): """Context manager to edit a file "in place", use it as: - with edit("/etc/hosts") as i, o: + with edit("/etc/hosts") as (i, o): for line in i: o.write(line.replace("localhoat", "localhost")) """ @@ -376,7 +375,7 @@ def edit(file: Path): def setup_switchers( versions: Iterable[Version], languages: Iterable[Language], html_root: Path ): - """Setup cross-links between cpython versions: + """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ @@ -437,7 +436,7 @@ def build_robots_txt( ): """Disallow crawl of EOL versions in robots.txt.""" if not www_root.exists(): - logging.info("Skipping robots.txt generation (www root does not even exists).") + logging.info("Skipping robots.txt generation (www root does not even exist).") return robots_file = www_root / "robots.txt" with open(HERE / "templates" / "robots.txt", encoding="UTF-8") as template_file: @@ -457,7 +456,7 @@ def build_sitemap( ): """Build a sitemap with all live versions and translations.""" if not www_root.exists(): - logging.info("Skipping sitemap generation (www root does not even exists).") + logging.info("Skipping sitemap generation (www root does not even exist).") return with open(HERE / "templates" / "sitemap.xml", encoding="UTF-8") as template_file: template = jinja2.Template(template_file.read()) @@ -472,7 +471,7 @@ def build_sitemap( def build_404(www_root: Path, group): """Build a nice 404 error page to display in case PDFs are not built yet.""" if not www_root.exists(): - logging.info("Skipping 404 page generation (www root does not even exists).") + logging.info("Skipping 404 page generation (www root does not even exist).") return not_found_file = www_root / "404.html" shutil.copyfile(HERE / "templates" / "404.html", not_found_file) @@ -550,7 +549,7 @@ def parse_args(): ) parser.add_argument( "--skip-cache-invalidation", - help="Skip fastly cache invalidation.", + help="Skip Fastly cache invalidation.", action="store_true", ) parser.add_argument( @@ -598,7 +597,7 @@ def parse_args(): def setup_logging(log_directory: Path): - """Setup logging to stderr if ran by a human, or to a file if ran from a cron.""" + """Setup logging to stderr if run by a human, or to a file if run from a cron.""" if sys.stderr.isatty(): logging.basicConfig( format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stderr @@ -615,7 +614,7 @@ def setup_logging(log_directory: Path): @dataclass class DocBuilder: - """Builder for a cpython version and a language.""" + """Builder for a CPython version and a language.""" version: Version versions: Iterable[Version] @@ -634,7 +633,7 @@ class DocBuilder: def full_build(self): """Tell if a full build is needed. - A full build is slow, it builds pdf, txt, epub, texinfo, and + A full build is slow; it builds pdf, txt, epub, texinfo, and archives everything. A partial build only builds HTML and does not archive, it's @@ -664,7 +663,7 @@ def run(self) -> bool: @property def checkout(self) -> Path: - """Path to cpython git clone.""" + """Path to CPython git clone.""" return self.build_root / "cpython" def clone_translation(self): @@ -687,7 +686,7 @@ def translation_repo(self): @property def translation_branch(self): - """Some cpython versions may be untranslated, being either too old or + """Some CPython versions may be untranslated, being either too old or too new. This function looks for remote branches on the given repo, and @@ -745,7 +744,7 @@ def build(self): python = self.venv / "bin" / "python" sphinxbuild = self.venv / "bin" / "sphinx-build" blurb = self.venv / "bin" / "blurb" - # Disable cpython switchers, we handle them now: + # Disable CPython switchers, we handle them now: def is_mac(): return platform.system() == 'Darwin' @@ -955,7 +954,7 @@ def load_state(self) -> dict: return {} def save_state(self, build_duration: float): - """Save current cpython sha1 and current translation sha1. + """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. """ @@ -979,7 +978,7 @@ def save_state(self, build_duration: float): def symlink(www_root: Path, language: Language, directory: str, name: str, group: str, skip_cache_invalidation: bool): """Used by major_symlinks and dev_symlink to maintain symlinks.""" - if language.tag == "en": # english is rooted on /, no /en/ + if language.tag == "en": # English is rooted on /, no /en/ path = www_root else: path = www_root / language.tag @@ -1000,7 +999,7 @@ def symlink(www_root: Path, language: Language, directory: str, name: str, group def major_symlinks( www_root: Path, group, versions: Iterable[Version], languages: Iterable[Language], skip_cache_invalidation: bool ): - """Maintains the /2/ and /3/ symlinks for each languages. + """Maintains the /2/ and /3/ symlinks for each language. Like: - /3/ → /3.9/ @@ -1014,7 +1013,7 @@ def major_symlinks( def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidation: bool): - """Maintains the /dev/ symlinks for each languages. + """Maintains the /dev/ symlinks for each language. Like: - /dev/ → /3.11/ @@ -1029,7 +1028,7 @@ def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidat def purge(*paths): """Remove one or many paths from docs.python.org's CDN. - To be used when a file change, so the CDN fetch the new one. + To be used when a file changes, so the CDN fetches the new one. """ base = "https://docs.python.org/" for path in paths: @@ -1041,7 +1040,7 @@ def purge(*paths): def purge_path(www_root: Path, path: Path): """Recursively remove a path from docs.python.org's CDN. - To be used when a directory change, so the CDN fetch the new one. + To be used when a directory changes, so the CDN fetches the new one. """ purge(*[file.relative_to(www_root) for file in path.glob("**/*")]) purge(path.relative_to(www_root)) @@ -1101,7 +1100,7 @@ def parse_languages_from_config(): def build_docs(args) -> bool: - """Build all docs (each languages and each versions).""" + """Build all docs (each language and each version).""" versions = parse_versions_from_devguide() languages = parse_languages_from_config() todo = [ diff --git a/check_versions.py b/check_versions.py index 630eb6e..0ca0770 100644 --- a/check_versions.py +++ b/check_versions.py @@ -20,12 +20,12 @@ def parse_args(): description="""Check the version of our build in different branches Hint: Use with | column -t""" ) - parser.add_argument("cpython_clone", help="Path to a clone of cpython", type=Path) + parser.add_argument("cpython_clone", help="Path to a clone of CPython", type=Path) return parser.parse_args() def remote_by_url(repo: git.Repo, url_pattern: str): - """Find a remote of repo matching the regex url_pattern.""" + """Find a remote in the repo that matches the URL pattern.""" for remote in repo.remotes: for url in remote.urls: if re.search(url_pattern, url): From dd8a562a500e2f41ef378ae1ebf19076115fc0b1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 14 Aug 2024 18:17:21 +0300 Subject: [PATCH 010/176] Show elapsed time for build and publishing --- .github/workflows/lint.yml | 22 ++++++++++++++++++ .github/workflows/test.yml | 39 +++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 47 ++++++++++++++++++++++++++++++++++++++ build_docs.py | 25 +++++++++++++++++--- pyproject.toml | 7 ++++++ tests/test_build_docs.py | 21 +++++++++++++++++ tox.ini | 25 ++++++++++++++++++++ 7 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/test.yml create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml create mode 100644 tests/test_build_docs.py create mode 100644 tox.ini diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..d553e49 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,22 @@ +name: Lint + +on: [push, pull_request, workflow_dispatch] + +env: + FORCE_COLOR: 1 + PIP_DISABLE_PIP_VERSION_CHECK: 1 + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + cache: pip + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..668336d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,39 @@ +name: Test + +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + PIP_DISABLE_PIP_VERSION_CHECK: 1 + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Install uv + uses: hynek/setup-cached-uv@v2 + + - name: Install dependencies + run: | + uv pip install --system -U tox-uv + + - name: Tox tests + run: | + tox -e py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..db47c50 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,47 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: forbid-submodules + - id: requirements-txt-fixer + - id: trailing-whitespace + + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.28.6 + hooks: + - id: check-github-workflows + + - repo: https://github.com/rhysd/actionlint + rev: v1.7.1 + hooks: + - id: actionlint + + - repo: https://github.com/tox-dev/pyproject-fmt + rev: 2.1.3 + hooks: + - id: pyproject-fmt + + - repo: https://github.com/abravalheri/validate-pyproject + rev: v0.18 + hooks: + - id: validate-pyproject + + - repo: https://github.com/tox-dev/tox-ini-fmt + rev: 1.3.1 + hooks: + - id: tox-ini-fmt + + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes + +ci: + autoupdate_schedule: quarterly diff --git a/build_docs.py b/build_docs.py index 212989e..701beb1 100755 --- a/build_docs.py +++ b/build_docs.py @@ -28,7 +28,7 @@ import json import logging import logging.handlers -from functools import total_ordering +from functools import total_ordering, cache from os import readlink import platform import re @@ -702,6 +702,7 @@ def translation_branch(self): def build(self): """Build this version/language doc.""" logging.info("Build start.") + start_time = perf_counter() sphinxopts = list(self.language.sphinxopts) sphinxopts.extend(["-q"]) if self.language.tag != "en": @@ -778,7 +779,7 @@ def is_mac(): setup_switchers( self.versions, self.languages, self.checkout / "Doc" / "build" / "html" ) - logging.info("Build done.") + logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) def build_venv(self): """Build a venv for the specific Python version. @@ -800,6 +801,7 @@ def build_venv(self): def copy_build_to_webroot(self): """Copy a given build to the appropriate webroot with appropriate rights.""" logging.info("Publishing start.") + start_time = perf_counter() self.www_root.mkdir(parents=True, exist_ok=True) if self.language.tag == "en": target = self.www_root / self.version.name @@ -912,7 +914,9 @@ def copy_build_to_webroot(self): purge(*prefixes) for prefix in prefixes: purge(*[prefix + p for p in changed]) - logging.info("Publishing done") + logging.info( + "Publishing done (%s).", format_seconds(perf_counter() - start_time) + ) def should_rebuild(self): state = self.load_state() @@ -1147,6 +1151,21 @@ def build_docs(args) -> bool: return all_built_successfully +@cache +def format_seconds(seconds: float) -> str: + hours, remainder = divmod(seconds, 3600) + minutes, seconds = divmod(remainder, 60) + seconds = round(seconds) + + match (hours, minutes, seconds): + case 0, 0, s: + return f"{s}s" + case 0, m, s: + return f"{m}m {s}s" + case h, m, s: + return f"{h}h {m}m {s}s" + + def main(): """Script entry point.""" args = parse_args() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6c56bf0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.pytest.ini_options] +pythonpath = [ + ".", +] +testpaths = [ + "tests", +] diff --git a/tests/test_build_docs.py b/tests/test_build_docs.py new file mode 100644 index 0000000..975e03c --- /dev/null +++ b/tests/test_build_docs.py @@ -0,0 +1,21 @@ +import pytest + +from build_docs import format_seconds + + +@pytest.mark.parametrize( + "seconds, expected", + [ + (0.4, "0s"), + (0.5, "0s"), + (0.6, "1s"), + (1.5, "2s"), + (30, "30s"), + (60, "1m 0s"), + (185, "3m 5s"), + (454, "7m 34s"), + (7456, "2h 4m 16s"), + ], +) +def test_format_seconds(seconds: int, expected: str) -> None: + assert format_seconds(seconds) == expected diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..40a034d --- /dev/null +++ b/tox.ini @@ -0,0 +1,25 @@ +[tox] +requires = + tox>=4.2 +env_list = + lint + py{313, 312, 311, 310} + +[testenv] +package = wheel +wheel_build_env = .pkg +skip_install = true +deps = + -r requirements.txt + pytest +commands = + {envpython} -m pytest {posargs} + +[testenv:lint] +skip_install = true +deps = + pre-commit +pass_env = + PRE_COMMIT_COLOR +commands = + pre-commit run --all-files --show-diff-on-failure From 3ef52a6613cac9a265c0920c24aa9178b92f3b4c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:20:07 +0300 Subject: [PATCH 011/176] Fix hours and minutes for float input --- build_docs.py | 2 +- tests/test_build_docs.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 701beb1..0e67d7b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1155,7 +1155,7 @@ def build_docs(args) -> bool: def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) - seconds = round(seconds) + hours, minutes, seconds = int(hours), int(minutes), round(seconds) match (hours, minutes, seconds): case 0, 0, s: diff --git a/tests/test_build_docs.py b/tests/test_build_docs.py index 975e03c..4457e95 100644 --- a/tests/test_build_docs.py +++ b/tests/test_build_docs.py @@ -15,7 +15,12 @@ (185, "3m 5s"), (454, "7m 34s"), (7456, "2h 4m 16s"), + (30.1, "30s"), + (60.2, "1m 0s"), + (185.3, "3m 5s"), + (454.4, "7m 34s"), + (7456.5, "2h 4m 16s"), ], ) -def test_format_seconds(seconds: int, expected: str) -> None: +def test_format_seconds(seconds: float, expected: str) -> None: assert format_seconds(seconds) == expected From 6c194eb105f49728cc19d58f3c999eb8dbfc1ebf Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 15 Aug 2024 17:04:34 +0300 Subject: [PATCH 012/176] Remove functools.cache --- build_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0e67d7b..c72c56c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -28,7 +28,7 @@ import json import logging import logging.handlers -from functools import total_ordering, cache +from functools import total_ordering from os import readlink import platform import re @@ -1151,7 +1151,6 @@ def build_docs(args) -> bool: return all_built_successfully -@cache def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) From 56d72d43e5759cc0ed600827b56e81d8310bcaca Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 24 Aug 2024 14:36:09 +0200 Subject: [PATCH 013/176] Update the version tables in the README (#189) * Update README tables * Remove EOL versions from the language table --- README.md | 40 ++++++++-------- build_docs.py | 4 +- check_versions.py | 119 ++++++++++++++++++++++++---------------------- 3 files changed, 85 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 3e70fa4..d5beb39 100644 --- a/README.md +++ b/README.md @@ -23,34 +23,32 @@ of Sphinx we're using where: Sphinx configuration in various branches: - ========= ============= ============= ================== ================== - version travis azure requirements.txt conf.py - ========= ============= ============= ================== ================== - 2.7 sphinx~=2.0.1 ø ø needs_sphinx='1.2' - 3.5 sphinx==1.8.2 ø ø needs_sphinx='1.8' - 3.6 sphinx==1.8.2 sphinx==1.8.2 ø needs_sphinx='1.2' - 3.7 ø ø ø ø - 3.8 ø ø sphinx==2.4.4 needs_sphinx='1.8' - 3.9 ø ø sphinx==2.4.4 needs_sphinx='1.8' - 3.1 ø ø sphinx==3.4.3 needs_sphinx='3.2' - 3.11 ø ø sphinx==4.5.0 needs_sphinx='4.2' - 3.12 ø ø sphinx==4.5.0 needs_sphinx='4.2' - 3.13 ø ø sphinx==6.2.1 needs_sphinx='4.2' - ========= ============= ============= ================== ================== + ========= ============= ================== ==================== + version travis requirements.txt conf.py + ========= ============= ================== ==================== + 2.7 sphinx~=2.0.1 ø needs_sphinx='1.2' + 3.5 sphinx==1.8.2 ø needs_sphinx='1.8' + 3.6 sphinx==1.8.2 ø needs_sphinx='1.2' + 3.7 sphinx==1.8.2 sphinx==2.3.1 needs_sphinx="1.6.6" + 3.8 ø sphinx==2.4.4 needs_sphinx='1.8' + 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' + 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' + 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' + 3.12 ø sphinx~=8.0.0 needs_sphinx='6.2.1' + 3.13 ø sphinx~=8.0.0 needs_sphinx='6.2.1' + 3.14 ø sphinx~=8.0.0 needs_sphinx='6.2.1' + ========= ============= ================== ==================== Sphinx build as seen on docs.python.org: ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= version en es fr id it ja ko pl pt-br tr uk zh-cn zh-tw ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= - 2.7 ø 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 - 3.5 ø 1.8.4 1.8.4 1.8.4 1.8.4 ø 1.8.4 1.8.4 ø 1.8.4 1.8.4 1.8.4 1.8.4 - 3.6 ø 2.3.1 2.3.1 2.3.1 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 2.3.1 2.3.1 2.3.1 - 3.7 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 3.8 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 - 3.11 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 - 3.12 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 - 3.13 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 + 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 + 3.12 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 + 3.13 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 + 3.14 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= diff --git a/build_docs.py b/build_docs.py index 212989e..93dcac4 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1079,7 +1079,9 @@ def parse_versions_from_devguide(): "python/devguide/main/include/release-cycle.json", timeout=30, ).json() - return [Version.from_json(name, release) for name, release in releases.items()] + versions = [Version.from_json(name, release) for name, release in releases.items()] + versions.sort(key=Version.as_tuple) + return versions def parse_languages_from_config(): diff --git a/check_versions.py b/check_versions.py index 630eb6e..a1900c3 100644 --- a/check_versions.py +++ b/check_versions.py @@ -13,6 +13,8 @@ import build_docs logger = logging.getLogger(__name__) +VERSIONS = build_docs.parse_versions_from_devguide() +LANGUAGES = build_docs.parse_languages_from_config() def parse_args(): @@ -24,102 +26,107 @@ def parse_args(): return parser.parse_args() -def remote_by_url(repo: git.Repo, url_pattern: str): +def find_upstream_remote_name(repo: git.Repo) -> str: """Find a remote of repo matching the regex url_pattern.""" for remote in repo.remotes: for url in remote.urls: - if re.search(url_pattern, url): - return remote + if "github.com/python" in url: + return f"{remote.name}/" def find_sphinx_spec(text: str): if found := re.search( - """sphinx[=<>~]{1,2}[0-9.]{3,}|needs_sphinx = [0-9.'"]*""", text, flags=re.I + """sphinx[=<>~]{1,2}[0-9.]{3,}|needs_sphinx = [0-9.'"]*""", + text, + flags=re.IGNORECASE, ): return found.group(0).replace(" ", "") return "ø" -def find_sphinx_in_file(repo: git.Repo, branch, filename): - upstream = remote_by_url(repo, "github.com.python").name - # Just in case you don't use origin/: - branch = branch.replace("origin/", upstream + "/") - try: - return find_sphinx_spec(repo.git.show(f"{branch}:{filename}")) - except git.exc.GitCommandError: - return "ø" +def find_sphinx_in_files(repo: git.Repo, branch_or_tag, filenames): + upstream = find_upstream_remote_name(repo) + # Just in case you don't use upstream/: + branch_or_tag = branch_or_tag.replace("upstream/", upstream) + specs = [] + for filename in filenames: + try: + blob = repo.git.show(f"{branch_or_tag}:{filename}") + except git.exc.GitCommandError: + specs.append("ø") + else: + specs.append(find_sphinx_spec(blob)) + return specs CONF_FILES = { "travis": ".travis.yml", - "azure": ".azure-pipelines/docs-steps.yml", "requirements.txt": "Doc/requirements.txt", "conf.py": "Doc/conf.py", } -def search_sphinx_versions_in_cpython(repo: git.Repo): - repo.git.fetch("https://github.com/python/cpython") - table = [] - for version in sorted(build_docs.VERSIONS): - table.append( - [ - version.name, - *[ - find_sphinx_in_file(repo, version.branch_or_tag, filename) - for filename in CONF_FILES.values() - ], - ] - ) - print(tabulate(table, headers=["version", *CONF_FILES.keys()], tablefmt="rst")) +def branch_or_tag_for(version): + if version.status == "EOL": + return f"tags/{version.branch_or_tag}" + return f"upstream/{version.branch_or_tag}" -async def get_version_in_prod(language, version): - url = f"https://docs.python.org/{language}/{version}/".replace("/en/", "/") +def search_sphinx_versions_in_cpython(repo: git.Repo): + repo.git.fetch("https://github.com/python/cpython") + filenames = CONF_FILES.values() + table = [ + [ + version.name, + *find_sphinx_in_files(repo, branch_or_tag_for(version), filenames), + ] + for version in VERSIONS + ] + headers = ["version", *CONF_FILES.keys()] + print(tabulate(table, headers=headers, tablefmt="rst", disable_numparse=True)) + + +async def get_version_in_prod(language: str, version: str) -> str: + if language == "en": + url = f"https://docs.python.org/{version}/" + else: + url = f"https://docs.python.org/{language}/{version}/" async with httpx.AsyncClient() as client: try: - response = await client.get(url, timeout=10) + response = await client.get(url, timeout=5) except httpx.ConnectTimeout: return "(timeout)" - text = response.text.encode("ASCII", errors="ignore").decode("ASCII") + # Python 2.6--3.7: sphinx.pocoo.org + # from Python 3.8: www.sphinx-doc.org if created_using := re.search( - r"(?:sphinx.pocoo.org|www.sphinx-doc.org).*?([0-9.]+[0-9])", text, flags=re.M + r"(?:sphinx.pocoo.org|www.sphinx-doc.org).*?([0-9.]+[0-9])", response.text ): return created_using.group(1) return "ø" async def which_sphinx_is_used_in_production(): - table = [] - for version in sorted(build_docs.VERSIONS): - table.append( - [ - version.name, - *await asyncio.gather( - *[ - get_version_in_prod(language.tag, version.name) - for language in build_docs.LANGUAGES - ] - ), - ] - ) - print( - tabulate( - table, - disable_numparse=True, - headers=[ - "version", - *[language.tag for language in sorted(build_docs.LANGUAGES)], - ], - tablefmt="rst", - ) - ) + table = [ + [ + version.name, + *await asyncio.gather( + *[ + get_version_in_prod(language.tag, version.name) + for language in LANGUAGES + ] + ), + ] + for version in VERSIONS + ] + headers = ["version", *[language.tag for language in LANGUAGES]] + print(tabulate(table, headers=headers, tablefmt="rst", disable_numparse=True)) def main(): logging.basicConfig(level=logging.INFO) logging.getLogger("charset_normalizer").setLevel(logging.WARNING) logging.getLogger("asyncio").setLevel(logging.WARNING) + logging.getLogger("httpx").setLevel(logging.WARNING) args = parse_args() repo = git.Repo(args.cpython_clone) print("Sphinx configuration in various branches:", end="\n\n") From 124693f58f4cc22b9a5d40aaab73f1cb26195cb0 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:57:12 +0300 Subject: [PATCH 014/176] Keep build_docs() near main() --- build_docs.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/build_docs.py b/build_docs.py index c72c56c..006b050 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1104,6 +1104,20 @@ def parse_languages_from_config(): return languages +def format_seconds(seconds: float) -> str: + hours, remainder = divmod(seconds, 3600) + minutes, seconds = divmod(remainder, 60) + hours, minutes, seconds = int(hours), int(minutes), round(seconds) + + match (hours, minutes, seconds): + case 0, 0, s: + return f"{s}s" + case 0, m, s: + return f"{m}m {s}s" + case h, m, s: + return f"{h}h {m}m {s}s" + + def build_docs(args) -> bool: """Build all docs (each languages and each versions).""" versions = parse_versions_from_devguide() @@ -1151,20 +1165,6 @@ def build_docs(args) -> bool: return all_built_successfully -def format_seconds(seconds: float) -> str: - hours, remainder = divmod(seconds, 3600) - minutes, seconds = divmod(remainder, 60) - hours, minutes, seconds = int(hours), int(minutes), round(seconds) - - match (hours, minutes, seconds): - case 0, 0, s: - return f"{s}s" - case 0, m, s: - return f"{m}m {s}s" - case h, m, s: - return f"{h}h {m}m {s}s" - - def main(): """Script entry point.""" args = parse_args() From 21d094e2a2e97985ad3690f8066f5f080ec19a16 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 25 Aug 2024 13:01:47 +0300 Subject: [PATCH 015/176] Show elapsed time for full docs build --- build_docs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build_docs.py b/build_docs.py index 006b050..35b794e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1120,6 +1120,8 @@ def format_seconds(seconds: float) -> str: def build_docs(args) -> bool: """Build all docs (each languages and each versions).""" + logging.info("Full build start.") + start_time = perf_counter() versions = parse_versions_from_devguide() languages = parse_languages_from_config() todo = [ @@ -1162,6 +1164,8 @@ def build_docs(args) -> bool: dev_symlink(args.www_root, args.group, versions, languages, args.skip_cache_invalidation) proofread_canonicals(args.www_root, args.skip_cache_invalidation) + logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) + return all_built_successfully From ed42020f66c254dd5bed910b96e5f58602892810 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 26 Aug 2024 08:41:15 +0300 Subject: [PATCH 016/176] Build each docs version with fresh commit --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 93dcac4..b20e489 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1117,7 +1117,6 @@ def build_docs(args) -> bool: cpython_repo = Repository( "https://github.com/python/cpython.git", args.build_root / "cpython" ) - cpython_repo.update() while todo: version, language = todo.pop() logging.root.handlers[0].setFormatter( @@ -1129,6 +1128,7 @@ def build_docs(args) -> bool: with sentry_sdk.configure_scope() as scope: scope.set_tag("version", version.name) scope.set_tag("language", language.tag) + cpython_repo.update() builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) From a21b2bb0840a17a11ce1edf8de4fbc4ed333b46d Mon Sep 17 00:00:00 2001 From: Ege Akman Date: Tue, 10 Sep 2024 13:05:45 -0400 Subject: [PATCH 017/176] Update .gitignore Co-authored-by: Ezio Melotti --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 79fca5f..aaefb08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /build_root/ /logs/ /www/ +# temporary lock file created while building the docs build_docs.lock From f2d2ffb83ce8f3dac961a75d2069033b668b6dc7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:22:21 +0300 Subject: [PATCH 018/176] Update pre-commit --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db47c50..dba593f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.28.6 + rev: 0.29.2 hooks: - id: check-github-workflows @@ -24,17 +24,17 @@ repos: - id: actionlint - repo: https://github.com/tox-dev/pyproject-fmt - rev: 2.1.3 + rev: 2.2.3 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.18 + rev: v0.19 hooks: - id: validate-pyproject - repo: https://github.com/tox-dev/tox-ini-fmt - rev: 1.3.1 + rev: 1.4.0 hooks: - id: tox-ini-fmt From 652c561ac48ef8f3924ad188482058aa7be1ac11 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:44:46 +0300 Subject: [PATCH 019/176] Simplify uv + tox-uv setup --- .github/workflows/test.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 668336d..b2686a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,10 +30,6 @@ jobs: - name: Install uv uses: hynek/setup-cached-uv@v2 - - name: Install dependencies - run: | - uv pip install --system -U tox-uv - - name: Tox tests run: | - tox -e py + uvx --with tox-uv tox -e py From c6a8855c0fba7ba7408fc4b1969ca90e3ce43c1e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:18:54 +0300 Subject: [PATCH 020/176] Using uv not pip --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2686a7..d74e607 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,6 @@ permissions: env: FORCE_COLOR: 1 - PIP_DISABLE_PIP_VERSION_CHECK: 1 jobs: test: From b0921bd556d9b25637019184085e5e13e2830fc7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:42:11 +0300 Subject: [PATCH 021/176] Reformat pyproject.toml --- pyproject.toml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6c56bf0..e85ab2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,3 @@ [tool.pytest.ini_options] -pythonpath = [ - ".", -] -testpaths = [ - "tests", -] +pythonpath = [ "." ] +testpaths = [ "tests" ] From 6360ad789b2c3235086ec4d8e16f5bf7576cbb3b Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:53:02 +0100 Subject: [PATCH 022/176] Improve type hints (use Sequence) --- build_docs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index a3971a5..961c48c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -21,6 +21,7 @@ """ from argparse import ArgumentParser +from collections.abc import Sequence from contextlib import suppress, contextmanager from dataclasses import dataclass import filecmp @@ -372,7 +373,7 @@ def edit(file: Path): def setup_switchers( - versions: Iterable[Version], languages: Iterable[Language], html_root: Path + versions: Sequence[Version], languages: Sequence[Language], html_root: Path ): """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher @@ -617,9 +618,9 @@ class DocBuilder: """Builder for a CPython version and a language.""" version: Version - versions: Iterable[Version] + versions: Sequence[Version] language: Language - languages: Iterable[Language] + languages: Sequence[Language] cpython_repo: Repository build_root: Path www_root: Path @@ -1127,7 +1128,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: return versions -def parse_languages_from_config(): +def parse_languages_from_config() -> list[Language]: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) languages = [] From 2828c2b2f1112b038638a73153c5cbf77a4306f6 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:54:44 +0100 Subject: [PATCH 023/176] Remove needless sorts --- build_docs.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/build_docs.py b/build_docs.py index 961c48c..a77371b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -182,9 +182,7 @@ def setup_indexsidebar(self, versions, dest_path): sidebar_file.write( sidebar_template.render( current_version=self, - versions=sorted( - versions, key=lambda v: version_to_tuple(v.name), reverse=True - ), + versions=versions[::-1], ) ) @@ -339,12 +337,7 @@ def locate_nearest_version(available_versions, target_version): '3.7' """ - available_versions_tuples = sorted( - [ - version_to_tuple(available_version) - for available_version in set(available_versions) - ] - ) + available_versions_tuples = sorted(map(version_to_tuple, set(available_versions))) target_version_tuple = version_to_tuple(target_version) try: found = available_versions_tuples[ @@ -402,11 +395,7 @@ def setup_switchers( OrderedDict( [ (version.name, version.picker_label) - for version in sorted( - versions, - key=lambda v: version_to_tuple(v.name), - reverse=True, - ) + for version in reversed(versions) ] ) ), From 988128e0d6e7f4b35ee968be1a36695f328d898f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:55:10 +0100 Subject: [PATCH 024/176] Run languages in definition order --- build_docs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index a77371b..42e5ad0 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1156,10 +1156,13 @@ def build_docs(args) -> bool: http = urllib3.PoolManager() versions = parse_versions_from_devguide(http) languages = parse_languages_from_config() + # Reverse languages but not versions, because we take version-language + # pairs from the end of the list, effectively reversing it. + # This runs languages in config.toml order and versions newest first. todo = [ (version, language) for version in Version.filter(versions, args.branch) - for language in Language.filter(languages, args.languages) + for language in reversed(Language.filter(languages, args.languages)) ] del args.branch del args.languages From 4eb435ed09d5914e0f24672321f695b6c34c6fcb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:58:24 +0100 Subject: [PATCH 025/176] Remove OrderedDict --- build_docs.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/build_docs.py b/build_docs.py index 42e5ad0..a62efce 100755 --- a/build_docs.py +++ b/build_docs.py @@ -37,7 +37,6 @@ import subprocess import sys from bisect import bisect_left as bisect -from collections import OrderedDict from datetime import datetime as dt, timezone from pathlib import Path from string import Template @@ -372,6 +371,9 @@ def setup_switchers( - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ + languages_map = dict(sorted((l.tag, l.name) for l in languages if l.in_prod)) + versions_map = {v.name: v.picker_label for v in reversed(versions)} + with open( HERE / "templates" / "switchers.js", encoding="UTF-8" ) as switchers_template_file: @@ -380,25 +382,8 @@ def setup_switchers( switchers_path.write_text( template.safe_substitute( { - "LANGUAGES": json.dumps( - OrderedDict( - sorted( - [ - (language.tag, language.name) - for language in languages - if language.in_prod - ] - ) - ) - ), - "VERSIONS": json.dumps( - OrderedDict( - [ - (version.name, version.picker_label) - for version in reversed(versions) - ] - ) - ), + "LANGUAGES": json.dumps(languages_map), + "VERSIONS": json.dumps(versions_map), } ), encoding="UTF-8", From a00bf2bde0569ba9fcb9b21e612e737714f1055b Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:16:00 +0100 Subject: [PATCH 026/176] Use keyword-arguments --- build_docs.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index a62efce..2ebc312 100755 --- a/build_docs.py +++ b/build_docs.py @@ -381,10 +381,8 @@ def setup_switchers( switchers_path = html_root / "_static" / "switchers.js" switchers_path.write_text( template.safe_substitute( - { - "LANGUAGES": json.dumps(languages_map), - "VERSIONS": json.dumps(versions_map), - } + LANGUAGES=json.dumps(languages_map), + VERSIONS=json.dumps(versions_map), ), encoding="UTF-8", ) From 04cc37b17d86a868e7109554ef3b28b061b8594f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:41:44 +0100 Subject: [PATCH 027/176] Add the check-times script (#196) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .pre-commit-config.yaml | 5 +++ build_docs.py | 2 +- check_times.py | 87 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 check_times.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dba593f..c3d2f5f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,6 +13,11 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.7 + hooks: + - id: ruff-format + - repo: https://github.com/python-jsonschema/check-jsonschema rev: 0.29.2 hooks: diff --git a/build_docs.py b/build_docs.py index 2ebc312..8f5f382 100755 --- a/build_docs.py +++ b/build_docs.py @@ -721,7 +721,7 @@ def build(self): # Disable CPython switchers, we handle them now: def is_mac(): - return platform.system() == 'Darwin' + return platform.system() == "Darwin" run( ["sed", "-i"] diff --git a/check_times.py b/check_times.py new file mode 100644 index 0000000..4b6665d --- /dev/null +++ b/check_times.py @@ -0,0 +1,87 @@ +"""Check the frequency of the rebuild loop. + +This must be run in a directory that has the ``docsbuild.log*`` files. +For example: + +.. code-block:: bash + + $ scp "adam@docs.nyc1.psf.io:/var/log/docsbuild/docsbuild.log*" docsbuild-logs + $ python check_times.py +""" + +import datetime as dt +import gzip +from pathlib import Path + +from build_docs import format_seconds + + +def get_lines() -> list[str]: + lines = [] + zipped_logs = list(Path.cwd().glob("docsbuild.log.*.gz")) + zipped_logs.sort(key=lambda p: int(p.name.split(".")[-2]), reverse=True) + for logfile in zipped_logs: + with gzip.open(logfile, "rt", encoding="utf-8") as f: + lines += f.readlines() + with open("docsbuild.log", encoding="utf-8") as f: + lines += f.readlines() + return lines + + +def calc_time(lines: list[str]) -> None: + start = end = language = version = start_timestamp = None + reason = lang_ver = "" + + print("Start | Version | Language | Build | Trigger") + print(":-- | :--: | :--: | --: | :--:") + + for line in lines: + line = line.strip() + + if ": Should rebuild: " in line: + if "no previous state found" in line: + reason = "brand new" + elif "new translations" in line: + reason = "translation" + elif "Doc/ has changed" in line: + reason = "docs" + else: + reason = "" + lang_ver = line.split(" ")[3].removesuffix(":") + + if line.endswith("Build start."): + timestamp = line[:23].replace(",", ".") + language, version = line.split(" ")[3].removesuffix(":").split("/") + start = dt.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") + start_timestamp = f"{line[:16]} UTC" + + if start and ": Build done " in line: + timestamp = line[:23].replace(",", ".") + language, version = line.split(" ")[3].removesuffix(":").split("/") + end = dt.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") + + if start and end: + duration = (end - start).total_seconds() + fmt_duration = format_seconds(duration) + if lang_ver != f"{language}/{version}": + reason = "" + print( + f"{start_timestamp: <20} | {version: <7} | {language: <8} | {fmt_duration :<14} | {reason}" + ) + start = end = start_timestamp = None + + if ": Full build done" in line: + timestamp = f"{line[:16]} UTC" + _, fmt_duration = line.removesuffix(").").split("(") + print( + f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration :<14} | -----------" + ) + + if start and end is None: + print( + f"{start_timestamp: <20} | {version: <7} | {language: <8} | In progress... | {reason}" + ) + + +if __name__ == "__main__": + calc_time(get_lines()) From 8c93ad15bf75ffee72ff22c0f62e7e0727a7bb46 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 21:33:12 +0100 Subject: [PATCH 028/176] Use pathlib more (#197) --- build_docs.py | 80 ++++++++++++++++++++++----------------------------- 1 file changed, 35 insertions(+), 45 deletions(-) diff --git a/build_docs.py b/build_docs.py index 8f5f382..255f45d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -20,6 +20,8 @@ """ +from __future__ import annotations + from argparse import ArgumentParser from collections.abc import Sequence from contextlib import suppress, contextmanager @@ -171,19 +173,15 @@ def picker_label(self): return f"pre ({self.name})" return self.name - def setup_indexsidebar(self, versions, dest_path): + def setup_indexsidebar(self, versions: Sequence[Version], dest_path: Path): """Build indexsidebar.html for Sphinx.""" - with open( - HERE / "templates" / "indexsidebar.html", encoding="UTF-8" - ) as sidebar_template_file: - sidebar_template = jinja2.Template(sidebar_template_file.read()) - with open(dest_path, "w", encoding="UTF-8") as sidebar_file: - sidebar_file.write( - sidebar_template.render( - current_version=self, - versions=versions[::-1], - ) - ) + template_path = HERE / "templates" / "indexsidebar.html" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render( + current_version=self, + versions=versions[::-1], + ) + dest_path.write_text(rendered_template, encoding="UTF-8") @classmethod def from_json(cls, name, values): @@ -374,19 +372,17 @@ def setup_switchers( languages_map = dict(sorted((l.tag, l.name) for l in languages if l.in_prod)) versions_map = {v.name: v.picker_label for v in reversed(versions)} - with open( - HERE / "templates" / "switchers.js", encoding="UTF-8" - ) as switchers_template_file: - template = Template(switchers_template_file.read()) + switchers_template_file = HERE / "templates" / "switchers.js" switchers_path = html_root / "_static" / "switchers.js" - switchers_path.write_text( - template.safe_substitute( - LANGUAGES=json.dumps(languages_map), - VERSIONS=json.dumps(versions_map), - ), - encoding="UTF-8", + + template = Template(switchers_template_file.read_text(encoding="UTF-8")) + rendered_template = template.safe_substitute( + LANGUAGES=json.dumps(languages_map), + VERSIONS=json.dumps(versions_map), ) - for file in Path(html_root).glob("**/*.html"): + switchers_path.write_text(rendered_template, encoding="UTF-8") + + for file in html_root.glob("**/*.html"): depth = len(file.relative_to(html_root).parts) - 1 src = f"{'../' * depth}_static/switchers.js" script = f' \n' @@ -411,15 +407,13 @@ def build_robots_txt( if not www_root.exists(): logging.info("Skipping robots.txt generation (www root does not even exist).") return - robots_file = www_root / "robots.txt" - with open(HERE / "templates" / "robots.txt", encoding="UTF-8") as template_file: - template = jinja2.Template(template_file.read()) - with open(robots_file, "w", encoding="UTF-8") as robots_txt_file: - robots_txt_file.write( - template.render(languages=languages, versions=versions) + "\n" - ) - robots_file.chmod(0o775) - run(["chgrp", group, robots_file]) + template_path = HERE / "templates" / "robots.txt" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render(languages=languages, versions=versions) + robots_path = www_root / "robots.txt" + robots_path.write_text(rendered_template + "\n", encoding="UTF-8") + robots_path.chmod(0o775) + run(["chgrp", group, robots_path]) if not skip_cache_invalidation: purge(http, "robots.txt") @@ -431,14 +425,13 @@ def build_sitemap( if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exist).") return - with open(HERE / "templates" / "sitemap.xml", encoding="UTF-8") as template_file: - template = jinja2.Template(template_file.read()) - sitemap_file = www_root / "sitemap.xml" - sitemap_file.write_text( - template.render(languages=languages, versions=versions) + "\n", encoding="UTF-8" - ) - sitemap_file.chmod(0o664) - run(["chgrp", group, sitemap_file]) + template_path = HERE / "templates" / "sitemap.xml" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render(languages=languages, versions=versions) + sitemap_path = www_root / "sitemap.xml" + sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") + sitemap_path.chmod(0o664) + run(["chgrp", group, sitemap_path]) def build_404(www_root: Path, group): @@ -867,10 +860,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: [ "cp", "-a", - *[ - str(dist) - for dist in (Path(self.checkout) / "Doc" / "dist").glob("*") - ], + *(self.checkout / "Doc" / "dist").glob("*"), target / "archives", ] ) @@ -972,7 +962,7 @@ def symlink( directory_path = path / directory if not directory_path.exists(): return # No touching link, dest doc not built yet. - if link.exists() and readlink(str(link)) == directory: + if link.exists() and readlink(link) == directory: return # Link is already pointing to right doc. if link.exists(): link.unlink() From 50e99b54038464b03a92a39ab026910bb3fb48b1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 25 Sep 2024 07:05:11 +0100 Subject: [PATCH 029/176] Fix directories for suggested use pattern --- check_times.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/check_times.py b/check_times.py index 4b6665d..f9ba49a 100644 --- a/check_times.py +++ b/check_times.py @@ -5,7 +5,8 @@ .. code-block:: bash - $ scp "adam@docs.nyc1.psf.io:/var/log/docsbuild/docsbuild.log*" docsbuild-logs + $ mkdir -p docsbuild-logs + $ scp "adam@docs.nyc1.psf.io:/var/log/docsbuild/docsbuild.log*" docsbuild-logs/ $ python check_times.py """ @@ -15,15 +16,17 @@ from build_docs import format_seconds +LOGS_ROOT = Path('docsbuild-logs').resolve() + def get_lines() -> list[str]: lines = [] - zipped_logs = list(Path.cwd().glob("docsbuild.log.*.gz")) + zipped_logs = list(LOGS_ROOT.glob("docsbuild.log.*.gz")) zipped_logs.sort(key=lambda p: int(p.name.split(".")[-2]), reverse=True) for logfile in zipped_logs: with gzip.open(logfile, "rt", encoding="utf-8") as f: lines += f.readlines() - with open("docsbuild.log", encoding="utf-8") as f: + with open(LOGS_ROOT / "docsbuild.log", encoding="utf-8") as f: lines += f.readlines() return lines From bce1bed404a59e4e1cefe2dbd428294e359c0edf Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 25 Sep 2024 07:11:21 +0100 Subject: [PATCH 030/176] fixup! Fix directories for suggested use pattern --- check_times.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_times.py b/check_times.py index f9ba49a..3b3e53c 100644 --- a/check_times.py +++ b/check_times.py @@ -16,7 +16,7 @@ from build_docs import format_seconds -LOGS_ROOT = Path('docsbuild-logs').resolve() +LOGS_ROOT = Path("docsbuild-logs").resolve() def get_lines() -> list[str]: From b87d94f3ee6690832015079aec7a1ddc0c92ac5e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:06:52 +0100 Subject: [PATCH 031/176] Log updates to ``state.toml`` --- build_docs.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/build_docs.py b/build_docs.py index 255f45d..67bde35 100755 --- a/build_docs.py +++ b/build_docs.py @@ -610,6 +610,7 @@ def full_build(self): def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() + start_timestamp = dt.now(tz=timezone.utc).replace(microsecond=0) logging.info("Running.") try: self.cpython_repo.switch(self.version.branch_or_tag) @@ -619,7 +620,10 @@ def run(self, http: urllib3.PoolManager) -> bool: self.build_venv() self.build() self.copy_build_to_webroot(http) - self.save_state(build_duration=perf_counter() - start_time) + self.save_state( + build_start=start_timestamp, + build_duration=perf_counter() - start_time, + ) except Exception as err: logging.exception("Badly handled exception, human, please help.") if sentry_sdk: @@ -921,7 +925,7 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_duration: float): + def save_state(self, build_start: dt, build_duration: float): """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -932,17 +936,23 @@ def save_state(self, build_duration: float): except FileNotFoundError: states = tomlkit.document() - state = {} - state["cpython_sha"] = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() + key = f"/{self.language.tag}/{self.version.name}/" + state = { + "last_build_start": build_start, + "last_build_duration": round(build_duration, 0), + "cpython_sha": self.cpython_repo.run("rev-parse", "HEAD").stdout.strip(), + } if self.language.tag != "en": state["translation_sha"] = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() - state["last_build"] = dt.now(timezone.utc) - state["last_build_duration"] = build_duration - states[f"/{self.language.tag}/{self.version.name}/"] = state + states[key] = state state_file.write_text(tomlkit.dumps(states), encoding="UTF-8") + tbl = tomlkit.inline_table() + tbl |= state + logging.info("Saved new rebuild state for %s: %s", key, tbl.as_string()) + def symlink( www_root: Path, From 33c527278f953ff60348ab8fb16e4415edaae8f1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 25 Sep 2024 22:13:57 +0100 Subject: [PATCH 032/176] Update build_docs.py Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- build_docs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 67bde35..5e9eb6b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -949,9 +949,9 @@ def save_state(self, build_start: dt, build_duration: float): states[key] = state state_file.write_text(tomlkit.dumps(states), encoding="UTF-8") - tbl = tomlkit.inline_table() - tbl |= state - logging.info("Saved new rebuild state for %s: %s", key, tbl.as_string()) + table = tomlkit.inline_table() + table |= state + logging.info("Saved new rebuild state for %s: %s", key, table.as_string()) def symlink( From 094e52f3cf51e7c97f84b8cf9cac738334108bca Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 26 Sep 2024 17:40:27 +0100 Subject: [PATCH 033/176] Include trigger reason --- build_docs.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/build_docs.py b/build_docs.py index 5e9eb6b..58836e4 100755 --- a/build_docs.py +++ b/build_docs.py @@ -616,13 +616,14 @@ def run(self, http: urllib3.PoolManager) -> bool: self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() - if self.should_rebuild(): + if trigger_reason := self.should_rebuild(): self.build_venv() self.build() self.copy_build_to_webroot(http) self.save_state( build_start=start_timestamp, build_duration=perf_counter() - start_time, + trigger=trigger_reason, ) except Exception as err: logging.exception("Badly handled exception, human, please help.") @@ -889,7 +890,7 @@ def should_rebuild(self): state = self.load_state() if not state: logging.info("Should rebuild: no previous state found.") - return True + return "no previous state" cpython_sha = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() if self.language.tag != "en": translation_sha = self.translation_repo.run( @@ -901,7 +902,7 @@ def should_rebuild(self): state["translation_sha"], translation_sha, ) - return True + return "new translations" if cpython_sha != state["cpython_sha"]: diff = self.cpython_repo.run( "diff", "--name-only", state["cpython_sha"], cpython_sha @@ -912,7 +913,7 @@ def should_rebuild(self): state["cpython_sha"], cpython_sha, ) - return True + return "Doc/ has changed" logging.info("Nothing changed, no rebuild needed.") return False @@ -925,7 +926,7 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_start: dt, build_duration: float): + def save_state(self, build_start: dt, build_duration: float, trigger: str): """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -940,6 +941,7 @@ def save_state(self, build_start: dt, build_duration: float): state = { "last_build_start": build_start, "last_build_duration": round(build_duration, 0), + "triggered_by": trigger, "cpython_sha": self.cpython_repo.run("rev-parse", "HEAD").stdout.strip(), } if self.language.tag != "en": From 5d26f06479cb78912fdbf69a538068aad5f0fd79 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 26 Sep 2024 23:08:04 +0100 Subject: [PATCH 034/176] Add ``--select-output`` (#199) --- .gitignore | 2 ++ build_docs.py | 98 +++++++++++++++++++++++++++------------------------ 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index aaefb08..c257236 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ /www/ # temporary lock file created while building the docs build_docs.lock +build_docs_archives.lock +build_docs_html.lock # Created by https://www.gitignore.io/api/python diff --git a/build_docs.py b/build_docs.py index 58836e4..0805139 100755 --- a/build_docs.py +++ b/build_docs.py @@ -22,7 +22,7 @@ from __future__ import annotations -from argparse import ArgumentParser +from argparse import ArgumentParser, Namespace from collections.abc import Sequence from contextlib import suppress, contextmanager from dataclasses import dataclass @@ -44,7 +44,7 @@ from string import Template from textwrap import indent from time import perf_counter, sleep -from typing import Iterable +from typing import Iterable, Literal from urllib.parse import urljoin import jinja2 @@ -487,11 +487,16 @@ def parse_args(): parser = ArgumentParser( description="Runs a build of the Python docs for various branches." ) + parser.add_argument( + "--select-output", + choices=("no-html", "only-html"), + help="Choose what outputs to build.", + ) parser.add_argument( "-q", "--quick", action="store_true", - help="Make HTML files only (Makefile rules suffixed with -html).", + help="Run a quick build (only HTML files).", ) parser.add_argument( "-b", @@ -589,6 +594,7 @@ class DocBuilder: cpython_repo: Repository build_root: Path www_root: Path + select_output: Literal["no-html", "only-html"] | None quick: bool group: str log_directory: Path @@ -596,16 +602,10 @@ class DocBuilder: theme: Path @property - def full_build(self): - """Tell if a full build is needed. - - A full build is slow; it builds pdf, txt, epub, texinfo, and - archives everything. - - A partial build only builds HTML and does not archive, it's - fast. - """ - return not self.quick and not self.language.html_only + def html_only(self): + return ( + self.select_output == "only-html" or self.quick or self.language.html_only + ) def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" @@ -635,7 +635,7 @@ def run(self, http: urllib3.PoolManager) -> bool: @property def checkout(self) -> Path: """Path to CPython git clone.""" - return self.build_root / "cpython" + return self.build_root / _checkout_name(self.select_output) def clone_translation(self): self.translation_repo.update() @@ -703,15 +703,13 @@ def build(self): if self.version.status == "EOL": sphinxopts.append("-D html_context.outdated=1") - maketarget = ( - "autobuild-" - + ( - "dev" - if self.version.status in ("in development", "pre-release") - else "stable" - ) - + ("" if self.full_build else "-html") - ) + + if self.version.status in ("in development", "pre-release"): + maketarget = "autobuild-dev" + else: + maketarget = "autobuild-stable" + if self.html_only: + maketarget += "-html" logging.info("Running make %s", maketarget) python = self.venv / "bin" / "python" sphinxbuild = self.venv / "bin" / "sphinx-build" @@ -820,28 +818,18 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: ";", ] ) - if self.full_build: - run( - [ - "rsync", - "-a", - "--delete-delay", - "--filter", - "P archives/", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - ] - ) - else: - run( - [ - "rsync", - "-a", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - ] - ) - if self.full_build: + run( + [ + "rsync", + "-a", + "--delete-delay", + "--filter", + "P archives/", + str(self.checkout / "Doc" / "build" / "html") + "/", + target, + ] + ) + if not self.quick: logging.debug("Copying dist files.") run( [ @@ -1153,7 +1141,8 @@ def build_docs(args) -> bool: del args.languages all_built_successfully = True cpython_repo = Repository( - "https://github.com/python/cpython.git", args.build_root / "cpython" + "https://github.com/python/cpython.git", + args.build_root / _checkout_name(args.select_output), ) while todo: version, language = todo.pop() @@ -1208,13 +1197,28 @@ def build_docs(args) -> bool: return all_built_successfully +def _checkout_name(select_output: str | None) -> str: + if select_output is not None: + return f"cpython-{select_output}" + return "cpython" + + def main(): """Script entry point.""" args = parse_args() setup_logging(args.log_directory) + if args.select_output is None: + build_docs_with_lock(args, "build_docs.lock") + elif args.select_output == "no-html": + build_docs_with_lock(args, "build_docs_archives.lock") + elif args.select_output == "only-html": + build_docs_with_lock(args, "build_docs_html.lock") + + +def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: try: - lock = zc.lockfile.LockFile(HERE / "build_docs.lock") + lock = zc.lockfile.LockFile(HERE / lockfile_name) except zc.lockfile.LockError: logging.info("Another builder is running... dying...") return EX_FAILURE From a884721350970f08edfce348da2ba4c7d4ae55bc Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:35:55 +0100 Subject: [PATCH 035/176] Set the logging filename based upon ``--select-output`` --- build_docs.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0805139..17e4f66 100755 --- a/build_docs.py +++ b/build_docs.py @@ -567,18 +567,19 @@ def parse_args(): return args -def setup_logging(log_directory: Path): +def setup_logging(log_directory: Path, select_output: str | None): """Setup logging to stderr if run by a human, or to a file if run from a cron.""" + log_format = "%(asctime)s %(levelname)s: %(message)s" if sys.stderr.isatty(): - logging.basicConfig( - format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stderr - ) + logging.basicConfig(format=log_format, stream=sys.stderr) else: log_directory.mkdir(parents=True, exist_ok=True) - handler = logging.handlers.WatchedFileHandler(log_directory / "docsbuild.log") - handler.setFormatter( - logging.Formatter("%(asctime)s %(levelname)s: %(message)s") - ) + if select_output is None: + filename = log_directory / "docsbuild.log" + else: + filename = log_directory / f"docsbuild-{select_output}.log" + handler = logging.handlers.WatchedFileHandler(filename) + handler.setFormatter(logging.Formatter(log_format)) logging.getLogger().addHandler(handler) logging.getLogger().setLevel(logging.DEBUG) @@ -1206,7 +1207,7 @@ def _checkout_name(select_output: str | None) -> str: def main(): """Script entry point.""" args = parse_args() - setup_logging(args.log_directory) + setup_logging(args.log_directory, args.select_output) if args.select_output is None: build_docs_with_lock(args, "build_docs.lock") From c422c3c9714dcb5170c34c6789482d33e96a5838 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:26:39 +0100 Subject: [PATCH 036/176] Log the output of the sphinx ``make`` command --- build_docs.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0805139..3f18a62 100755 --- a/build_docs.py +++ b/build_docs.py @@ -220,7 +220,7 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" cmd = [str(arg) for arg in cmd] cmdstring = shlex.join(cmd) - logging.debug("Run: %r", cmdstring) + logging.debug("Run: '%s'", cmdstring) result = subprocess.run( cmd, cwd=cwd, @@ -242,6 +242,21 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: return result +def run_with_logging(cmd, cwd=None): + """Like subprocess.check_call, with logging before the command execution.""" + cmd = list(map(str, cmd)) + logging.debug("Run: %s", shlex.join(cmd)) + with subprocess.Popen(cmd, cwd=cwd, encoding="utf-8") as p: + try: + for line in p.stdout: + logging.debug(">>>> %s", line.rstrip()) + except: + p.kill() + raise + if return_code := p.poll(): + raise subprocess.CalledProcessError(return_code, cmd[0]) + + def changed_files(left, right): """Compute a list of different files between left and right, recursively. Resulting paths are relative to left. @@ -728,7 +743,7 @@ def is_mac(): self.versions, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", ) - run( + run_with_logging( [ "make", "-C", From 3bda3f4d0f7d073811e63dc97906af6d2ef16c01 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:32:56 +0100 Subject: [PATCH 037/176] Require tomlkit>=0.13 (#204) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e208755..0cac810 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ jinja2 sentry-sdk>=2 -tomlkit +tomlkit>=0.13 urllib3>=2 zc.lockfile From 3b9d1facdba73b32b1140267bcd85c3b6e345e01 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 28 Sep 2024 14:09:14 +0100 Subject: [PATCH 038/176] Fix ``TypeError: 'NoneType' object is not iterable`` --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 861eace..a6e82ff 100755 --- a/build_docs.py +++ b/build_docs.py @@ -248,7 +248,7 @@ def run_with_logging(cmd, cwd=None): logging.debug("Run: %s", shlex.join(cmd)) with subprocess.Popen(cmd, cwd=cwd, encoding="utf-8") as p: try: - for line in p.stdout: + for line in (p.stdout or ()): logging.debug(">>>> %s", line.rstrip()) except: p.kill() From 5f97ccaab09db6d6d19ab26699834462fd3dc619 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 28 Sep 2024 14:20:24 +0100 Subject: [PATCH 039/176] Fix ``run_with_logging()``: add missing ``subprocess.PIPE`` --- build_docs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index a6e82ff..c5b13f9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -246,7 +246,14 @@ def run_with_logging(cmd, cwd=None): """Like subprocess.check_call, with logging before the command execution.""" cmd = list(map(str, cmd)) logging.debug("Run: %s", shlex.join(cmd)) - with subprocess.Popen(cmd, cwd=cwd, encoding="utf-8") as p: + with subprocess.Popen( + cmd, + cwd=cwd, + stdin=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + encoding="utf-8", + ) as p: try: for line in (p.stdout or ()): logging.debug(">>>> %s", line.rstrip()) From 992d2df9fdd7fc0c73f7b74a8dcebcd5bf26e6c1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 28 Sep 2024 14:22:53 +0100 Subject: [PATCH 040/176] Fix lint failure (ruff format) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index c5b13f9..12c1319 100755 --- a/build_docs.py +++ b/build_docs.py @@ -255,7 +255,7 @@ def run_with_logging(cmd, cwd=None): encoding="utf-8", ) as p: try: - for line in (p.stdout or ()): + for line in p.stdout or (): logging.debug(">>>> %s", line.rstrip()) except: p.kill() From caa4a29fd0d2cbd837e907d2686faf3debeabe73 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:14:36 +0100 Subject: [PATCH 041/176] Only run HTML-specific tasks in builds that output HTML (#207) --- build_docs.py | 112 +++++++++++++++++++++++++++----------------------- 1 file changed, 60 insertions(+), 52 deletions(-) diff --git a/build_docs.py b/build_docs.py index 12c1319..033ef0f 100755 --- a/build_docs.py +++ b/build_docs.py @@ -32,7 +32,6 @@ import logging.handlers from functools import total_ordering from os import readlink -import platform import re import shlex import shutil @@ -630,6 +629,11 @@ def html_only(self): self.select_output == "only-html" or self.quick or self.language.html_only ) + @property + def includes_html(self): + """Does the build we are running include HTML output?""" + return self.select_output != "no-html" + def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() @@ -737,20 +741,18 @@ def build(self): python = self.venv / "bin" / "python" sphinxbuild = self.venv / "bin" / "sphinx-build" blurb = self.venv / "bin" / "blurb" - # Disable CPython switchers, we handle them now: - - def is_mac(): - return platform.system() == "Darwin" - run( - ["sed", "-i"] - + ([""] if is_mac() else []) - + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] - ) - self.version.setup_indexsidebar( - self.versions, - self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", - ) + if self.includes_html: + # Disable CPython switchers, we handle them now: + run( + ["sed", "-i"] + + ([""] if sys.platform == "darwin" else []) + + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] + ) + self.version.setup_indexsidebar( + self.versions, + self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", + ) run_with_logging( [ "make", @@ -767,9 +769,10 @@ def is_mac(): ) run(["mkdir", "-p", self.log_directory]) run(["chgrp", "-R", self.group, self.log_directory]) - setup_switchers( - self.versions, self.languages, self.checkout / "Doc" / "build" / "html" - ) + if self.includes_html: + setup_switchers( + self.versions, self.languages, self.checkout / "Doc" / "build" / "html" + ) logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) def build_venv(self): @@ -817,42 +820,47 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: except subprocess.CalledProcessError as err: logging.warning("Can't change group of %s: %s", target, str(err)) - changed = changed_files(self.checkout / "Doc" / "build" / "html", target) - logging.info("Copying HTML files to %s", target) - run( - [ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "build" / "html/", - ] - ) - run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) - run( - [ - "find", - self.checkout / "Doc" / "build" / "html", - "-type", - "d", - "-exec", - "chmod", - "o+x", - "{}", - ";", - ] - ) - run( - [ - "rsync", - "-a", - "--delete-delay", - "--filter", - "P archives/", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - ] - ) + changed = [] + if self.includes_html: + # Copy built HTML files to webroot (default /srv/docs.python.org) + changed = changed_files(self.checkout / "Doc" / "build" / "html", target) + logging.info("Copying HTML files to %s", target) + run( + [ + "chown", + "-R", + ":" + self.group, + self.checkout / "Doc" / "build" / "html/", + ] + ) + run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) + run( + [ + "find", + self.checkout / "Doc" / "build" / "html", + "-type", + "d", + "-exec", + "chmod", + "o+x", + "{}", + ";", + ] + ) + run( + [ + "rsync", + "-a", + "--delete-delay", + "--filter", + "P archives/", + str(self.checkout / "Doc" / "build" / "html") + "/", + target, + ] + ) + if not self.quick: + # Copy archive files to /archives/ logging.debug("Copying dist files.") run( [ From be14d9554893b106e852cebda5e0149d89445201 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:25:29 +0100 Subject: [PATCH 042/176] Set the ``state.toml`` filename based upon ``--select-output`` (#206) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- build_docs.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 033ef0f..51bfe42 100755 --- a/build_docs.py +++ b/build_docs.py @@ -937,7 +937,10 @@ def should_rebuild(self): return False def load_state(self) -> dict: - state_file = self.build_root / "state.toml" + if self.select_output is not None: + state_file = self.build_root / f"state-{self.select_output}.toml" + else: + state_file = self.build_root / "state.toml" try: return tomlkit.loads(state_file.read_text(encoding="UTF-8"))[ f"/{self.language.tag}/{self.version.name}/" @@ -950,7 +953,10 @@ def save_state(self, build_start: dt, build_duration: float, trigger: str): Using this we can deduce if a rebuild is needed or not. """ - state_file = self.build_root / "state.toml" + if self.select_output is not None: + state_file = self.build_root / f"state-{self.select_output}.toml" + else: + state_file = self.build_root / "state.toml" try: states = tomlkit.parse(state_file.read_text(encoding="UTF-8")) except FileNotFoundError: From caee2be0bf028abe1644c49a9997f5814aa56957 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:26:47 +0100 Subject: [PATCH 043/176] Use saved state information in ``check_times.py`` (#205) --- check_times.py | 84 +++++++++++++++++++++++--------------------------- 1 file changed, 38 insertions(+), 46 deletions(-) diff --git a/check_times.py b/check_times.py index 3b3e53c..07a1af6 100644 --- a/check_times.py +++ b/check_times.py @@ -1,17 +1,17 @@ """Check the frequency of the rebuild loop. -This must be run in a directory that has the ``docsbuild.log*`` files. +This must be run in a directory that has the ``docsbuild*`` log files. For example: .. code-block:: bash $ mkdir -p docsbuild-logs - $ scp "adam@docs.nyc1.psf.io:/var/log/docsbuild/docsbuild.log*" docsbuild-logs/ + $ scp "adam@docs.nyc1.psf.io:/var/log/docsbuild/docsbuild*" docsbuild-logs/ $ python check_times.py """ -import datetime as dt import gzip +import tomllib from pathlib import Path from build_docs import format_seconds @@ -19,21 +19,21 @@ LOGS_ROOT = Path("docsbuild-logs").resolve() -def get_lines() -> list[str]: +def get_lines(filename: str = "docsbuild.log") -> list[str]: lines = [] - zipped_logs = list(LOGS_ROOT.glob("docsbuild.log.*.gz")) + zipped_logs = list(LOGS_ROOT.glob(f"{filename}.*.gz")) zipped_logs.sort(key=lambda p: int(p.name.split(".")[-2]), reverse=True) for logfile in zipped_logs: with gzip.open(logfile, "rt", encoding="utf-8") as f: lines += f.readlines() - with open(LOGS_ROOT / "docsbuild.log", encoding="utf-8") as f: + with open(LOGS_ROOT / filename, encoding="utf-8") as f: lines += f.readlines() return lines def calc_time(lines: list[str]) -> None: - start = end = language = version = start_timestamp = None - reason = lang_ver = "" + in_progress = False + in_progress_line = "" print("Start | Version | Language | Build | Trigger") print(":-- | :--: | :--: | --: | :--:") @@ -41,50 +41,42 @@ def calc_time(lines: list[str]) -> None: for line in lines: line = line.strip() - if ": Should rebuild: " in line: - if "no previous state found" in line: - reason = "brand new" - elif "new translations" in line: - reason = "translation" - elif "Doc/ has changed" in line: - reason = "docs" - else: - reason = "" - lang_ver = line.split(" ")[3].removesuffix(":") - - if line.endswith("Build start."): - timestamp = line[:23].replace(",", ".") - language, version = line.split(" ")[3].removesuffix(":").split("/") - start = dt.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") - start_timestamp = f"{line[:16]} UTC" - - if start and ": Build done " in line: - timestamp = line[:23].replace(",", ".") - language, version = line.split(" ")[3].removesuffix(":").split("/") - end = dt.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") - - if start and end: - duration = (end - start).total_seconds() - fmt_duration = format_seconds(duration) - if lang_ver != f"{language}/{version}": - reason = "" + if "Saved new rebuild state for" in line: + _, state = line.split("Saved new rebuild state for", 1) + key, state_toml = state.strip().split(": ", 1) + language, version = key.strip("/").split("/", 1) + state_data = tomllib.loads(f"t = {state_toml}")["t"] + start = state_data["last_build_start"] + fmt_duration = format_seconds(state_data["last_build_duration"]) + reason = state_data["triggered_by"] print( - f"{start_timestamp: <20} | {version: <7} | {language: <8} | {fmt_duration :<14} | {reason}" + f"{start:%Y-%m-%d %H:%M UTC} | {version: <7} | {language: <8} | {fmt_duration :<14} | {reason}" ) - start = end = start_timestamp = None - if ": Full build done" in line: - timestamp = f"{line[:16]} UTC" - _, fmt_duration = line.removesuffix(").").split("(") - print( - f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration :<14} | -----------" - ) + if line.endswith("Build start."): + in_progress = True + in_progress_line = line + + if in_progress and ": Build done " in line: + in_progress = False - if start and end is None: + if in_progress: + start_timestamp = f"{in_progress_line[:16]} UTC" + language, version = in_progress_line.split(" ")[3].removesuffix(":").split("/") print( - f"{start_timestamp: <20} | {version: <7} | {language: <8} | In progress... | {reason}" + f"{start_timestamp: <20} | {version: <7} | {language: <8} | In progress... | ..." ) + print() + if __name__ == "__main__": - calc_time(get_lines()) + print("Build times (HTML only)") + print("=======================") + print() + calc_time(get_lines("docsbuild-only-html.log")) + + print("Build times (no HTML)") + print("=====================") + print() + calc_time(get_lines("docsbuild-no-html.log")) From fb5c33d3bae3952e12a45b7f19113352514a231c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:30:52 +0100 Subject: [PATCH 044/176] Skip non-HTML builds for HTML-only languages (#208) --- build_docs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_docs.py b/build_docs.py index 51bfe42..e7b8129 100755 --- a/build_docs.py +++ b/build_docs.py @@ -640,6 +640,9 @@ def run(self, http: urllib3.PoolManager) -> bool: start_timestamp = dt.now(tz=timezone.utc).replace(microsecond=0) logging.info("Running.") try: + if self.language.html_only and not self.includes_html: + logging.info("Skipping non-HTML build (language is HTML-only).") + return True self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() From 38645de81da2d368744400bb78650db3012d6cb9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 2 Oct 2024 22:56:55 +0100 Subject: [PATCH 045/176] Include full build duration (#210) Inadvertently removed. --- check_times.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/check_times.py b/check_times.py index 07a1af6..e561b09 100644 --- a/check_times.py +++ b/check_times.py @@ -60,6 +60,13 @@ def calc_time(lines: list[str]) -> None: if in_progress and ": Build done " in line: in_progress = False + if ": Full build done" in line: + timestamp = f"{line[:16]} UTC" + _, fmt_duration = line.removesuffix(").").split("(") + print( + f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration :<14} | -----------" + ) + if in_progress: start_timestamp = f"{in_progress_line[:16]} UTC" language, version = in_progress_line.split(" ")[3].removesuffix(":").split("/") From a1c7fb8395538eb850ccf018de47c86486f05b3e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 3 Oct 2024 00:12:51 +0100 Subject: [PATCH 046/176] Rebuild on changes to Misc/NEWS.d/ (#211) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index e7b8129..5d3ed4d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -929,7 +929,7 @@ def should_rebuild(self): diff = self.cpython_repo.run( "diff", "--name-only", state["cpython_sha"], cpython_sha ).stdout - if "Doc/" in diff: + if "Doc/" in diff or "Misc/NEWS.d/" in diff: logging.info( "Should rebuild: Doc/ has changed (from %s to %s)", state["cpython_sha"], From 566ca52820fc47b4dd4eb2ba28ecde449a488bec Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 3 Oct 2024 00:13:25 +0100 Subject: [PATCH 047/176] Run ``sphinx-build`` with the default verbosity level (#212) This lets us see which steps are taking more time in sphinx. Quiet mode was originally added in 8d82617d78e25448b379003a811629b7329209b1. --- build_docs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 5d3ed4d..27fee1e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -704,7 +704,6 @@ def build(self): logging.info("Build start.") start_time = perf_counter() sphinxopts = list(self.language.sphinxopts) - sphinxopts.extend(["-q"]) if self.language.tag != "en": locale_dirs = self.build_root / self.version.name / "locale" sphinxopts.extend( From 3a11a361dd38134a9d5c995a1c9a0b415c831e5d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 6 Oct 2024 16:35:38 +0100 Subject: [PATCH 048/176] Harmonise run() and run_with_logging() (#213) --- build_docs.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/build_docs.py b/build_docs.py index 27fee1e..c7122af 100755 --- a/build_docs.py +++ b/build_docs.py @@ -41,7 +41,6 @@ from datetime import datetime as dt, timezone from pathlib import Path from string import Template -from textwrap import indent from time import perf_counter, sleep from typing import Iterable, Literal from urllib.parse import urljoin @@ -217,7 +216,7 @@ def filter(languages, language_tags=None): def run(cmd, cwd=None) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" - cmd = [str(arg) for arg in cmd] + cmd = list(map(str, cmd)) cmdstring = shlex.join(cmd) logging.debug("Run: '%s'", cmdstring) result = subprocess.run( @@ -233,9 +232,9 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: if result.returncode: # Log last 20 lines, those are likely the interesting ones. logging.error( - "Run: %r KO:\n%s", + "Run: '%s' KO:\n%s", cmdstring, - indent("\n".join(result.stdout.split("\n")[-20:]), " "), + "\n".join(f" {line}" for line in result.stdout.split("\n")[-20:]), ) result.check_returncode() return result @@ -244,7 +243,7 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: def run_with_logging(cmd, cwd=None): """Like subprocess.check_call, with logging before the command execution.""" cmd = list(map(str, cmd)) - logging.debug("Run: %s", shlex.join(cmd)) + logging.debug("Run: '%s'", shlex.join(cmd)) with subprocess.Popen( cmd, cwd=cwd, @@ -255,7 +254,7 @@ def run_with_logging(cmd, cwd=None): ) as p: try: for line in p.stdout or (): - logging.debug(">>>> %s", line.rstrip()) + logging.debug(">>>> %s", line.rstrip()) except: p.kill() raise From dcb1aa07594eb6600cb40fb7536351caa8ac3403 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:57:33 +0100 Subject: [PATCH 049/176] Add 3.8 to robots.txt (#214) --- templates/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/robots.txt b/templates/robots.txt index 635bfdc..a7de4a6 100644 --- a/templates/robots.txt +++ b/templates/robots.txt @@ -23,3 +23,4 @@ Disallow: /3.4/ Disallow: /3.5/ Disallow: /3.6/ Disallow: /3.7/ +Disallow: /3.8/ From 65c2c305b0d6863f871bb954814cf061ca5855ec Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 9 Oct 2024 21:43:37 +0100 Subject: [PATCH 050/176] Simplify copying robots.txt (#215) --- build_docs.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/build_docs.py b/build_docs.py index c7122af..6541070 100755 --- a/build_docs.py +++ b/build_docs.py @@ -415,23 +415,19 @@ def setup_switchers( ofile.write(line) -def build_robots_txt( - versions: Iterable[Version], - languages: Iterable[Language], +def copy_robots_txt( www_root: Path, group, skip_cache_invalidation, http: urllib3.PoolManager, ) -> None: - """Disallow crawl of EOL versions in robots.txt.""" + """Copy robots.txt to www_root.""" if not www_root.exists(): - logging.info("Skipping robots.txt generation (www root does not even exist).") + logging.info("Skipping copying robots.txt (www root does not even exist).") return template_path = HERE / "templates" / "robots.txt" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render(languages=languages, versions=versions) robots_path = www_root / "robots.txt" - robots_path.write_text(rendered_template + "\n", encoding="UTF-8") + shutil.copyfile(template_path, robots_path) robots_path.chmod(0o775) run(["chgrp", group, robots_path]) if not skip_cache_invalidation: @@ -1204,9 +1200,7 @@ def build_docs(args) -> bool: build_sitemap(versions, languages, args.www_root, args.group) build_404(args.www_root, args.group) - build_robots_txt( - versions, - languages, + copy_robots_txt( args.www_root, args.group, args.skip_cache_invalidation, From 743a3a35fb04dd8b67ea8b0fdfc4fd47e876e774 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 10 Oct 2024 20:56:15 +0100 Subject: [PATCH 051/176] Simplify `Version.requirements` (#218) --- build_docs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 6541070..ffae5be 100755 --- a/build_docs.py +++ b/build_docs.py @@ -110,10 +110,8 @@ def requirements(self): """ if self.name == "3.5": return ["jieba", "blurb", "sphinx==1.8.4", "jinja2<3.1", "docutils<=0.17.1"] - if self.name in ("3.7", "3.6", "2.7"): + if self.name in {"3.7", "3.6", "2.7"}: return ["jieba", "blurb", "sphinx==2.3.1", "jinja2<3.1", "docutils<=0.17.1"] - if self.name == ("3.8", "3.9"): - return ["jieba", "blurb", "sphinx==2.4.4", "jinja2<3.1", "docutils<=0.17.1"] return [ "jieba", # To improve zh search. From 134f57cda62cbfb70340daaf80d27cd0c9472f22 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:00:15 +0100 Subject: [PATCH 052/176] Use PyStemmer (#217) PyStemmer exposes bindings to libstemmer_c, the core Snowball library written in C. This can improve performance of word stemming. --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index ffae5be..a744a42 100755 --- a/build_docs.py +++ b/build_docs.py @@ -115,6 +115,7 @@ def requirements(self): return [ "jieba", # To improve zh search. + "PyStemmer~=2.2.0", # To improve performance for word stemming. "-rrequirements.txt", ] From 5ac8ae9ae00951246c746616434f3e20ccac8d9a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 10 Oct 2024 23:20:29 +0100 Subject: [PATCH 053/176] Add an HTML-only (English) build variant (#219) --- build_docs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index a744a42..207250b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -504,7 +504,7 @@ def parse_args(): ) parser.add_argument( "--select-output", - choices=("no-html", "only-html"), + choices=("no-html", "only-html", "only-html-en"), help="Choose what outputs to build.", ) parser.add_argument( @@ -610,7 +610,7 @@ class DocBuilder: cpython_repo: Repository build_root: Path www_root: Path - select_output: Literal["no-html", "only-html"] | None + select_output: Literal["no-html", "only-html", "only-html-en"] | None quick: bool group: str log_directory: Path @@ -620,7 +620,9 @@ class DocBuilder: @property def html_only(self): return ( - self.select_output == "only-html" or self.quick or self.language.html_only + self.select_output in {"only-html", "only-html-en"} + or self.quick + or self.language.html_only ) @property @@ -1245,6 +1247,8 @@ def main(): build_docs_with_lock(args, "build_docs_archives.lock") elif args.select_output == "only-html": build_docs_with_lock(args, "build_docs_html.lock") + elif args.select_output == "only-html-en": + build_docs_with_lock(args, "build_docs_html_en.lock") def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: From 2406204b5e24743309a2257de4e66ffda9056975 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Oct 2024 03:43:46 +0100 Subject: [PATCH 054/176] Add English HTML-only to check_times --- check_times.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/check_times.py b/check_times.py index e561b09..9310cd4 100644 --- a/check_times.py +++ b/check_times.py @@ -78,6 +78,11 @@ def calc_time(lines: list[str]) -> None: if __name__ == "__main__": + print("Build times (HTML only; English)") + print("=======================") + print() + calc_time(get_lines("docsbuild-only-html-en.log")) + print("Build times (HTML only)") print("=======================") print() From e6c58e56af21837e6874f34457b3db1ca93ba3dd Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Oct 2024 03:49:26 +0100 Subject: [PATCH 055/176] Update versions in README.md --- README.md | 13 ++++++------- check_versions.py | 4 +++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d5beb39..5176cd0 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ of Sphinx we're using where: 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' - 3.12 ø sphinx~=8.0.0 needs_sphinx='6.2.1' - 3.13 ø sphinx~=8.0.0 needs_sphinx='6.2.1' - 3.14 ø sphinx~=8.0.0 needs_sphinx='6.2.1' + 3.12 ø sphinx~=8.1.0 needs_sphinx='6.2.1' + 3.13 ø sphinx~=8.1.0 needs_sphinx='6.2.1' + 3.14 ø sphinx~=8.1.0 needs_sphinx='6.2.1' ========= ============= ================== ==================== Sphinx build as seen on docs.python.org: @@ -44,11 +44,10 @@ of Sphinx we're using where: ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= version en es fr id it ja ko pl pt-br tr uk zh-cn zh-tw ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= - 3.8 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 - 3.12 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 - 3.13 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 - 3.14 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 + 3.12 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 + 3.13 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 + 3.14 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= diff --git a/check_versions.py b/check_versions.py index 63ffab5..70cade9 100644 --- a/check_versions.py +++ b/check_versions.py @@ -7,13 +7,15 @@ import re import httpx +import urllib3 from tabulate import tabulate import git import build_docs logger = logging.getLogger(__name__) -VERSIONS = build_docs.parse_versions_from_devguide() +http = urllib3.PoolManager() +VERSIONS = build_docs.parse_versions_from_devguide(http) LANGUAGES = build_docs.parse_languages_from_config() From cfae93d691b94533af11dcd38bb98f2ecae00926 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 20 Oct 2024 01:36:49 +0100 Subject: [PATCH 056/176] Update versions in README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5176cd0..056f73d 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ of Sphinx we're using where: 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' - 3.12 ø sphinx~=8.1.0 needs_sphinx='6.2.1' - 3.13 ø sphinx~=8.1.0 needs_sphinx='6.2.1' - 3.14 ø sphinx~=8.1.0 needs_sphinx='6.2.1' + 3.12 ø sphinx~=8.1.0 needs_sphinx='7.2.6' + 3.13 ø sphinx~=8.1.0 needs_sphinx='7.2.6' + 3.14 ø sphinx~=8.1.0 needs_sphinx='7.2.6' ========= ============= ================== ==================== Sphinx build as seen on docs.python.org: From a97ad6839868ba9781dc14fc1e0f5aa73fe63db8 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 20 Oct 2024 01:41:18 +0100 Subject: [PATCH 057/176] Update versions in README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 056f73d..767014c 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ of Sphinx we're using where: 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 - 3.12 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 - 3.13 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 - 3.14 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 + 3.12 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 + 3.13 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 + 3.14 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= From 949821476e5adc509873c7efba17872e97917b50 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:47:26 +0100 Subject: [PATCH 058/176] Add logging in post-build tasks (#221) --- build_docs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build_docs.py b/build_docs.py index 207250b..a7b4758 100755 --- a/build_docs.py +++ b/build_docs.py @@ -424,6 +424,7 @@ def copy_robots_txt( if not www_root.exists(): logging.info("Skipping copying robots.txt (www root does not even exist).") return + logging.info("Copying robots.txt...") template_path = HERE / "templates" / "robots.txt" robots_path = www_root / "robots.txt" shutil.copyfile(template_path, robots_path) @@ -440,6 +441,7 @@ def build_sitemap( if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exist).") return + logging.info("Starting sitemap generation...") template_path = HERE / "templates" / "sitemap.xml" template = jinja2.Template(template_path.read_text(encoding="UTF-8")) rendered_template = template.render(languages=languages, versions=versions) @@ -454,6 +456,7 @@ def build_404(www_root: Path, group): if not www_root.exists(): logging.info("Skipping 404 page generation (www root does not even exist).") return + logging.info("Copying 404 page...") not_found_file = www_root / "404.html" shutil.copyfile(HERE / "templates" / "404.html", not_found_file) not_found_file.chmod(0o664) @@ -1022,6 +1025,7 @@ def major_symlinks( - /fr/3/ → /fr/3.9/ - /es/3/ → /es/3.9/ """ + logging.info("Creating major version symlinks...") current_stable = Version.current_stable(versions).name for language in languages: symlink( @@ -1051,6 +1055,7 @@ def dev_symlink( - /fr/dev/ → /fr/3.11/ - /es/dev/ → /es/3.11/ """ + logging.info("Creating development version symlinks...") current_dev = Version.current_dev(versions).name for language in languages: symlink( @@ -1096,6 +1101,7 @@ def proofread_canonicals( - /3.11/whatsnew/3.11.html typically would link to /3/whatsnew/3.11.html, which may not exist yet. """ + logging.info("Checking canonical links...") canonical_re = re.compile( """""" ) From bbf1112ac03e2045dfbcd76d04c99c5765fd35db Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:27:39 +0100 Subject: [PATCH 059/176] Purge using Surrogate-Key headers (#220) --- build_docs.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/build_docs.py b/build_docs.py index a7b4758..fefcb15 100755 --- a/build_docs.py +++ b/build_docs.py @@ -31,7 +31,7 @@ import logging import logging.handlers from functools import total_ordering -from os import readlink +from os import getenv, readlink import re import shlex import shutil @@ -895,13 +895,8 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: logging.info("%s files changed", len(changed)) if changed and not self.skip_cache_invalidation: - targets_dir = str(self.www_root) - prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout - prefixes = prefixes.replace(targets_dir + "/", "") - prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix] - purge(http, *prefixes) - for prefix in prefixes: - purge(http, *[prefix + p for p in changed]) + surrogate_key = f"{self.language.tag}/{self.version.name}" + purge_surrogate_key(http, surrogate_key) logging.info( "Publishing done (%s).", format_seconds(perf_counter() - start_time) ) @@ -1007,7 +1002,8 @@ def symlink( link.symlink_to(directory) run(["chown", "-h", ":" + group, str(link)]) if not skip_cache_invalidation: - purge_path(http, www_root, link) + surrogate_key = f"{language.tag}/{name}" + purge_surrogate_key(http, surrogate_key) def major_symlinks( @@ -1081,14 +1077,25 @@ def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: http.request("PURGE", url, timeout=30) -def purge_path(http: urllib3.PoolManager, www_root: Path, path: Path) -> None: - """Recursively remove a path from docs.python.org's CDN. +def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: + """Remove paths from docs.python.org's CDN. + All paths matching the given 'Surrogate-Key' will be removed. + This is set by the Nginx server for every language-version pair. To be used when a directory changes, so the CDN fetches the new one. + + https://www.fastly.com/documentation/reference/api/purging/#purge-tag """ - purge(http, *[file.relative_to(www_root) for file in path.glob("**/*")]) - purge(http, path.relative_to(www_root)) - purge(http, str(path.relative_to(www_root)) + "/") + service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") + fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") + + logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) + http.request( + "POST", + f"https://api.fastly.com/service/{service_id}/purge/{surrogate_key}", + headers={"Fastly-Key": fastly_key}, + timeout=30, + ) def proofread_canonicals( From d708c717b4cbd22d12adb125d5354e7e11c21343 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 24 Oct 2024 19:19:14 +0100 Subject: [PATCH 060/176] Always purge symlinks --- build_docs.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/build_docs.py b/build_docs.py index fefcb15..9f26bf9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -995,12 +995,14 @@ def symlink( directory_path = path / directory if not directory_path.exists(): return # No touching link, dest doc not built yet. - if link.exists() and readlink(link) == directory: - return # Link is already pointing to right doc. - if link.exists(): - link.unlink() - link.symlink_to(directory) - run(["chown", "-h", ":" + group, str(link)]) + + link_exists = link.exists() + if not link_exists or readlink(link) != directory: + # Link does not exist or points to the wrong target. + if link_exists: + link.unlink() + link.symlink_to(directory) + run(["chown", "-h", f":{group}", str(link)]) if not skip_cache_invalidation: surrogate_key = f"{language.tag}/{name}" purge_surrogate_key(http, surrogate_key) From e0c1fed030ce4e1d92426f2e45e615b3be621ac6 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 24 Oct 2024 20:15:07 +0100 Subject: [PATCH 061/176] Simplify Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- build_docs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 9f26bf9..658ba63 100755 --- a/build_docs.py +++ b/build_docs.py @@ -996,10 +996,9 @@ def symlink( if not directory_path.exists(): return # No touching link, dest doc not built yet. - link_exists = link.exists() - if not link_exists or readlink(link) != directory: + if not link.exists() or readlink(link) != directory: # Link does not exist or points to the wrong target. - if link_exists: + if link.exists(): link.unlink() link.symlink_to(directory) run(["chown", "-h", f":{group}", str(link)]) From 36f8c1ebf61f5d3c970566e2da0a273ff29ca3d2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:29:32 +0000 Subject: [PATCH 062/176] Further simplify symlink removal (#224) Co-authored-by: Ezio Melotti --- build_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 658ba63..6596a25 100755 --- a/build_docs.py +++ b/build_docs.py @@ -998,8 +998,7 @@ def symlink( if not link.exists() or readlink(link) != directory: # Link does not exist or points to the wrong target. - if link.exists(): - link.unlink() + link.unlink(missing_ok=True) link.symlink_to(directory) run(["chown", "-h", f":{group}", str(link)]) if not skip_cache_invalidation: From 1807c70aff5a050077ede04fe65c7689110bf5f1 Mon Sep 17 00:00:00 2001 From: Kerim Kabirov Date: Mon, 28 Oct 2024 16:31:05 +0100 Subject: [PATCH 063/176] Add context labels to the version switcher (#223) --- templates/switchers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 29204ae..999ca10 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -26,7 +26,7 @@ } function build_version_select(release) { - let buf = ['']; const major_minor = release.split(".").slice(0, 2).join("."); Object.entries(all_versions).forEach(function([version, title]) { @@ -42,7 +42,7 @@ } function build_language_select(current_language) { - let buf = ['']; Object.entries(all_languages).forEach(function([language, title]) { if (language === current_language) { From 7e274eb87a36acf9ca35e3f1671ee4eb2d87d9f3 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:47:52 +0000 Subject: [PATCH 064/176] Remove String.startsWith polyfill --- templates/switchers.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 999ca10..ad31a03 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,15 +1,6 @@ (function() { 'use strict'; - if (!String.prototype.startsWith) { - Object.defineProperty(String.prototype, 'startsWith', { - value: function(search, rawPos) { - const pos = rawPos > 0 ? rawPos|0 : 0; - return this.substring(pos, pos + search.length) === search; - } - }); - } - // Parses versions in URL segments like: // "3", "dev", "release/2.7" or "3.6rc2" const version_regexs = [ From 27e2dc7fbf61c48bd58c79152b5a0480c2d02ef1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:53:09 +0000 Subject: [PATCH 065/176] Remove create_placeholders_if_missing() The placeholders have been in the theme since v2021.5 --- templates/switchers.js | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index ad31a03..ffedb7d 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -132,41 +132,11 @@ return '' } - function create_placeholders_if_missing() { - const version_segment = version_segment_from_url(); - const language_segment = language_segment_from_url(); - const index = "/" + language_segment + version_segment; - - if (document.querySelectorAll('.version_switcher_placeholder').length > 0) { - return; - } - - const html = ' \ - \ -Documentation »'; - - const probable_places = [ - "body>div.related>ul>li:not(.right):contains('Documentation'):first", - "body>div.related>ul>li:not(.right):contains('documentation'):first", - ]; - - for (let i = 0; i < probable_places.length; i++) { - let probable_place = $(probable_places[i]); - if (probable_place.length == 1) { - probable_place.html(html); - document.getElementById('indexlink').href = index; - return; - } - } - } - document.addEventListener('DOMContentLoaded', function() { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - create_placeholders_if_missing(); - let placeholders = document.querySelectorAll('.version_switcher_placeholder'); placeholders.forEach(function(placeholder) { placeholder.innerHTML = version_select; From 75594be5806eb6ed91aecac46a9a4036f734dc02 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:54:10 +0000 Subject: [PATCH 066/176] Use exact comparison operators --- templates/switchers.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index ffedb7d..664db12 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -55,7 +55,7 @@ function navigate_to_first_existing(urls) { // Navigate to the first existing URL in urls. const url = urls.shift(); - if (urls.length == 0 || url.startsWith("file:///")) { + if (urls.length === 0 || url.startsWith("file:///")) { window.location.href = url; return; } @@ -79,7 +79,7 @@ const current_version = version_segment_from_url(); const new_url = url.replace('/' + current_language + current_version, '/' + current_language + selected_version); - if (new_url != url) { + if (new_url !== url) { navigate_to_first_existing([ new_url, url.replace('/' + current_language + current_version, @@ -96,11 +96,11 @@ const url = window.location.href; const current_language = language_segment_from_url(); const current_version = version_segment_from_url(); - if (selected_language == 'en/') // Special 'default' case for English. + if (selected_language === 'en/') // Special 'default' case for English. selected_language = ''; let new_url = url.replace('/' + current_language + current_version, '/' + selected_language + current_version); - if (new_url != url) { + if (new_url !== url) { navigate_to_first_existing([ new_url, '/' From 8a71e4273a287deb62e8ed84cc9d837a1a13a0f4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:56:33 +0000 Subject: [PATCH 067/176] Remove an unneeded variable --- templates/switchers.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 664db12..70997e1 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -135,10 +135,9 @@ document.addEventListener('DOMContentLoaded', function() { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - let placeholders = document.querySelectorAll('.version_switcher_placeholder'); - placeholders.forEach(function(placeholder) { + const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); + document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { placeholder.innerHTML = version_select; let selectElement = placeholder.querySelector('select'); @@ -146,9 +145,7 @@ }); const language_select = build_language_select(current_language); - - placeholders = document.querySelectorAll('.language_switcher_placeholder'); - placeholders.forEach(function(placeholder) { + document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { placeholder.innerHTML = language_select; let selectElement = placeholder.querySelector('select'); From ce8b4742aa957612ce6734b78bdf6e242a6bd43c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:57:55 +0000 Subject: [PATCH 068/176] Remove anonymous function (IIFE) --- templates/switchers.js | 284 ++++++++++++++++++++--------------------- 1 file changed, 141 insertions(+), 143 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 70997e1..d436d5e 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,155 +1,153 @@ -(function() { - 'use strict'; - - // Parses versions in URL segments like: - // "3", "dev", "release/2.7" or "3.6rc2" - const version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; - - const all_versions = $VERSIONS; - const all_languages = $LANGUAGES; - - function quote_attr(str) { - return '"' + str.replace('"', '\\"') + '"'; - } +'use strict'; + +// Parses versions in URL segments like: +// "3", "dev", "release/2.7" or "3.6rc2" +const version_regexs = [ + '(?:\\d)', + '(?:\\d\\.\\d[\\w\\d\\.]*)', + '(?:dev)', + '(?:release/\\d.\\d[\\x\\d\\.]*)']; + +const all_versions = $VERSIONS; +const all_languages = $LANGUAGES; + +function quote_attr(str) { + return '"' + str.replace('"', '\\"') + '"'; +} + +function build_version_select(release) { + let buf = ['']; - const major_minor = release.split(".").slice(0, 2).join("."); + buf.push(''); + return buf.join(''); +} - Object.entries(all_versions).forEach(function([version, title]) { - if (version === major_minor) { - buf.push(''); - } else { - buf.push(''); - } - }); +function build_language_select(current_language) { + let buf = [''); - return buf.join(''); + Object.entries(all_languages).forEach(function([language, title]) { + if (language === current_language) { + buf.push(''); + } else { + buf.push(''); + } + }); + if (!(current_language in all_languages)) { + // In case we're browsing a language that is not yet in all_languages. + buf.push(''); + all_languages[current_language] = current_language; } - - function build_language_select(current_language) { - let buf = [''); + return buf.join(''); +} + +function navigate_to_first_existing(urls) { + // Navigate to the first existing URL in urls. + const url = urls.shift(); + if (urls.length === 0 || url.startsWith("file:///")) { + window.location.href = url; + return; + } + fetch(url) + .then(function(response) { + if (response.ok) { + window.location.href = url; } else { - buf.push(''); + navigate_to_first_existing(urls); } + }) + .catch(function(error) { + navigate_to_first_existing(urls); }); - if (!(current_language in all_languages)) { - // In case we're browsing a language that is not yet in all_languages. - buf.push(''); - all_languages[current_language] = current_language; - } - buf.push(''); - return buf.join(''); - } - - function navigate_to_first_existing(urls) { - // Navigate to the first existing URL in urls. - const url = urls.shift(); - if (urls.length === 0 || url.startsWith("file:///")) { - window.location.href = url; - return; - } - fetch(url) - .then(function(response) { - if (response.ok) { - window.location.href = url; - } else { - navigate_to_first_existing(urls); - } - }) - .catch(function(error) { - navigate_to_first_existing(urls); - }); - } - - function on_version_switch() { - const selected_version = this.options[this.selectedIndex].value + '/'; - const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); - const new_url = url.replace('/' + current_language + current_version, - '/' + current_language + selected_version); - if (new_url !== url) { - navigate_to_first_existing([ - new_url, - url.replace('/' + current_language + current_version, - '/' + selected_version), - '/' + current_language + selected_version, - '/' + selected_version, - '/' - ]); - } - } - - function on_language_switch() { - let selected_language = this.options[this.selectedIndex].value + '/'; - const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); - if (selected_language === 'en/') // Special 'default' case for English. - selected_language = ''; - let new_url = url.replace('/' + current_language + current_version, - '/' + selected_language + current_version); - if (new_url !== url) { - navigate_to_first_existing([ - new_url, - '/' - ]); - } - } - - // Returns the path segment of the language as a string, like 'fr/' - // or '' if not found. - function language_segment_from_url() { - const path = window.location.pathname; - const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' - const match = path.match(language_regexp); - if (match !== null) - return match[1]; - return ''; +} + +function on_version_switch() { + const selected_version = this.options[this.selectedIndex].value + '/'; + const url = window.location.href; + const current_language = language_segment_from_url(); + const current_version = version_segment_from_url(); + const new_url = url.replace('/' + current_language + current_version, + '/' + current_language + selected_version); + if (new_url !== url) { + navigate_to_first_existing([ + new_url, + url.replace('/' + current_language + current_version, + '/' + selected_version), + '/' + current_language + selected_version, + '/' + selected_version, + '/' + ]); } - - // Returns the path segment of the version as a string, like '3.6/' - // or '' if not found. - function version_segment_from_url() { - const path = window.location.pathname; - const language_segment = language_segment_from_url(); - const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - const version_regexp = language_segment + '(' + version_segment + ')'; - const match = path.match(version_regexp); - if (match !== null) - return match[1]; - return '' +} + +function on_language_switch() { + let selected_language = this.options[this.selectedIndex].value + '/'; + const url = window.location.href; + const current_language = language_segment_from_url(); + const current_version = version_segment_from_url(); + if (selected_language === 'en/') // Special 'default' case for English. + selected_language = ''; + let new_url = url.replace('/' + current_language + current_version, + '/' + selected_language + current_version); + if (new_url !== url) { + navigate_to_first_existing([ + new_url, + '/' + ]); } +} + +// Returns the path segment of the language as a string, like 'fr/' +// or '' if not found. +function language_segment_from_url() { + const path = window.location.pathname; + const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' + const match = path.match(language_regexp); + if (match !== null) + return match[1]; + return ''; +} + +// Returns the path segment of the version as a string, like '3.6/' +// or '' if not found. +function version_segment_from_url() { + const path = window.location.pathname; + const language_segment = language_segment_from_url(); + const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; + const version_regexp = language_segment + '(' + version_segment + ')'; + const match = path.match(version_regexp); + if (match !== null) + return match[1]; + return '' +} + +document.addEventListener('DOMContentLoaded', function() { + const language_segment = language_segment_from_url(); + const current_language = language_segment.replace(/\/+$/g, '') || 'en'; + + const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); + document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { + placeholder.innerHTML = version_select; + + let selectElement = placeholder.querySelector('select'); + selectElement.addEventListener('change', on_version_switch); + }); - document.addEventListener('DOMContentLoaded', function() { - const language_segment = language_segment_from_url(); - const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - - const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = version_select; - - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_version_switch); - }); - - const language_select = build_language_select(current_language); - document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = language_select; + const language_select = build_language_select(current_language); + document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { + placeholder.innerHTML = language_select; - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_language_switch); - }); + let selectElement = placeholder.querySelector('select'); + selectElement.addEventListener('change', on_language_switch); }); -})(); +}); From cdcd52305ee0b7b6d88f2cf859ae5d2f37e2ddd3 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:59:28 +0000 Subject: [PATCH 069/176] Name the initialisation function --- templates/switchers.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index d436d5e..8bdf3d5 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -130,8 +130,7 @@ function version_segment_from_url() { return match[1]; return '' } - -document.addEventListener('DOMContentLoaded', function() { +const _initialise_switchers = () => { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; @@ -150,4 +149,10 @@ document.addEventListener('DOMContentLoaded', function() { let selectElement = placeholder.querySelector('select'); selectElement.addEventListener('change', on_language_switch); }); -}); +}; + +if (document.readyState !== 'loading') { + _initialise_switchers(); +} else { + document.addEventListener('DOMContentLoaded', _initialise_switchers); +} From 84a3f9c5d8794fa20e738ddf195d2f6c0abb36f7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:09:38 +0000 Subject: [PATCH 070/176] Construct DOM nodes instead of parsing HTML --- templates/switchers.js | 90 ++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 8bdf3d5..094c2ce 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -15,41 +15,45 @@ function quote_attr(str) { return '"' + str.replace('"', '\\"') + '"'; } -function build_version_select(release) { - let buf = [''); - return buf.join(''); -} + select.add(option); + } -function build_language_select(current_language) { - let buf = [''); - return buf.join(''); -} + + const select = document.createElement('select'); + select.className = 'language-select'; + + for (const [language, title] in all_languages) { + const option = document.createElement('option'); + option.value = language; + option.text = title; + if (language === current_language) option.selected = true; + select.add(option); + } + + return select; +}; function navigate_to_first_existing(urls) { // Navigate to the first existing URL in urls. @@ -134,21 +138,23 @@ const _initialise_switchers = () => { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = version_select; - - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_version_switch); - }); - - const language_select = build_language_select(current_language); - document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = language_select; + const version_select = _create_version_select(DOCUMENTATION_OPTIONS.VERSION); + document + .querySelectorAll('.version_switcher_placeholder') + .forEach((placeholder) => { + const s = version_select.cloneNode(true); + s.addEventListener('change', on_version_switch); + placeholder.append(s); + }); - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_language_switch); - }); + const language_select = _create_language_select(current_language); + document + .querySelectorAll('.language_switcher_placeholder') + .forEach((placeholder) => { + const s = language_select.cloneNode(true); + s.addEventListener('change', on_language_switch); + placeholder.append(s); + }); }; if (document.readyState !== 'loading') { From 2a5adf99c1af5d8b2f1874c696651ca72c2a0518 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:18:22 +0000 Subject: [PATCH 071/176] Use arrow functions --- templates/switchers.js | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 094c2ce..b5b46ba 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -55,7 +55,7 @@ const _create_language_select = (current_language) => { return select; }; -function navigate_to_first_existing(urls) { +const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. const url = urls.shift(); if (urls.length === 0 || url.startsWith("file:///")) { @@ -63,19 +63,20 @@ function navigate_to_first_existing(urls) { return; } fetch(url) - .then(function(response) { + .then((response) => { if (response.ok) { window.location.href = url; } else { navigate_to_first_existing(urls); } }) - .catch(function(error) { + .catch((err) => { + void err; navigate_to_first_existing(urls); }); -} +}; -function on_version_switch() { +const _on_version_switch = () => { const selected_version = this.options[this.selectedIndex].value + '/'; const url = window.location.href; const current_language = language_segment_from_url(); @@ -83,7 +84,7 @@ function on_version_switch() { const new_url = url.replace('/' + current_language + current_version, '/' + current_language + selected_version); if (new_url !== url) { - navigate_to_first_existing([ + _navigate_to_first_existing([ new_url, url.replace('/' + current_language + current_version, '/' + selected_version), @@ -92,9 +93,9 @@ function on_version_switch() { '/' ]); } -} +}; -function on_language_switch() { +const _on_language_switch = () => { let selected_language = this.options[this.selectedIndex].value + '/'; const url = window.location.href; const current_language = language_segment_from_url(); @@ -104,12 +105,12 @@ function on_language_switch() { let new_url = url.replace('/' + current_language + current_version, '/' + selected_language + current_version); if (new_url !== url) { - navigate_to_first_existing([ + _navigate_to_first_existing([ new_url, '/' ]); } -} +}; // Returns the path segment of the language as a string, like 'fr/' // or '' if not found. @@ -143,7 +144,7 @@ const _initialise_switchers = () => { .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { const s = version_select.cloneNode(true); - s.addEventListener('change', on_version_switch); + s.addEventListener('change', _on_version_switch); placeholder.append(s); }); @@ -152,7 +153,7 @@ const _initialise_switchers = () => { .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { const s = language_select.cloneNode(true); - s.addEventListener('change', on_language_switch); + s.addEventListener('change', _on_language_switch); placeholder.append(s); }); }; From b4b36c0983a8693cdfdd743268630678c737b158 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:18:37 +0000 Subject: [PATCH 072/176] Remove unused quote_attr() function --- templates/switchers.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index b5b46ba..d70fc8f 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -11,10 +11,6 @@ const version_regexs = [ const all_versions = $VERSIONS; const all_languages = $LANGUAGES; -function quote_attr(str) { - return '"' + str.replace('"', '\\"') + '"'; -} - const _create_version_select = (release) => { const major_minor = release.split('.').slice(0, 2).join('.'); const select = document.createElement('select'); From 2cac9f627ffc927ade8c09b3b0eec45181ff7c46 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:25:08 +0000 Subject: [PATCH 073/176] Run prettier --- templates/switchers.js | 44 +++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index d70fc8f..1b7a71d 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -6,7 +6,8 @@ const version_regexs = [ '(?:\\d)', '(?:\\d\\.\\d[\\w\\d\\.]*)', '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; + '(?:release/\\d.\\d[\\x\\d\\.]*)', +]; const all_versions = $VERSIONS; const all_languages = $LANGUAGES; @@ -54,7 +55,7 @@ const _create_language_select = (current_language) => { const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. const url = urls.shift(); - if (urls.length === 0 || url.startsWith("file:///")) { + if (urls.length === 0 || url.startsWith('file:///')) { window.location.href = url; return; } @@ -77,16 +78,20 @@ const _on_version_switch = () => { const url = window.location.href; const current_language = language_segment_from_url(); const current_version = version_segment_from_url(); - const new_url = url.replace('/' + current_language + current_version, - '/' + current_language + selected_version); + const new_url = url.replace( + '/' + current_language + current_version, + '/' + current_language + selected_version, + ); if (new_url !== url) { _navigate_to_first_existing([ new_url, - url.replace('/' + current_language + current_version, - '/' + selected_version), + url.replace( + '/' + current_language + current_version, + '/' + selected_version, + ), '/' + current_language + selected_version, '/' + selected_version, - '/' + '/', ]); } }; @@ -96,15 +101,15 @@ const _on_language_switch = () => { const url = window.location.href; const current_language = language_segment_from_url(); const current_version = version_segment_from_url(); - if (selected_language === 'en/') // Special 'default' case for English. + if (selected_language === 'en/') + // Special 'default' case for English. selected_language = ''; - let new_url = url.replace('/' + current_language + current_version, - '/' + selected_language + current_version); + let new_url = url.replace( + '/' + current_language + current_version, + '/' + selected_language + current_version, + ); if (new_url !== url) { - _navigate_to_first_existing([ - new_url, - '/' - ]); + _navigate_to_first_existing([new_url, '/']); } }; @@ -112,10 +117,10 @@ const _on_language_switch = () => { // or '' if not found. function language_segment_from_url() { const path = window.location.pathname; - const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' + const language_regexp = + '/((?:' + Object.keys(all_languages).join('|') + ')/)'; const match = path.match(language_regexp); - if (match !== null) - return match[1]; + if (match !== null) return match[1]; return ''; } @@ -127,9 +132,8 @@ function version_segment_from_url() { const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; const version_regexp = language_segment + '(' + version_segment + ')'; const match = path.match(version_regexp); - if (match !== null) - return match[1]; - return '' + if (match !== null) return match[1]; + return ''; } const _initialise_switchers = () => { const language_segment = language_segment_from_url(); From 7b5e7730954e7b788195ecddb12f8f7056b92675 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:29:30 +0000 Subject: [PATCH 074/176] Make the regular expression for version parts a constant --- templates/switchers.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 1b7a71d..6b12e32 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,13 +1,14 @@ 'use strict'; // Parses versions in URL segments like: -// "3", "dev", "release/2.7" or "3.6rc2" -const version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)', -]; +const _VERSION_PATTERN = ( + '((?:' + + '(?:\\d)' // e.g. "3" + +'|(?:\\d\\.\\d[\\w\\d\\.]*)' // e.g. "3.6rc2" + +'|(?:dev)' // e.g. "dev" + +'|(?:release/\\d.\\d[\\x\\d\\.]*)'// e.g. "release/2.7" + + ')/)' +); const all_versions = $VERSIONS; const all_languages = $LANGUAGES; @@ -128,9 +129,7 @@ function language_segment_from_url() { // or '' if not found. function version_segment_from_url() { const path = window.location.pathname; - const language_segment = language_segment_from_url(); - const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - const version_regexp = language_segment + '(' + version_segment + ')'; + const version_regexp = language_segment_from_url() + _VERSION_PATTERN; const match = path.match(version_regexp); if (match !== null) return match[1]; return ''; From ab3b07962977ace9fea0c8c4034d53cb129872d0 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:05:32 +0000 Subject: [PATCH 075/176] Use constants from DOCUMENTATION_OPTIONS where possible --- templates/switchers.js | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 6b12e32..f98de05 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,5 +1,18 @@ 'use strict'; +const _CURRENT_VERSION = DOCUMENTATION_OPTIONS.VERSION; +const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; +const _CURRENT_PREFIX = (() => { + // Sphinx 7.2+ defines the content root data attribute in the HTML element. + const _CONTENT_ROOT = document.documentElement.dataset.content_root; + if (_CONTENT_ROOT !== undefined) { + return new URL(_CONTENT_ROOT, window.location).pathname; + } + // Fallback for older versions of Sphinx (used in Python 3.10 and older). + const _NUM_PREFIX_PARTS = _CURRENT_LANGUAGE === 'en' ? 2 : 3; + return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; +})(); + // Parses versions in URL segments like: const _VERSION_PATTERN = ( '((?:' @@ -77,20 +90,15 @@ const _navigate_to_first_existing = (urls) => { const _on_version_switch = () => { const selected_version = this.options[this.selectedIndex].value + '/'; const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); const new_url = url.replace( - '/' + current_language + current_version, - '/' + current_language + selected_version, + _CURRENT_PREFIX, + '/' + _CURRENT_LANGUAGE + selected_version, ); if (new_url !== url) { _navigate_to_first_existing([ new_url, - url.replace( - '/' + current_language + current_version, - '/' + selected_version, - ), - '/' + current_language + selected_version, + url.replace(_CURRENT_PREFIX, '/' + selected_version), + '/' + _CURRENT_LANGUAGE + selected_version, '/' + selected_version, '/', ]); @@ -100,14 +108,12 @@ const _on_version_switch = () => { const _on_language_switch = () => { let selected_language = this.options[this.selectedIndex].value + '/'; const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); if (selected_language === 'en/') // Special 'default' case for English. selected_language = ''; let new_url = url.replace( - '/' + current_language + current_version, - '/' + selected_language + current_version, + _CURRENT_PREFIX, + '/' + selected_language + _CURRENT_VERSION, ); if (new_url !== url) { _navigate_to_first_existing([new_url, '/']); @@ -135,10 +141,7 @@ function version_segment_from_url() { return ''; } const _initialise_switchers = () => { - const language_segment = language_segment_from_url(); - const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - - const version_select = _create_version_select(DOCUMENTATION_OPTIONS.VERSION); + const version_select = _create_version_select(_CURRENT_VERSION); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { @@ -147,7 +150,7 @@ const _initialise_switchers = () => { placeholder.append(s); }); - const language_select = _create_language_select(current_language); + const language_select = _create_language_select(_CURRENT_LANGUAGE); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { From a5936462d9811cd843ddb9337208272d996a098d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:06:10 +0000 Subject: [PATCH 076/176] Remove now-unused segment_from_url() functions --- templates/switchers.js | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index f98de05..600c955 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -13,16 +13,6 @@ const _CURRENT_PREFIX = (() => { return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; })(); -// Parses versions in URL segments like: -const _VERSION_PATTERN = ( - '((?:' - + '(?:\\d)' // e.g. "3" - +'|(?:\\d\\.\\d[\\w\\d\\.]*)' // e.g. "3.6rc2" - +'|(?:dev)' // e.g. "dev" - +'|(?:release/\\d.\\d[\\x\\d\\.]*)'// e.g. "release/2.7" - + ')/)' -); - const all_versions = $VERSIONS; const all_languages = $LANGUAGES; @@ -120,26 +110,6 @@ const _on_language_switch = () => { } }; -// Returns the path segment of the language as a string, like 'fr/' -// or '' if not found. -function language_segment_from_url() { - const path = window.location.pathname; - const language_regexp = - '/((?:' + Object.keys(all_languages).join('|') + ')/)'; - const match = path.match(language_regexp); - if (match !== null) return match[1]; - return ''; -} - -// Returns the path segment of the version as a string, like '3.6/' -// or '' if not found. -function version_segment_from_url() { - const path = window.location.pathname; - const version_regexp = language_segment_from_url() + _VERSION_PATTERN; - const match = path.match(version_regexp); - if (match !== null) return match[1]; - return ''; -} const _initialise_switchers = () => { const version_select = _create_version_select(_CURRENT_VERSION); document From b62ea7a325fb2dc53350f07be9c0129de1f11c6a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:08:12 +0000 Subject: [PATCH 077/176] Use the event argument of select element callback functions --- templates/switchers.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 600c955..dbd4809 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -77,8 +77,8 @@ const _navigate_to_first_existing = (urls) => { }); }; -const _on_version_switch = () => { - const selected_version = this.options[this.selectedIndex].value + '/'; +const _on_version_switch = (event) => { + const selected_version = event.target.value + '/'; const url = window.location.href; const new_url = url.replace( _CURRENT_PREFIX, @@ -95,8 +95,8 @@ const _on_version_switch = () => { } }; -const _on_language_switch = () => { - let selected_language = this.options[this.selectedIndex].value + '/'; +const _on_language_switch = (event) => { + let selected_language = event.target.value + '/'; const url = window.location.href; if (selected_language === 'en/') // Special 'default' case for English. From e5b53bbf6d545d0d95cff2b9f74c1f602efb2511 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:43:21 +0000 Subject: [PATCH 078/176] Improve logic for the callback functions --- templates/switchers.js | 90 ++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index dbd4809..ababf04 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -58,55 +58,67 @@ const _create_language_select = (current_language) => { const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. - const url = urls.shift(); - if (urls.length === 0 || url.startsWith('file:///')) { - window.location.href = url; - return; + for (const url of urls) { + if (url.startsWith('file:///')) { + window.location.href = url; + return; + } + fetch(url) + .then((response) => { + if (response.ok) { + window.location.href = url; + return url; + } + }) + .catch((err) => { + console.error(`Error when fetching '${url}'!`); + console.error(err); + }); } - fetch(url) - .then((response) => { - if (response.ok) { - window.location.href = url; - } else { - navigate_to_first_existing(urls); - } - }) - .catch((err) => { - void err; - navigate_to_first_existing(urls); - }); + + // if all else fails, redirect to the d.p.o root + window.location.href = '/'; + return '/'; }; const _on_version_switch = (event) => { - const selected_version = event.target.value + '/'; - const url = window.location.href; - const new_url = url.replace( - _CURRENT_PREFIX, - '/' + _CURRENT_LANGUAGE + selected_version, - ); - if (new_url !== url) { + const selected_version = event.target.value; + // English has no language prefix. + const new_prefix_en = `/${selected_version}/`; + const new_prefix = + _CURRENT_LANGUAGE === 'en' + ? new_prefix_en + : `/${_CURRENT_LANGUAGE}/${selected_version}/`; + if (_CURRENT_PREFIX !== new_prefix) { + // Try the following pages in order: + // 1. The current page in the current language with the new version + // 2. The current page in English with the new version + // 3. The documentation home in the current language with the new version + // 4. The documentation home in English with the new version _navigate_to_first_existing([ - new_url, - url.replace(_CURRENT_PREFIX, '/' + selected_version), - '/' + _CURRENT_LANGUAGE + selected_version, - '/' + selected_version, - '/', + window.location.href.replace(_CURRENT_PREFIX, new_prefix), + window.location.href.replace(_CURRENT_PREFIX, new_prefix_en), + new_prefix, + new_prefix_en, ]); } }; const _on_language_switch = (event) => { - let selected_language = event.target.value + '/'; - const url = window.location.href; - if (selected_language === 'en/') - // Special 'default' case for English. - selected_language = ''; - let new_url = url.replace( - _CURRENT_PREFIX, - '/' + selected_language + _CURRENT_VERSION, - ); - if (new_url !== url) { - _navigate_to_first_existing([new_url, '/']); + const selected_language = event.target.value; + // English has no language prefix. + const new_prefix = + selected_language === 'en' + ? `/${_CURRENT_VERSION}/` + : `/${selected_language}/${_CURRENT_VERSION}/`; + if (_CURRENT_PREFIX !== new_prefix) { + // Try the following pages in order: + // 1. The current page in the new language with the current version + // 2. The documentation home in the new language with the current version + _navigate_to_first_existing([ + window.location.href.replace(_CURRENT_PREFIX, new_prefix), + new_prefix, + ]); } }; From 1179ee9d3b62ee38f978ffa6b97e311b2775d230 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:49:33 +0000 Subject: [PATCH 079/176] Iterate over arrays with for...of --- templates/switchers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index ababf04..af73ced 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -21,7 +21,7 @@ const _create_version_select = (release) => { const select = document.createElement('select'); select.className = 'version-select'; - for (const [version, title] in all_versions) { + for (const [version, title] of Object.entries(all_versions)) { const option = document.createElement('option'); option.value = version; if (version === major_minor) { @@ -45,7 +45,7 @@ const _create_language_select = (current_language) => { const select = document.createElement('select'); select.className = 'language-select'; - for (const [language, title] in all_languages) { + for (const [language, title] of Object.entries(all_languages)) { const option = document.createElement('option'); option.value = language; option.text = title; From d1b7ad8450b1ef58b307c61882c0336c60ef1276 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:52:54 +0000 Subject: [PATCH 080/176] Fix _CURRENT_VERSION for pre-releases --- templates/switchers.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/switchers.js b/templates/switchers.js index af73ced..afa26f4 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,6 +1,7 @@ 'use strict'; -const _CURRENT_VERSION = DOCUMENTATION_OPTIONS.VERSION; +const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ''; +const _CURRENT_VERSION = _CURRENT_RELEASE.split('.', 2).join('.'); const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; const _CURRENT_PREFIX = (() => { // Sphinx 7.2+ defines the content root data attribute in the HTML element. From 7efb6bf363a2f22338d0f9732e39895fa28abbe7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:00:10 +0000 Subject: [PATCH 081/176] Use _CURRENT_RELEASE in _create_version_select() --- templates/switchers.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index afa26f4..774f04e 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -17,16 +17,15 @@ const _CURRENT_PREFIX = (() => { const all_versions = $VERSIONS; const all_languages = $LANGUAGES; -const _create_version_select = (release) => { - const major_minor = release.split('.').slice(0, 2).join('.'); +const _create_version_select = () => { const select = document.createElement('select'); select.className = 'version-select'; for (const [version, title] of Object.entries(all_versions)) { const option = document.createElement('option'); option.value = version; - if (version === major_minor) { - option.text = release; + if (version === _CURRENT_VERSION) { + option.text = _CURRENT_RELEASE; option.selected = true; } else { option.text = title; @@ -124,7 +123,7 @@ const _on_language_switch = (event) => { }; const _initialise_switchers = () => { - const version_select = _create_version_select(_CURRENT_VERSION); + const version_select = _create_version_select(); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { From 359c393d92dbec5f383b974652f19259edf9fdbb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:02:14 +0000 Subject: [PATCH 082/176] Use _CURRENT_LANGUAGE in _create_language_select() --- templates/switchers.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 774f04e..740cc12 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -36,10 +36,10 @@ const _create_version_select = () => { return select; }; -const _create_language_select = (current_language) => { - if (!(current_language in all_languages)) { +const _create_language_select = () => { + if (!(_CURRENT_LANGUAGE in all_languages)) { // In case we are browsing a language that is not yet in all_languages. - all_languages[current_language] = current_language; + all_languages[_CURRENT_LANGUAGE] = _CURRENT_LANGUAGE; } const select = document.createElement('select'); @@ -49,7 +49,7 @@ const _create_language_select = (current_language) => { const option = document.createElement('option'); option.value = language; option.text = title; - if (language === current_language) option.selected = true; + if (language === _CURRENT_LANGUAGE) option.selected = true; select.add(option); } @@ -132,7 +132,7 @@ const _initialise_switchers = () => { placeholder.append(s); }); - const language_select = _create_language_select(_CURRENT_LANGUAGE); + const language_select = _create_language_select(); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { From e55b26c44c6410d2720d8d4b0b73d5140a29721e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:05:57 +0000 Subject: [PATCH 083/176] Add better handling for file URIs --- templates/switchers.js | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 740cc12..025daed 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,9 +1,14 @@ 'use strict'; +// File URIs must begin with either one or three forward slashes +const _is_file_uri = (uri) => uri.startsWith('file:/'); + +const _IS_LOCAL = _is_file_uri(window.location.href); const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ''; const _CURRENT_VERSION = _CURRENT_RELEASE.split('.', 2).join('.'); const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; const _CURRENT_PREFIX = (() => { + if (_IS_LOCAL) return null; // Sphinx 7.2+ defines the content root data attribute in the HTML element. const _CONTENT_ROOT = document.documentElement.dataset.content_root; if (_CONTENT_ROOT !== undefined) { @@ -20,6 +25,10 @@ const all_languages = $LANGUAGES; const _create_version_select = () => { const select = document.createElement('select'); select.className = 'version-select'; + if (_IS_LOCAL) { + select.disabled = true; + select.title = 'Version switching is disabled in local builds'; + } for (const [version, title] of Object.entries(all_versions)) { const option = document.createElement('option'); @@ -44,6 +53,10 @@ const _create_language_select = () => { const select = document.createElement('select'); select.className = 'language-select'; + if (_IS_LOCAL) { + select.disabled = true; + select.title = 'Language switching is disabled in local builds'; + } for (const [language, title] of Object.entries(all_languages)) { const option = document.createElement('option'); @@ -59,10 +72,6 @@ const _create_language_select = () => { const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { - if (url.startsWith('file:///')) { - window.location.href = url; - return; - } fetch(url) .then((response) => { if (response.ok) { @@ -82,6 +91,8 @@ const _navigate_to_first_existing = (urls) => { }; const _on_version_switch = (event) => { + if (_IS_LOCAL) return; + const selected_version = event.target.value; // English has no language prefix. const new_prefix_en = `/${selected_version}/`; @@ -105,6 +116,8 @@ const _on_version_switch = (event) => { }; const _on_language_switch = (event) => { + if (_IS_LOCAL) return; + const selected_language = event.target.value; // English has no language prefix. const new_prefix = From 4c495ccf8bb0eb512a4b7acba7b6d682babdf91c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:35:03 +0000 Subject: [PATCH 084/176] Remove placeholder classes when initialisation is complete --- templates/switchers.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 025daed..c6dd30e 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -143,6 +143,7 @@ const _initialise_switchers = () => { const s = version_select.cloneNode(true); s.addEventListener('change', _on_version_switch); placeholder.append(s); + placeholder.classList.remove('version_switcher_placeholder'); }); const language_select = _create_language_select(); @@ -152,6 +153,7 @@ const _initialise_switchers = () => { const s = language_select.cloneNode(true); s.addEventListener('change', _on_language_switch); placeholder.append(s); + placeholder.classList.remove('language_switcher_placeholder'); }); }; From 3031085ac20893336bd9f317b66d97db0ce21af1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:46:44 +0000 Subject: [PATCH 085/176] Use a Map for versions and languages --- templates/switchers.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index c6dd30e..2e4fd76 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -19,10 +19,10 @@ const _CURRENT_PREFIX = (() => { return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; })(); -const all_versions = $VERSIONS; -const all_languages = $LANGUAGES; +const _ALL_VERSIONS = new Map(Object.entries($VERSIONS)); +const _ALL_LANGUAGES = new Map(Object.entries($LANGUAGES)); -const _create_version_select = () => { +const _create_version_select = (versions) => { const select = document.createElement('select'); select.className = 'version-select'; if (_IS_LOCAL) { @@ -30,7 +30,7 @@ const _create_version_select = () => { select.title = 'Version switching is disabled in local builds'; } - for (const [version, title] of Object.entries(all_versions)) { + for (const [version, title] of versions) { const option = document.createElement('option'); option.value = version; if (version === _CURRENT_VERSION) { @@ -45,10 +45,10 @@ const _create_version_select = () => { return select; }; -const _create_language_select = () => { - if (!(_CURRENT_LANGUAGE in all_languages)) { - // In case we are browsing a language that is not yet in all_languages. - all_languages[_CURRENT_LANGUAGE] = _CURRENT_LANGUAGE; +const _create_language_select = (languages) => { + if (!languages.has(_CURRENT_LANGUAGE)) { + // In case we are browsing a language that is not yet in languages. + languages.set(_CURRENT_LANGUAGE, _CURRENT_LANGUAGE); } const select = document.createElement('select'); @@ -58,7 +58,7 @@ const _create_language_select = () => { select.title = 'Language switching is disabled in local builds'; } - for (const [language, title] of Object.entries(all_languages)) { + for (const [language, title] of languages) { const option = document.createElement('option'); option.value = language; option.text = title; @@ -136,7 +136,10 @@ const _on_language_switch = (event) => { }; const _initialise_switchers = () => { - const version_select = _create_version_select(); + const versions = _ALL_VERSIONS; + const languages = _ALL_LANGUAGES; + + const version_select = _create_version_select(versions); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { @@ -146,7 +149,7 @@ const _initialise_switchers = () => { placeholder.classList.remove('version_switcher_placeholder'); }); - const language_select = _create_language_select(); + const language_select = _create_language_select(languages); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { From 8cb67060a4fd9e4508b006df3052d91c2d545a3a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:51:02 +0000 Subject: [PATCH 086/176] Add JSDoc comments --- templates/switchers.js | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 2e4fd76..cd4cbc2 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -22,6 +22,11 @@ const _CURRENT_PREFIX = (() => { const _ALL_VERSIONS = new Map(Object.entries($VERSIONS)); const _ALL_LANGUAGES = new Map(Object.entries($LANGUAGES)); +/** + * @param {Map} versions + * @returns {HTMLSelectElement} + * @private + */ const _create_version_select = (versions) => { const select = document.createElement('select'); select.className = 'version-select'; @@ -45,6 +50,11 @@ const _create_version_select = (versions) => { return select; }; +/** + * @param {Map} languages + * @returns {HTMLSelectElement} + * @private + */ const _create_language_select = (languages) => { if (!languages.has(_CURRENT_LANGUAGE)) { // In case we are browsing a language that is not yet in languages. @@ -69,6 +79,11 @@ const _create_language_select = (languages) => { return select; }; +/** + * Change the current page to the first existing URL in the list. + * @param {Array} urls + * @private + */ const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { @@ -90,6 +105,12 @@ const _navigate_to_first_existing = (urls) => { return '/'; }; +/** + * Callback for the version switcher. + * @param {Event} event + * @returns {void} + * @private + */ const _on_version_switch = (event) => { if (_IS_LOCAL) return; @@ -115,6 +136,12 @@ const _on_version_switch = (event) => { } }; +/** + * Callback for the language switcher. + * @param {Event} event + * @returns {void} + * @private + */ const _on_language_switch = (event) => { if (_IS_LOCAL) return; @@ -135,6 +162,11 @@ const _on_language_switch = (event) => { } }; +/** + * Initialisation function for the version and language switchers. + * @returns {void} + * @private + */ const _initialise_switchers = () => { const versions = _ALL_VERSIONS; const languages = _ALL_LANGUAGES; From 58c9634bcad42f8114e8665955d73dcbfbfab02e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 29 Oct 2024 22:22:46 +0000 Subject: [PATCH 087/176] Convert promises to async-await (#227) --- templates/switchers.js | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index cd4cbc2..b479b3c 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -84,20 +84,18 @@ const _create_language_select = (languages) => { * @param {Array} urls * @private */ -const _navigate_to_first_existing = (urls) => { +const _navigate_to_first_existing = async (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { - fetch(url) - .then((response) => { - if (response.ok) { - window.location.href = url; - return url; - } - }) - .catch((err) => { - console.error(`Error when fetching '${url}'!`); - console.error(err); - }); + try { + const response = await fetch(url, { method: 'HEAD' }); + if (response.ok) { + window.location.href = url; + return url; + } + } catch (err) { + console.error(`Error when fetching '${url}': ${err}`); + } } // if all else fails, redirect to the d.p.o root @@ -111,7 +109,7 @@ const _navigate_to_first_existing = (urls) => { * @returns {void} * @private */ -const _on_version_switch = (event) => { +const _on_version_switch = async (event) => { if (_IS_LOCAL) return; const selected_version = event.target.value; @@ -127,7 +125,7 @@ const _on_version_switch = (event) => { // 2. The current page in English with the new version // 3. The documentation home in the current language with the new version // 4. The documentation home in English with the new version - _navigate_to_first_existing([ + await _navigate_to_first_existing([ window.location.href.replace(_CURRENT_PREFIX, new_prefix), window.location.href.replace(_CURRENT_PREFIX, new_prefix_en), new_prefix, @@ -142,7 +140,7 @@ const _on_version_switch = (event) => { * @returns {void} * @private */ -const _on_language_switch = (event) => { +const _on_language_switch = async (event) => { if (_IS_LOCAL) return; const selected_language = event.target.value; @@ -155,7 +153,7 @@ const _on_language_switch = (event) => { // Try the following pages in order: // 1. The current page in the new language with the current version // 2. The documentation home in the new language with the current version - _navigate_to_first_existing([ + await _navigate_to_first_existing([ window.location.href.replace(_CURRENT_PREFIX, new_prefix), new_prefix, ]); From c1fc85cfe4aeb2e99cc09096e28b4d8ca76e3db2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 29 Oct 2024 22:32:11 +0000 Subject: [PATCH 088/176] Always create new select nodes (#228) --- templates/switchers.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index b479b3c..dd28044 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -169,21 +169,19 @@ const _initialise_switchers = () => { const versions = _ALL_VERSIONS; const languages = _ALL_LANGUAGES; - const version_select = _create_version_select(versions); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { - const s = version_select.cloneNode(true); + const s = _create_version_select(versions); s.addEventListener('change', _on_version_switch); placeholder.append(s); placeholder.classList.remove('version_switcher_placeholder'); }); - const language_select = _create_language_select(languages); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { - const s = language_select.cloneNode(true); + const s = _create_language_select(languages); s.addEventListener('change', _on_language_switch); placeholder.append(s); placeholder.classList.remove('language_switcher_placeholder'); From eff0dd956eb3e58cb294100aea1bf0bf12a53ba9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:36:29 +0000 Subject: [PATCH 089/176] Run prettier in GitHub Actions (#232) --- .github/workflows/lint.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d553e49..26f2cc1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ permissions: contents: read jobs: - lint: + pre-commit: runs-on: ubuntu-latest steps: @@ -20,3 +20,14 @@ jobs: python-version: "3.x" cache: pip - uses: pre-commit/action@v3.0.1 + + prettier: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Lint with Prettier + run: npx prettier templates/switchers.js --check --single-quote From 49641c17b23dac6ecf3fefbe4483dca530dfd19f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:42:25 +0000 Subject: [PATCH 090/176] Create switchers maps from arrays of pairs (#231) --- build_docs.py | 8 ++++---- templates/switchers.js | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build_docs.py b/build_docs.py index 6596a25..f14cc2b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -388,16 +388,16 @@ def setup_switchers( - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - languages_map = dict(sorted((l.tag, l.name) for l in languages if l.in_prod)) - versions_map = {v.name: v.picker_label for v in reversed(versions)} + language_pairs = sorted((l.tag, l.name) for l in languages if l.in_prod) + version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] switchers_template_file = HERE / "templates" / "switchers.js" switchers_path = html_root / "_static" / "switchers.js" template = Template(switchers_template_file.read_text(encoding="UTF-8")) rendered_template = template.safe_substitute( - LANGUAGES=json.dumps(languages_map), - VERSIONS=json.dumps(versions_map), + LANGUAGES=json.dumps(language_pairs), + VERSIONS=json.dumps(version_pairs), ) switchers_path.write_text(rendered_template, encoding="UTF-8") diff --git a/templates/switchers.js b/templates/switchers.js index dd28044..d1a4768 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -19,8 +19,8 @@ const _CURRENT_PREFIX = (() => { return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; })(); -const _ALL_VERSIONS = new Map(Object.entries($VERSIONS)); -const _ALL_LANGUAGES = new Map(Object.entries($LANGUAGES)); +const _ALL_VERSIONS = new Map($VERSIONS); +const _ALL_LANGUAGES = new Map($LANGUAGES); /** * @param {Map} versions From 91d0994a794d255c767e42873ffee6502c88b91a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:45:36 +0000 Subject: [PATCH 091/176] Keep switcher placeholder classes --- templates/switchers.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index d1a4768..3eefa71 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -175,7 +175,6 @@ const _initialise_switchers = () => { const s = _create_version_select(versions); s.addEventListener('change', _on_version_switch); placeholder.append(s); - placeholder.classList.remove('version_switcher_placeholder'); }); document @@ -184,7 +183,6 @@ const _initialise_switchers = () => { const s = _create_language_select(languages); s.addEventListener('change', _on_language_switch); placeholder.append(s); - placeholder.classList.remove('language_switcher_placeholder'); }); }; From cc4f5531c333c0beaaa4e89e528787e877db6c43 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 31 Oct 2024 18:13:40 +0200 Subject: [PATCH 092/176] Run prettier via pre-commit (#234) --- .github/workflows/lint.yml | 13 +--------- .pre-commit-config.yaml | 20 ++++++++++----- templates/switchers.js | 52 +++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 45 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 26f2cc1..d553e49 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ permissions: contents: read jobs: - pre-commit: + lint: runs-on: ubuntu-latest steps: @@ -20,14 +20,3 @@ jobs: python-version: "3.x" cache: pip - uses: pre-commit/action@v3.0.1 - - prettier: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "22" - - name: Lint with Prettier - run: npx prettier templates/switchers.js --check --single-quote diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c3d2f5f..60a928c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -14,35 +14,41 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.7 + rev: v0.7.1 hooks: - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.2 + rev: 0.29.4 hooks: - id: check-github-workflows - repo: https://github.com/rhysd/actionlint - rev: v1.7.1 + rev: v1.7.3 hooks: - id: actionlint - repo: https://github.com/tox-dev/pyproject-fmt - rev: 2.2.3 + rev: v2.5.0 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.19 + rev: v0.22 hooks: - id: validate-pyproject - repo: https://github.com/tox-dev/tox-ini-fmt - rev: 1.4.0 + rev: 1.4.1 hooks: - id: tox-ini-fmt + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.3.3 + hooks: + - id: prettier + files: templates/switchers.js + - repo: meta hooks: - id: check-hooks-apply diff --git a/templates/switchers.js b/templates/switchers.js index 3eefa71..44c71bb 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,12 +1,12 @@ -'use strict'; +"use strict"; // File URIs must begin with either one or three forward slashes -const _is_file_uri = (uri) => uri.startsWith('file:/'); +const _is_file_uri = (uri) => uri.startsWith("file:/"); const _IS_LOCAL = _is_file_uri(window.location.href); -const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ''; -const _CURRENT_VERSION = _CURRENT_RELEASE.split('.', 2).join('.'); -const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; +const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ""; +const _CURRENT_VERSION = _CURRENT_RELEASE.split(".", 2).join("."); +const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || "en"; const _CURRENT_PREFIX = (() => { if (_IS_LOCAL) return null; // Sphinx 7.2+ defines the content root data attribute in the HTML element. @@ -15,8 +15,8 @@ const _CURRENT_PREFIX = (() => { return new URL(_CONTENT_ROOT, window.location).pathname; } // Fallback for older versions of Sphinx (used in Python 3.10 and older). - const _NUM_PREFIX_PARTS = _CURRENT_LANGUAGE === 'en' ? 2 : 3; - return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; + const _NUM_PREFIX_PARTS = _CURRENT_LANGUAGE === "en" ? 2 : 3; + return window.location.pathname.split("/", _NUM_PREFIX_PARTS).join("/") + "/"; })(); const _ALL_VERSIONS = new Map($VERSIONS); @@ -28,15 +28,15 @@ const _ALL_LANGUAGES = new Map($LANGUAGES); * @private */ const _create_version_select = (versions) => { - const select = document.createElement('select'); - select.className = 'version-select'; + const select = document.createElement("select"); + select.className = "version-select"; if (_IS_LOCAL) { select.disabled = true; - select.title = 'Version switching is disabled in local builds'; + select.title = "Version switching is disabled in local builds"; } for (const [version, title] of versions) { - const option = document.createElement('option'); + const option = document.createElement("option"); option.value = version; if (version === _CURRENT_VERSION) { option.text = _CURRENT_RELEASE; @@ -61,15 +61,15 @@ const _create_language_select = (languages) => { languages.set(_CURRENT_LANGUAGE, _CURRENT_LANGUAGE); } - const select = document.createElement('select'); - select.className = 'language-select'; + const select = document.createElement("select"); + select.className = "language-select"; if (_IS_LOCAL) { select.disabled = true; - select.title = 'Language switching is disabled in local builds'; + select.title = "Language switching is disabled in local builds"; } for (const [language, title] of languages) { - const option = document.createElement('option'); + const option = document.createElement("option"); option.value = language; option.text = title; if (language === _CURRENT_LANGUAGE) option.selected = true; @@ -88,7 +88,7 @@ const _navigate_to_first_existing = async (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { try { - const response = await fetch(url, { method: 'HEAD' }); + const response = await fetch(url, { method: "HEAD" }); if (response.ok) { window.location.href = url; return url; @@ -99,8 +99,8 @@ const _navigate_to_first_existing = async (urls) => { } // if all else fails, redirect to the d.p.o root - window.location.href = '/'; - return '/'; + window.location.href = "/"; + return "/"; }; /** @@ -116,7 +116,7 @@ const _on_version_switch = async (event) => { // English has no language prefix. const new_prefix_en = `/${selected_version}/`; const new_prefix = - _CURRENT_LANGUAGE === 'en' + _CURRENT_LANGUAGE === "en" ? new_prefix_en : `/${_CURRENT_LANGUAGE}/${selected_version}/`; if (_CURRENT_PREFIX !== new_prefix) { @@ -146,7 +146,7 @@ const _on_language_switch = async (event) => { const selected_language = event.target.value; // English has no language prefix. const new_prefix = - selected_language === 'en' + selected_language === "en" ? `/${_CURRENT_VERSION}/` : `/${selected_language}/${_CURRENT_VERSION}/`; if (_CURRENT_PREFIX !== new_prefix) { @@ -170,24 +170,24 @@ const _initialise_switchers = () => { const languages = _ALL_LANGUAGES; document - .querySelectorAll('.version_switcher_placeholder') + .querySelectorAll(".version_switcher_placeholder") .forEach((placeholder) => { const s = _create_version_select(versions); - s.addEventListener('change', _on_version_switch); + s.addEventListener("change", _on_version_switch); placeholder.append(s); }); document - .querySelectorAll('.language_switcher_placeholder') + .querySelectorAll(".language_switcher_placeholder") .forEach((placeholder) => { const s = _create_language_select(languages); - s.addEventListener('change', _on_language_switch); + s.addEventListener("change", _on_language_switch); placeholder.append(s); }); }; -if (document.readyState !== 'loading') { +if (document.readyState !== "loading") { _initialise_switchers(); } else { - document.addEventListener('DOMContentLoaded', _initialise_switchers); + document.addEventListener("DOMContentLoaded", _initialise_switchers); } From b2b548353b50a3316ba098b89382fe6b8f8474a9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 31 Oct 2024 23:04:00 +0000 Subject: [PATCH 093/176] Update JSDoc comments in switchers.js --- templates/switchers.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 44c71bb..7a4fb63 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -104,7 +104,7 @@ const _navigate_to_first_existing = async (urls) => { }; /** - * Callback for the version switcher. + * Navigate to the selected version. * @param {Event} event * @returns {void} * @private @@ -135,7 +135,7 @@ const _on_version_switch = async (event) => { }; /** - * Callback for the language switcher. + * Navigate to the selected language. * @param {Event} event * @returns {void} * @private @@ -161,7 +161,7 @@ const _on_language_switch = async (event) => { }; /** - * Initialisation function for the version and language switchers. + * Set up the version and language switchers. * @returns {void} * @private */ From b5f5a52faf28ab13990ebbc577c1a3c206343850 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:15:46 +0200 Subject: [PATCH 094/176] Add instructions on manually rebuilding a branch --- README.md | 4 ++-- RELEASING.md | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 RELEASING.md diff --git a/README.md b/README.md index 767014c..b9881e7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This repository contains scripts for automatically building the Python documentation on [docs.python.org](https://docs.python.org). -# How to test it? +## How to test it? The following command should build all maintained versions and translations in `./www`, beware it can take a few hours: @@ -15,7 +15,7 @@ If you don't need to build all translations of all branches, add `--language en --branch main`. -# Check current version +## Check current version Install `tools_requirements.txt` then run `python check_versions.py ../cpython/` (pointing to a real CPython clone) to see which version diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..abe72cd --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,16 @@ +# Manually rebuild a branch + +Docs for [feature and bugfix branches](https://devguide.python.org/versions/) are +automatically built from a cron. + +Manual rebuilds are needed for new security releases, +and to add the end-of-life banner for newly end-of-life branches. + +To manually rebuild a branch, for example 3.11: + +```shell +ssh docs.nyc1.psf.io +sudo su --shell=/bin/bash docsbuild +screen -DUR # Rejoin screen session if it exists, otherwise create a new one +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 +``` From 10064ee59d8c6f2fbad2391055d746f1d5deb991 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 10 Dec 2024 23:10:01 +0200 Subject: [PATCH 095/176] Move to README --- README.md | 17 +++++++++++++++++ RELEASING.md | 16 ---------------- 2 files changed, 17 insertions(+), 16 deletions(-) delete mode 100644 RELEASING.md diff --git a/README.md b/README.md index b9881e7..1b76bcd 100644 --- a/README.md +++ b/README.md @@ -51,3 +51,20 @@ of Sphinx we're using where: 3.13 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 3.14 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= + +## Manually rebuild a branch + +Docs for [feature and bugfix branches](https://devguide.python.org/versions/) are +automatically built from a cron. + +Manual rebuilds are needed for new security releases, +and to add the end-of-life banner for newly end-of-life branches. + +To manually rebuild a branch, for example 3.11: + +```shell +ssh docs.nyc1.psf.io +sudo su --shell=/bin/bash docsbuild +screen -DUR # Rejoin screen session if it exists, otherwise create a new one +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 +``` diff --git a/RELEASING.md b/RELEASING.md deleted file mode 100644 index abe72cd..0000000 --- a/RELEASING.md +++ /dev/null @@ -1,16 +0,0 @@ -# Manually rebuild a branch - -Docs for [feature and bugfix branches](https://devguide.python.org/versions/) are -automatically built from a cron. - -Manual rebuilds are needed for new security releases, -and to add the end-of-life banner for newly end-of-life branches. - -To manually rebuild a branch, for example 3.11: - -```shell -ssh docs.nyc1.psf.io -sudo su --shell=/bin/bash docsbuild -screen -DUR # Rejoin screen session if it exists, otherwise create a new one -/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 -``` From e4a8aff9772738a63d0945042777d18c3d926930 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 28 Dec 2024 12:30:57 +0000 Subject: [PATCH 096/176] Enable the Polish translation in the language switcher (#237) --- config.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/config.toml b/config.toml index 3716d7f..3679c73 100644 --- a/config.toml +++ b/config.toml @@ -69,7 +69,6 @@ sphinxopts = [ [languages.pl] name = "Polish" -in_prod = false [languages.pt_BR] name = "Brazilian Portuguese" From b3f238f367d70c4f8cab4d3163d1a5eecc771c2f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 17 Jan 2025 12:51:16 +0000 Subject: [PATCH 097/176] Enable translation_progress_classes (#239) --- build_docs.py | 26 ++++++++++++++------------ config.toml | 1 - 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/build_docs.py b/build_docs.py index f14cc2b..b5cf717 100755 --- a/build_docs.py +++ b/build_docs.py @@ -192,7 +192,7 @@ def __gt__(self, other): return self.as_tuple() > other.as_tuple() -@dataclass(frozen=True, order=True) +@dataclass(order=True, frozen=True, kw_only=True) class Language: iso639_tag: str name: str @@ -710,6 +710,7 @@ def build(self): f"-D locale_dirs={locale_dirs}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", + "-D translation_progress_classes=1", ) ) if self.language.tag == "ja": @@ -1141,19 +1142,20 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: def parse_languages_from_config() -> list[Language]: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) - languages = [] defaults = config["defaults"] - for iso639_tag, section in config["languages"].items(): - languages.append( - Language( - iso639_tag, - section["name"], - section.get("in_prod", defaults["in_prod"]), - sphinxopts=section.get("sphinxopts", defaults["sphinxopts"]), - html_only=section.get("html_only", defaults["html_only"]), - ) + default_in_prod = defaults.get("in_prod", True) + default_sphinxopts = defaults.get("sphinxopts", []) + default_html_only = defaults.get("html_only", False) + return [ + Language( + iso639_tag=iso639_tag, + name=section["name"], + in_prod=section.get("in_prod", default_in_prod), + sphinxopts=section.get("sphinxopts", default_sphinxopts), + html_only=section.get("html_only", default_html_only), ) - return languages + for iso639_tag, section in config["languages"].items() + ] def format_seconds(seconds: float) -> str: diff --git a/config.toml b/config.toml index 3679c73..b0994ad 100644 --- a/config.toml +++ b/config.toml @@ -33,7 +33,6 @@ in_prod = false [languages.it] name = "Italian" -in_prod = true [languages.ja] name = "Japanese" From be699c482265555fda16dc8dccf1bcef25d23fb8 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 18 Jan 2025 15:38:54 +0000 Subject: [PATCH 098/176] Disable translation_progress_classes Sphinx does not properly convert command-line overrides to Boolean values, so ``-D translation_progress_classes=1`` fails with: Sphinx parallel build error: sphinx.errors.ConfigError: translation_progress_classes must be True, False, "translated" or "untranslated" --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index b5cf717..40384b8 100755 --- a/build_docs.py +++ b/build_docs.py @@ -710,7 +710,7 @@ def build(self): f"-D locale_dirs={locale_dirs}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", - "-D translation_progress_classes=1", + # "-D translation_progress_classes=1", ) ) if self.language.tag == "ja": From 4e7299dfd3719b08d42a162c0bbb9181218b5596 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 18 Jan 2025 18:47:45 +0000 Subject: [PATCH 099/176] Identify the language correctly for older versions (#240) --- templates/switchers.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/switchers.js b/templates/switchers.js index 7a4fb63..774366f 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -6,7 +6,13 @@ const _is_file_uri = (uri) => uri.startsWith("file:/"); const _IS_LOCAL = _is_file_uri(window.location.href); const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ""; const _CURRENT_VERSION = _CURRENT_RELEASE.split(".", 2).join("."); -const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || "en"; +const _CURRENT_LANGUAGE = (() => { + const _LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || "en"; + // Python 2.7 and 3.5--3.10 use ``LANGUAGE: 'None'`` for English + // in ``documentation_options.js``. + if (_LANGUAGE === "none") return "en"; + return _LANGUAGE; +})(); const _CURRENT_PREFIX = (() => { if (_IS_LOCAL) return null; // Sphinx 7.2+ defines the content root data attribute in the HTML element. From 2efa5bce54a2d09573e62a580047b5c7419bbedc Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 20 Jan 2025 23:35:16 +0000 Subject: [PATCH 100/176] Install matplotlib for opengraph preview images (#242) --- build_docs.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 40384b8..a6b08bb 100755 --- a/build_docs.py +++ b/build_docs.py @@ -745,6 +745,10 @@ def build(self): blurb = self.venv / "bin" / "blurb" if self.includes_html: + # Define a tag to enable opengraph socialcards previews + # (used in Doc/conf.py and requires matplotlib) + sphinxopts.append("-t create-social-cards") + # Disable CPython switchers, we handle them now: run( ["sed", "-i"] @@ -783,13 +787,17 @@ def build_venv(self): So we can reuse them from builds to builds, while they contain different Sphinx versions. """ + requirements = [self.theme] + self.version.requirements + if self.includes_html: + # opengraph previews + requirements.append("matplotlib>=3") + venv_path = self.build_root / ("venv-" + self.version.name) run([sys.executable, "-m", "venv", venv_path]) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] + ["--upgrade-strategy=eager"] - + [self.theme] - + self.version.requirements, + + requirements, cwd=self.checkout / "Doc", ) run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) From 6aa487d97044d111c27deae12baa85755351e4fb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 19 Feb 2025 03:23:46 +0000 Subject: [PATCH 101/176] Include languages' translated names in the switcher (#245) Co-authored-by: Ezio Melotti Co-authored-by: Rafael Fontenelle Co-authored-by: W. H. Wang --- build_docs.py | 11 ++++++++++- config.toml | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index a6b08bb..8b7f3f9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -196,6 +196,7 @@ def __gt__(self, other): class Language: iso639_tag: str name: str + translated_name: str in_prod: bool sphinxopts: tuple html_only: bool = False @@ -204,6 +205,12 @@ class Language: def tag(self): return self.iso639_tag.replace("_", "-").lower() + @property + def switcher_label(self): + if self.translated_name: + return f"{self.name} | {self.translated_name}" + return self.name + @staticmethod def filter(languages, language_tags=None): """Filter a sequence of languages according to --languages.""" @@ -388,7 +395,7 @@ def setup_switchers( - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - language_pairs = sorted((l.tag, l.name) for l in languages if l.in_prod) + language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] switchers_template_file = HERE / "templates" / "switchers.js" @@ -1151,6 +1158,7 @@ def parse_languages_from_config() -> list[Language]: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) defaults = config["defaults"] + default_translated_name = defaults.get("translated_name", "") default_in_prod = defaults.get("in_prod", True) default_sphinxopts = defaults.get("sphinxopts", []) default_html_only = defaults.get("html_only", False) @@ -1158,6 +1166,7 @@ def parse_languages_from_config() -> list[Language]: Language( iso639_tag=iso639_tag, name=section["name"], + translated_name=section.get("translated_name", default_translated_name), in_prod=section.get("in_prod", default_in_prod), sphinxopts=section.get("sphinxopts", default_sphinxopts), html_only=section.get("html_only", default_html_only), diff --git a/config.toml b/config.toml index b0994ad..489c774 100644 --- a/config.toml +++ b/config.toml @@ -1,5 +1,12 @@ +# name: the English name for the language. +# translated_name: the 'local' name for the language. +# in_prod: If true, include in the language switcher. +# html_only: If true, only create HTML files. +# sphinxopts: Extra options to pass to SPHINXOPTS in the Makefile. + [defaults] # name has no default, it is mandatory. +translated_name = "" in_prod = true html_only = false sphinxopts = [ @@ -13,6 +20,7 @@ name = "English" [languages.es] name = "Spanish" +translated_name = "español" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -21,6 +29,7 @@ sphinxopts = [ [languages.fr] name = "French" +translated_name = "français" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -29,13 +38,16 @@ sphinxopts = [ [languages.id] name = "Indonesian" +translated_name = "Indonesia" in_prod = false [languages.it] name = "Italian" +translated_name = "italiano" [languages.ja] name = "Japanese" +translated_name = "日本語" sphinxopts = [ '-D latex_engine=lualatex', '-D latex_elements.inputenc=', @@ -59,6 +71,7 @@ sphinxopts = [ [languages.ko] name = "Korean" +translated_name = "한국어" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -68,20 +81,25 @@ sphinxopts = [ [languages.pl] name = "Polish" +translated_name = "polski" [languages.pt_BR] name = "Brazilian Portuguese" +translated_name = "Português brasileiro" [languages.tr] name = "Turkish" +translated_name = "Türkçe" [languages.uk] name = "Ukrainian" +translated_name = "українська" in_prod = false html_only = true [languages.zh_CN] name = "Simplified Chinese" +translated_name = "简体中文" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -90,6 +108,7 @@ sphinxopts = [ [languages.zh_TW] name = "Traditional Chinese" +translated_name = "繁體中文" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', From b66ca348eeea7d97eed0617cee091162320a8704 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 19 Feb 2025 12:53:56 +0000 Subject: [PATCH 102/176] Re-enable translation_progress_classes We now use Sphinx 8.2, which should properly convert command-line overrides to Boolean values. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 8b7f3f9..ca57ca5 100755 --- a/build_docs.py +++ b/build_docs.py @@ -717,7 +717,7 @@ def build(self): f"-D locale_dirs={locale_dirs}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", - # "-D translation_progress_classes=1", + "-D translation_progress_classes=1", ) ) if self.language.tag == "ja": From 273cd3c5ab4242c37117a15938e439f289b4c995 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:21:34 +0200 Subject: [PATCH 103/176] Test Python 3.14 & remove Python 3.10-3.12 (#251) --- .github/workflows/test.yml | 2 +- build_docs.py | 8 ++++---- tox.ini | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d74e607..ebff180 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.13", "3.14"] os: [ubuntu-latest] steps: diff --git a/build_docs.py b/build_docs.py index ca57ca5..fb51cf9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -23,7 +23,7 @@ from __future__ import annotations from argparse import ArgumentParser, Namespace -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from contextlib import suppress, contextmanager from dataclasses import dataclass import filecmp @@ -42,7 +42,7 @@ from pathlib import Path from string import Template from time import perf_counter, sleep -from typing import Iterable, Literal +from typing import Literal from urllib.parse import urljoin import jinja2 @@ -479,7 +479,7 @@ def version_info(): """Handler for --version.""" try: platex_version = head( - subprocess.check_output(["platex", "--version"], universal_newlines=True), + subprocess.check_output(["platex", "--version"], text=True), lines=3, ) except FileNotFoundError: @@ -487,7 +487,7 @@ def version_info(): try: xelatex_version = head( - subprocess.check_output(["xelatex", "--version"], universal_newlines=True), + subprocess.check_output(["xelatex", "--version"], text=True), lines=2, ) except FileNotFoundError: diff --git a/tox.ini b/tox.ini index 40a034d..56c6420 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ requires = tox>=4.2 env_list = lint - py{313, 312, 311, 310} + py{314, 313} [testenv] package = wheel From 6902d1c2302411dc560655f7948bd2dee6dcf264 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 20 Feb 2025 15:15:09 +0200 Subject: [PATCH 104/176] Update pre-commit (#249) --- .github/workflows/lint.yml | 7 ++++--- .github/workflows/test.yml | 5 +++-- .pre-commit-config.yaml | 17 +++++++++++------ build_docs.py | 5 ++--- check_times.py | 4 ++-- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d553e49..cd9d07e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,19 +2,20 @@ name: Lint on: [push, pull_request, workflow_dispatch] +permissions: {} + env: FORCE_COLOR: 1 PIP_DISABLE_PIP_VERSION_CHECK: 1 -permissions: - contents: read - jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ebff180..2976bae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,7 @@ name: Test on: [push, pull_request, workflow_dispatch] -permissions: - contents: read +permissions: {} env: FORCE_COLOR: 1 @@ -19,6 +18,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60a928c..44948ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,37 +14,42 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.9.6 hooks: - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.4 + rev: 0.31.1 hooks: - id: check-github-workflows - repo: https://github.com/rhysd/actionlint - rev: v1.7.3 + rev: v1.7.7 hooks: - id: actionlint + - repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v1.3.1 + hooks: + - id: zizmor + - repo: https://github.com/tox-dev/pyproject-fmt rev: v2.5.0 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.22 + rev: v0.23 hooks: - id: validate-pyproject - repo: https://github.com/tox-dev/tox-ini-fmt - rev: 1.4.1 + rev: 1.5.0 hooks: - id: tox-ini-fmt - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.3.3 + rev: v3.5.1 hooks: - id: prettier files: templates/switchers.js diff --git a/build_docs.py b/build_docs.py index fb51cf9..7da3520 100755 --- a/build_docs.py +++ b/build_docs.py @@ -86,7 +86,7 @@ def __init__(self, name, *, status, branch_or_tag=None): if status not in self.STATUSES: raise ValueError( "Version status expected to be one of: " - f"{', '.join(self.STATUSES|set(self.SYNONYMS.keys()))}, got {status!r}." + f"{', '.join(self.STATUSES | set(self.SYNONYMS.keys()))}, got {status!r}." ) self.name = name self.branch_or_tag = branch_or_tag @@ -732,8 +732,7 @@ def build(self): shell=True, ) subprocess.check_output( - "sed -i s/\N{REPLACEMENT CHARACTER}/?/g " - f"{self.checkout}/Doc/**/*.rst", + f"sed -i s/\N{REPLACEMENT CHARACTER}/?/g {self.checkout}/Doc/**/*.rst", shell=True, ) diff --git a/check_times.py b/check_times.py index 9310cd4..2b3d2f9 100644 --- a/check_times.py +++ b/check_times.py @@ -50,7 +50,7 @@ def calc_time(lines: list[str]) -> None: fmt_duration = format_seconds(state_data["last_build_duration"]) reason = state_data["triggered_by"] print( - f"{start:%Y-%m-%d %H:%M UTC} | {version: <7} | {language: <8} | {fmt_duration :<14} | {reason}" + f"{start:%Y-%m-%d %H:%M UTC} | {version: <7} | {language: <8} | {fmt_duration:<14} | {reason}" ) if line.endswith("Build start."): @@ -64,7 +64,7 @@ def calc_time(lines: list[str]) -> None: timestamp = f"{line[:16]} UTC" _, fmt_duration = line.removesuffix(").").split("(") print( - f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration :<14} | -----------" + f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration:<14} | -----------" ) if in_progress: From 7583b1744007e74cf9aafce98fa59677a75ce28d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 5 Apr 2025 16:29:14 +0100 Subject: [PATCH 105/176] Define ``ogp_site_url`` for social media cards (#252) --- build_docs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 7da3520..23fd6c0 100755 --- a/build_docs.py +++ b/build_docs.py @@ -751,9 +751,15 @@ def build(self): blurb = self.venv / "bin" / "blurb" if self.includes_html: + site_url = self.version.url + if self.language.tag != "en": + site_url += f"{self.language.tag}/" # Define a tag to enable opengraph socialcards previews # (used in Doc/conf.py and requires matplotlib) - sphinxopts.append("-t create-social-cards") + sphinxopts += ( + "-t create-social-cards", + f"-D ogp_site_url={site_url}", + ) # Disable CPython switchers, we handle them now: run( From 667cd01ec2b694fdecae6a3416f16774d51d390a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 01:09:44 +0100 Subject: [PATCH 106/176] Sort imports --- build_docs.py | 12 ++++++------ check_versions.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build_docs.py b/build_docs.py index 23fd6c0..1ab2fdd 100755 --- a/build_docs.py +++ b/build_docs.py @@ -22,23 +22,23 @@ from __future__ import annotations -from argparse import ArgumentParser, Namespace -from collections.abc import Iterable, Sequence -from contextlib import suppress, contextmanager -from dataclasses import dataclass import filecmp import json import logging import logging.handlers -from functools import total_ordering -from os import getenv, readlink import re import shlex import shutil import subprocess import sys +from argparse import ArgumentParser, Namespace from bisect import bisect_left as bisect +from collections.abc import Iterable, Sequence +from contextlib import contextmanager, suppress +from dataclasses import dataclass from datetime import datetime as dt, timezone +from functools import total_ordering +from os import getenv, readlink from pathlib import Path from string import Template from time import perf_counter, sleep diff --git a/check_versions.py b/check_versions.py index 70cade9..343c85a 100644 --- a/check_versions.py +++ b/check_versions.py @@ -1,15 +1,15 @@ #!/usr/bin/env python -from pathlib import Path import argparse import asyncio import logging import re +from pathlib import Path +import git import httpx import urllib3 from tabulate import tabulate -import git import build_docs From 62903155bd36c7ce09e8136b5d804552f4ba477c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 01:50:33 +0100 Subject: [PATCH 107/176] Move ``parse_args()`` and ``setup_logging()`` after ``main()`` --- build_docs.py | 206 +++++++++++++++++++++++++------------------------- 1 file changed, 103 insertions(+), 103 deletions(-) diff --git a/build_docs.py b/build_docs.py index 1ab2fdd..3da84df 100755 --- a/build_docs.py +++ b/build_docs.py @@ -506,109 +506,6 @@ def version_info(): ) -def parse_args(): - """Parse command-line arguments.""" - - parser = ArgumentParser( - description="Runs a build of the Python docs for various branches." - ) - parser.add_argument( - "--select-output", - choices=("no-html", "only-html", "only-html-en"), - help="Choose what outputs to build.", - ) - parser.add_argument( - "-q", - "--quick", - action="store_true", - help="Run a quick build (only HTML files).", - ) - parser.add_argument( - "-b", - "--branch", - metavar="3.12", - help="Version to build (defaults to all maintained branches).", - ) - parser.add_argument( - "-r", - "--build-root", - type=Path, - help="Path to a directory containing a checkout per branch.", - default=Path("/srv/docsbuild"), - ) - parser.add_argument( - "-w", - "--www-root", - type=Path, - help="Path where generated files will be copied.", - default=Path("/srv/docs.python.org"), - ) - parser.add_argument( - "--skip-cache-invalidation", - help="Skip Fastly cache invalidation.", - action="store_true", - ) - parser.add_argument( - "--group", - help="Group files on targets and www-root file should get.", - default="docs", - ) - parser.add_argument( - "--log-directory", - type=Path, - help="Directory used to store logs.", - default=Path("/var/log/docsbuild/"), - ) - parser.add_argument( - "--languages", - nargs="*", - help="Language translation, as a PEP 545 language tag like" - " 'fr' or 'pt-br'. " - "Builds all available languages by default.", - metavar="fr", - ) - parser.add_argument( - "--version", - action="store_true", - help="Get build_docs and dependencies version info", - ) - parser.add_argument( - "--theme", - default="python-docs-theme", - help="Python package to use for python-docs-theme: Useful to test branches:" - " --theme git+https://github.com/obulat/python-docs-theme@master", - ) - args = parser.parse_args() - if args.version: - version_info() - sys.exit(0) - del args.version - if args.log_directory: - args.log_directory = args.log_directory.resolve() - if args.build_root: - args.build_root = args.build_root.resolve() - if args.www_root: - args.www_root = args.www_root.resolve() - return args - - -def setup_logging(log_directory: Path, select_output: str | None): - """Setup logging to stderr if run by a human, or to a file if run from a cron.""" - log_format = "%(asctime)s %(levelname)s: %(message)s" - if sys.stderr.isatty(): - logging.basicConfig(format=log_format, stream=sys.stderr) - else: - log_directory.mkdir(parents=True, exist_ok=True) - if select_output is None: - filename = log_directory / "docsbuild.log" - else: - filename = log_directory / f"docsbuild-{select_output}.log" - handler = logging.handlers.WatchedFileHandler(filename) - handler.setFormatter(logging.Formatter(log_format)) - logging.getLogger().addHandler(handler) - logging.getLogger().setLevel(logging.DEBUG) - - @dataclass class DocBuilder: """Builder for a CPython version and a language.""" @@ -1288,6 +1185,109 @@ def main(): build_docs_with_lock(args, "build_docs_html_en.lock") +def parse_args(): + """Parse command-line arguments.""" + + parser = ArgumentParser( + description="Runs a build of the Python docs for various branches." + ) + parser.add_argument( + "--select-output", + choices=("no-html", "only-html", "only-html-en"), + help="Choose what outputs to build.", + ) + parser.add_argument( + "-q", + "--quick", + action="store_true", + help="Run a quick build (only HTML files).", + ) + parser.add_argument( + "-b", + "--branch", + metavar="3.12", + help="Version to build (defaults to all maintained branches).", + ) + parser.add_argument( + "-r", + "--build-root", + type=Path, + help="Path to a directory containing a checkout per branch.", + default=Path("/srv/docsbuild"), + ) + parser.add_argument( + "-w", + "--www-root", + type=Path, + help="Path where generated files will be copied.", + default=Path("/srv/docs.python.org"), + ) + parser.add_argument( + "--skip-cache-invalidation", + help="Skip Fastly cache invalidation.", + action="store_true", + ) + parser.add_argument( + "--group", + help="Group files on targets and www-root file should get.", + default="docs", + ) + parser.add_argument( + "--log-directory", + type=Path, + help="Directory used to store logs.", + default=Path("/var/log/docsbuild/"), + ) + parser.add_argument( + "--languages", + nargs="*", + help="Language translation, as a PEP 545 language tag like" + " 'fr' or 'pt-br'. " + "Builds all available languages by default.", + metavar="fr", + ) + parser.add_argument( + "--version", + action="store_true", + help="Get build_docs and dependencies version info", + ) + parser.add_argument( + "--theme", + default="python-docs-theme", + help="Python package to use for python-docs-theme: Useful to test branches:" + " --theme git+https://github.com/obulat/python-docs-theme@master", + ) + args = parser.parse_args() + if args.version: + version_info() + sys.exit(0) + del args.version + if args.log_directory: + args.log_directory = args.log_directory.resolve() + if args.build_root: + args.build_root = args.build_root.resolve() + if args.www_root: + args.www_root = args.www_root.resolve() + return args + + +def setup_logging(log_directory: Path, select_output: str | None): + """Setup logging to stderr if run by a human, or to a file if run from a cron.""" + log_format = "%(asctime)s %(levelname)s: %(message)s" + if sys.stderr.isatty(): + logging.basicConfig(format=log_format, stream=sys.stderr) + else: + log_directory.mkdir(parents=True, exist_ok=True) + if select_output is None: + filename = log_directory / "docsbuild.log" + else: + filename = log_directory / f"docsbuild-{select_output}.log" + handler = logging.handlers.WatchedFileHandler(filename) + handler.setFormatter(logging.Formatter(log_format)) + logging.getLogger().addHandler(handler) + logging.getLogger().setLevel(logging.DEBUG) + + def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: try: lock = zc.lockfile.LockFile(HERE / lockfile_name) From 8d0b9c3237cc34e9ebc0480cb85bd3345f8da3a9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 01:52:49 +0100 Subject: [PATCH 108/176] Move ``build_docs()`` after ``build_docs_with_lock()`` --- build_docs.py | 608 +++++++++++++++++++++++++------------------------- 1 file changed, 304 insertions(+), 304 deletions(-) diff --git a/build_docs.py b/build_docs.py index 3da84df..fa0ce44 100755 --- a/build_docs.py +++ b/build_docs.py @@ -421,55 +421,6 @@ def setup_switchers( ofile.write(line) -def copy_robots_txt( - www_root: Path, - group, - skip_cache_invalidation, - http: urllib3.PoolManager, -) -> None: - """Copy robots.txt to www_root.""" - if not www_root.exists(): - logging.info("Skipping copying robots.txt (www root does not even exist).") - return - logging.info("Copying robots.txt...") - template_path = HERE / "templates" / "robots.txt" - robots_path = www_root / "robots.txt" - shutil.copyfile(template_path, robots_path) - robots_path.chmod(0o775) - run(["chgrp", group, robots_path]) - if not skip_cache_invalidation: - purge(http, "robots.txt") - - -def build_sitemap( - versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group -): - """Build a sitemap with all live versions and translations.""" - if not www_root.exists(): - logging.info("Skipping sitemap generation (www root does not even exist).") - return - logging.info("Starting sitemap generation...") - template_path = HERE / "templates" / "sitemap.xml" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render(languages=languages, versions=versions) - sitemap_path = www_root / "sitemap.xml" - sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") - sitemap_path.chmod(0o664) - run(["chgrp", group, sitemap_path]) - - -def build_404(www_root: Path, group): - """Build a nice 404 error page to display in case PDFs are not built yet.""" - if not www_root.exists(): - logging.info("Skipping 404 page generation (www root does not even exist).") - return - logging.info("Copying 404 page...") - not_found_file = www_root / "404.html" - shutil.copyfile(HERE / "templates" / "404.html", not_found_file) - not_found_file.chmod(0o664) - run(["chgrp", group, not_found_file]) - - def head(text, lines=10): """Return the first *lines* lines from the given text.""" return "\n".join(text.split("\n")[:lines]) @@ -895,188 +846,6 @@ def save_state(self, build_start: dt, build_duration: float, trigger: str): logging.info("Saved new rebuild state for %s: %s", key, table.as_string()) -def symlink( - www_root: Path, - language: Language, - directory: str, - name: str, - group: str, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Used by major_symlinks and dev_symlink to maintain symlinks.""" - if language.tag == "en": # English is rooted on /, no /en/ - path = www_root - else: - path = www_root / language.tag - link = path / name - directory_path = path / directory - if not directory_path.exists(): - return # No touching link, dest doc not built yet. - - if not link.exists() or readlink(link) != directory: - # Link does not exist or points to the wrong target. - link.unlink(missing_ok=True) - link.symlink_to(directory) - run(["chown", "-h", f":{group}", str(link)]) - if not skip_cache_invalidation: - surrogate_key = f"{language.tag}/{name}" - purge_surrogate_key(http, surrogate_key) - - -def major_symlinks( - www_root: Path, - group: str, - versions: Iterable[Version], - languages: Iterable[Language], - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /2/ and /3/ symlinks for each language. - - Like: - - /3/ → /3.9/ - - /fr/3/ → /fr/3.9/ - - /es/3/ → /es/3.9/ - """ - logging.info("Creating major version symlinks...") - current_stable = Version.current_stable(versions).name - for language in languages: - symlink( - www_root, - language, - current_stable, - "3", - group, - skip_cache_invalidation, - http, - ) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) - - -def dev_symlink( - www_root: Path, - group, - versions, - languages, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /dev/ symlinks for each language. - - Like: - - /dev/ → /3.11/ - - /fr/dev/ → /fr/3.11/ - - /es/dev/ → /es/3.11/ - """ - logging.info("Creating development version symlinks...") - current_dev = Version.current_dev(versions).name - for language in languages: - symlink( - www_root, - language, - current_dev, - "dev", - group, - skip_cache_invalidation, - http, - ) - - -def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: - """Remove one or many paths from docs.python.org's CDN. - - To be used when a file changes, so the CDN fetches the new one. - """ - base = "https://docs.python.org/" - for path in paths: - url = urljoin(base, str(path)) - logging.debug("Purging %s from CDN", url) - http.request("PURGE", url, timeout=30) - - -def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: - """Remove paths from docs.python.org's CDN. - - All paths matching the given 'Surrogate-Key' will be removed. - This is set by the Nginx server for every language-version pair. - To be used when a directory changes, so the CDN fetches the new one. - - https://www.fastly.com/documentation/reference/api/purging/#purge-tag - """ - service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") - fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") - - logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) - http.request( - "POST", - f"https://api.fastly.com/service/{service_id}/purge/{surrogate_key}", - headers={"Fastly-Key": fastly_key}, - timeout=30, - ) - - -def proofread_canonicals( - www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager -) -> None: - """In www_root we check that all canonical links point to existing contents. - - It can happen that a canonical is "broken": - - - /3.11/whatsnew/3.11.html typically would link to - /3/whatsnew/3.11.html, which may not exist yet. - """ - logging.info("Checking canonical links...") - canonical_re = re.compile( - """""" - ) - for file in www_root.glob("**/*.html"): - html = file.read_text(encoding="UTF-8", errors="surrogateescape") - canonical = canonical_re.search(html) - if not canonical: - continue - target = canonical.group(1) - if not (www_root / target).exists(): - logging.info("Removing broken canonical from %s to %s", file, target) - html = html.replace(canonical.group(0), "") - file.write_text(html, encoding="UTF-8", errors="surrogateescape") - if not skip_cache_invalidation: - purge(http, str(file).replace("/srv/docs.python.org/", "")) - - -def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: - releases = http.request( - "GET", - "https://raw.githubusercontent.com/" - "python/devguide/main/include/release-cycle.json", - timeout=30, - ).json() - versions = [Version.from_json(name, release) for name, release in releases.items()] - versions.sort(key=Version.as_tuple) - return versions - - -def parse_languages_from_config() -> list[Language]: - """Read config.toml to discover languages to build.""" - config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) - defaults = config["defaults"] - default_translated_name = defaults.get("translated_name", "") - default_in_prod = defaults.get("in_prod", True) - default_sphinxopts = defaults.get("sphinxopts", []) - default_html_only = defaults.get("html_only", False) - return [ - Language( - iso639_tag=iso639_tag, - name=section["name"], - translated_name=section.get("translated_name", default_translated_name), - in_prod=section.get("in_prod", default_in_prod), - sphinxopts=section.get("sphinxopts", default_sphinxopts), - html_only=section.get("html_only", default_html_only), - ) - for iso639_tag, section in config["languages"].items() - ] - - def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) @@ -1091,79 +860,6 @@ def format_seconds(seconds: float) -> str: return f"{h}h {m}m {s}s" -def build_docs(args) -> bool: - """Build all docs (each language and each version).""" - logging.info("Full build start.") - start_time = perf_counter() - http = urllib3.PoolManager() - versions = parse_versions_from_devguide(http) - languages = parse_languages_from_config() - # Reverse languages but not versions, because we take version-language - # pairs from the end of the list, effectively reversing it. - # This runs languages in config.toml order and versions newest first. - todo = [ - (version, language) - for version in Version.filter(versions, args.branch) - for language in reversed(Language.filter(languages, args.languages)) - ] - del args.branch - del args.languages - all_built_successfully = True - cpython_repo = Repository( - "https://github.com/python/cpython.git", - args.build_root / _checkout_name(args.select_output), - ) - while todo: - version, language = todo.pop() - logging.root.handlers[0].setFormatter( - logging.Formatter( - f"%(asctime)s %(levelname)s {language.tag}/{version.name}: %(message)s" - ) - ) - if sentry_sdk: - scope = sentry_sdk.get_isolation_scope() - scope.set_tag("version", version.name) - scope.set_tag("language", language.tag) - cpython_repo.update() - builder = DocBuilder( - version, versions, language, languages, cpython_repo, **vars(args) - ) - all_built_successfully &= builder.run(http) - logging.root.handlers[0].setFormatter( - logging.Formatter("%(asctime)s %(levelname)s: %(message)s") - ) - - build_sitemap(versions, languages, args.www_root, args.group) - build_404(args.www_root, args.group) - copy_robots_txt( - args.www_root, - args.group, - args.skip_cache_invalidation, - http, - ) - major_symlinks( - args.www_root, - args.group, - versions, - languages, - args.skip_cache_invalidation, - http, - ) - dev_symlink( - args.www_root, - args.group, - versions, - languages, - args.skip_cache_invalidation, - http, - ) - proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) - - logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - - return all_built_successfully - - def _checkout_name(select_output: str | None) -> str: if select_output is not None: return f"cpython-{select_output}" @@ -1301,5 +997,309 @@ def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: lock.close() +def build_docs(args) -> bool: + """Build all docs (each language and each version).""" + logging.info("Full build start.") + start_time = perf_counter() + http = urllib3.PoolManager() + versions = parse_versions_from_devguide(http) + languages = parse_languages_from_config() + # Reverse languages but not versions, because we take version-language + # pairs from the end of the list, effectively reversing it. + # This runs languages in config.toml order and versions newest first. + todo = [ + (version, language) + for version in Version.filter(versions, args.branch) + for language in reversed(Language.filter(languages, args.languages)) + ] + del args.branch + del args.languages + all_built_successfully = True + cpython_repo = Repository( + "https://github.com/python/cpython.git", + args.build_root / _checkout_name(args.select_output), + ) + while todo: + version, language = todo.pop() + logging.root.handlers[0].setFormatter( + logging.Formatter( + f"%(asctime)s %(levelname)s {language.tag}/{version.name}: %(message)s" + ) + ) + if sentry_sdk: + scope = sentry_sdk.get_isolation_scope() + scope.set_tag("version", version.name) + scope.set_tag("language", language.tag) + cpython_repo.update() + builder = DocBuilder( + version, versions, language, languages, cpython_repo, **vars(args) + ) + all_built_successfully &= builder.run(http) + logging.root.handlers[0].setFormatter( + logging.Formatter("%(asctime)s %(levelname)s: %(message)s") + ) + + build_sitemap(versions, languages, args.www_root, args.group) + build_404(args.www_root, args.group) + copy_robots_txt( + args.www_root, + args.group, + args.skip_cache_invalidation, + http, + ) + major_symlinks( + args.www_root, + args.group, + versions, + languages, + args.skip_cache_invalidation, + http, + ) + dev_symlink( + args.www_root, + args.group, + versions, + languages, + args.skip_cache_invalidation, + http, + ) + proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) + + logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) + + return all_built_successfully + + +def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: + releases = http.request( + "GET", + "https://raw.githubusercontent.com/" + "python/devguide/main/include/release-cycle.json", + timeout=30, + ).json() + versions = [Version.from_json(name, release) for name, release in releases.items()] + versions.sort(key=Version.as_tuple) + return versions + + +def parse_languages_from_config() -> list[Language]: + """Read config.toml to discover languages to build.""" + config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) + defaults = config["defaults"] + default_translated_name = defaults.get("translated_name", "") + default_in_prod = defaults.get("in_prod", True) + default_sphinxopts = defaults.get("sphinxopts", []) + default_html_only = defaults.get("html_only", False) + return [ + Language( + iso639_tag=iso639_tag, + name=section["name"], + translated_name=section.get("translated_name", default_translated_name), + in_prod=section.get("in_prod", default_in_prod), + sphinxopts=section.get("sphinxopts", default_sphinxopts), + html_only=section.get("html_only", default_html_only), + ) + for iso639_tag, section in config["languages"].items() + ] + + +def build_sitemap( + versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group +): + """Build a sitemap with all live versions and translations.""" + if not www_root.exists(): + logging.info("Skipping sitemap generation (www root does not even exist).") + return + logging.info("Starting sitemap generation...") + template_path = HERE / "templates" / "sitemap.xml" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render(languages=languages, versions=versions) + sitemap_path = www_root / "sitemap.xml" + sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") + sitemap_path.chmod(0o664) + run(["chgrp", group, sitemap_path]) + + +def build_404(www_root: Path, group): + """Build a nice 404 error page to display in case PDFs are not built yet.""" + if not www_root.exists(): + logging.info("Skipping 404 page generation (www root does not even exist).") + return + logging.info("Copying 404 page...") + not_found_file = www_root / "404.html" + shutil.copyfile(HERE / "templates" / "404.html", not_found_file) + not_found_file.chmod(0o664) + run(["chgrp", group, not_found_file]) + + +def copy_robots_txt( + www_root: Path, + group, + skip_cache_invalidation, + http: urllib3.PoolManager, +) -> None: + """Copy robots.txt to www_root.""" + if not www_root.exists(): + logging.info("Skipping copying robots.txt (www root does not even exist).") + return + logging.info("Copying robots.txt...") + template_path = HERE / "templates" / "robots.txt" + robots_path = www_root / "robots.txt" + shutil.copyfile(template_path, robots_path) + robots_path.chmod(0o775) + run(["chgrp", group, robots_path]) + if not skip_cache_invalidation: + purge(http, "robots.txt") + + +def major_symlinks( + www_root: Path, + group: str, + versions: Iterable[Version], + languages: Iterable[Language], + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Maintains the /2/ and /3/ symlinks for each language. + + Like: + - /3/ → /3.9/ + - /fr/3/ → /fr/3.9/ + - /es/3/ → /es/3.9/ + """ + logging.info("Creating major version symlinks...") + current_stable = Version.current_stable(versions).name + for language in languages: + symlink( + www_root, + language, + current_stable, + "3", + group, + skip_cache_invalidation, + http, + ) + symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) + + +def dev_symlink( + www_root: Path, + group, + versions, + languages, + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Maintains the /dev/ symlinks for each language. + + Like: + - /dev/ → /3.11/ + - /fr/dev/ → /fr/3.11/ + - /es/dev/ → /es/3.11/ + """ + logging.info("Creating development version symlinks...") + current_dev = Version.current_dev(versions).name + for language in languages: + symlink( + www_root, + language, + current_dev, + "dev", + group, + skip_cache_invalidation, + http, + ) + + +def symlink( + www_root: Path, + language: Language, + directory: str, + name: str, + group: str, + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Used by major_symlinks and dev_symlink to maintain symlinks.""" + if language.tag == "en": # English is rooted on /, no /en/ + path = www_root + else: + path = www_root / language.tag + link = path / name + directory_path = path / directory + if not directory_path.exists(): + return # No touching link, dest doc not built yet. + + if not link.exists() or readlink(link) != directory: + # Link does not exist or points to the wrong target. + link.unlink(missing_ok=True) + link.symlink_to(directory) + run(["chown", "-h", f":{group}", str(link)]) + if not skip_cache_invalidation: + surrogate_key = f"{language.tag}/{name}" + purge_surrogate_key(http, surrogate_key) + + +def proofread_canonicals( + www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager +) -> None: + """In www_root we check that all canonical links point to existing contents. + + It can happen that a canonical is "broken": + + - /3.11/whatsnew/3.11.html typically would link to + /3/whatsnew/3.11.html, which may not exist yet. + """ + logging.info("Checking canonical links...") + canonical_re = re.compile( + """""" + ) + for file in www_root.glob("**/*.html"): + html = file.read_text(encoding="UTF-8", errors="surrogateescape") + canonical = canonical_re.search(html) + if not canonical: + continue + target = canonical.group(1) + if not (www_root / target).exists(): + logging.info("Removing broken canonical from %s to %s", file, target) + html = html.replace(canonical.group(0), "") + file.write_text(html, encoding="UTF-8", errors="surrogateescape") + if not skip_cache_invalidation: + purge(http, str(file).replace("/srv/docs.python.org/", "")) + + +def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: + """Remove one or many paths from docs.python.org's CDN. + + To be used when a file changes, so the CDN fetches the new one. + """ + base = "https://docs.python.org/" + for path in paths: + url = urljoin(base, str(path)) + logging.debug("Purging %s from CDN", url) + http.request("PURGE", url, timeout=30) + + +def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: + """Remove paths from docs.python.org's CDN. + + All paths matching the given 'Surrogate-Key' will be removed. + This is set by the Nginx server for every language-version pair. + To be used when a directory changes, so the CDN fetches the new one. + + https://www.fastly.com/documentation/reference/api/purging/#purge-tag + """ + service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") + fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") + + logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) + http.request( + "POST", + f"https://api.fastly.com/service/{service_id}/purge/{surrogate_key}", + headers={"Fastly-Key": fastly_key}, + timeout=30, + ) + + if __name__ == "__main__": sys.exit(main()) From e096701ea6bf82281dc1c73936bf8ca2d8b39bee Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:17:57 +0100 Subject: [PATCH 109/176] Improve type annotations for versions and languages (#253) --- build_docs.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index fa0ce44..21b3e6a 100755 --- a/build_docs.py +++ b/build_docs.py @@ -33,7 +33,6 @@ import sys from argparse import ArgumentParser, Namespace from bisect import bisect_left as bisect -from collections.abc import Iterable, Sequence from contextlib import contextmanager, suppress from dataclasses import dataclass from datetime import datetime as dt, timezone @@ -42,7 +41,6 @@ from pathlib import Path from string import Template from time import perf_counter, sleep -from typing import Literal from urllib.parse import urljoin import jinja2 @@ -50,6 +48,14 @@ import urllib3 import zc.lockfile +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Sequence + from typing import Literal, TypeAlias + + Versions: TypeAlias = Sequence["Version"] + Languages: TypeAlias = Sequence["Language"] + try: from os import EX_OK, EX_SOFTWARE as EX_FAILURE except ImportError: @@ -170,7 +176,7 @@ def picker_label(self): return f"pre ({self.name})" return self.name - def setup_indexsidebar(self, versions: Sequence[Version], dest_path: Path): + def setup_indexsidebar(self, versions: Versions, dest_path: Path): """Build indexsidebar.html for Sphinx.""" template_path = HERE / "templates" / "indexsidebar.html" template = jinja2.Template(template_path.read_text(encoding="UTF-8")) @@ -388,9 +394,7 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers( - versions: Sequence[Version], languages: Sequence[Language], html_root: Path -): +def setup_switchers(versions: Versions, languages: Languages, html_root: Path): """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher @@ -462,9 +466,9 @@ class DocBuilder: """Builder for a CPython version and a language.""" version: Version - versions: Sequence[Version] + versions: Versions language: Language - languages: Sequence[Language] + languages: Languages cpython_repo: Repository build_root: Path www_root: Path @@ -1070,7 +1074,7 @@ def build_docs(args) -> bool: return all_built_successfully -def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: +def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: releases = http.request( "GET", "https://raw.githubusercontent.com/" @@ -1082,7 +1086,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: return versions -def parse_languages_from_config() -> list[Language]: +def parse_languages_from_config() -> Languages: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) defaults = config["defaults"] @@ -1103,9 +1107,7 @@ def parse_languages_from_config() -> list[Language]: ] -def build_sitemap( - versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group -): +def build_sitemap(versions: Versions, languages: Languages, www_root: Path, group): """Build a sitemap with all live versions and translations.""" if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exist).") @@ -1155,8 +1157,8 @@ def copy_robots_txt( def major_symlinks( www_root: Path, group: str, - versions: Iterable[Version], - languages: Iterable[Language], + versions: Versions, + languages: Languages, skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: From 044aba7d5a27748d6d28f043446b3334fe76dd68 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:36:45 +0100 Subject: [PATCH 110/176] Create a dataclass for Versions (#254) --- build_docs.py | 123 ++++++++++++++++++++++++++++---------------------- 1 file changed, 69 insertions(+), 54 deletions(-) diff --git a/build_docs.py b/build_docs.py index 21b3e6a..3c8d435 100755 --- a/build_docs.py +++ b/build_docs.py @@ -50,10 +50,9 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Iterator, Sequence from typing import Literal, TypeAlias - Versions: TypeAlias = Sequence["Version"] Languages: TypeAlias = Sequence["Language"] try: @@ -71,6 +70,57 @@ HERE = Path(__file__).resolve().parent +@dataclass(frozen=True, slots=True) +class Versions: + _seq: Sequence[Version] + + def __iter__(self) -> Iterator[Version]: + return iter(self._seq) + + def __reversed__(self) -> Iterator[Version]: + return reversed(self._seq) + + @classmethod + def from_json(cls, data) -> Versions: + versions = sorted( + [Version.from_json(name, release) for name, release in data.items()], + key=Version.as_tuple, + ) + return cls(versions) + + def filter(self, branch: str = "") -> Sequence[Version]: + """Filter the given versions. + + If *branch* is given, only *versions* matching *branch* are returned. + + Else all live versions are returned (this means no EOL and no + security-fixes branches). + """ + if branch: + return [v for v in self if branch in (v.name, v.branch_or_tag)] + return [v for v in self if v.status not in {"EOL", "security-fixes"}] + + @property + def current_stable(self) -> Version: + """Find the current stable CPython version.""" + return max((v for v in self if v.status == "stable"), key=Version.as_tuple) + + @property + def current_dev(self) -> Version: + """Find the current CPython version in development.""" + return max(self, key=Version.as_tuple) + + def setup_indexsidebar(self, current: Version, dest_path: Path) -> None: + """Build indexsidebar.html for Sphinx.""" + template_path = HERE / "templates" / "indexsidebar.html" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render( + current_version=current, + versions=list(reversed(self)), + ) + dest_path.write_text(rendered_template, encoding="UTF-8") + + @total_ordering class Version: """Represents a CPython version and its documentation build dependencies.""" @@ -101,6 +151,17 @@ def __init__(self, name, *, status, branch_or_tag=None): def __repr__(self): return f"Version({self.name})" + def __eq__(self, other): + return self.name == other.name + + def __gt__(self, other): + return self.as_tuple() > other.as_tuple() + + @classmethod + def from_json(cls, name, values): + """Loads a version from devguide's json representation.""" + return cls(name, status=values["status"], branch_or_tag=values["branch"]) + @property def requirements(self): """Generate the right requirements for this version. @@ -144,29 +205,6 @@ def title(self): """The title of this version's doc, for the sidebar.""" return f"Python {self.name} ({self.status})" - @staticmethod - def filter(versions, branch=None): - """Filter the given versions. - - If *branch* is given, only *versions* matching *branch* are returned. - - Else all live versions are returned (this means no EOL and no - security-fixes branches). - """ - if branch: - return [v for v in versions if branch in (v.name, v.branch_or_tag)] - return [v for v in versions if v.status not in ("EOL", "security-fixes")] - - @staticmethod - def current_stable(versions): - """Find the current stable CPython version.""" - return max((v for v in versions if v.status == "stable"), key=Version.as_tuple) - - @staticmethod - def current_dev(versions): - """Find the current CPython version in development.""" - return max(versions, key=Version.as_tuple) - @property def picker_label(self): """Forge the label of a version picker.""" @@ -176,27 +214,6 @@ def picker_label(self): return f"pre ({self.name})" return self.name - def setup_indexsidebar(self, versions: Versions, dest_path: Path): - """Build indexsidebar.html for Sphinx.""" - template_path = HERE / "templates" / "indexsidebar.html" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render( - current_version=self, - versions=versions[::-1], - ) - dest_path.write_text(rendered_template, encoding="UTF-8") - - @classmethod - def from_json(cls, name, values): - """Loads a version from devguide's json representation.""" - return cls(name, status=values["status"], branch_or_tag=values["branch"]) - - def __eq__(self, other): - return self.name == other.name - - def __gt__(self, other): - return self.as_tuple() > other.as_tuple() - @dataclass(order=True, frozen=True, kw_only=True) class Language: @@ -619,8 +636,8 @@ def build(self): + ([""] if sys.platform == "darwin" else []) + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] ) - self.version.setup_indexsidebar( - self.versions, + self.versions.setup_indexsidebar( + self.version, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", ) run_with_logging( @@ -1013,7 +1030,7 @@ def build_docs(args) -> bool: # This runs languages in config.toml order and versions newest first. todo = [ (version, language) - for version in Version.filter(versions, args.branch) + for version in versions.filter(args.branch) for language in reversed(Language.filter(languages, args.languages)) ] del args.branch @@ -1081,9 +1098,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: "python/devguide/main/include/release-cycle.json", timeout=30, ).json() - versions = [Version.from_json(name, release) for name, release in releases.items()] - versions.sort(key=Version.as_tuple) - return versions + return Versions.from_json(releases) def parse_languages_from_config() -> Languages: @@ -1170,7 +1185,7 @@ def major_symlinks( - /es/3/ → /es/3.9/ """ logging.info("Creating major version symlinks...") - current_stable = Version.current_stable(versions).name + current_stable = versions.current_stable.name for language in languages: symlink( www_root, @@ -1200,7 +1215,7 @@ def dev_symlink( - /es/dev/ → /es/3.11/ """ logging.info("Creating development version symlinks...") - current_dev = Version.current_dev(versions).name + current_dev = versions.current_dev.name for language in languages: symlink( www_root, From a8070868f67096d5c05122c29d408ac4c1f1083d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:51:46 +0100 Subject: [PATCH 111/176] Create a dataclass for Languages (#255) --- build_docs.py | 70 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/build_docs.py b/build_docs.py index 3c8d435..ebb5a94 100755 --- a/build_docs.py +++ b/build_docs.py @@ -51,9 +51,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: from collections.abc import Iterator, Sequence - from typing import Literal, TypeAlias - - Languages: TypeAlias = Sequence["Language"] + from typing import Literal try: from os import EX_OK, EX_SOFTWARE as EX_FAILURE @@ -215,13 +213,50 @@ def picker_label(self): return self.name +@dataclass(frozen=True, slots=True) +class Languages: + _seq: Sequence[Language] + + def __iter__(self) -> Iterator[Language]: + return iter(self._seq) + + def __reversed__(self) -> Iterator[Language]: + return reversed(self._seq) + + @classmethod + def from_json(cls, defaults, languages) -> Languages: + default_translated_name = defaults.get("translated_name", "") + default_in_prod = defaults.get("in_prod", True) + default_sphinxopts = defaults.get("sphinxopts", []) + default_html_only = defaults.get("html_only", False) + langs = [ + Language( + iso639_tag=iso639_tag, + name=section["name"], + translated_name=section.get("translated_name", default_translated_name), + in_prod=section.get("in_prod", default_in_prod), + sphinxopts=section.get("sphinxopts", default_sphinxopts), + html_only=section.get("html_only", default_html_only), + ) + for iso639_tag, section in languages.items() + ] + return cls(langs) + + def filter(self, language_tags: Sequence[str] = ()) -> Sequence[Language]: + """Filter a sequence of languages according to --languages.""" + if language_tags: + language_tags = frozenset(language_tags) + return [l for l in self if l.tag in language_tags] + return list(self) + + @dataclass(order=True, frozen=True, kw_only=True) class Language: iso639_tag: str name: str translated_name: str in_prod: bool - sphinxopts: tuple + sphinxopts: Sequence[str] html_only: bool = False @property @@ -234,14 +269,6 @@ def switcher_label(self): return f"{self.name} | {self.translated_name}" return self.name - @staticmethod - def filter(languages, language_tags=None): - """Filter a sequence of languages according to --languages.""" - if language_tags: - languages_dict = {language.tag: language for language in languages} - return [languages_dict[tag] for tag in language_tags] - return languages - def run(cmd, cwd=None) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" @@ -1031,7 +1058,7 @@ def build_docs(args) -> bool: todo = [ (version, language) for version in versions.filter(args.branch) - for language in reversed(Language.filter(languages, args.languages)) + for language in reversed(languages.filter(args.languages)) ] del args.branch del args.languages @@ -1104,22 +1131,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: def parse_languages_from_config() -> Languages: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) - defaults = config["defaults"] - default_translated_name = defaults.get("translated_name", "") - default_in_prod = defaults.get("in_prod", True) - default_sphinxopts = defaults.get("sphinxopts", []) - default_html_only = defaults.get("html_only", False) - return [ - Language( - iso639_tag=iso639_tag, - name=section["name"], - translated_name=section.get("translated_name", default_translated_name), - in_prod=section.get("in_prod", default_in_prod), - sphinxopts=section.get("sphinxopts", default_sphinxopts), - html_only=section.get("html_only", default_html_only), - ) - for iso639_tag, section in config["languages"].items() - ] + return Languages.from_json(config["defaults"], config["languages"]) def build_sitemap(versions: Versions, languages: Languages, www_root: Path, group): From 1a8a586d37840fa5ed43aed225fd9dcb6e8ca07f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:59:27 +0100 Subject: [PATCH 112/176] Convert some relative imports to absolute (#256) --- build_docs.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/build_docs.py b/build_docs.py index ebb5a94..a19f2f8 100755 --- a/build_docs.py +++ b/build_docs.py @@ -22,22 +22,22 @@ from __future__ import annotations +import argparse +import dataclasses +import datetime as dt import filecmp import json import logging import logging.handlers +import os import re import shlex import shutil import subprocess import sys -from argparse import ArgumentParser, Namespace from bisect import bisect_left as bisect from contextlib import contextmanager, suppress -from dataclasses import dataclass -from datetime import datetime as dt, timezone from functools import total_ordering -from os import getenv, readlink from pathlib import Path from string import Template from time import perf_counter, sleep @@ -68,7 +68,7 @@ HERE = Path(__file__).resolve().parent -@dataclass(frozen=True, slots=True) +@dataclasses.dataclass(frozen=True, slots=True) class Versions: _seq: Sequence[Version] @@ -213,7 +213,7 @@ def picker_label(self): return self.name -@dataclass(frozen=True, slots=True) +@dataclasses.dataclass(frozen=True, slots=True) class Languages: _seq: Sequence[Language] @@ -250,7 +250,7 @@ def filter(self, language_tags: Sequence[str] = ()) -> Sequence[Language]: return list(self) -@dataclass(order=True, frozen=True, kw_only=True) +@dataclasses.dataclass(order=True, frozen=True, kw_only=True) class Language: iso639_tag: str name: str @@ -337,7 +337,7 @@ def traverse(dircmp_result): return changed -@dataclass +@dataclasses.dataclass class Repository: """Git repository abstraction for our specific needs.""" @@ -505,7 +505,7 @@ def version_info(): ) -@dataclass +@dataclasses.dataclass class DocBuilder: """Builder for a CPython version and a language.""" @@ -539,7 +539,7 @@ def includes_html(self): def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() - start_timestamp = dt.now(tz=timezone.utc).replace(microsecond=0) + start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) logging.info("Running.") try: if self.language.html_only and not self.includes_html: @@ -861,7 +861,7 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_start: dt, build_duration: float, trigger: str): + def save_state(self, build_start: dt.datetime, build_duration: float, trigger: str): """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -932,8 +932,9 @@ def main(): def parse_args(): """Parse command-line arguments.""" - parser = ArgumentParser( - description="Runs a build of the Python docs for various branches." + parser = argparse.ArgumentParser( + description="Runs a build of the Python docs for various branches.", + allow_abbrev=False, ) parser.add_argument( "--select-output", @@ -1032,7 +1033,7 @@ def setup_logging(log_directory: Path, select_output: str | None): logging.getLogger().setLevel(logging.DEBUG) -def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: +def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: try: lock = zc.lockfile.LockFile(HERE / lockfile_name) except zc.lockfile.LockError: @@ -1045,7 +1046,7 @@ def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: lock.close() -def build_docs(args) -> bool: +def build_docs(args: argparse.Namespace) -> bool: """Build all docs (each language and each version).""" logging.info("Full build start.") start_time = perf_counter() @@ -1259,7 +1260,7 @@ def symlink( if not directory_path.exists(): return # No touching link, dest doc not built yet. - if not link.exists() or readlink(link) != directory: + if not link.exists() or os.readlink(link) != directory: # Link does not exist or points to the wrong target. link.unlink(missing_ok=True) link.symlink_to(directory) @@ -1318,8 +1319,8 @@ def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: https://www.fastly.com/documentation/reference/api/purging/#purge-tag """ - service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") - fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") + service_id = os.environ.get("FASTLY_SERVICE_ID", "__UNSET__") + fastly_key = os.environ.get("FASTLY_TOKEN", "__UNSET__") logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) http.request( From a6c458d9ffaa135ec4a317012c21e9eae04814eb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 03:25:25 +0100 Subject: [PATCH 113/176] Only recreate a symlink when the relevant version has changed (#257) --- build_docs.py | 109 +++++++++++++++++++++----------------------------- 1 file changed, 45 insertions(+), 64 deletions(-) diff --git a/build_docs.py b/build_docs.py index a19f2f8..d03d687 100755 --- a/build_docs.py +++ b/build_docs.py @@ -50,7 +50,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Iterator, Sequence + from collections.abc import Iterator, Sequence, Set from typing import Literal try: @@ -1063,7 +1063,9 @@ def build_docs(args: argparse.Namespace) -> bool: ] del args.branch del args.languages - all_built_successfully = True + + build_succeeded = set() + build_failed = set() cpython_repo = Repository( "https://github.com/python/cpython.git", args.build_root / _checkout_name(args.select_output), @@ -1083,7 +1085,12 @@ def build_docs(args: argparse.Namespace) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - all_built_successfully &= builder.run(http) + built_successfully = builder.run(http) + if built_successfully: + build_succeeded.add((version.name, language.tag)) + else: + build_failed.add((version.name, language.tag)) + logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") ) @@ -1096,19 +1103,12 @@ def build_docs(args: argparse.Namespace) -> bool: args.skip_cache_invalidation, http, ) - major_symlinks( - args.www_root, - args.group, - versions, - languages, - args.skip_cache_invalidation, - http, - ) - dev_symlink( + make_symlinks( args.www_root, args.group, versions, languages, + build_succeeded, args.skip_cache_invalidation, http, ) @@ -1116,7 +1116,7 @@ def build_docs(args: argparse.Namespace) -> bool: logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - return all_built_successfully + return len(build_failed) == 0 def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: @@ -1182,68 +1182,46 @@ def copy_robots_txt( purge(http, "robots.txt") -def major_symlinks( +def make_symlinks( www_root: Path, group: str, versions: Versions, languages: Languages, + successful_builds: Set[tuple[str, str]], skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: - """Maintains the /2/ and /3/ symlinks for each language. + """Maintains the /2/, /3/, and /dev/ symlinks for each language. Like: - - /3/ → /3.9/ - - /fr/3/ → /fr/3.9/ - - /es/3/ → /es/3.9/ + - /2/ → /2.7/ + - /3/ → /3.12/ + - /dev/ → /3.14/ + - /fr/3/ → /fr/3.12/ + - /es/dev/ → /es/3.14/ """ - logging.info("Creating major version symlinks...") - current_stable = versions.current_stable.name - for language in languages: - symlink( - www_root, - language, - current_stable, - "3", - group, - skip_cache_invalidation, - http, - ) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) - - -def dev_symlink( - www_root: Path, - group, - versions, - languages, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /dev/ symlinks for each language. - - Like: - - /dev/ → /3.11/ - - /fr/dev/ → /fr/3.11/ - - /es/dev/ → /es/3.11/ - """ - logging.info("Creating development version symlinks...") - current_dev = versions.current_dev.name - for language in languages: - symlink( - www_root, - language, - current_dev, - "dev", - group, - skip_cache_invalidation, - http, - ) + logging.info("Creating major and development version symlinks...") + for symlink_name, symlink_target in ( + ("3", versions.current_stable.name), + ("2", "2.7"), + ("dev", versions.current_dev.name), + ): + for language in languages: + if (symlink_target, language.tag) in successful_builds: + symlink( + www_root, + language.tag, + symlink_target, + symlink_name, + group, + skip_cache_invalidation, + http, + ) def symlink( www_root: Path, - language: Language, + language_tag: str, directory: str, name: str, group: str, @@ -1251,10 +1229,13 @@ def symlink( http: urllib3.PoolManager, ) -> None: """Used by major_symlinks and dev_symlink to maintain symlinks.""" - if language.tag == "en": # English is rooted on /, no /en/ + msg = "Creating symlink from %s to %s" + if language_tag == "en": # English is rooted on /, no /en/ path = www_root + logging.debug(msg, name, directory) else: - path = www_root / language.tag + path = www_root / language_tag + logging.debug(msg, f"{language_tag}/{name}", f"{language_tag}/{directory}") link = path / name directory_path = path / directory if not directory_path.exists(): @@ -1266,7 +1247,7 @@ def symlink( link.symlink_to(directory) run(["chown", "-h", f":{group}", str(link)]) if not skip_cache_invalidation: - surrogate_key = f"{language.tag}/{name}" + surrogate_key = f"{language_tag}/{name}" purge_surrogate_key(http, surrogate_key) From 3d068f403607669b9408115dce9918a9b915dc62 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 05:41:28 +0100 Subject: [PATCH 114/176] Adjust symlink log message (#259) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index d03d687..b6a58a9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1229,7 +1229,7 @@ def symlink( http: urllib3.PoolManager, ) -> None: """Used by major_symlinks and dev_symlink to maintain symlinks.""" - msg = "Creating symlink from %s to %s" + msg = "Creating symlink from /%s/ to /%s/" if language_tag == "en": # English is rooted on /, no /en/ path = www_root logging.debug(msg, name, directory) From 0ef872fd7e7b00e6e1193a73ebaf37d7eff7b444 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 05:43:29 +0100 Subject: [PATCH 115/176] Skip CDN requests when secrets are unset (#260) --- build_docs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index b6a58a9..4d7bf8d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1300,8 +1300,13 @@ def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: https://www.fastly.com/documentation/reference/api/purging/#purge-tag """ - service_id = os.environ.get("FASTLY_SERVICE_ID", "__UNSET__") - fastly_key = os.environ.get("FASTLY_TOKEN", "__UNSET__") + unset = "__UNSET__" + service_id = os.environ.get("FASTLY_SERVICE_ID", unset) + fastly_key = os.environ.get("FASTLY_TOKEN", unset) + + if service_id == unset or fastly_key == unset: + logging.info("CDN secrets not set, skipping Surrogate-Key purge") + return logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) http.request( From 6d537969095c21f34406ff954535020b690f0d3d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 06:00:37 +0100 Subject: [PATCH 116/176] Add ``--language`` as a synonym of ``--languages`` --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 4d7bf8d..8f5d763 100755 --- a/build_docs.py +++ b/build_docs.py @@ -984,7 +984,7 @@ def parse_args(): default=Path("/var/log/docsbuild/"), ) parser.add_argument( - "--languages", + "--languages", "--language", nargs="*", help="Language translation, as a PEP 545 language tag like" " 'fr' or 'pt-br'. " From 77a363be7d4e4574bb950e02cb504434bdb2e830 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 06:29:32 +0100 Subject: [PATCH 117/176] Add Ruff configuration file (#262) --- .pre-commit-config.yaml | 2 +- .ruff.toml | 7 ++ build_docs.py | 143 ++++++++++++++++++---------------------- check_versions.py | 10 ++- 4 files changed, 76 insertions(+), 86 deletions(-) create mode 100644 .ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 44948ef..4ad4370 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.6 + rev: v0.11.5 hooks: - id: ruff-format diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..e837d03 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,7 @@ +target-version = "py313" # Pin Ruff to Python 3.13 +line-length = 88 +output-format = "full" + +[format] +preview = true +docstring-code-format = true diff --git a/build_docs.py b/build_docs.py index 8f5d763..0ca807e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -608,14 +608,12 @@ def build(self): sphinxopts = list(self.language.sphinxopts) if self.language.tag != "en": locale_dirs = self.build_root / self.version.name / "locale" - sphinxopts.extend( - ( - f"-D locale_dirs={locale_dirs}", - f"-D language={self.language.iso639_tag}", - "-D gettext_compact=0", - "-D translation_progress_classes=1", - ) - ) + sphinxopts.extend(( + f"-D locale_dirs={locale_dirs}", + f"-D language={self.language.iso639_tag}", + "-D gettext_compact=0", + "-D translation_progress_classes=1", + )) if self.language.tag == "ja": # Since luatex doesn't support \ufffd, replace \ufffd with '?'. # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b @@ -667,20 +665,18 @@ def build(self): self.version, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", ) - run_with_logging( - [ - "make", - "-C", - self.checkout / "Doc", - "PYTHON=" + str(python), - "SPHINXBUILD=" + str(sphinxbuild), - "BLURB=" + str(blurb), - "VENVDIR=" + str(self.venv), - "SPHINXOPTS=" + " ".join(sphinxopts), - "SPHINXERRORHANDLING=", - maketarget, - ] - ) + run_with_logging([ + "make", + "-C", + self.checkout / "Doc", + "PYTHON=" + str(python), + "SPHINXBUILD=" + str(sphinxbuild), + "BLURB=" + str(blurb), + "VENVDIR=" + str(self.venv), + "SPHINXOPTS=" + " ".join(sphinxopts), + "SPHINXERRORHANDLING=", + maketarget, + ]) run(["mkdir", "-p", self.log_directory]) run(["chgrp", "-R", self.group, self.log_directory]) if self.includes_html: @@ -743,69 +739,57 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: # Copy built HTML files to webroot (default /srv/docs.python.org) changed = changed_files(self.checkout / "Doc" / "build" / "html", target) logging.info("Copying HTML files to %s", target) - run( - [ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "build" / "html/", - ] - ) + run([ + "chown", + "-R", + ":" + self.group, + self.checkout / "Doc" / "build" / "html/", + ]) run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) - run( - [ - "find", - self.checkout / "Doc" / "build" / "html", - "-type", - "d", - "-exec", - "chmod", - "o+x", - "{}", - ";", - ] - ) - run( - [ - "rsync", - "-a", - "--delete-delay", - "--filter", - "P archives/", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - ] - ) + run([ + "find", + self.checkout / "Doc" / "build" / "html", + "-type", + "d", + "-exec", + "chmod", + "o+x", + "{}", + ";", + ]) + run([ + "rsync", + "-a", + "--delete-delay", + "--filter", + "P archives/", + str(self.checkout / "Doc" / "build" / "html") + "/", + target, + ]) if not self.quick: # Copy archive files to /archives/ logging.debug("Copying dist files.") - run( - [ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "dist", - ] - ) - run( - [ - "chmod", - "-R", - "o+r", - self.checkout / "Doc" / "dist", - ] - ) + run([ + "chown", + "-R", + ":" + self.group, + self.checkout / "Doc" / "dist", + ]) + run([ + "chmod", + "-R", + "o+r", + self.checkout / "Doc" / "dist", + ]) run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) run(["chown", ":" + self.group, target / "archives"]) - run( - [ - "cp", - "-a", - *(self.checkout / "Doc" / "dist").glob("*"), - target / "archives", - ] - ) + run([ + "cp", + "-a", + *(self.checkout / "Doc" / "dist").glob("*"), + target / "archives", + ]) changed.append("archives/") for file in (target / "archives").iterdir(): changed.append("archives/" + file.name) @@ -984,7 +968,8 @@ def parse_args(): default=Path("/var/log/docsbuild/"), ) parser.add_argument( - "--languages", "--language", + "--languages", + "--language", nargs="*", help="Language translation, as a PEP 545 language tag like" " 'fr' or 'pt-br'. " diff --git a/check_versions.py b/check_versions.py index 343c85a..1a1016f 100644 --- a/check_versions.py +++ b/check_versions.py @@ -111,12 +111,10 @@ async def which_sphinx_is_used_in_production(): table = [ [ version.name, - *await asyncio.gather( - *[ - get_version_in_prod(language.tag, version.name) - for language in LANGUAGES - ] - ), + *await asyncio.gather(*[ + get_version_in_prod(language.tag, version.name) + for language in LANGUAGES + ]), ] for version in VERSIONS ] From a6a666d9f5007351803eb83b4525864d314732f9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 07:17:01 +0100 Subject: [PATCH 118/176] Account for skipped builds in ``build_docs()`` (#261) --- build_docs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0ca807e..08ce611 100755 --- a/build_docs.py +++ b/build_docs.py @@ -536,7 +536,7 @@ def includes_html(self): """Does the build we are running include HTML output?""" return self.select_output != "no-html" - def run(self, http: urllib3.PoolManager) -> bool: + def run(self, http: urllib3.PoolManager) -> bool | None: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) @@ -544,7 +544,7 @@ def run(self, http: urllib3.PoolManager) -> bool: try: if self.language.html_only and not self.includes_html: logging.info("Skipping non-HTML build (language is HTML-only).") - return True + return None # skipped self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() @@ -557,6 +557,8 @@ def run(self, http: urllib3.PoolManager) -> bool: build_duration=perf_counter() - start_time, trigger=trigger_reason, ) + else: + return None # skipped except Exception as err: logging.exception("Badly handled exception, human, please help.") if sentry_sdk: @@ -1073,7 +1075,7 @@ def build_docs(args: argparse.Namespace) -> bool: built_successfully = builder.run(http) if built_successfully: build_succeeded.add((version.name, language.tag)) - else: + elif built_successfully is not None: build_failed.add((version.name, language.tag)) logging.root.handlers[0].setFormatter( From e80b7296bdff2d7034d397bdd73b3852e8e1c4c1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 14:10:00 +0100 Subject: [PATCH 119/176] Improve performance for ``proofread_canonicals()`` (#258) --- build_docs.py | 51 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index 08ce611..63b561c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -23,6 +23,7 @@ from __future__ import annotations import argparse +import concurrent.futures import dataclasses import datetime as dt import filecmp @@ -1249,21 +1250,41 @@ def proofread_canonicals( /3/whatsnew/3.11.html, which may not exist yet. """ logging.info("Checking canonical links...") - canonical_re = re.compile( - """""" - ) - for file in www_root.glob("**/*.html"): - html = file.read_text(encoding="UTF-8", errors="surrogateescape") - canonical = canonical_re.search(html) - if not canonical: - continue - target = canonical.group(1) - if not (www_root / target).exists(): - logging.info("Removing broken canonical from %s to %s", file, target) - html = html.replace(canonical.group(0), "") - file.write_text(html, encoding="UTF-8", errors="surrogateescape") - if not skip_cache_invalidation: - purge(http, str(file).replace("/srv/docs.python.org/", "")) + worker_count = (os.cpu_count() or 1) + 2 + with concurrent.futures.ThreadPoolExecutor(worker_count) as executor: + futures = { + executor.submit(_check_canonical_rel, file, www_root) + for file in www_root.glob("**/*.html") + } + paths_to_purge = { + res.relative_to(www_root) # strip the leading /srv/docs.python.org + for fut in concurrent.futures.as_completed(futures) + if (res := fut.result()) is not None + } + if not skip_cache_invalidation: + purge(http, *paths_to_purge) + + +def _check_canonical_rel(file: Path, www_root: Path): + # Check for a canonical relation link in the HTML. + # If one exists, ensure that the target exists + # or otherwise remove the canonical link element. + prefix = b'' + pfx_len = len(prefix) + sfx_len = len(suffix) + html = file.read_bytes() + try: + start = html.index(prefix) + end = html.index(suffix, start + pfx_len) + except ValueError: + return None + target = html[start + pfx_len : end].decode(errors="surrogateescape") + if (www_root / target).exists(): + return None + logging.info("Removing broken canonical from %s to %s", file, target) + file.write_bytes(html[:start] + html[end + sfx_len :]) + return file def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: From 946b6bc68b239931579e56f43f45f6e15d3e60c4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:19:21 +0100 Subject: [PATCH 120/176] Account for recent removal of self-closing tags (#264) --- build_docs.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/build_docs.py b/build_docs.py index 63b561c..ff9eb59 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1265,25 +1265,26 @@ def proofread_canonicals( purge(http, *paths_to_purge) +# Python 3.12 onwards doesn't use self-closing tags for +_canonical_re = re.compile( + b"""""" +) + + def _check_canonical_rel(file: Path, www_root: Path): # Check for a canonical relation link in the HTML. # If one exists, ensure that the target exists # or otherwise remove the canonical link element. - prefix = b'' - pfx_len = len(prefix) - sfx_len = len(suffix) html = file.read_bytes() - try: - start = html.index(prefix) - end = html.index(suffix, start + pfx_len) - except ValueError: + canonical = _canonical_re.search(html) + if canonical is None: return None - target = html[start + pfx_len : end].decode(errors="surrogateescape") + target = canonical[1].decode(encoding="UTF-8", errors="surrogateescape") if (www_root / target).exists(): return None logging.info("Removing broken canonical from %s to %s", file, target) - file.write_bytes(html[:start] + html[end + sfx_len :]) + start, end = canonical.span() + file.write_bytes(html[:start] + html[end:]) return file From 2113fd7daaed924663c986384048defcab68b342 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 11 Apr 2025 17:46:23 +0300 Subject: [PATCH 121/176] Allow passing multiple branches to build via CLI (#235) --- .coveragerc | 7 ++++ .github/workflows/test.yml | 7 ++++ README.md | 7 +++- build_docs.py | 19 +++++----- tests/test_build_docs_versions.py | 62 +++++++++++++++++++++++++++++++ tox.ini | 13 ++++++- 6 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 .coveragerc create mode 100644 tests/test_build_docs_versions.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..0f12707 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +# .coveragerc to control coverage.py + +[report] +# Regexes for lines to exclude from consideration +exclude_also = + # Don't complain if non-runnable code isn't run: + if __name__ == .__main__.: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2976bae..e272e76 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,3 +33,10 @@ jobs: - name: Tox tests run: | uvx --with tox-uv tox -e py + + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + flags: ${{ matrix.os }} + name: ${{ matrix.os }} Python ${{ matrix.python-version }} + token: ${{ secrets.CODECOV_ORG_TOKEN }} diff --git a/README.md b/README.md index 1b76bcd..d78bdb7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ +# docsbuild-scripts + +[![GitHub Actions status](https://github.com/python/docsbuild-scripts/actions/workflows/test.yml/badge.svg)](https://github.com/python/docsbuild-scripts/actions/workflows/test.yml) +[![Codecov](https://codecov.io/gh/python/docsbuild-scripts/branch/main/graph/badge.svg)](https://codecov.io/gh/python/docsbuild-scripts) + This repository contains scripts for automatically building the Python documentation on [docs.python.org](https://docs.python.org). @@ -12,7 +17,7 @@ python3 ./build_docs.py --quick --build-root ./build_root --www-root ./www --log ``` If you don't need to build all translations of all branches, add -`--language en --branch main`. +`--languages en --branches main`. ## Check current version diff --git a/build_docs.py b/build_docs.py index ff9eb59..d4578eb 100755 --- a/build_docs.py +++ b/build_docs.py @@ -87,16 +87,17 @@ def from_json(cls, data) -> Versions: ) return cls(versions) - def filter(self, branch: str = "") -> Sequence[Version]: + def filter(self, branches: Sequence[str] = ()) -> Sequence[Version]: """Filter the given versions. - If *branch* is given, only *versions* matching *branch* are returned. + If *branches* is given, only *versions* matching *branches* are returned. Else all live versions are returned (this means no EOL and no security-fixes branches). """ - if branch: - return [v for v in self if branch in (v.name, v.branch_or_tag)] + if branches: + branches = frozenset(branches) + return [v for v in self if {v.name, v.branch_or_tag} & branches] return [v for v in self if v.status not in {"EOL", "security-fixes"}] @property @@ -936,9 +937,10 @@ def parse_args(): ) parser.add_argument( "-b", - "--branch", + "--branches", + nargs="*", metavar="3.12", - help="Version to build (defaults to all maintained branches).", + help="Versions to build (defaults to all maintained branches).", ) parser.add_argument( "-r", @@ -972,7 +974,6 @@ def parse_args(): ) parser.add_argument( "--languages", - "--language", nargs="*", help="Language translation, as a PEP 545 language tag like" " 'fr' or 'pt-br'. " @@ -1046,10 +1047,10 @@ def build_docs(args: argparse.Namespace) -> bool: # This runs languages in config.toml order and versions newest first. todo = [ (version, language) - for version in versions.filter(args.branch) + for version in versions.filter(args.branches) for language in reversed(languages.filter(args.languages)) ] - del args.branch + del args.branches del args.languages build_succeeded = set() diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py new file mode 100644 index 0000000..5ed9bcb --- /dev/null +++ b/tests/test_build_docs_versions.py @@ -0,0 +1,62 @@ +from build_docs import Versions, Version + + +def test_filter_default() -> None: + # Arrange + versions = Versions([ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + Version("3.11", status="security"), + Version("3.10", status="security"), + Version("3.9", status="security"), + ]) + + # Act + filtered = versions.filter() + + # Assert + assert filtered == [ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + ] + + +def test_filter_one() -> None: + # Arrange + versions = Versions([ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + Version("3.11", status="security"), + Version("3.10", status="security"), + Version("3.9", status="security"), + ]) + + # Act + filtered = versions.filter(["3.13"]) + + # Assert + assert filtered == [Version("3.13", status="security")] + + +def test_filter_multiple() -> None: + # Arrange + versions = Versions([ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + Version("3.11", status="security"), + Version("3.10", status="security"), + Version("3.9", status="security"), + ]) + + # Act + filtered = versions.filter(["3.13", "3.14"]) + + # Assert + assert filtered == [ + Version("3.14", status="feature"), + Version("3.13", status="security"), + ] diff --git a/tox.ini b/tox.ini index 56c6420..12efcdf 100644 --- a/tox.ini +++ b/tox.ini @@ -12,8 +12,19 @@ skip_install = true deps = -r requirements.txt pytest + pytest-cov +pass_env = + FORCE_COLOR +set_env = + COVERAGE_CORE = sysmon commands = - {envpython} -m pytest {posargs} + {envpython} -m pytest \ + --cov . \ + --cov tests \ + --cov-report html \ + --cov-report term \ + --cov-report xml \ + {posargs} [testenv:lint] skip_install = true From bd0e222e4e0844f2790d61e5d66e0243b97073c8 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Fri, 11 Apr 2025 14:49:11 -0400 Subject: [PATCH 122/176] Move environment variables to a configuration file (#269) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- build_docs.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 48 insertions(+) diff --git a/build_docs.py b/build_docs.py index d4578eb..22fc5bd 100755 --- a/build_docs.py +++ b/build_docs.py @@ -5,6 +5,26 @@ Without any arguments builds docs for all active versions and languages. +Environment variables for: + +- `SENTRY_DSN` (Error reporting) +- `FASTLY_SERVICE_ID` / `FASTLY_TOKEN` (CDN purges) +- `PYTHON_DOCS_ENABLE_ANALYTICS` (Enable Plausible for online docs) + +are read from the site configuration path for your platform +(/etc/xdg/docsbuild-scripts on linux) if available, +and can be overriden by writing a file to the user config dir +for your platform ($HOME/.config/docsbuild-scripts on linux). +The contents of the file is parsed as toml: + +```toml +[env] +SENTRY_DSN = "https://0a0a0a0a0a0a0a0a0a0a0a@sentry.io/69420" +FASTLY_SERVICE_ID = "deadbeefdeadbeefdead" +FASTLY_TOKEN = "secureme!" +PYTHON_DOCS_ENABLE_ANALYTICS = "1" +``` + Languages are stored in `config.toml` while versions are discovered from the devguide. @@ -48,6 +68,7 @@ import tomlkit import urllib3 import zc.lockfile +from platformdirs import user_config_path, site_config_path TYPE_CHECKING = False if TYPE_CHECKING: @@ -906,6 +927,7 @@ def main(): """Script entry point.""" args = parse_args() setup_logging(args.log_directory, args.select_output) + load_environment_variables() if args.select_output is None: build_docs_with_lock(args, "build_docs.lock") @@ -1022,6 +1044,31 @@ def setup_logging(log_directory: Path, select_output: str | None): logging.getLogger().setLevel(logging.DEBUG) +def load_environment_variables() -> None: + _user_config_path = user_config_path("docsbuild-scripts") + _site_config_path = site_config_path("docsbuild-scripts") + if _user_config_path.is_file(): + ENV_CONF_FILE = _user_config_path + elif _site_config_path.is_file(): + ENV_CONF_FILE = _site_config_path + else: + logging.info( + "No environment variables configured. " + f"Configure in {_site_config_path} or {_user_config_path}." + ) + return + + logging.info(f"Reading environment variables from {ENV_CONF_FILE}.") + if ENV_CONF_FILE == _site_config_path: + logging.info(f"You can override settings in {_user_config_path}.") + elif _site_config_path.is_file(): + logging.info(f"Overriding {_site_config_path}.") + with open(ENV_CONF_FILE, "r") as f: + for key, value in tomlkit.parse(f.read()).get("env", {}).items(): + logging.debug(f"Setting {key} in environment.") + os.environ[key] = value + + def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: try: lock = zc.lockfile.LockFile(HERE / lockfile_name) diff --git a/requirements.txt b/requirements.txt index 0cac810..535b36a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ jinja2 +platformdirs sentry-sdk>=2 tomlkit>=0.13 urllib3>=2 From 10e20e45c8aaa473b667ec9e2bb4f3a277685488 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:25:54 +0100 Subject: [PATCH 123/176] Require imghdr for building Python 3.8-3.10 (#267) --- build_docs.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/build_docs.py b/build_docs.py index 22fc5bd..1b790c6 100755 --- a/build_docs.py +++ b/build_docs.py @@ -196,23 +196,30 @@ def requirements(self): See https://github.com/python/cpython/issues/91483 """ - if self.name == "3.5": - return ["jieba", "blurb", "sphinx==1.8.4", "jinja2<3.1", "docutils<=0.17.1"] - if self.name in {"3.7", "3.6", "2.7"}: - return ["jieba", "blurb", "sphinx==2.3.1", "jinja2<3.1", "docutils<=0.17.1"] - - return [ + dependencies = [ "jieba", # To improve zh search. "PyStemmer~=2.2.0", # To improve performance for word stemming. "-rrequirements.txt", ] + if self.as_tuple() >= (3, 11): + return dependencies + if self.as_tuple() >= (3, 8): + # Restore the imghdr module for Python 3.8-3.10. + return dependencies + ["standard-imghdr"] + + # Requirements/constraints for Python 3.7 and older, pre-requirements.txt + reqs = ["jieba", "blurb", "jinja2<3.1", "docutils<=0.17.1", "standard-imghdr"] + if self.name in {"3.7", "3.6", "2.7"}: + return reqs + ["sphinx==2.3.1"] + if self.name == "3.5": + return reqs + ["sphinx==1.8.4"] @property def changefreq(self): """Estimate this version change frequency, for the sitemap.""" return {"EOL": "never", "security-fixes": "yearly"}.get(self.status, "daily") - def as_tuple(self): + def as_tuple(self) -> tuple[int, ...]: """This version name as tuple, for easy comparisons.""" return version_to_tuple(self.name) @@ -407,7 +414,7 @@ def update(self): self.clone() or self.fetch() -def version_to_tuple(version): +def version_to_tuple(version) -> tuple[int, ...]: """Transform a version string to a tuple, for easy comparisons.""" return tuple(int(part) for part in version.split(".")) From 776f413c7e174017d59aa38ff9f75dc135f749e1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:30:16 +0100 Subject: [PATCH 124/176] Add ``--force`` to always rebuild (#268) --- README.md | 2 +- build_docs.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d78bdb7..2c84353 100644 --- a/README.md +++ b/README.md @@ -71,5 +71,5 @@ To manually rebuild a branch, for example 3.11: ssh docs.nyc1.psf.io sudo su --shell=/bin/bash docsbuild screen -DUR # Rejoin screen session if it exists, otherwise create a new one -/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --force --branch 3.11 ``` diff --git a/build_docs.py b/build_docs.py index 1b790c6..9727a85 100755 --- a/build_docs.py +++ b/build_docs.py @@ -566,7 +566,7 @@ def includes_html(self): """Does the build we are running include HTML output?""" return self.select_output != "no-html" - def run(self, http: urllib3.PoolManager) -> bool | None: + def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) @@ -578,7 +578,7 @@ def run(self, http: urllib3.PoolManager) -> bool | None: self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() - if trigger_reason := self.should_rebuild(): + if trigger_reason := self.should_rebuild(force_build): self.build_venv() self.build() self.copy_build_to_webroot(http) @@ -834,7 +834,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: "Publishing done (%s).", format_seconds(perf_counter() - start_time) ) - def should_rebuild(self): + def should_rebuild(self, force: bool): state = self.load_state() if not state: logging.info("Should rebuild: no previous state found.") @@ -862,6 +862,9 @@ def should_rebuild(self): cpython_sha, ) return "Doc/ has changed" + if force: + logging.info("Should rebuild: forced.") + return "forced" logging.info("Nothing changed, no rebuild needed.") return False @@ -985,6 +988,12 @@ def parse_args(): help="Path where generated files will be copied.", default=Path("/srv/docs.python.org"), ) + parser.add_argument( + "--force", + action="store_true", + help="Always build the chosen languages and versions, " + "regardless of existing state.", + ) parser.add_argument( "--skip-cache-invalidation", help="Skip Fastly cache invalidation.", @@ -1128,7 +1137,7 @@ def build_docs(args: argparse.Namespace) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - built_successfully = builder.run(http) + built_successfully = builder.run(http, force_build=args.force) if built_successfully: build_succeeded.add((version.name, language.tag)) elif built_successfully is not None: From 82e2a41997ce7bb02b113d429b894119ac2c8437 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:38:18 +0100 Subject: [PATCH 125/176] Delete ``force`` from ``args`` namespace (#270) --- build_docs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 9727a85..309d786 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1115,6 +1115,8 @@ def build_docs(args: argparse.Namespace) -> bool: ] del args.branches del args.languages + force_build = args.force + del args.force build_succeeded = set() build_failed = set() @@ -1137,7 +1139,7 @@ def build_docs(args: argparse.Namespace) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - built_successfully = builder.run(http, force_build=args.force) + built_successfully = builder.run(http, force_build=force_build) if built_successfully: build_succeeded.add((version.name, language.tag)) elif built_successfully is not None: From 4a5d2ca799ae7a7b9240cde7a987b8c63da070b1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 12 Apr 2025 02:14:44 +0300 Subject: [PATCH 126/176] Test for GNU sed instead of macOS (#263) --- build_docs.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 309d786..97eee6d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -687,10 +687,25 @@ def build(self): f"-D ogp_site_url={site_url}", ) + def is_gnu_sed() -> bool: + """Check if we are using GNU sed.""" + try: + subprocess.run( + ["sed", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + return True + except subprocess.CalledProcessError: + return False + except FileNotFoundError: + return False + # Disable CPython switchers, we handle them now: run( ["sed", "-i"] - + ([""] if sys.platform == "darwin" else []) + + ([] if is_gnu_sed() else [""]) + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] ) self.versions.setup_indexsidebar( From c341248dbc162b2ab9c527d95e8de0a9b6958285 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 00:19:50 +0100 Subject: [PATCH 127/176] Ensure that `Doc/dist` exists (#271) --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index 97eee6d..c0a954e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -817,6 +817,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: if not self.quick: # Copy archive files to /archives/ logging.debug("Copying dist files.") + (self.checkout / "Doc" / "dist").mkdir(exist_ok=True) run([ "chown", "-R", From 7a9bca9945f149486a366d4050f658f3f34dc81b Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 01:14:16 +0100 Subject: [PATCH 128/176] Remove use of the ``sed`` command (#272) --- build_docs.py | 42 ++++++------------------------------------ 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/build_docs.py b/build_docs.py index c0a954e..ac4f351 100755 --- a/build_docs.py +++ b/build_docs.py @@ -646,21 +646,6 @@ def build(self): "-D gettext_compact=0", "-D translation_progress_classes=1", )) - if self.language.tag == "ja": - # Since luatex doesn't support \ufffd, replace \ufffd with '?'. - # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b - # Luatex already fixed this issue, so we can remove this once Texlive - # is updated. - # (https://github.com/TeX-Live/luatex/commit/af5faf1) - subprocess.check_output( - "sed -i s/\N{REPLACEMENT CHARACTER}/?/g " - f"{locale_dirs}/ja/LC_MESSAGES/**/*.po", - shell=True, - ) - subprocess.check_output( - f"sed -i s/\N{REPLACEMENT CHARACTER}/?/g {self.checkout}/Doc/**/*.rst", - shell=True, - ) if self.version.status == "EOL": sphinxopts.append("-D html_context.outdated=1") @@ -687,27 +672,12 @@ def build(self): f"-D ogp_site_url={site_url}", ) - def is_gnu_sed() -> bool: - """Check if we are using GNU sed.""" - try: - subprocess.run( - ["sed", "--version"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=True, - ) - return True - except subprocess.CalledProcessError: - return False - except FileNotFoundError: - return False - - # Disable CPython switchers, we handle them now: - run( - ["sed", "-i"] - + ([] if is_gnu_sed() else [""]) - + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] - ) + if self.version.as_tuple() < (3, 8): + # Disable CPython switchers, we handle them now: + text = (self.checkout / "Doc" / "Makefile").read_text(encoding="utf-8") + text = text.replace(" -A switchers=1", "") + (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") + self.versions.setup_indexsidebar( self.version, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", From feef5f169a1a2155097d3b44eb649f34405b0653 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 01:38:36 +0100 Subject: [PATCH 129/176] Only copy archive files if the dist directory exists (#273) --- build_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index ac4f351..654c945 100755 --- a/build_docs.py +++ b/build_docs.py @@ -784,10 +784,9 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: target, ]) - if not self.quick: + if not self.quick and (self.checkout / "Doc" / "dist").is_dir(): # Copy archive files to /archives/ logging.debug("Copying dist files.") - (self.checkout / "Doc" / "dist").mkdir(exist_ok=True) run([ "chown", "-R", From 76192128301d3b553b9b6f287b880d454c245606 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 03:05:54 +0100 Subject: [PATCH 130/176] Add more type hints (#274) --- build_docs.py | 109 ++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 48 deletions(-) diff --git a/build_docs.py b/build_docs.py index 654c945..0eb900e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -72,7 +72,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Iterator, Sequence, Set + from collections.abc import Collection, Iterator, Sequence, Set from typing import Literal try: @@ -101,7 +101,7 @@ def __reversed__(self) -> Iterator[Version]: return reversed(self._seq) @classmethod - def from_json(cls, data) -> Versions: + def from_json(cls, data: dict) -> Versions: versions = sorted( [Version.from_json(name, release) for name, release in data.items()], key=Version.as_tuple, @@ -158,7 +158,9 @@ class Version: "prerelease": "pre-release", } - def __init__(self, name, *, status, branch_or_tag=None): + def __init__( + self, name: str, *, status: str, branch_or_tag: str | None = None + ) -> None: status = self.SYNONYMS.get(status, status) if status not in self.STATUSES: raise ValueError( @@ -169,22 +171,22 @@ def __init__(self, name, *, status, branch_or_tag=None): self.branch_or_tag = branch_or_tag self.status = status - def __repr__(self): + def __repr__(self) -> str: return f"Version({self.name})" - def __eq__(self, other): + def __eq__(self, other: Version) -> bool: return self.name == other.name - def __gt__(self, other): + def __gt__(self, other: Version) -> bool: return self.as_tuple() > other.as_tuple() @classmethod - def from_json(cls, name, values): + def from_json(cls, name: str, values: dict) -> Version: """Loads a version from devguide's json representation.""" return cls(name, status=values["status"], branch_or_tag=values["branch"]) @property - def requirements(self): + def requirements(self) -> list[str]: """Generate the right requirements for this version. Since CPython 3.8 a Doc/requirements.txt file can be used. @@ -213,9 +215,10 @@ def requirements(self): return reqs + ["sphinx==2.3.1"] if self.name == "3.5": return reqs + ["sphinx==1.8.4"] + raise ValueError("unreachable") @property - def changefreq(self): + def changefreq(self) -> str: """Estimate this version change frequency, for the sitemap.""" return {"EOL": "never", "security-fixes": "yearly"}.get(self.status, "daily") @@ -224,17 +227,17 @@ def as_tuple(self) -> tuple[int, ...]: return version_to_tuple(self.name) @property - def url(self): + def url(self) -> str: """The doc URL of this version in production.""" return f"https://docs.python.org/{self.name}/" @property - def title(self): + def title(self) -> str: """The title of this version's doc, for the sidebar.""" return f"Python {self.name} ({self.status})" @property - def picker_label(self): + def picker_label(self) -> str: """Forge the label of a version picker.""" if self.status == "in development": return f"dev ({self.name})" @@ -254,7 +257,7 @@ def __reversed__(self) -> Iterator[Language]: return reversed(self._seq) @classmethod - def from_json(cls, defaults, languages) -> Languages: + def from_json(cls, defaults: dict, languages: dict) -> Languages: default_translated_name = defaults.get("translated_name", "") default_in_prod = defaults.get("in_prod", True) default_sphinxopts = defaults.get("sphinxopts", []) @@ -290,17 +293,19 @@ class Language: html_only: bool = False @property - def tag(self): + def tag(self) -> str: return self.iso639_tag.replace("_", "-").lower() @property - def switcher_label(self): + def switcher_label(self) -> str: if self.translated_name: return f"{self.name} | {self.translated_name}" return self.name -def run(cmd, cwd=None) -> subprocess.CompletedProcess: +def run( + cmd: Sequence[str | Path], cwd: Path | None = None +) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" cmd = list(map(str, cmd)) cmdstring = shlex.join(cmd) @@ -326,7 +331,7 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: return result -def run_with_logging(cmd, cwd=None): +def run_with_logging(cmd: Sequence[str | Path], cwd: Path | None = None) -> None: """Like subprocess.check_call, with logging before the command execution.""" cmd = list(map(str, cmd)) logging.debug("Run: '%s'", shlex.join(cmd)) @@ -348,13 +353,13 @@ def run_with_logging(cmd, cwd=None): raise subprocess.CalledProcessError(return_code, cmd[0]) -def changed_files(left, right): +def changed_files(left: Path, right: Path) -> list[str]: """Compute a list of different files between left and right, recursively. Resulting paths are relative to left. """ changed = [] - def traverse(dircmp_result): + def traverse(dircmp_result: filecmp.dircmp) -> None: base = Path(dircmp_result.left).relative_to(left) for file in dircmp_result.diff_files: changed.append(str(base / file)) @@ -374,11 +379,11 @@ class Repository: remote: str directory: Path - def run(self, *args): + def run(self, *args: str) -> subprocess.CompletedProcess: """Run git command in the clone repository.""" return run(("git", "-C", self.directory) + args) - def get_ref(self, pattern): + def get_ref(self, pattern: str) -> str: """Return the reference of a given tag or branch.""" try: # Maybe it's a branch @@ -387,7 +392,7 @@ def get_ref(self, pattern): # Maybe it's a tag return self.run("show-ref", "-s", "tags/" + pattern).stdout.strip() - def fetch(self): + def fetch(self) -> subprocess.CompletedProcess: """Try (and retry) to run git fetch.""" try: return self.run("fetch") @@ -396,12 +401,12 @@ def fetch(self): sleep(5) return self.run("fetch") - def switch(self, branch_or_tag): + def switch(self, branch_or_tag: str) -> None: """Reset and cleans the repository to the given branch or tag.""" self.run("reset", "--hard", self.get_ref(branch_or_tag), "--") self.run("clean", "-dfqx") - def clone(self): + def clone(self) -> bool: """Maybe clone the repository, if not already cloned.""" if (self.directory / ".git").is_dir(): return False # Already cloned @@ -410,21 +415,23 @@ def clone(self): run(["git", "clone", self.remote, self.directory]) return True - def update(self): + def update(self) -> None: self.clone() or self.fetch() -def version_to_tuple(version) -> tuple[int, ...]: +def version_to_tuple(version: str) -> tuple[int, ...]: """Transform a version string to a tuple, for easy comparisons.""" return tuple(int(part) for part in version.split(".")) -def tuple_to_version(version_tuple): +def tuple_to_version(version_tuple: tuple[int, ...]) -> str: """Reverse version_to_tuple.""" return ".".join(str(part) for part in version_tuple) -def locate_nearest_version(available_versions, target_version): +def locate_nearest_version( + available_versions: Collection[str], target_version: str +) -> str: """Look for the nearest version of target_version in available_versions. Versions are to be given as tuples, like (3, 7) for 3.7. @@ -468,7 +475,7 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers(versions: Versions, languages: Languages, html_root: Path): +def setup_switchers(versions: Versions, languages: Languages, html_root: Path) -> None: """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher @@ -499,12 +506,12 @@ def setup_switchers(versions: Versions, languages: Languages, html_root: Path): ofile.write(line) -def head(text, lines=10): +def head(text: str, lines: int = 10) -> str: """Return the first *lines* lines from the given text.""" return "\n".join(text.split("\n")[:lines]) -def version_info(): +def version_info() -> None: """Handler for --version.""" try: platex_version = head( @@ -554,7 +561,7 @@ class DocBuilder: theme: Path @property - def html_only(self): + def html_only(self) -> bool: return ( self.select_output in {"only-html", "only-html-en"} or self.quick @@ -562,7 +569,7 @@ def html_only(self): ) @property - def includes_html(self): + def includes_html(self) -> bool: """Does the build we are running include HTML output?""" return self.select_output != "no-html" @@ -601,12 +608,12 @@ def checkout(self) -> Path: """Path to CPython git clone.""" return self.build_root / _checkout_name(self.select_output) - def clone_translation(self): + def clone_translation(self) -> None: self.translation_repo.update() self.translation_repo.switch(self.translation_branch) @property - def translation_repo(self): + def translation_repo(self) -> Repository: """See PEP 545 for translations repository naming convention.""" locale_repo = f"https://github.com/python/python-docs-{self.language.tag}.git" @@ -620,7 +627,7 @@ def translation_repo(self): return Repository(locale_repo, locale_clone_dir) @property - def translation_branch(self): + def translation_branch(self) -> str: """Some CPython versions may be untranslated, being either too old or too new. @@ -633,7 +640,7 @@ def translation_branch(self): branches = re.findall(r"/([0-9]+\.[0-9]+)$", remote_branches, re.M) return locate_nearest_version(branches, self.version.name) - def build(self): + def build(self) -> None: """Build this version/language doc.""" logging.info("Build start.") start_time = perf_counter() @@ -702,7 +709,7 @@ def build(self): ) logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) - def build_venv(self): + def build_venv(self) -> None: """Build a venv for the specific Python version. So we can reuse them from builds to builds, while they contain @@ -819,7 +826,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: "Publishing done (%s).", format_seconds(perf_counter() - start_time) ) - def should_rebuild(self, force: bool): + def should_rebuild(self, force: bool) -> str | Literal[False]: state = self.load_state() if not state: logging.info("Should rebuild: no previous state found.") @@ -865,7 +872,9 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_start: dt.datetime, build_duration: float, trigger: str): + def save_state( + self, build_start: dt.datetime, build_duration: float, trigger: str + ) -> None: """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -911,6 +920,8 @@ def format_seconds(seconds: float) -> str: case h, m, s: return f"{h}h {m}m {s}s" + raise ValueError("unreachable") + def _checkout_name(select_output: str | None) -> str: if select_output is not None: @@ -918,7 +929,7 @@ def _checkout_name(select_output: str | None) -> str: return "cpython" -def main(): +def main() -> None: """Script entry point.""" args = parse_args() setup_logging(args.log_directory, args.select_output) @@ -934,7 +945,7 @@ def main(): build_docs_with_lock(args, "build_docs_html_en.lock") -def parse_args(): +def parse_args() -> argparse.Namespace: """Parse command-line arguments.""" parser = argparse.ArgumentParser( @@ -1028,7 +1039,7 @@ def parse_args(): return args -def setup_logging(log_directory: Path, select_output: str | None): +def setup_logging(log_directory: Path, select_output: str | None) -> None: """Setup logging to stderr if run by a human, or to a file if run from a cron.""" log_format = "%(asctime)s %(levelname)s: %(message)s" if sys.stderr.isatty(): @@ -1174,7 +1185,9 @@ def parse_languages_from_config() -> Languages: return Languages.from_json(config["defaults"], config["languages"]) -def build_sitemap(versions: Versions, languages: Languages, www_root: Path, group): +def build_sitemap( + versions: Versions, languages: Languages, www_root: Path, group: str +) -> None: """Build a sitemap with all live versions and translations.""" if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exist).") @@ -1189,7 +1202,7 @@ def build_sitemap(versions: Versions, languages: Languages, www_root: Path, grou run(["chgrp", group, sitemap_path]) -def build_404(www_root: Path, group): +def build_404(www_root: Path, group: str) -> None: """Build a nice 404 error page to display in case PDFs are not built yet.""" if not www_root.exists(): logging.info("Skipping 404 page generation (www root does not even exist).") @@ -1203,8 +1216,8 @@ def build_404(www_root: Path, group): def copy_robots_txt( www_root: Path, - group, - skip_cache_invalidation, + group: str, + skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: """Copy robots.txt to www_root.""" @@ -1322,7 +1335,7 @@ def proofread_canonicals( ) -def _check_canonical_rel(file: Path, www_root: Path): +def _check_canonical_rel(file: Path, www_root: Path) -> Path | None: # Check for a canonical relation link in the HTML. # If one exists, ensure that the target exists # or otherwise remove the canonical link element. From 2fee30f3edc36216b0e329a93703d038b960d446 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:05:11 +0100 Subject: [PATCH 131/176] Refresh versions table in README (#276) --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2c84353..8f914c6 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,9 @@ of Sphinx we're using where: 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' - 3.12 ø sphinx~=8.1.0 needs_sphinx='7.2.6' - 3.13 ø sphinx~=8.1.0 needs_sphinx='7.2.6' - 3.14 ø sphinx~=8.1.0 needs_sphinx='7.2.6' + 3.12 ø sphinx~=8.2.0 needs_sphinx='8.2.0' + 3.13 ø sphinx~=8.2.0 needs_sphinx='8.2.0' + 3.14 ø sphinx~=8.2.0 needs_sphinx='8.2.0' ========= ============= ================== ==================== Sphinx build as seen on docs.python.org: @@ -52,9 +52,9 @@ of Sphinx we're using where: 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 - 3.12 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 - 3.13 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 - 3.14 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 + 3.12 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 + 3.13 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 + 3.14 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= ## Manually rebuild a branch From 57136bc88571e8115b30f94ca9adaebb2cd7cd38 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:06:16 +0100 Subject: [PATCH 132/176] Use Ruff linting (#277) --- .pre-commit-config.yaml | 1 + .ruff.toml | 22 ++++++++++++++++ build_docs.py | 42 ++++++++++++++++--------------- tests/test_build_docs.py | 2 +- tests/test_build_docs_versions.py | 2 +- 5 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4ad4370..c0eb053 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.11.5 hooks: + - id: ruff - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema diff --git a/.ruff.toml b/.ruff.toml index e837d03..6862f11 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -5,3 +5,25 @@ output-format = "full" [format] preview = true docstring-code-format = true + +[lint] +preview = true +select = [ + "C4", # flake8-comprehensions + "B", # flake8-bugbear + "E", # pycodestyle + "F", # pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "I", # isort + "N", # pep8-naming + "PERF", # perflint + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "TC", # flake8-type-checking + "UP", # pyupgrade + "W", # pycodestyle +] +ignore = [ + "E501", # Ignore line length errors (we use auto-formatting) +] diff --git a/build_docs.py b/build_docs.py index 0eb900e..7329bbe 100755 --- a/build_docs.py +++ b/build_docs.py @@ -65,10 +65,10 @@ from urllib.parse import urljoin import jinja2 +import platformdirs import tomlkit import urllib3 import zc.lockfile -from platformdirs import user_config_path, site_config_path TYPE_CHECKING = False if TYPE_CHECKING: @@ -76,7 +76,8 @@ from typing import Literal try: - from os import EX_OK, EX_SOFTWARE as EX_FAILURE + from os import EX_OK + from os import EX_SOFTWARE as EX_FAILURE except ImportError: EX_OK, EX_FAILURE = 0, 1 @@ -279,7 +280,7 @@ def filter(self, language_tags: Sequence[str] = ()) -> Sequence[Language]: """Filter a sequence of languages according to --languages.""" if language_tags: language_tags = frozenset(language_tags) - return [l for l in self if l.tag in language_tags] + return [l for l in self if l.tag in language_tags] # NoQA: E741 return list(self) @@ -480,7 +481,7 @@ def setup_switchers(versions: Versions, languages: Languages, html_root: Path) - - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) + language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] switchers_template_file = HERE / "templates" / "switchers.js" @@ -1057,28 +1058,29 @@ def setup_logging(log_directory: Path, select_output: str | None) -> None: def load_environment_variables() -> None: - _user_config_path = user_config_path("docsbuild-scripts") - _site_config_path = site_config_path("docsbuild-scripts") - if _user_config_path.is_file(): - ENV_CONF_FILE = _user_config_path - elif _site_config_path.is_file(): - ENV_CONF_FILE = _site_config_path + dbs_user_config = platformdirs.user_config_path("docsbuild-scripts") + dbs_site_config = platformdirs.site_config_path("docsbuild-scripts") + if dbs_user_config.is_file(): + env_conf_file = dbs_user_config + elif dbs_site_config.is_file(): + env_conf_file = dbs_site_config else: logging.info( "No environment variables configured. " - f"Configure in {_site_config_path} or {_user_config_path}." + f"Configure in {dbs_site_config} or {dbs_user_config}." ) return - logging.info(f"Reading environment variables from {ENV_CONF_FILE}.") - if ENV_CONF_FILE == _site_config_path: - logging.info(f"You can override settings in {_user_config_path}.") - elif _site_config_path.is_file(): - logging.info(f"Overriding {_site_config_path}.") - with open(ENV_CONF_FILE, "r") as f: - for key, value in tomlkit.parse(f.read()).get("env", {}).items(): - logging.debug(f"Setting {key} in environment.") - os.environ[key] = value + logging.info(f"Reading environment variables from {env_conf_file}.") + if env_conf_file == dbs_site_config: + logging.info(f"You can override settings in {dbs_user_config}.") + elif dbs_site_config.is_file(): + logging.info(f"Overriding {dbs_site_config}.") + + env_config = env_conf_file.read_text(encoding="utf-8") + for key, value in tomlkit.parse(env_config).get("env", {}).items(): + logging.debug(f"Setting {key} in environment.") + os.environ[key] = value def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: diff --git a/tests/test_build_docs.py b/tests/test_build_docs.py index 4457e95..028da90 100644 --- a/tests/test_build_docs.py +++ b/tests/test_build_docs.py @@ -4,7 +4,7 @@ @pytest.mark.parametrize( - "seconds, expected", + ("seconds", "expected"), [ (0.4, "0s"), (0.5, "0s"), diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py index 5ed9bcb..662838e 100644 --- a/tests/test_build_docs_versions.py +++ b/tests/test_build_docs_versions.py @@ -1,4 +1,4 @@ -from build_docs import Versions, Version +from build_docs import Version, Versions def test_filter_default() -> None: From 3deb0c3248f9016419868ed25a87d617ad786d04 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:10:16 +0100 Subject: [PATCH 133/176] Use strict mode for ``flake8-type-checking`` --- .ruff.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ruff.toml b/.ruff.toml index 6862f11..52976fe 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -27,3 +27,7 @@ select = [ ignore = [ "E501", # Ignore line length errors (we use auto-formatting) ] + +[lint.flake8-type-checking] +exempt-modules = [] +strict = true From 8c8b000889e51c078045a61b822ced8e6f226609 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:25:36 +0100 Subject: [PATCH 134/176] Enable flake8-logging-format in Ruff (#278) --- .ruff.toml | 1 + build_docs.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 52976fe..47cbf74 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -15,6 +15,7 @@ select = [ "F", # pyflakes "FA", # flake8-future-annotations "FLY", # flynt + "G", # flake8-logging-format "I", # isort "N", # pep8-naming "PERF", # perflint diff --git a/build_docs.py b/build_docs.py index 7329bbe..a204560 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1066,20 +1066,21 @@ def load_environment_variables() -> None: env_conf_file = dbs_site_config else: logging.info( - "No environment variables configured. " - f"Configure in {dbs_site_config} or {dbs_user_config}." + "No environment variables configured. Configure in %s or %s.", + dbs_site_config, + dbs_user_config, ) return - logging.info(f"Reading environment variables from {env_conf_file}.") + logging.info("Reading environment variables from %s.", env_conf_file) if env_conf_file == dbs_site_config: - logging.info(f"You can override settings in {dbs_user_config}.") + logging.info("You can override settings in %s.", dbs_user_config) elif dbs_site_config.is_file(): - logging.info(f"Overriding {dbs_site_config}.") + logging.info("Overriding %s.", dbs_site_config) env_config = env_conf_file.read_text(encoding="utf-8") for key, value in tomlkit.parse(env_config).get("env", {}).items(): - logging.debug(f"Setting {key} in environment.") + logging.debug("Setting %s in environment.", key) os.environ[key] = value From 88404600ea006ee14ea1bdbe716c925acba67bbb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 15:46:32 +0100 Subject: [PATCH 135/176] Add a simple integration test (#275) --- .github/workflows/test.yml | 87 ++++++++++++++++++++++++++------------ build_docs.py | 2 +- 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e272e76..d7ebf7f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,9 @@ name: Test -on: [push, pull_request, workflow_dispatch] +on: + push: + pull_request: + workflow_dispatch: permissions: {} @@ -8,35 +11,63 @@ env: FORCE_COLOR: 1 jobs: - test: - runs-on: ${{ matrix.os }} + integration: + name: Integration test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Set up requirements + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Build documentation + run: > + python ./build_docs.py + --quick + --build-root ./build_root + --www-root ./www + --log-directory ./logs + --group "$(id -g)" + --skip-cache-invalidation + --languages en + --branches 3.14 + + unit: + name: Unit tests + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.13", "3.14"] - os: [ubuntu-latest] + python-version: + - "3.13" + - "3.14" steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - - - name: Install uv - uses: hynek/setup-cached-uv@v2 - - - name: Tox tests - run: | - uvx --with tox-uv tox -e py - - - name: Upload coverage - uses: codecov/codecov-action@v5 - with: - flags: ${{ matrix.os }} - name: ${{ matrix.os }} Python ${{ matrix.python-version }} - token: ${{ secrets.CODECOV_ORG_TOKEN }} + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Install uv + uses: hynek/setup-cached-uv@v2 + + - name: Tox tests + run: uvx --with tox-uv tox -e py + + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + name: Python ${{ matrix.python-version }} + token: ${{ secrets.CODECOV_ORG_TOKEN }} diff --git a/build_docs.py b/build_docs.py index a204560..2301918 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1043,7 +1043,7 @@ def parse_args() -> argparse.Namespace: def setup_logging(log_directory: Path, select_output: str | None) -> None: """Setup logging to stderr if run by a human, or to a file if run from a cron.""" log_format = "%(asctime)s %(levelname)s: %(message)s" - if sys.stderr.isatty(): + if sys.stderr.isatty() or "CI" in os.environ: logging.basicConfig(format=log_format, stream=sys.stderr) else: log_directory.mkdir(parents=True, exist_ok=True) From 5cfd94e84ead25ce6baa3d859a53c30ea37eac7e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 15:51:56 +0100 Subject: [PATCH 136/176] Convert ``Version`` to a dataclass (#279) --- build_docs.py | 53 +++++++++++++------------------ tests/test_build_docs_versions.py | 48 ++++++++++++++-------------- 2 files changed, 46 insertions(+), 55 deletions(-) diff --git a/build_docs.py b/build_docs.py index 2301918..1efdabf 100755 --- a/build_docs.py +++ b/build_docs.py @@ -58,7 +58,6 @@ import sys from bisect import bisect_left as bisect from contextlib import contextmanager, suppress -from functools import total_ordering from pathlib import Path from string import Template from time import perf_counter, sleep @@ -103,11 +102,23 @@ def __reversed__(self) -> Iterator[Version]: @classmethod def from_json(cls, data: dict) -> Versions: - versions = sorted( - [Version.from_json(name, release) for name, release in data.items()], - key=Version.as_tuple, - ) - return cls(versions) + """Load versions from the devguide's JSON representation.""" + permitted = ", ".join(sorted(Version.STATUSES | Version.SYNONYMS.keys())) + + versions = [] + for name, release in data.items(): + branch = release["branch"] + status = release["status"] + status = Version.SYNONYMS.get(status, status) + if status not in Version.STATUSES: + msg = ( + f"Saw invalid version status {status!r}, " + f"expected to be one of {permitted}." + ) + raise ValueError(msg) + versions.append(Version(name=name, status=status, branch_or_tag=branch)) + + return cls(sorted(versions, key=Version.as_tuple)) def filter(self, branches: Sequence[str] = ()) -> Sequence[Version]: """Filter the given versions. @@ -143,10 +154,14 @@ def setup_indexsidebar(self, current: Version, dest_path: Path) -> None: dest_path.write_text(rendered_template, encoding="UTF-8") -@total_ordering +@dataclasses.dataclass(frozen=True, kw_only=True, slots=True) class Version: """Represents a CPython version and its documentation build dependencies.""" + name: str + status: Literal["EOL", "security-fixes", "stable", "pre-release", "in development"] + branch_or_tag: str + STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"} # Those synonyms map branch status vocabulary found in the devguide @@ -159,33 +174,9 @@ class Version: "prerelease": "pre-release", } - def __init__( - self, name: str, *, status: str, branch_or_tag: str | None = None - ) -> None: - status = self.SYNONYMS.get(status, status) - if status not in self.STATUSES: - raise ValueError( - "Version status expected to be one of: " - f"{', '.join(self.STATUSES | set(self.SYNONYMS.keys()))}, got {status!r}." - ) - self.name = name - self.branch_or_tag = branch_or_tag - self.status = status - - def __repr__(self) -> str: - return f"Version({self.name})" - def __eq__(self, other: Version) -> bool: return self.name == other.name - def __gt__(self, other: Version) -> bool: - return self.as_tuple() > other.as_tuple() - - @classmethod - def from_json(cls, name: str, values: dict) -> Version: - """Loads a version from devguide's json representation.""" - return cls(name, status=values["status"], branch_or_tag=values["branch"]) - @property def requirements(self) -> list[str]: """Generate the right requirements for this version. diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py index 662838e..42b5392 100644 --- a/tests/test_build_docs_versions.py +++ b/tests/test_build_docs_versions.py @@ -4,12 +4,12 @@ def test_filter_default() -> None: # Arrange versions = Versions([ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), - Version("3.11", status="security"), - Version("3.10", status="security"), - Version("3.9", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.11", status="security-fixes", branch_or_tag=""), + Version(name="3.10", status="security-fixes", branch_or_tag=""), + Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) # Act @@ -17,39 +17,39 @@ def test_filter_default() -> None: # Assert assert filtered == [ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), ] def test_filter_one() -> None: # Arrange versions = Versions([ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), - Version("3.11", status="security"), - Version("3.10", status="security"), - Version("3.9", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.11", status="security-fixes", branch_or_tag=""), + Version(name="3.10", status="security-fixes", branch_or_tag=""), + Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) # Act filtered = versions.filter(["3.13"]) # Assert - assert filtered == [Version("3.13", status="security")] + assert filtered == [Version(name="3.13", status="security-fixes", branch_or_tag="")] def test_filter_multiple() -> None: # Arrange versions = Versions([ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), - Version("3.11", status="security"), - Version("3.10", status="security"), - Version("3.9", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.11", status="security-fixes", branch_or_tag=""), + Version(name="3.10", status="security-fixes", branch_or_tag=""), + Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) # Act @@ -57,6 +57,6 @@ def test_filter_multiple() -> None: # Assert assert filtered == [ - Version("3.14", status="feature"), - Version("3.13", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="security-fixes", branch_or_tag=""), ] From 9f3437707aa698ecc560a5d911cfbad9803541e7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 19:00:31 +0100 Subject: [PATCH 137/176] Extract ``switchers.js`` rendering into a new function. (#280) Previously, identical content for ``switchers.js`` was rendered for each version-language pair. This commit pre-renders the content of the file, then writing it to disk for each version-language pair. --- build_docs.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index 1efdabf..08f457e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -467,23 +467,13 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers(versions: Versions, languages: Languages, html_root: Path) -> None: +def setup_switchers(script_content: bytes, html_root: Path) -> None: """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 - version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] - - switchers_template_file = HERE / "templates" / "switchers.js" switchers_path = html_root / "_static" / "switchers.js" - - template = Template(switchers_template_file.read_text(encoding="UTF-8")) - rendered_template = template.safe_substitute( - LANGUAGES=json.dumps(language_pairs), - VERSIONS=json.dumps(version_pairs), - ) - switchers_path.write_text(rendered_template, encoding="UTF-8") + switchers_path.write_text(script_content, encoding="UTF-8") for file in html_root.glob("**/*.html"): depth = len(file.relative_to(html_root).parts) - 1 @@ -541,8 +531,8 @@ class DocBuilder: version: Version versions: Versions language: Language - languages: Languages cpython_repo: Repository + switchers_content: bytes build_root: Path www_root: Path select_output: Literal["no-html", "only-html", "only-html-en"] | None @@ -697,7 +687,7 @@ def build(self) -> None: run(["chgrp", "-R", self.group, self.log_directory]) if self.includes_html: setup_switchers( - self.versions, self.languages, self.checkout / "Doc" / "build" / "html" + self.switchers_content, self.checkout / "Doc" / "build" / "html" ) logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) @@ -1108,6 +1098,8 @@ def build_docs(args: argparse.Namespace) -> bool: force_build = args.force del args.force + switchers_content = render_switchers(versions, languages) + build_succeeded = set() build_failed = set() cpython_repo = Repository( @@ -1127,7 +1119,12 @@ def build_docs(args: argparse.Namespace) -> bool: scope.set_tag("language", language.tag) cpython_repo.update() builder = DocBuilder( - version, versions, language, languages, cpython_repo, **vars(args) + version, + versions, + language, + cpython_repo, + switchers_content, + **vars(args), ) built_successfully = builder.run(http, force_build=force_build) if built_successfully: @@ -1179,6 +1176,19 @@ def parse_languages_from_config() -> Languages: return Languages.from_json(config["defaults"], config["languages"]) +def render_switchers(versions: Versions, languages: Languages) -> bytes: + language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 + version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] + + switchers_template_file = HERE / "templates" / "switchers.js" + template = Template(switchers_template_file.read_text(encoding="UTF-8")) + rendered_template = template.safe_substitute( + LANGUAGES=json.dumps(language_pairs), + VERSIONS=json.dumps(version_pairs), + ) + return rendered_template.encode("UTF-8") + + def build_sitemap( versions: Versions, languages: Languages, www_root: Path, group: str ) -> None: From 638d5e6634869c769ac49ac655c92c47d9f4a662 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 19:02:28 +0100 Subject: [PATCH 138/176] Correct the type annotation for ``DocBuilder.theme`` (#281) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 08f457e..07c98ae 100755 --- a/build_docs.py +++ b/build_docs.py @@ -540,7 +540,7 @@ class DocBuilder: group: str log_directory: Path skip_cache_invalidation: bool - theme: Path + theme: str @property def html_only(self) -> bool: From f30fec8a7a811e7f325cadd705f7db4578e2d7a4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 22:07:23 +0100 Subject: [PATCH 139/176] Skip checking canonical links if nothing was built (#282) --- build_docs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index 07c98ae..faa8648 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1101,7 +1101,7 @@ def build_docs(args: argparse.Namespace) -> bool: switchers_content = render_switchers(versions, languages) build_succeeded = set() - build_failed = set() + any_build_failed = False cpython_repo = Repository( "https://github.com/python/cpython.git", args.build_root / _checkout_name(args.select_output), @@ -1130,7 +1130,7 @@ def build_docs(args: argparse.Namespace) -> bool: if built_successfully: build_succeeded.add((version.name, language.tag)) elif built_successfully is not None: - build_failed.add((version.name, language.tag)) + any_build_failed = True logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") @@ -1153,11 +1153,13 @@ def build_docs(args: argparse.Namespace) -> bool: args.skip_cache_invalidation, http, ) - proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) + if build_succeeded: + # Only check canonicals if at least one version was built. + proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - return len(build_failed) == 0 + return any_build_failed def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: From 982c0f8b71f6fa10a8378eb3e3a60ed50fea4e5a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 13 Apr 2025 11:32:47 +0100 Subject: [PATCH 140/176] Fix TypeError in ``setup_switchers()`` (#284) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index faa8648..fef379e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -473,7 +473,7 @@ def setup_switchers(script_content: bytes, html_root: Path) -> None: - Cross-link various versions in a version switcher """ switchers_path = html_root / "_static" / "switchers.js" - switchers_path.write_text(script_content, encoding="UTF-8") + switchers_path.write_bytes(script_content) for file in html_root.glob("**/*.html"): depth = len(file.relative_to(html_root).parts) - 1 From cc5e153b4afc6bbc7dd5aac99ee8f396d9b9b7a4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 13 Apr 2025 13:50:33 +0100 Subject: [PATCH 141/176] Ensure exit status is returned by ``build_docs.py`` (#286) --- build_docs.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/build_docs.py b/build_docs.py index fef379e..e40259c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -911,20 +911,21 @@ def _checkout_name(select_output: str | None) -> str: return "cpython" -def main() -> None: +def main() -> int: """Script entry point.""" args = parse_args() setup_logging(args.log_directory, args.select_output) load_environment_variables() if args.select_output is None: - build_docs_with_lock(args, "build_docs.lock") - elif args.select_output == "no-html": - build_docs_with_lock(args, "build_docs_archives.lock") - elif args.select_output == "only-html": - build_docs_with_lock(args, "build_docs_html.lock") - elif args.select_output == "only-html-en": - build_docs_with_lock(args, "build_docs_html_en.lock") + return build_docs_with_lock(args, "build_docs.lock") + if args.select_output == "no-html": + return build_docs_with_lock(args, "build_docs_archives.lock") + if args.select_output == "only-html": + return build_docs_with_lock(args, "build_docs_html.lock") + if args.select_output == "only-html-en": + return build_docs_with_lock(args, "build_docs_html_en.lock") + return EX_FAILURE def parse_args() -> argparse.Namespace: @@ -1073,12 +1074,12 @@ def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: return EX_FAILURE try: - return EX_OK if build_docs(args) else EX_FAILURE + return build_docs(args) finally: lock.close() -def build_docs(args: argparse.Namespace) -> bool: +def build_docs(args: argparse.Namespace) -> int: """Build all docs (each language and each version).""" logging.info("Full build start.") start_time = perf_counter() @@ -1159,7 +1160,7 @@ def build_docs(args: argparse.Namespace) -> bool: logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - return any_build_failed + return EX_FAILURE if any_build_failed else EX_OK def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: @@ -1397,4 +1398,4 @@ def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) From 1e856ea3a38944e9743c11110d67599c3159cb07 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 13 Apr 2025 19:08:49 +0300 Subject: [PATCH 142/176] Run unit tests on three operating systems (#285) Co-authored-by: Ezio Melotti --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7ebf7f..8a9324f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,13 +41,12 @@ jobs: unit: name: Unit tests - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: - - "3.13" - - "3.14" + python-version: ["3.13", "3.14"] + os: [windows-latest, macos-latest, ubuntu-latest] steps: - uses: actions/checkout@v4 @@ -69,5 +68,6 @@ jobs: - name: Upload coverage uses: codecov/codecov-action@v5 with: + flags: ${{ matrix.os }} name: Python ${{ matrix.python-version }} token: ${{ secrets.CODECOV_ORG_TOKEN }} From 9f45c4e38487289db9dacaea600ecec2720b1507 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:44:32 +0300 Subject: [PATCH 143/176] Test the ``Version`` class (#287) --- tests/test_build_docs_version.py | 88 ++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/test_build_docs_version.py diff --git a/tests/test_build_docs_version.py b/tests/test_build_docs_version.py new file mode 100644 index 0000000..bd5e644 --- /dev/null +++ b/tests/test_build_docs_version.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import pytest + +from build_docs import Version + + +def test_equality() -> None: + # Arrange + version1 = Version(name="3.13", status="stable", branch_or_tag="3.13") + version2 = Version(name="3.13", status="stable", branch_or_tag="3.13") + + # Act / Assert + assert version1 == version2 + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("3.13", "-rrequirements.txt"), + ("3.10", "standard-imghdr"), + ("3.7", "sphinx==2.3.1"), + ("3.5", "sphinx==1.8.4"), + ], +) +def test_requirements(name: str, expected: str) -> None: + # Arrange + version = Version(name=name, status="stable", branch_or_tag="") + + # Act / Assert + assert expected in version.requirements + + +def test_requirements_error() -> None: + # Arrange + version = Version(name="2.8", status="ex-release", branch_or_tag="") + + # Act / Assert + with pytest.raises(ValueError, match="unreachable"): + _ = version.requirements + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + ("EOL", "never"), + ("security-fixes", "yearly"), + ("stable", "daily"), + ], +) +def test_changefreq(status: str, expected: str) -> None: + # Arrange + version = Version(name="3.13", status=status, branch_or_tag="") + + # Act / Assert + assert version.changefreq == expected + + +def test_url() -> None: + # Arrange + version = Version(name="3.13", status="stable", branch_or_tag="") + + # Act / Assert + assert version.url == "https://docs.python.org/3.13/" + + +def test_title() -> None: + # Arrange + version = Version(name="3.14", status="in development", branch_or_tag="") + + # Act / Assert + assert version.title == "Python 3.14 (in development)" + + +@pytest.mark.parametrize( + ("name", "status", "expected"), + [ + ("3.15", "in development", "dev (3.15)"), + ("3.14", "pre-release", "pre (3.14)"), + ("3.13", "stable", "3.13"), + ], +) +def test_picker_label(name: str, status: str, expected: str) -> None: + # Arrange + version = Version(name=name, status=status, branch_or_tag="") + + # Act / Assert + assert version.picker_label == expected From 9bcde435e586cca62c694678b0966b4b84d93047 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:45:44 +0100 Subject: [PATCH 144/176] Simplify generation of ``indexsidebar.html`` (#283) Remove the double-rendering of ``indexsidebar.html``. For end-of-life versions, change the sidebar to contain a link to the current stable version. For non-EOL versions, overwrite the new ``_docs_by_version.html`` file with the list of links. --- build_docs.py | 40 +++++++++++++++++++-------------- templates/_docs_by_version.html | 11 +++++++++ templates/indexsidebar.html | 23 +++++-------------- 3 files changed, 39 insertions(+), 35 deletions(-) create mode 100644 templates/_docs_by_version.html diff --git a/build_docs.py b/build_docs.py index e40259c..9b498fe 100755 --- a/build_docs.py +++ b/build_docs.py @@ -143,16 +143,6 @@ def current_dev(self) -> Version: """Find the current CPython version in development.""" return max(self, key=Version.as_tuple) - def setup_indexsidebar(self, current: Version, dest_path: Path) -> None: - """Build indexsidebar.html for Sphinx.""" - template_path = HERE / "templates" / "indexsidebar.html" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render( - current_version=current, - versions=list(reversed(self)), - ) - dest_path.write_text(rendered_template, encoding="UTF-8") - @dataclasses.dataclass(frozen=True, kw_only=True, slots=True) class Version: @@ -529,9 +519,9 @@ class DocBuilder: """Builder for a CPython version and a language.""" version: Version - versions: Versions language: Language cpython_repo: Repository + docs_by_version_content: bytes switchers_content: bytes build_root: Path www_root: Path @@ -667,10 +657,7 @@ def build(self) -> None: text = text.replace(" -A switchers=1", "") (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") - self.versions.setup_indexsidebar( - self.version, - self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", - ) + self.setup_indexsidebar() run_with_logging([ "make", "-C", @@ -713,6 +700,18 @@ def build_venv(self) -> None: run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) self.venv = venv_path + def setup_indexsidebar(self) -> None: + """Copy indexsidebar.html for Sphinx.""" + tmpl_src = HERE / "templates" + tmpl_dst = self.checkout / "Doc" / "tools" / "templates" + dbv_path = tmpl_dst / "_docs_by_version.html" + + shutil.copy(tmpl_src / "indexsidebar.html", tmpl_dst / "indexsidebar.html") + if self.version.status != "EOL": + dbv_path.write_bytes(self.docs_by_version_content) + else: + shutil.copy(tmpl_src / "_docs_by_version.html", dbv_path) + def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: """Copy a given build to the appropriate webroot with appropriate rights.""" logging.info("Publishing start.") @@ -1099,6 +1098,7 @@ def build_docs(args: argparse.Namespace) -> int: force_build = args.force del args.force + docs_by_version_content = render_docs_by_version(versions).encode() switchers_content = render_switchers(versions, languages) build_succeeded = set() @@ -1118,12 +1118,12 @@ def build_docs(args: argparse.Namespace) -> int: scope = sentry_sdk.get_isolation_scope() scope.set_tag("version", version.name) scope.set_tag("language", language.tag) - cpython_repo.update() + cpython_repo.update() builder = DocBuilder( version, - versions, language, cpython_repo, + docs_by_version_content, switchers_content, **vars(args), ) @@ -1179,6 +1179,12 @@ def parse_languages_from_config() -> Languages: return Languages.from_json(config["defaults"], config["languages"]) +def render_docs_by_version(versions: Versions) -> str: + """Generate content for _docs_by_version.html.""" + links = [f'
  • {v.title}
    • ' for v in reversed(versions)] + return "\n".join(links) + + def render_switchers(versions: Versions, languages: Languages) -> bytes: language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] diff --git a/templates/_docs_by_version.html b/templates/_docs_by_version.html new file mode 100644 index 0000000..1a84cfb --- /dev/null +++ b/templates/_docs_by_version.html @@ -0,0 +1,11 @@ +{# +This file is only used in indexsidebar.html, where it is included in the docs +by version list. For non-end-of-life branches, build_docs.py overwrites this +list with the full list of versions. + +Keep the following two files synchronised: +* cpython/Doc/tools/templates/_docs_by_version.html +* docsbuild-scripts/templates/_docs_by_version.html +#} +
  • {% trans %}Stable{% endtrans %}
    • +
  • {% trans %}In development{% endtrans %}
    • diff --git a/templates/indexsidebar.html b/templates/indexsidebar.html index 3a56219..eea29e2 100644 --- a/templates/indexsidebar.html +++ b/templates/indexsidebar.html @@ -1,30 +1,17 @@ -{# -Beware, this file is rendered twice via Jinja2: -- First by build_docs.py, given 'current_version' and 'versions'. -- A 2nd time by Sphinx. -#} - -{% raw %}

    {% trans %}Download{% endtrans %}

    {% trans %}Download these documents{% endtrans %}

    -{% endraw %} -{% if current_version.status != "EOL" %} -{% raw %}

    {% trans %}Docs by version{% endtrans %}

    {% endraw %} +

    {% trans %}Docs by version{% endtrans %}

    -{% endif %} -{% raw %}

    {% trans %}Other resources{% endtrans %}

    -{% endraw %} From efd1e17c947f426af9301ec0387f2e56bf3f86bf Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:45:53 +0100 Subject: [PATCH 145/176] Use ``venv.create()`` to create virtual environments (#289) --- build_docs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 9b498fe..733473e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -56,6 +56,7 @@ import shutil import subprocess import sys +import venv from bisect import bisect_left as bisect from contextlib import contextmanager, suppress from pathlib import Path @@ -690,7 +691,7 @@ def build_venv(self) -> None: requirements.append("matplotlib>=3") venv_path = self.build_root / ("venv-" + self.version.name) - run([sys.executable, "-m", "venv", venv_path]) + venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] + ["--upgrade-strategy=eager"] From 776e64996819c11cec1a4154342630f3e7ea4c1f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:56:15 +0100 Subject: [PATCH 146/176] Create a function to perform ``chgrp`` operations (#290) --- build_docs.py | 70 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/build_docs.py b/build_docs.py index 733473e..2f164f8 100755 --- a/build_docs.py +++ b/build_docs.py @@ -671,8 +671,8 @@ def build(self) -> None: "SPHINXERRORHANDLING=", maketarget, ]) - run(["mkdir", "-p", self.log_directory]) - run(["chgrp", "-R", self.group, self.log_directory]) + self.log_directory.mkdir(parents=True, exist_ok=True) + chgrp(self.log_directory, group=self.group, recursive=True) if self.includes_html: setup_switchers( self.switchers_content, self.checkout / "Doc" / "build" / "html" @@ -723,10 +723,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: else: language_dir = self.www_root / self.language.tag language_dir.mkdir(parents=True, exist_ok=True) - try: - run(["chgrp", "-R", self.group, language_dir]) - except subprocess.CalledProcessError as err: - logging.warning("Can't change group of %s: %s", language_dir, str(err)) + chgrp(language_dir, group=self.group, recursive=True) language_dir.chmod(0o775) target = language_dir / self.version.name @@ -735,22 +732,18 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: target.chmod(0o775) except PermissionError as err: logging.warning("Can't change mod of %s: %s", target, str(err)) - try: - run(["chgrp", "-R", self.group, target]) - except subprocess.CalledProcessError as err: - logging.warning("Can't change group of %s: %s", target, str(err)) + chgrp(target, group=self.group, recursive=True) changed = [] if self.includes_html: # Copy built HTML files to webroot (default /srv/docs.python.org) changed = changed_files(self.checkout / "Doc" / "build" / "html", target) logging.info("Copying HTML files to %s", target) - run([ - "chown", - "-R", - ":" + self.group, + chgrp( self.checkout / "Doc" / "build" / "html/", - ]) + group=self.group, + recursive=True, + ) run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) run([ "find", @@ -776,12 +769,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: if not self.quick and (self.checkout / "Doc" / "dist").is_dir(): # Copy archive files to /archives/ logging.debug("Copying dist files.") - run([ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "dist", - ]) + chgrp(self.checkout / "Doc" / "dist", group=self.group, recursive=True) run([ "chmod", "-R", @@ -789,7 +777,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: self.checkout / "Doc" / "dist", ]) run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) - run(["chown", ":" + self.group, target / "archives"]) + chgrp(target / "archives", group=self.group) run([ "cp", "-a", @@ -889,6 +877,36 @@ def save_state( logging.info("Saved new rebuild state for %s: %s", key, table.as_string()) +def chgrp( + path: Path, + /, + group: int | str | None, + *, + recursive: bool = False, + follow_symlinks: bool = True, +) -> None: + if sys.platform == "win32": + return + + from grp import getgrnam + + try: + try: + group_id = int(group) + except ValueError: + group_id = getgrnam(group)[2] + except (LookupError, TypeError, ValueError): + return + + try: + os.chown(path, -1, group_id, follow_symlinks=follow_symlinks) + if recursive: + for p in path.rglob("*"): + os.chown(p, -1, group_id, follow_symlinks=follow_symlinks) + except OSError as err: + logging.warning("Can't change group of %s: %s", path, str(err)) + + def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) @@ -1213,7 +1231,7 @@ def build_sitemap( sitemap_path = www_root / "sitemap.xml" sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") sitemap_path.chmod(0o664) - run(["chgrp", group, sitemap_path]) + chgrp(sitemap_path, group=group) def build_404(www_root: Path, group: str) -> None: @@ -1225,7 +1243,7 @@ def build_404(www_root: Path, group: str) -> None: not_found_file = www_root / "404.html" shutil.copyfile(HERE / "templates" / "404.html", not_found_file) not_found_file.chmod(0o664) - run(["chgrp", group, not_found_file]) + chgrp(not_found_file, group=group) def copy_robots_txt( @@ -1243,7 +1261,7 @@ def copy_robots_txt( robots_path = www_root / "robots.txt" shutil.copyfile(template_path, robots_path) robots_path.chmod(0o775) - run(["chgrp", group, robots_path]) + chgrp(robots_path, group=group) if not skip_cache_invalidation: purge(http, "robots.txt") @@ -1311,7 +1329,7 @@ def symlink( # Link does not exist or points to the wrong target. link.unlink(missing_ok=True) link.symlink_to(directory) - run(["chown", "-h", f":{group}", str(link)]) + chgrp(link, group=group, follow_symlinks=False) if not skip_cache_invalidation: surrogate_key = f"{language_tag}/{name}" purge_surrogate_key(http, surrogate_key) From 6164fac89acf0e3890d70bdd60320cd9b5de77e2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 21:29:27 +0100 Subject: [PATCH 147/176] Create a function for ``chmod`` operations (#291) --- build_docs.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/build_docs.py b/build_docs.py index 2f164f8..d1afdf2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -54,6 +54,7 @@ import re import shlex import shutil +import stat import subprocess import sys import venv @@ -744,18 +745,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: group=self.group, recursive=True, ) - run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) - run([ - "find", - self.checkout / "Doc" / "build" / "html", - "-type", - "d", - "-exec", - "chmod", - "o+x", - "{}", - ";", - ]) + chmod_make_readable(self.checkout / "Doc" / "build" / "html") run([ "rsync", "-a", @@ -770,12 +760,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: # Copy archive files to /archives/ logging.debug("Copying dist files.") chgrp(self.checkout / "Doc" / "dist", group=self.group, recursive=True) - run([ - "chmod", - "-R", - "o+r", - self.checkout / "Doc" / "dist", - ]) + chmod_make_readable(self.checkout / "Doc" / "dist") run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) chgrp(target / "archives", group=self.group) run([ @@ -907,6 +892,18 @@ def chgrp( logging.warning("Can't change group of %s: %s", path, str(err)) +def chmod_make_readable(path: Path, /, mode: int = stat.S_IROTH) -> None: + if not path.is_dir(): + raise ValueError + + path.chmod(path.stat().st_mode | stat.S_IROTH | stat.S_IXOTH) # o+rx + for p in path.rglob("*"): + if p.is_dir(): + p.chmod(p.stat().st_mode | stat.S_IROTH | stat.S_IXOTH) # o+rx + else: + p.chmod(p.stat().st_mode | stat.S_IROTH) # o+r + + def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) From e1dcdfcaaac76fbb6a4608e169bef9d2e5f6f5a2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 22:53:25 +0100 Subject: [PATCH 148/176] Remove subprocess calls for copying archive files (#292) --- .github/workflows/test.yml | 6 ++++++ build_docs.py | 27 ++++++++++++++------------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a9324f..850fec7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,12 @@ jobs: --languages en --branches 3.14 + - name: Upload documentation + uses: actions/upload-artifact@v4 + with: + name: www-root + path: ./www + unit: name: Unit tests runs-on: ${{ matrix.os }} diff --git a/build_docs.py b/build_docs.py index d1afdf2..83bc26c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -756,22 +756,23 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: target, ]) - if not self.quick and (self.checkout / "Doc" / "dist").is_dir(): + dist_dir = self.checkout / "Doc" / "dist" + if dist_dir.is_dir(): # Copy archive files to /archives/ logging.debug("Copying dist files.") - chgrp(self.checkout / "Doc" / "dist", group=self.group, recursive=True) - chmod_make_readable(self.checkout / "Doc" / "dist") - run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) - chgrp(target / "archives", group=self.group) - run([ - "cp", - "-a", - *(self.checkout / "Doc" / "dist").glob("*"), - target / "archives", - ]) + chgrp(dist_dir, group=self.group, recursive=True) + chmod_make_readable(dist_dir) + archives_dir = target / "archives" + archives_dir.mkdir(parents=True, exist_ok=True) + archives_dir.chmod( + archives_dir.stat().st_mode | stat.S_IROTH | stat.S_IXOTH + ) + chgrp(archives_dir, group=self.group) + for dist_file in dist_dir.iterdir(): + shutil.copy2(dist_file, archives_dir / dist_file.name) changed.append("archives/") - for file in (target / "archives").iterdir(): - changed.append("archives/" + file.name) + for file in archives_dir.iterdir(): + changed.append(f"archives/{file.name}") logging.info("%s files changed", len(changed)) if changed and not self.skip_cache_invalidation: From d6f84292abef7150bdd0a9467d0fca08df7188f5 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 22:58:51 +0100 Subject: [PATCH 149/176] Set explicit artefact retention time --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 850fec7..b64e8a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,7 @@ jobs: with: name: www-root path: ./www + retention-days: 2 unit: name: Unit tests From 783e70445fa48a8d3d7268e75eb45b2926b04c05 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 23:22:12 +0100 Subject: [PATCH 150/176] Convert ``changed`` to an integer count --- build_docs.py | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/build_docs.py b/build_docs.py index 83bc26c..d20aea4 100755 --- a/build_docs.py +++ b/build_docs.py @@ -337,23 +337,15 @@ def run_with_logging(cmd: Sequence[str | Path], cwd: Path | None = None) -> None raise subprocess.CalledProcessError(return_code, cmd[0]) -def changed_files(left: Path, right: Path) -> list[str]: - """Compute a list of different files between left and right, recursively. - Resulting paths are relative to left. - """ - changed = [] +def changed_files(left: Path, right: Path) -> int: + """Compute the number of different files in the two directory trees.""" - def traverse(dircmp_result: filecmp.dircmp) -> None: - base = Path(dircmp_result.left).relative_to(left) - for file in dircmp_result.diff_files: - changed.append(str(base / file)) - if file == "index.html": - changed.append(str(base) + "/") - for dircomp in dircmp_result.subdirs.values(): - traverse(dircomp) + def traverse(dircmp_result: filecmp.dircmp) -> int: + changed = len(dircmp_result.diff_files) + changed += sum(map(traverse, dircmp_result.subdirs.values())) + return changed - traverse(filecmp.dircmp(left, right)) - return changed + return traverse(filecmp.dircmp(left, right)) @dataclasses.dataclass @@ -735,10 +727,10 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: logging.warning("Can't change mod of %s: %s", target, str(err)) chgrp(target, group=self.group, recursive=True) - changed = [] + changed = 0 if self.includes_html: # Copy built HTML files to webroot (default /srv/docs.python.org) - changed = changed_files(self.checkout / "Doc" / "build" / "html", target) + changed += changed_files(self.checkout / "Doc" / "build" / "html", target) logging.info("Copying HTML files to %s", target) chgrp( self.checkout / "Doc" / "build" / "html/", @@ -768,13 +760,12 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: archives_dir.stat().st_mode | stat.S_IROTH | stat.S_IXOTH ) chgrp(archives_dir, group=self.group) + changed += 1 for dist_file in dist_dir.iterdir(): shutil.copy2(dist_file, archives_dir / dist_file.name) - changed.append("archives/") - for file in archives_dir.iterdir(): - changed.append(f"archives/{file.name}") + changed += 1 - logging.info("%s files changed", len(changed)) + logging.info("%s files changed", changed) if changed and not self.skip_cache_invalidation: surrogate_key = f"{self.language.tag}/{self.version.name}" purge_surrogate_key(http, surrogate_key) From 9082a8eed43f3c2ec4f8101176d3932b4949ecde Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 23:32:48 +0100 Subject: [PATCH 151/176] Use f-strings instead of string concatenation --- build_docs.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build_docs.py b/build_docs.py index d20aea4..847bfe2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -363,10 +363,10 @@ def get_ref(self, pattern: str) -> str: """Return the reference of a given tag or branch.""" try: # Maybe it's a branch - return self.run("show-ref", "-s", "origin/" + pattern).stdout.strip() + return self.run("show-ref", "-s", f"origin/{pattern}").stdout.strip() except subprocess.CalledProcessError: # Maybe it's a tag - return self.run("show-ref", "-s", "tags/" + pattern).stdout.strip() + return self.run("show-ref", "-s", f"tags/{pattern}").stdout.strip() def fetch(self) -> subprocess.CompletedProcess: """Try (and retry) to run git fetch.""" @@ -656,11 +656,11 @@ def build(self) -> None: "make", "-C", self.checkout / "Doc", - "PYTHON=" + str(python), - "SPHINXBUILD=" + str(sphinxbuild), - "BLURB=" + str(blurb), - "VENVDIR=" + str(self.venv), - "SPHINXOPTS=" + " ".join(sphinxopts), + f"PYTHON={python}", + f"SPHINXBUILD={sphinxbuild}", + f"BLURB={blurb}", + f"VENVDIR={self.venv}", + f"SPHINXOPTS={' '.join(sphinxopts)}", "SPHINXERRORHANDLING=", maketarget, ]) @@ -683,7 +683,7 @@ def build_venv(self) -> None: # opengraph previews requirements.append("matplotlib>=3") - venv_path = self.build_root / ("venv-" + self.version.name) + venv_path = self.build_root / f"venv-{self.version.name}" venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] From fe84a0bb4c442bcdb32154768c160468b1e90d6e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 23:41:47 +0100 Subject: [PATCH 152/176] Use tuples for subprocess argument lists --- build_docs.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/build_docs.py b/build_docs.py index 847bfe2..d31ebdd 100755 --- a/build_docs.py +++ b/build_docs.py @@ -388,7 +388,7 @@ def clone(self) -> bool: return False # Already cloned logging.info("Cloning %s into %s", self.remote, self.directory) self.directory.mkdir(mode=0o775, parents=True, exist_ok=True) - run(["git", "clone", self.remote, self.directory]) + run(("git", "clone", self.remote, self.directory)) return True def update(self) -> None: @@ -481,7 +481,7 @@ def version_info() -> None: """Handler for --version.""" try: platex_version = head( - subprocess.check_output(["platex", "--version"], text=True), + subprocess.check_output(("platex", "--version"), text=True), lines=3, ) except FileNotFoundError: @@ -489,7 +489,7 @@ def version_info() -> None: try: xelatex_version = head( - subprocess.check_output(["xelatex", "--version"], text=True), + subprocess.check_output(("xelatex", "--version"), text=True), lines=2, ) except FileNotFoundError: @@ -652,7 +652,7 @@ def build(self) -> None: (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") self.setup_indexsidebar() - run_with_logging([ + run_with_logging(( "make", "-C", self.checkout / "Doc", @@ -663,7 +663,7 @@ def build(self) -> None: f"SPHINXOPTS={' '.join(sphinxopts)}", "SPHINXERRORHANDLING=", maketarget, - ]) + )) self.log_directory.mkdir(parents=True, exist_ok=True) chgrp(self.log_directory, group=self.group, recursive=True) if self.includes_html: @@ -678,7 +678,7 @@ def build_venv(self) -> None: So we can reuse them from builds to builds, while they contain different Sphinx versions. """ - requirements = [self.theme] + self.version.requirements + requirements = list(self.version.requirements) if self.includes_html: # opengraph previews requirements.append("matplotlib>=3") @@ -686,12 +686,19 @@ def build_venv(self) -> None: venv_path = self.build_root / f"venv-{self.version.name}" venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( - [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] - + ["--upgrade-strategy=eager"] - + requirements, + ( + venv_path / "bin" / "python", + "-m", + "pip", + "install", + "--upgrade", + "--upgrade-strategy=eager", + self.theme, + *requirements, + ), cwd=self.checkout / "Doc", ) - run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) + run((venv_path / "bin" / "python", "-m", "pip", "freeze", "--all")) self.venv = venv_path def setup_indexsidebar(self) -> None: @@ -738,7 +745,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: recursive=True, ) chmod_make_readable(self.checkout / "Doc" / "build" / "html") - run([ + run(( "rsync", "-a", "--delete-delay", @@ -746,7 +753,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: "P archives/", str(self.checkout / "Doc" / "build" / "html") + "/", target, - ]) + )) dist_dir = self.checkout / "Doc" / "dist" if dist_dir.is_dir(): From d1f0418ea5aadb14a670151eb16ce0e2ba4d6cda Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 00:02:39 +0100 Subject: [PATCH 153/176] Add more helper properties in the ``Language`` class (#293) --- build_docs.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/build_docs.py b/build_docs.py index d31ebdd..bd94ba1 100755 --- a/build_docs.py +++ b/build_docs.py @@ -280,6 +280,14 @@ class Language: def tag(self) -> str: return self.iso639_tag.replace("_", "-").lower() + @property + def is_translation(self) -> bool: + return self.tag != "en" + + @property + def locale_repo_url(self) -> str: + return f"https://github.com/python/python-docs-{self.tag}.git" + @property def switcher_label(self) -> str: if self.translated_name: @@ -549,7 +557,7 @@ def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: logging.info("Skipping non-HTML build (language is HTML-only).") return None # skipped self.cpython_repo.switch(self.version.branch_or_tag) - if self.language.tag != "en": + if self.language.is_translation: self.clone_translation() if trigger_reason := self.should_rebuild(force_build): self.build_venv() @@ -569,6 +577,10 @@ def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: return False return True + @property + def locale_dir(self) -> Path: + return self.build_root / self.version.name / "locale" + @property def checkout(self) -> Path: """Path to CPython git clone.""" @@ -582,15 +594,8 @@ def clone_translation(self) -> None: def translation_repo(self) -> Repository: """See PEP 545 for translations repository naming convention.""" - locale_repo = f"https://github.com/python/python-docs-{self.language.tag}.git" - locale_clone_dir = ( - self.build_root - / self.version.name - / "locale" - / self.language.iso639_tag - / "LC_MESSAGES" - ) - return Repository(locale_repo, locale_clone_dir) + locale_clone_dir = self.locale_dir / self.language.iso639_tag / "LC_MESSAGES" + return Repository(self.language.locale_repo_url, locale_clone_dir) @property def translation_branch(self) -> str: @@ -611,10 +616,9 @@ def build(self) -> None: logging.info("Build start.") start_time = perf_counter() sphinxopts = list(self.language.sphinxopts) - if self.language.tag != "en": - locale_dirs = self.build_root / self.version.name / "locale" + if self.language.is_translation: sphinxopts.extend(( - f"-D locale_dirs={locale_dirs}", + f"-D locale_dirs={self.locale_dir}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", "-D translation_progress_classes=1", @@ -636,7 +640,7 @@ def build(self) -> None: if self.includes_html: site_url = self.version.url - if self.language.tag != "en": + if self.language.is_translation: site_url += f"{self.language.tag}/" # Define a tag to enable opengraph socialcards previews # (used in Doc/conf.py and requires matplotlib) @@ -718,7 +722,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: logging.info("Publishing start.") start_time = perf_counter() self.www_root.mkdir(parents=True, exist_ok=True) - if self.language.tag == "en": + if not self.language.is_translation: target = self.www_root / self.version.name else: language_dir = self.www_root / self.language.tag @@ -786,7 +790,7 @@ def should_rebuild(self, force: bool) -> str | Literal[False]: logging.info("Should rebuild: no previous state found.") return "no previous state" cpython_sha = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() - if self.language.tag != "en": + if self.language.is_translation: translation_sha = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() @@ -849,7 +853,7 @@ def save_state( "triggered_by": trigger, "cpython_sha": self.cpython_repo.run("rev-parse", "HEAD").stdout.strip(), } - if self.language.tag != "en": + if self.language.is_translation: state["translation_sha"] = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() From c64294658cf6ceffd6c169a95646ee14ef148b3f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 00:53:19 +0100 Subject: [PATCH 154/176] Constrain blurb to version 1.1 or older (#294) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index bd94ba1..d95b408 100755 --- a/build_docs.py +++ b/build_docs.py @@ -194,7 +194,7 @@ def requirements(self) -> list[str]: return dependencies + ["standard-imghdr"] # Requirements/constraints for Python 3.7 and older, pre-requirements.txt - reqs = ["jieba", "blurb", "jinja2<3.1", "docutils<=0.17.1", "standard-imghdr"] + reqs = ["jieba", "blurb<1.2", "jinja2<3.1", "docutils<0.18", "standard-imghdr"] if self.name in {"3.7", "3.6", "2.7"}: return reqs + ["sphinx==2.3.1"] if self.name == "3.5": From d53c8a6e4d64571d4902523d8d42ed4af5cb2907 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 01:10:42 +0100 Subject: [PATCH 155/176] Add constraints for sphinxcontrib dependencies (#295) --- build_docs.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index d95b408..e1ca9a3 100755 --- a/build_docs.py +++ b/build_docs.py @@ -183,9 +183,9 @@ def requirements(self) -> list[str]: """ dependencies = [ + "-rrequirements.txt", "jieba", # To improve zh search. "PyStemmer~=2.2.0", # To improve performance for word stemming. - "-rrequirements.txt", ] if self.as_tuple() >= (3, 11): return dependencies @@ -194,7 +194,19 @@ def requirements(self) -> list[str]: return dependencies + ["standard-imghdr"] # Requirements/constraints for Python 3.7 and older, pre-requirements.txt - reqs = ["jieba", "blurb<1.2", "jinja2<3.1", "docutils<0.18", "standard-imghdr"] + reqs = [ + "blurb<1.2", + "docutils<=0.17.1", + "jieba", + "jinja2<3.1", + "sphinxcontrib-applehelp<=1.0.2", + "sphinxcontrib-devhelp<=1.0.2", + "sphinxcontrib-htmlhelp<=2.0", + "sphinxcontrib-jsmath<=1.0.1", + "sphinxcontrib-qthelp<=1.0.3", + "sphinxcontrib-serializinghtml<=1.1.5", + "standard-imghdr", + ] if self.name in {"3.7", "3.6", "2.7"}: return reqs + ["sphinx==2.3.1"] if self.name == "3.5": From 3254c48f3cdab522896cc8d5d9d654dde11f3614 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 01:13:23 +0100 Subject: [PATCH 156/176] Constrain alabaster to version 0.7.12 or older --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index e1ca9a3..c257e2f 100755 --- a/build_docs.py +++ b/build_docs.py @@ -195,6 +195,7 @@ def requirements(self) -> list[str]: # Requirements/constraints for Python 3.7 and older, pre-requirements.txt reqs = [ + "alabaster<0.7.12", "blurb<1.2", "docutils<=0.17.1", "jieba", From 9a365d64d329aa3c286691dccd7ae475fc425829 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:24:10 +0100 Subject: [PATCH 157/176] Constrain python-docs-theme to version 2023.3.1 or older --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index c257e2f..6895618 100755 --- a/build_docs.py +++ b/build_docs.py @@ -200,6 +200,7 @@ def requirements(self) -> list[str]: "docutils<=0.17.1", "jieba", "jinja2<3.1", + "python-docs-theme<=2023.3.1", "sphinxcontrib-applehelp<=1.0.2", "sphinxcontrib-devhelp<=1.0.2", "sphinxcontrib-htmlhelp<=2.0", From c215a786f60a9e24311b038155f5646e548287e2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:25:43 +0100 Subject: [PATCH 158/176] Restore _create_placeholders_if_missing() (#296) --- templates/switchers.js | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 774366f..324fd65 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -28,6 +28,30 @@ const _CURRENT_PREFIX = (() => { const _ALL_VERSIONS = new Map($VERSIONS); const _ALL_LANGUAGES = new Map($LANGUAGES); +/** + * Required for Python 3.7 and earlier. + * @returns {void} + * @private + */ +const _create_placeholders_if_missing = () => { + if (document.querySelectorAll(".version_switcher_placeholder").length) return; + + const items = document.querySelectorAll("body>div.related>ul>li:not(.right)"); + for (const item of items) { + if (item.innerText.toLowerCase().includes("documentation")) { + const container = document.createElement("li"); + container.className = "switchers"; + for (const placeholder_name of ["language", "version"]) { + const placeholder = document.createElement("div"); + placeholder.className = `${placeholder_name}_switcher_placeholder`; + container.appendChild(placeholder); + } + item.parentElement.insertBefore(container, item); + return; + } + } +}; + /** * @param {Map} versions * @returns {HTMLSelectElement} @@ -175,6 +199,8 @@ const _initialise_switchers = () => { const versions = _ALL_VERSIONS; const languages = _ALL_LANGUAGES; + _create_placeholders_if_missing(); + document .querySelectorAll(".version_switcher_placeholder") .forEach((placeholder) => { From 65735049bc9de117d9493a7677259cecc1f4de22 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:32:19 +0100 Subject: [PATCH 159/176] Require pipes for building Python 3.5 (#297) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 6895618..c75f096 100755 --- a/build_docs.py +++ b/build_docs.py @@ -212,7 +212,7 @@ def requirements(self) -> list[str]: if self.name in {"3.7", "3.6", "2.7"}: return reqs + ["sphinx==2.3.1"] if self.name == "3.5": - return reqs + ["sphinx==1.8.4"] + return reqs + ["sphinx==1.8.4", "standard-pipes"] raise ValueError("unreachable") @property From c1be80ec82c318076b7c10b2e854674ca7d4ea62 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:49:00 +0100 Subject: [PATCH 160/176] Add inline styles in _create_placeholders_if_missing() --- templates/switchers.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 324fd65..e54a278 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -41,9 +41,12 @@ const _create_placeholders_if_missing = () => { if (item.innerText.toLowerCase().includes("documentation")) { const container = document.createElement("li"); container.className = "switchers"; + container.style.display = "inline-flex"; for (const placeholder_name of ["language", "version"]) { const placeholder = document.createElement("div"); placeholder.className = `${placeholder_name}_switcher_placeholder`; + placeholder.style.marginRight = "5px"; + placeholder.style.paddingLeft = "5px"; container.appendChild(placeholder); } item.parentElement.insertBefore(container, item); From 84a749ebd9f7957baee6121f79aa86f930456218 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 18 Apr 2025 22:27:20 +0300 Subject: [PATCH 161/176] Add more tests for `Versions` class (#288) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- .coveragerc | 1 + .pre-commit-config.yaml | 1 + tests/test_build_docs_versions.py | 104 +++++++++++++++++++++++------- 3 files changed, 82 insertions(+), 24 deletions(-) diff --git a/.coveragerc b/.coveragerc index 0f12707..f970781 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,3 +5,4 @@ exclude_also = # Don't complain if non-runnable code isn't run: if __name__ == .__main__.: + if TYPE_CHECKING: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c0eb053..869a979 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,7 @@ repos: rev: v0.11.5 hooks: - id: ruff + args: [--fix] - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py index 42b5392..1d8f6dc 100644 --- a/tests/test_build_docs_versions.py +++ b/tests/test_build_docs_versions.py @@ -1,9 +1,13 @@ +from __future__ import annotations + +import pytest + from build_docs import Version, Versions -def test_filter_default() -> None: - # Arrange - versions = Versions([ +@pytest.fixture +def versions() -> Versions: + return Versions([ Version(name="3.14", status="in development", branch_or_tag=""), Version(name="3.13", status="stable", branch_or_tag=""), Version(name="3.12", status="stable", branch_or_tag=""), @@ -12,28 +16,90 @@ def test_filter_default() -> None: Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) + +def test_reversed(versions: Versions) -> None: # Act - filtered = versions.filter() + output = list(reversed(versions)) # Assert - assert filtered == [ - Version(name="3.14", status="in development", branch_or_tag=""), + assert output[0].name == "3.9" + assert output[-1].name == "3.14" + + +def test_from_json() -> None: + # Arrange + json_data = { + "3.14": { + "branch": "main", + "pep": 745, + "status": "feature", + "first_release": "2025-10-01", + "end_of_life": "2030-10", + "release_manager": "Hugo van Kemenade", + }, + "3.13": { + "branch": "3.13", + "pep": 719, + "status": "bugfix", + "first_release": "2024-10-07", + "end_of_life": "2029-10", + "release_manager": "Thomas Wouters", + }, + } + + # Act + versions = list(Versions.from_json(json_data)) + + # Assert + assert versions == [ Version(name="3.13", status="stable", branch_or_tag=""), - Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.14", status="in development", branch_or_tag=""), ] -def test_filter_one() -> None: +def test_from_json_error() -> None: # Arrange - versions = Versions([ + json_data = {"2.8": {"branch": "2.8", "pep": 404, "status": "ex-release"}} + + # Act / Assert + with pytest.raises( + ValueError, + match="Saw invalid version status 'ex-release', expected to be one of", + ): + Versions.from_json(json_data) + + +def test_current_stable(versions) -> None: + # Act + current_stable = versions.current_stable + + # Assert + assert current_stable.name == "3.13" + assert current_stable.status == "stable" + + +def test_current_dev(versions) -> None: + # Act + current_dev = versions.current_dev + + # Assert + assert current_dev.name == "3.14" + assert current_dev.status == "in development" + + +def test_filter_default(versions) -> None: + # Act + filtered = versions.filter() + + # Assert + assert filtered == [ Version(name="3.14", status="in development", branch_or_tag=""), Version(name="3.13", status="stable", branch_or_tag=""), Version(name="3.12", status="stable", branch_or_tag=""), - Version(name="3.11", status="security-fixes", branch_or_tag=""), - Version(name="3.10", status="security-fixes", branch_or_tag=""), - Version(name="3.9", status="security-fixes", branch_or_tag=""), - ]) + ] + +def test_filter_one(versions) -> None: # Act filtered = versions.filter(["3.13"]) @@ -41,17 +107,7 @@ def test_filter_one() -> None: assert filtered == [Version(name="3.13", status="security-fixes", branch_or_tag="")] -def test_filter_multiple() -> None: - # Arrange - versions = Versions([ - Version(name="3.14", status="in development", branch_or_tag=""), - Version(name="3.13", status="stable", branch_or_tag=""), - Version(name="3.12", status="stable", branch_or_tag=""), - Version(name="3.11", status="security-fixes", branch_or_tag=""), - Version(name="3.10", status="security-fixes", branch_or_tag=""), - Version(name="3.9", status="security-fixes", branch_or_tag=""), - ]) - +def test_filter_multiple(versions) -> None: # Act filtered = versions.filter(["3.13", "3.14"]) From 0e83b79b2d2488ec9226f28c93e215b5217e6730 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 22 May 2025 14:35:36 +0200 Subject: [PATCH 162/176] =?UTF-8?q?Hello=20=E0=A6=AC=E0=A6=BE=E0=A6=82?= =?UTF-8?q?=E0=A6=B2=E0=A6=BE=20(#301)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config.toml b/config.toml index 489c774..f25052a 100644 --- a/config.toml +++ b/config.toml @@ -36,6 +36,11 @@ sphinxopts = [ '-D latex_elements.fontenc=\\usepackage{fontspec}', ] +[languages.bn_IN] +name = "Bengali" +translated_name = "বাংলা" +in_prod = false + [languages.id] name = "Indonesian" translated_name = "Indonesia" From 15f94c0b7caec8b12f4bab3227967fd859ff6bad Mon Sep 17 00:00:00 2001 From: Octavian Mustafa Date: Mon, 16 Jun 2025 21:44:24 +0300 Subject: [PATCH 163/176] Start building Romanian translations (#302) --- config.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config.toml b/config.toml index f25052a..d628ca0 100644 --- a/config.toml +++ b/config.toml @@ -92,6 +92,11 @@ translated_name = "polski" name = "Brazilian Portuguese" translated_name = "Português brasileiro" +[languages.ro] +name = "Romanian" +translated_name = "Românește" +in_prod = false + [languages.tr] name = "Turkish" translated_name = "Türkçe" From a601ce67c6c2f3be7fde3376d3e5d3851f19950b Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Fri, 20 Jun 2025 16:56:41 +0200 Subject: [PATCH 164/176] Add Greek translation (#248) --- config.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config.toml b/config.toml index d628ca0..4a5958f 100644 --- a/config.toml +++ b/config.toml @@ -15,6 +15,10 @@ sphinxopts = [ "-D latex_elements.fontenc=", ] +[languages.el] +name = "Greek" +translated_name = "Ελληνικά" + [languages.en] name = "English" From 599b4ffd7f1cdde02bd8d0bf7781599f631e0752 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 15 Aug 2025 18:36:46 +0300 Subject: [PATCH 165/176] Enable ``suggest_on_error`` for argparse (#303) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index c75f096..e2798a5 100755 --- a/build_docs.py +++ b/build_docs.py @@ -967,6 +967,7 @@ def parse_args() -> argparse.Namespace: description="Runs a build of the Python docs for various branches.", allow_abbrev=False, ) + parser.suggest_on_error = True parser.add_argument( "--select-output", choices=("no-html", "only-html", "only-html-en"), From 0f2598a66e0193d64b2bc49176d71162928aff5f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 15 Aug 2025 16:50:00 +0100 Subject: [PATCH 166/176] Add the ``BuildMetadata`` class (#299) --- build_docs.py | 156 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 106 insertions(+), 50 deletions(-) diff --git a/build_docs.py b/build_docs.py index e2798a5..20c3518 100755 --- a/build_docs.py +++ b/build_docs.py @@ -294,14 +294,6 @@ class Language: def tag(self) -> str: return self.iso639_tag.replace("_", "-").lower() - @property - def is_translation(self) -> bool: - return self.tag != "en" - - @property - def locale_repo_url(self) -> str: - return f"https://github.com/python/python-docs-{self.tag}.git" - @property def switcher_label(self) -> str: if self.translated_name: @@ -309,6 +301,75 @@ def switcher_label(self) -> str: return self.name +@dataclasses.dataclass(frozen=True, kw_only=True, slots=True) +class BuildMetadata: + _version: Version + _language: Language + + @property + def sphinxopts(self) -> Sequence[str]: + return self._language.sphinxopts + + @property + def iso639_tag(self) -> str: + return self._language.iso639_tag + + @property + def html_only(self) -> bool: + return self._language.html_only + + @property + def url(self): + """The URL of this version in production.""" + if self.is_translation: + return f"https://docs.python.org/{self.version}/{self.language}/" + return f"https://docs.python.org/{self.version}/" + + @property + def branch_or_tag(self) -> str: + return self._version.branch_or_tag + + @property + def status(self) -> str: + return self._version.status + + @property + def is_eol(self) -> bool: + return self._version.status == "EOL" + + @property + def dependencies(self) -> list[str]: + return self._version.requirements + + @property + def version(self): + return self._version.name + + @property + def version_tuple(self): + return self._version.as_tuple() + + @property + def language(self): + return self._language.tag + + @property + def is_translation(self): + return self.language != "en" + + @property + def slug(self) -> str: + return f"{self.language}/{self.version}" + + @property + def venv_name(self) -> str: + return f"venv-{self.version}" + + @property + def locale_repo_url(self) -> str: + return f"https://github.com/python/python-docs-{self.language}.git" + + def run( cmd: Sequence[str | Path], cwd: Path | None = None ) -> subprocess.CompletedProcess: @@ -534,8 +595,7 @@ def version_info() -> None: class DocBuilder: """Builder for a CPython version and a language.""" - version: Version - language: Language + build_meta: BuildMetadata cpython_repo: Repository docs_by_version_content: bytes switchers_content: bytes @@ -553,7 +613,7 @@ def html_only(self) -> bool: return ( self.select_output in {"only-html", "only-html-en"} or self.quick - or self.language.html_only + or self.build_meta.html_only ) @property @@ -567,11 +627,11 @@ def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) logging.info("Running.") try: - if self.language.html_only and not self.includes_html: + if self.build_meta.html_only and not self.includes_html: logging.info("Skipping non-HTML build (language is HTML-only).") return None # skipped - self.cpython_repo.switch(self.version.branch_or_tag) - if self.language.is_translation: + self.cpython_repo.switch(self.build_meta.branch_or_tag) + if self.build_meta.is_translation: self.clone_translation() if trigger_reason := self.should_rebuild(force_build): self.build_venv() @@ -593,7 +653,7 @@ def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: @property def locale_dir(self) -> Path: - return self.build_root / self.version.name / "locale" + return self.build_root / self.build_meta.version / "locale" @property def checkout(self) -> Path: @@ -608,8 +668,8 @@ def clone_translation(self) -> None: def translation_repo(self) -> Repository: """See PEP 545 for translations repository naming convention.""" - locale_clone_dir = self.locale_dir / self.language.iso639_tag / "LC_MESSAGES" - return Repository(self.language.locale_repo_url, locale_clone_dir) + locale_clone_dir = self.locale_dir / self.build_meta.iso639_tag / "LC_MESSAGES" + return Repository(self.build_meta.locale_repo_url, locale_clone_dir) @property def translation_branch(self) -> str: @@ -623,25 +683,25 @@ def translation_branch(self) -> str: """ remote_branches = self.translation_repo.run("branch", "-r").stdout branches = re.findall(r"/([0-9]+\.[0-9]+)$", remote_branches, re.M) - return locate_nearest_version(branches, self.version.name) + return locate_nearest_version(branches, self.build_meta.version) def build(self) -> None: """Build this version/language doc.""" logging.info("Build start.") start_time = perf_counter() - sphinxopts = list(self.language.sphinxopts) - if self.language.is_translation: + sphinxopts = list(self.build_meta.sphinxopts) + if self.build_meta.is_translation: sphinxopts.extend(( f"-D locale_dirs={self.locale_dir}", - f"-D language={self.language.iso639_tag}", + f"-D language={self.build_meta.iso639_tag}", "-D gettext_compact=0", "-D translation_progress_classes=1", )) - if self.version.status == "EOL": + if self.build_meta.is_eol: sphinxopts.append("-D html_context.outdated=1") - if self.version.status in ("in development", "pre-release"): + if self.build_meta.status in ("in development", "pre-release"): maketarget = "autobuild-dev" else: maketarget = "autobuild-stable" @@ -653,9 +713,7 @@ def build(self) -> None: blurb = self.venv / "bin" / "blurb" if self.includes_html: - site_url = self.version.url - if self.language.is_translation: - site_url += f"{self.language.tag}/" + site_url = self.build_meta.url # Define a tag to enable opengraph socialcards previews # (used in Doc/conf.py and requires matplotlib) sphinxopts += ( @@ -663,7 +721,7 @@ def build(self) -> None: f"-D ogp_site_url={site_url}", ) - if self.version.as_tuple() < (3, 8): + if self.build_meta.version_tuple < (3, 8): # Disable CPython switchers, we handle them now: text = (self.checkout / "Doc" / "Makefile").read_text(encoding="utf-8") text = text.replace(" -A switchers=1", "") @@ -696,12 +754,12 @@ def build_venv(self) -> None: So we can reuse them from builds to builds, while they contain different Sphinx versions. """ - requirements = list(self.version.requirements) + requirements = list(self.build_meta.dependencies) if self.includes_html: # opengraph previews requirements.append("matplotlib>=3") - venv_path = self.build_root / f"venv-{self.version.name}" + venv_path = self.build_root / self.build_meta.venv_name venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( ( @@ -726,7 +784,7 @@ def setup_indexsidebar(self) -> None: dbv_path = tmpl_dst / "_docs_by_version.html" shutil.copy(tmpl_src / "indexsidebar.html", tmpl_dst / "indexsidebar.html") - if self.version.status != "EOL": + if not self.build_meta.is_eol: dbv_path.write_bytes(self.docs_by_version_content) else: shutil.copy(tmpl_src / "_docs_by_version.html", dbv_path) @@ -736,14 +794,14 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: logging.info("Publishing start.") start_time = perf_counter() self.www_root.mkdir(parents=True, exist_ok=True) - if not self.language.is_translation: - target = self.www_root / self.version.name + if not self.build_meta.is_translation: + target = self.www_root / self.build_meta.version else: - language_dir = self.www_root / self.language.tag + language_dir = self.www_root / self.build_meta.language language_dir.mkdir(parents=True, exist_ok=True) chgrp(language_dir, group=self.group, recursive=True) language_dir.chmod(0o775) - target = language_dir / self.version.name + target = language_dir / self.build_meta.version target.mkdir(parents=True, exist_ok=True) try: @@ -792,8 +850,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: logging.info("%s files changed", changed) if changed and not self.skip_cache_invalidation: - surrogate_key = f"{self.language.tag}/{self.version.name}" - purge_surrogate_key(http, surrogate_key) + purge_surrogate_key(http, self.build_meta.slug) logging.info( "Publishing done (%s).", format_seconds(perf_counter() - start_time) ) @@ -804,7 +861,7 @@ def should_rebuild(self, force: bool) -> str | Literal[False]: logging.info("Should rebuild: no previous state found.") return "no previous state" cpython_sha = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() - if self.language.is_translation: + if self.build_meta.is_translation: translation_sha = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() @@ -839,7 +896,7 @@ def load_state(self) -> dict: state_file = self.build_root / "state.toml" try: return tomlkit.loads(state_file.read_text(encoding="UTF-8"))[ - f"/{self.language.tag}/{self.version.name}/" + f"/{self.build_meta.slug}/" ] except (KeyError, FileNotFoundError): return {} @@ -860,14 +917,14 @@ def save_state( except FileNotFoundError: states = tomlkit.document() - key = f"/{self.language.tag}/{self.version.name}/" + key = f"/{self.build_meta.slug}/" state = { "last_build_start": build_start, "last_build_duration": round(build_duration, 0), "triggered_by": trigger, "cpython_sha": self.cpython_repo.run("rev-parse", "HEAD").stdout.strip(), } - if self.language.is_translation: + if self.build_meta.is_translation: state["translation_sha"] = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() @@ -1123,7 +1180,7 @@ def build_docs(args: argparse.Namespace) -> int: # pairs from the end of the list, effectively reversing it. # This runs languages in config.toml order and versions newest first. todo = [ - (version, language) + BuildMetadata(_version=version, _language=language) for version in versions.filter(args.branches) for language in reversed(languages.filter(args.languages)) ] @@ -1142,20 +1199,19 @@ def build_docs(args: argparse.Namespace) -> int: args.build_root / _checkout_name(args.select_output), ) while todo: - version, language = todo.pop() + build_props = todo.pop() logging.root.handlers[0].setFormatter( logging.Formatter( - f"%(asctime)s %(levelname)s {language.tag}/{version.name}: %(message)s" + f"%(asctime)s %(levelname)s {build_props.slug}: %(message)s" ) ) if sentry_sdk: scope = sentry_sdk.get_isolation_scope() - scope.set_tag("version", version.name) - scope.set_tag("language", language.tag) + scope.set_tag("version", build_props.version) + scope.set_tag("language", build_props.language) cpython_repo.update() builder = DocBuilder( - version, - language, + build_props, cpython_repo, docs_by_version_content, switchers_content, @@ -1163,7 +1219,7 @@ def build_docs(args: argparse.Namespace) -> int: ) built_successfully = builder.run(http, force_build=force_build) if built_successfully: - build_succeeded.add((version.name, language.tag)) + build_succeeded.add(build_props.slug) elif built_successfully is not None: any_build_failed = True @@ -1286,7 +1342,7 @@ def make_symlinks( group: str, versions: Versions, languages: Languages, - successful_builds: Set[tuple[str, str]], + successful_builds: Set[str], skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: @@ -1306,7 +1362,7 @@ def make_symlinks( ("dev", versions.current_dev.name), ): for language in languages: - if (symlink_target, language.tag) in successful_builds: + if f"{language.tag}/{symlink_target}" in successful_builds: symlink( www_root, language.tag, From 5453926d03a56f7b74b1a5ee4e1ef482c61f3e26 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 15 Aug 2025 19:51:55 +0300 Subject: [PATCH 167/176] Add options to only show certain build times (#307) --- check_times.py | 48 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/check_times.py b/check_times.py index 2b3d2f9..e3bbaea 100644 --- a/check_times.py +++ b/check_times.py @@ -10,6 +10,7 @@ $ python check_times.py """ +import argparse import gzip import tomllib from pathlib import Path @@ -78,17 +79,36 @@ def calc_time(lines: list[str]) -> None: if __name__ == "__main__": - print("Build times (HTML only; English)") - print("=======================") - print() - calc_time(get_lines("docsbuild-only-html-en.log")) - - print("Build times (HTML only)") - print("=======================") - print() - calc_time(get_lines("docsbuild-only-html.log")) - - print("Build times (no HTML)") - print("=====================") - print() - calc_time(get_lines("docsbuild-no-html.log")) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ALL_BUILDS = ("no-html", "only-html", "only-html-en") + parser.add_argument( + "--select-output", + choices=ALL_BUILDS, + nargs="*", + help="Choose what builds to show (default: all).", + ) + args = parser.parse_args() + parser.suggest_on_error = True + + if not args.select_output: + args.select_output = ALL_BUILDS + + if "only-html-en" in args.select_output: + print("Build times (HTML only; English)") + print("=======================") + print() + calc_time(get_lines("docsbuild-only-html-en.log")) + + if "only-html" in args.select_output: + print("Build times (HTML only)") + print("=======================") + print() + calc_time(get_lines("docsbuild-only-html.log")) + + if "no-html" in args.select_output: + print("Build times (no HTML)") + print("=====================") + print() + calc_time(get_lines("docsbuild-no-html.log")) From 1860eda1aba9329d93591e4d10c0408d2c48bb6d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 15 Aug 2025 18:29:47 +0100 Subject: [PATCH 168/176] Only build HTML for non-switcher languages (#308) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 20c3518..93121df 100755 --- a/build_docs.py +++ b/build_docs.py @@ -316,7 +316,7 @@ def iso639_tag(self) -> str: @property def html_only(self) -> bool: - return self._language.html_only + return self._language.html_only or not self._language.in_prod @property def url(self): From 6f7b363186286d49c142b003153a57f728fd83bd Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 8 Sep 2025 22:12:53 +0300 Subject: [PATCH 169/176] Use ``--branches`` in the README's example (#312) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8f914c6..fb16b51 100644 --- a/README.md +++ b/README.md @@ -71,5 +71,5 @@ To manually rebuild a branch, for example 3.11: ssh docs.nyc1.psf.io sudo su --shell=/bin/bash docsbuild screen -DUR # Rejoin screen session if it exists, otherwise create a new one -/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --force --branch 3.11 +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --force --branches 3.11 ``` From 7787668fb2deaa8ea6febfb0f45667fcc489f04d Mon Sep 17 00:00:00 2001 From: Octavian Mustafa Date: Tue, 30 Sep 2025 11:25:59 +0300 Subject: [PATCH 170/176] Add Romanian to the language switcher (#310) --- config.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/config.toml b/config.toml index 4a5958f..1896b19 100644 --- a/config.toml +++ b/config.toml @@ -99,7 +99,6 @@ translated_name = "Português brasileiro" [languages.ro] name = "Romanian" translated_name = "Românește" -in_prod = false [languages.tr] name = "Turkish" From c08754f67818bba2795025b8e9d5f9ca9f0d7dfc Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 17 Oct 2025 09:47:16 +0300 Subject: [PATCH 171/176] Add `tox -e cog` to update README tables (#313) --- README.md | 90 +++++++++++++++++++++++++++++------------------ check_versions.py | 15 ++++---- tox.ini | 11 ++++++ 3 files changed, 76 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index fb16b51..8ba7070 100644 --- a/README.md +++ b/README.md @@ -20,42 +20,64 @@ If you don't need to build all translations of all branches, add `--languages en --branches main`. -## Check current version +## Sphinx versions + + +Sphinx configuration in various branches: + +| version | requirements.txt | conf.py | +|-----------|--------------------|----------------------| +| 2.6 | ø | ø | +| 2.7 | ø | ø | +| 3.0 | ø | ø | +| 3.1 | ø | ø | +| 3.2 | ø | ø | +| 3.3 | ø | ø | +| 3.4 | ø | needs_sphinx='1.2' | +| 3.5 | ø | ø | +| 3.6 | ø | ø | +| 3.7 | ø | ø | +| 3.8 | ø | ø | +| 3.9 | sphinx==2.4.4 | needs_sphinx='1.8' | +| 3.10 | sphinx==3.4.3 | needs_sphinx='3.2' | +| 3.11 | sphinx~=7.2.0 | needs_sphinx='4.2' | +| 3.12 | sphinx~=8.2.0 | needs_sphinx='8.2.0' | +| 3.13 | sphinx~=8.2.0 | needs_sphinx='8.2.0' | +| 3.14 | sphinx~=8.2.0 | needs_sphinx='8.2.0' | +| 3.15 | sphinx~=8.2.0 | needs_sphinx='8.2.0' | + +Sphinx build as seen on docs.python.org: + +| version | el | en | es | fr | bn-in | id | it | ja | ko | pl | pt-br | ro | tr | uk | zh-cn | zh-tw | +|-----------|-------|-------|-------|-------|---------|-------|-------|-------|-------|-------|---------|-------|-------|-------|---------|---------| +| 2.6 | ø | 0.6.5 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | +| 2.7 | ø | 2.3.1 | ø | 2.3.1 | ø | 2.3.1 | ø | 2.3.1 | 2.3.1 | ø | 2.3.1 | ø | ø | ø | 2.3.1 | 2.3.1 | +| 3.0 | ø | 0.6 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | +| 3.1 | ø | 0.6.5 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | +| 3.2 | ø | 1.0.7 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | +| 3.3 | ø | 1.2 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | +| 3.4 | ø | 1.2.3 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | +| 3.5 | ø | 1.8.4 | 1.8.4 | 1.8.4 | ø | 1.8.4 | ø | 1.8.4 | 1.8.4 | 1.8.4 | 1.8.4 | ø | ø | ø | 1.8.4 | 1.8.4 | +| 3.6 | ø | 2.3.1 | 2.3.1 | 2.3.1 | ø | 2.3.1 | ø | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | ø | ø | ø | 2.3.1 | 2.3.1 | +| 3.7 | ø | 2.3.1 | 2.3.1 | 2.3.1 | ø | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | ø | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | +| 3.8 | ø | 2.4.4 | 2.4.4 | 2.4.4 | ø | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | ø | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | +| 3.9 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | +| 3.10 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | +| 3.11 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | +| 3.12 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | +| 3.13 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | +| 3.14 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | +| 3.15 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | + Install `tools_requirements.txt` then run `python check_versions.py -../cpython/` (pointing to a real CPython clone) to see which version -of Sphinx we're using where: - - Sphinx configuration in various branches: - - ========= ============= ================== ==================== - version travis requirements.txt conf.py - ========= ============= ================== ==================== - 2.7 sphinx~=2.0.1 ø needs_sphinx='1.2' - 3.5 sphinx==1.8.2 ø needs_sphinx='1.8' - 3.6 sphinx==1.8.2 ø needs_sphinx='1.2' - 3.7 sphinx==1.8.2 sphinx==2.3.1 needs_sphinx="1.6.6" - 3.8 ø sphinx==2.4.4 needs_sphinx='1.8' - 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' - 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' - 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' - 3.12 ø sphinx~=8.2.0 needs_sphinx='8.2.0' - 3.13 ø sphinx~=8.2.0 needs_sphinx='8.2.0' - 3.14 ø sphinx~=8.2.0 needs_sphinx='8.2.0' - ========= ============= ================== ==================== - - Sphinx build as seen on docs.python.org: - - ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= - version en es fr id it ja ko pl pt-br tr uk zh-cn zh-tw - ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= - 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 - 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 - 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 - 3.12 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 - 3.13 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 - 3.14 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 - ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= +../cpython/` (pointing to a real CPython clone) to see which versions +of Sphinx we're using. + +Or run `tox -e cog` (with a clone at `../cpython`) to directly update these tables. ## Manually rebuild a branch diff --git a/check_versions.py b/check_versions.py index 1a1016f..e381424 100644 --- a/check_versions.py +++ b/check_versions.py @@ -62,7 +62,6 @@ def find_sphinx_in_files(repo: git.Repo, branch_or_tag, filenames): CONF_FILES = { - "travis": ".travis.yml", "requirements.txt": "Doc/requirements.txt", "conf.py": "Doc/conf.py", } @@ -85,7 +84,7 @@ def search_sphinx_versions_in_cpython(repo: git.Repo): for version in VERSIONS ] headers = ["version", *CONF_FILES.keys()] - print(tabulate(table, headers=headers, tablefmt="rst", disable_numparse=True)) + print(tabulate(table, headers=headers, tablefmt="github", disable_numparse=True)) async def get_version_in_prod(language: str, version: str) -> str: @@ -119,16 +118,15 @@ async def which_sphinx_is_used_in_production(): for version in VERSIONS ] headers = ["version", *[language.tag for language in LANGUAGES]] - print(tabulate(table, headers=headers, tablefmt="rst", disable_numparse=True)) + print(tabulate(table, headers=headers, tablefmt="github", disable_numparse=True)) -def main(): +def check_versions(cpython_clone: str) -> None: logging.basicConfig(level=logging.INFO) logging.getLogger("charset_normalizer").setLevel(logging.WARNING) logging.getLogger("asyncio").setLevel(logging.WARNING) logging.getLogger("httpx").setLevel(logging.WARNING) - args = parse_args() - repo = git.Repo(args.cpython_clone) + repo = git.Repo(cpython_clone) print("Sphinx configuration in various branches:", end="\n\n") search_sphinx_versions_in_cpython(repo) print() @@ -136,5 +134,10 @@ def main(): asyncio.run(which_sphinx_is_used_in_production()) +def main(): + args = parse_args() + check_versions(args.cpython_clone) + + if __name__ == "__main__": main() diff --git a/tox.ini b/tox.ini index 12efcdf..a4d051c 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ requires = tox>=4.2 env_list = + cog lint py{314, 313} @@ -26,6 +27,16 @@ commands = --cov-report xml \ {posargs} +[testenv:cog] +base_python = python3.13 +skip_install = true +deps = + -r requirements.txt + -r tools_requirements.txt + cogapp +commands = + cog -Pr README.md + [testenv:lint] skip_install = true deps = From 89b13f9dee65c03f11455340745cb3db1127db3e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 10 Nov 2025 16:32:14 +0000 Subject: [PATCH 172/176] Use ``release-cycle.json`` from peps.python.org (#315) --- build_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 93121df..3111d52 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1256,8 +1256,7 @@ def build_docs(args: argparse.Namespace) -> int: def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: releases = http.request( "GET", - "https://raw.githubusercontent.com/" - "python/devguide/main/include/release-cycle.json", + "https://peps.python.org/api/release-cycle.json", timeout=30, ).json() return Versions.from_json(releases) From 02a00426ffd16b647fc3a1f1cd820623530e851c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 10 Nov 2025 18:55:52 +0200 Subject: [PATCH 173/176] Note that ``release-cycle.json`` is now sourced from peps.python.org (#314) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- build_docs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 3111d52..973c9a0 100755 --- a/build_docs.py +++ b/build_docs.py @@ -26,7 +26,7 @@ ``` Languages are stored in `config.toml` while versions are discovered -from the devguide. +from peps.python.org (generated by `python-releases.toml`). -q selects "quick build", which means to build only HTML. @@ -1174,7 +1174,7 @@ def build_docs(args: argparse.Namespace) -> int: logging.info("Full build start.") start_time = perf_counter() http = urllib3.PoolManager() - versions = parse_versions_from_devguide(http) + versions = parse_versions_from_peps_site(http) languages = parse_languages_from_config() # Reverse languages but not versions, because we take version-language # pairs from the end of the list, effectively reversing it. @@ -1253,7 +1253,7 @@ def build_docs(args: argparse.Namespace) -> int: return EX_FAILURE if any_build_failed else EX_OK -def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: +def parse_versions_from_peps_site(http: urllib3.PoolManager) -> Versions: releases = http.request( "GET", "https://peps.python.org/api/release-cycle.json", From 7554a0378dfe6280d40fa9ae61636cef963843f5 Mon Sep 17 00:00:00 2001 From: Daniel Nylander Date: Wed, 19 Nov 2025 11:02:44 +0100 Subject: [PATCH 174/176] Adding Swedish to config.toml (#316) --- config.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config.toml b/config.toml index 1896b19..1be0745 100644 --- a/config.toml +++ b/config.toml @@ -100,6 +100,11 @@ translated_name = "Português brasileiro" name = "Romanian" translated_name = "Românește" +[languages.sv] +name = "Swedish" +translated_name = "Svenska" +in_prod = false + [languages.tr] name = "Turkish" translated_name = "Türkçe" From 83f43d3130b7d4dbb9b57ed1a8479cb28ca8e2fd Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Fri, 21 Nov 2025 18:25:07 +0000 Subject: [PATCH 175/176] Commit --- config.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.toml b/config.toml index 1be0745..7ef45b0 100644 --- a/config.toml +++ b/config.toml @@ -4,6 +4,9 @@ # html_only: If true, only create HTML files. # sphinxopts: Extra options to pass to SPHINXOPTS in the Makefile. +# Remember to update the Salt config with redirects for new translations! +# For example: https://github.com/python/psf-salt/commit/14bdc3ae054468092e5a8c3cdacfd02f43c32e19 + [defaults] # name has no default, it is mandatory. translated_name = "" From 4becd180ce7ea197b0a4d00b848cae6cc7a883e3 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 8 Dec 2025 07:50:21 +0000 Subject: [PATCH 176/176] Add labels to switchers (#319) --- templates/switchers.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index e54a278..0a042fd 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -63,6 +63,7 @@ const _create_placeholders_if_missing = () => { const _create_version_select = (versions) => { const select = document.createElement("select"); select.className = "version-select"; + select.setAttribute("aria-label", "Python version"); if (_IS_LOCAL) { select.disabled = true; select.title = "Version switching is disabled in local builds"; @@ -96,6 +97,7 @@ const _create_language_select = (languages) => { const select = document.createElement("select"); select.className = "language-select"; + select.setAttribute("aria-label", "Language"); if (_IS_LOCAL) { select.disabled = true; select.title = "Language switching is disabled in local builds";