From 30b98030d78db56b28f8590b877f3c33f79b913d Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Sun, 8 May 2022 17:59:21 +0200 Subject: [PATCH 001/218] Hello Python 3.12! --- build_docs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 72cc6be..79e6bb6 100755 --- a/build_docs.py +++ b/build_docs.py @@ -196,12 +196,19 @@ def __gt__(self, other): # Please keep the list in reverse-order for ease of editing. VERSIONS = [ Version( - "3.11", + "3.12", branch="origin/main", status="in development", sphinx_version="4.5.0", sphinxopts=["-j4"], ), + Version( + "3.11", + branch="origin/3.11", + status="pre-release", + sphinx_version="4.5.0", + sphinxopts=["-j4"], + ), Version( "3.10", branch="origin/3.10", From 2593117eaf2afab55203a0aaf6f32f56217b1763 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 16 May 2022 19:13:05 +0300 Subject: [PATCH 002/218] Remove -j4 Sphinx option as the Makefile has '-j auto' (#129) --- build_docs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 79e6bb6..c64789d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -200,21 +200,18 @@ def __gt__(self, other): branch="origin/main", status="in development", sphinx_version="4.5.0", - sphinxopts=["-j4"], ), Version( "3.11", branch="origin/3.11", status="pre-release", sphinx_version="4.5.0", - sphinxopts=["-j4"], ), Version( "3.10", branch="origin/3.10", status="stable", sphinx_version="3.2.1", - sphinxopts=["-j4"], ), Version( "3.9", From 83ea4847da6df475c99470f57d78a918b5aaaeea Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Mon, 23 May 2022 12:38:57 +0200 Subject: [PATCH 003/218] TIL about /2/, or forgot about it. --- build_docs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index c64789d..f6e0dfa 100755 --- a/build_docs.py +++ b/build_docs.py @@ -916,7 +916,7 @@ def copy_build_to_webroot(self): def symlink(www_root: Path, language: Language, directory: str, name: str, group: str): - """Used by slash_3_symlink and dev_symlink to maintain symlinks.""" + """Used by major_symlinks and dev_symlink to maintain symlinks.""" if language.tag == "en": # english is rooted on /, no /en/ path = www_root else: @@ -933,8 +933,8 @@ def symlink(www_root: Path, language: Language, directory: str, name: str, group run(["chown", "-h", ":" + group, str(link)]) -def slash_3_symlink(www_root: Path, group): - """Maintains the /3/ symlinks for each languages. +def major_symlinks(www_root: Path, group): + """Maintains the /2/ and /3/ symlinks for each languages. Like: - /3/ → /3.9/ @@ -944,6 +944,7 @@ def slash_3_symlink(www_root: Path, group): current_stable = Version.current_stable().name for language in LANGUAGES: symlink(www_root, language, current_stable, "3", group) + symlink(www_root, language, "2.7", "2", group) def dev_symlink(www_root: Path, group): @@ -988,7 +989,7 @@ def main(): build_sitemap(args.www_root) build_robots_txt(args.www_root, args.group, args.skip_cache_invalidation) - slash_3_symlink(args.www_root, args.group) + major_symlinks(args.www_root, args.group) dev_symlink(args.www_root, args.group) From 40ce6d110914b1b929db9f7655898aaf105624b8 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Mon, 23 May 2022 13:10:28 +0200 Subject: [PATCH 004/218] Find and remove broken canonicals. --- build_docs.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/build_docs.py b/build_docs.py index f6e0dfa..c800f25 100755 --- a/build_docs.py +++ b/build_docs.py @@ -959,6 +959,28 @@ def dev_symlink(www_root: Path, group): for language in LANGUAGES: symlink(www_root, language, current_dev, "dev", group) +def proofread_canonicals(www_root: Path) -> None: + """In www_root we check that all canonical links point to existing contents. + + It can happen that a canonical is "broken": + + - /3.11/whatsnew/3.11.html typically would link to + /3/whatsnew/3.11.html, which may not exist yet. + """ + canonical_re = re.compile( + """""" + ) + for file in www_root.glob("**/*.html"): + html = file.read_text(encoding="UTF-8") + canonical = canonical_re.search(html) + if not canonical: + continue + target = canonical.group(1) + if not (www_root / target).exists(): + logging.info("Removing broken canonical from %s to %s", file, target) + html = html.replace(canonical.group(0), "") + file.write_text(html, encoding="UTF-8") + def main(): """Script entry point.""" @@ -991,6 +1013,7 @@ def main(): build_robots_txt(args.www_root, args.group, args.skip_cache_invalidation) major_symlinks(args.www_root, args.group) dev_symlink(args.www_root, args.group) + proofread_canonicals(args.www_root) if __name__ == "__main__": From 7c4f1465f2e4fb6f3f153659f5226d2d87a468be Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 24 May 2022 09:41:14 +0200 Subject: [PATCH 005/218] Bump sphinx to fix missing parenthesis when argument is a tuple. See: https://github.com/python/cpython/issues/93108 --- build_docs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index c800f25..b5f03ee 100755 --- a/build_docs.py +++ b/build_docs.py @@ -211,7 +211,7 @@ def __gt__(self, other): "3.10", branch="origin/3.10", status="stable", - sphinx_version="3.2.1", + sphinx_version="3.4.3", ), Version( "3.9", @@ -959,6 +959,7 @@ def dev_symlink(www_root: Path, group): for language in LANGUAGES: symlink(www_root, language, current_dev, "dev", group) + def proofread_canonicals(www_root: Path) -> None: """In www_root we check that all canonical links point to existing contents. From 3319b5522c7448a26d6436dfe75240f890090f33 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 24 May 2022 10:20:13 +0200 Subject: [PATCH 006/218] Remove superfluous quote. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index b5f03ee..0e143da 100755 --- a/build_docs.py +++ b/build_docs.py @@ -483,7 +483,7 @@ def setup_switchers(html_root: Path): for file in Path(html_root).glob("**/*.html"): depth = len(file.relative_to(html_root).parts) - 1 src = f"{'../' * depth}_static/switchers.js" - script = f' \n"' + script = f' \n' with edit(file) as (ifile, ofile): for line in ifile: if line == script: From ca1f1e559663d5f86730fed0c03c8c463a0479f8 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 24 May 2022 10:54:33 +0200 Subject: [PATCH 007/218] 404 page for artifacts. --- build_docs.py | 9 ++++ templates/404.html | 107 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 templates/404.html diff --git a/build_docs.py b/build_docs.py index 0e143da..cb20d4f 100755 --- a/build_docs.py +++ b/build_docs.py @@ -531,6 +531,14 @@ def build_sitemap(www_root: Path): ) +def build_404(www_root: Path): + """Build a nice 404 error page to display in case PDFs are not built yet.""" + if not www_root.exists(): + logging.info("Skipping 404 page generation (www root does not even exists).") + return + shutil.copyfile(HERE / "templates" / "404.html", www_root / "404.html") + + def head(text, lines=10): """Return the first *lines* lines from the given text.""" return "\n".join(text.split("\n")[:lines]) @@ -1011,6 +1019,7 @@ def main(): lock.close() build_sitemap(args.www_root) + build_404(args.www_root) build_robots_txt(args.www_root, args.group, args.skip_cache_invalidation) major_symlinks(args.www_root, args.group) dev_symlink(args.www_root, args.group) diff --git a/templates/404.html b/templates/404.html new file mode 100644 index 0000000..512faf8 --- /dev/null +++ b/templates/404.html @@ -0,0 +1,107 @@ + + + + + + + Archive not found + + + + + + + + + + + + + + + +
+ + + +
+ +
+
+ + + +
+
+
+
+

404 — Archive Not Found

+

The archive you're trying to download has not been built yet.

+

Please try again later.

+
+
+
+
+
+
+ +
+ © Copyright 2001-2022, Python Software Foundation. +
+ This page is licensed under the Python Software Foundation License Version 2. +
+ Examples, recipes, and other code in the documentation are additionally licensed under the Zero Clause BSD License. +
+ See History and License for more information.
+
+ The Python Software Foundation is a non-profit corporation. + Please donate. +
+ Found a bug? +
+
+ + + From ae2a8fc29c1f52c77d4710c63a705be63717dfd4 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Wed, 25 May 2022 00:10:20 +0200 Subject: [PATCH 008/218] Very old files (Python <= 2.2) are stored in latin-1 with a correct Content-Type. --- build_docs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index cb20d4f..72ca847 100755 --- a/build_docs.py +++ b/build_docs.py @@ -980,7 +980,7 @@ def proofread_canonicals(www_root: Path) -> None: """""" ) for file in www_root.glob("**/*.html"): - html = file.read_text(encoding="UTF-8") + html = file.read_text(encoding="UTF-8", errors="surrogateescape") canonical = canonical_re.search(html) if not canonical: continue @@ -988,7 +988,7 @@ def proofread_canonicals(www_root: Path) -> None: if not (www_root / target).exists(): logging.info("Removing broken canonical from %s to %s", file, target) html = html.replace(canonical.group(0), "") - file.write_text(html, encoding="UTF-8") + file.write_text(html, encoding="UTF-8", errors="surrogateescape") def main(): From ac1adcb83baf60261f243a74dab1fd4b9fc6bbf7 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Wed, 25 May 2022 00:19:50 +0200 Subject: [PATCH 009/218] Invalidate when removing a canonical link. --- build_docs.py | 18 ++++++++---------- requirements.txt | 1 + 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/build_docs.py b/build_docs.py index 72ca847..09796f1 100755 --- a/build_docs.py +++ b/build_docs.py @@ -43,6 +43,7 @@ import zc.lockfile import jinja2 +import requests HERE = Path(__file__).resolve().parent @@ -508,14 +509,7 @@ def build_robots_txt(www_root: Path, group, skip_cache_invalidation): robots_file.chmod(0o775) run(["chgrp", group, robots_file]) if not skip_cache_invalidation: - run( - [ - "curl", - "--silent", - "-XPURGE", - "https://docs.python.org/robots.txt", - ] - ) + requests.request("PURGE", "https://docs.python.org/robots.txt") def build_sitemap(www_root: Path): @@ -968,7 +962,7 @@ def dev_symlink(www_root: Path, group): symlink(www_root, language, current_dev, "dev", group) -def proofread_canonicals(www_root: Path) -> None: +def proofread_canonicals(www_root: Path, skip_cache_invalidation: bool) -> None: """In www_root we check that all canonical links point to existing contents. It can happen that a canonical is "broken": @@ -989,6 +983,10 @@ def proofread_canonicals(www_root: Path) -> None: logging.info("Removing broken canonical from %s to %s", file, target) html = html.replace(canonical.group(0), "") file.write_text(html, encoding="UTF-8", errors="surrogateescape") + if not skip_cache_invalidation: + url = str(file).replace("/srv/", "https://") + logging.info("Purging %s from CDN", url) + requests.request("PURGE", url) def main(): @@ -1023,7 +1021,7 @@ def main(): build_robots_txt(args.www_root, args.group, args.skip_cache_invalidation) major_symlinks(args.www_root, args.group) dev_symlink(args.www_root, args.group) - proofread_canonicals(args.www_root) + proofread_canonicals(args.www_root, args.skip_cache_invalidation) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 3d99dc4..65ae7f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ jinja2 +requests sentry-sdk zc.lockfile From 258f3122f1a22bd5b97d42e40f6ac644889b0719 Mon Sep 17 00:00:00 2001 From: Dmytro Kazanzhy Date: Mon, 25 Jul 2022 10:24:32 +0300 Subject: [PATCH 010/218] Add Ukrainian to docsbuild (#131) --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index 09796f1..9e6804c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -295,6 +295,7 @@ def __gt__(self, other): Language("zh-cn", "zh_CN", "Simplified Chinese", True, XELATEX_WITH_CJK), Language("zh-tw", "zh_TW", "Traditional Chinese", True, XELATEX_WITH_CJK), Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), + Language("uk", "uk", "Ukrainian", False, XELATEX_DEFAULT), } From 681fd2c92d5b5f449cfdb5a92c31dff652fb5f05 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 7 Aug 2022 12:29:07 -0700 Subject: [PATCH 011/218] Shot in the dark fix for broken 3.10 render (#133) * Shot in the dark fix for broken 3.10 render --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 9e6804c..3e8cd98 100755 --- a/build_docs.py +++ b/build_docs.py @@ -108,7 +108,7 @@ def requirements(self): if version_to_tuple(self.sphinx_version) < (4, 5): # see https://github.com/python/cpython/issues/91294 reqs += ["jinja2<3.1"] - if version_to_tuple(self.sphinx_version) <= (3, 2, 1): + if version_to_tuple(self.sphinx_version) < (3, 5, 4): # see https://github.com/python/cpython/issues/91483 reqs += ["docutils<=0.17.1"] return reqs From 2d1c637fffab3f76ca79b2f48241a5a202561a1c Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 21 Aug 2022 09:57:09 -0700 Subject: [PATCH 012/218] Run pip freeze after installing requirements (#132) It appears that the venv keeps breaking: https://github.com/python/cpython/issues/91483 I'm not sure how to best help investigate, but adding some more logging seems like it could be useful :-) --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index 3e8cd98..3e343b0 100755 --- a/build_docs.py +++ b/build_docs.py @@ -786,6 +786,7 @@ def build_venv(self): + [self.theme] + self.version.requirements ) + run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) self.venv = venv_path def copy_build_to_webroot(self): From dc35d4f4f45f3a0d827cbc1bcf87e85c3ffda148 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 11 Sep 2022 08:41:33 -0400 Subject: [PATCH 013/218] move 3.9 from stable to security-fixes avoiding daily rebuilds (#135) --- README.md | 12 +++++++----- build_docs.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0620305..71156ab 100644 --- a/README.md +++ b/README.md @@ -29,11 +29,12 @@ of Sphinx we're using where: 2.7 sphinx~=2.0.1 ø ø needs_sphinx='1.2' ø ø 3.5 sphinx==1.8.2 ø ø needs_sphinx='1.8' ø ø 3.6 sphinx==1.8.2 sphinx==1.8.2 ø needs_sphinx='1.2' Sphinx==2.3.1 ø - 3.7 sphinx==1.8.2 sphinx==1.8.2 ø needs_sphinx="1.6.6" Sphinx==2.3.1 Sphinx==2.3.1 + 3.7 sphinx==1.8.2 sphinx==1.8.2 sphinx==2.3.1 needs_sphinx="1.6.6" ø Sphinx==2.3.1 3.8 ø ø sphinx==2.4.4 needs_sphinx='1.8' ø ø 3.9 ø ø sphinx==2.4.4 needs_sphinx='1.8' ø ø - 3.1 ø ø sphinx==3.2.1 needs_sphinx='1.8' ø ø - 3.11 ø ø sphinx==4.5.0 needs_sphinx='1.8' ø ø + 3.10 ø ø sphinx==3.4.3 needs_sphinx='3.2' ø ø + 3.11 ø ø sphinx==4.5.0 needs_sphinx='3.2' ø ø + 3.12 ø ø sphinx==4.5.0 needs_sphinx='3.2' ø ø ======== ============= ============= ================== ==================== ============= =============== Sphinx build as seen on docs.python.org: @@ -47,8 +48,9 @@ of Sphinx we're using where: 3.7 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 3.8 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 - 3.10 3.2.1 3.2.1 3.2.1 3.2.1 3.2.1 3.2.1 3.2.1 3.2.1 3.2.1 3.2.1 - 3.11 4.2.0 4.2.0 4.2.0 4.2.0 4.2.0 4.2.0 4.2.0 4.2.0 4.2.0 4.2.0 + 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 + 3.11 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 + 3.12 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 ======== ===== ===== ===== ===== ===== ===== ===== ======= ======= ======= diff --git a/build_docs.py b/build_docs.py index 3e343b0..d298e10 100755 --- a/build_docs.py +++ b/build_docs.py @@ -217,7 +217,7 @@ def __gt__(self, other): Version( "3.9", branch="origin/3.9", - status="stable", + status="security-fixes", sphinx_version="2.4.4", ), Version( From c37e6770ccd031e69967dcb32599e09c10a67f58 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 14 Jun 2022 09:45:18 +0200 Subject: [PATCH 014/218] Ensure correct mode and groupe. --- build_docs.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index d298e10..973d766 100755 --- a/build_docs.py +++ b/build_docs.py @@ -513,7 +513,7 @@ def build_robots_txt(www_root: Path, group, skip_cache_invalidation): requests.request("PURGE", "https://docs.python.org/robots.txt") -def build_sitemap(www_root: Path): +def build_sitemap(www_root: Path, group): """Build a sitemap with all live versions and translations.""" if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exists).") @@ -524,14 +524,19 @@ def build_sitemap(www_root: Path): sitemap_file.write( template.render(languages=LANGUAGES, versions=VERSIONS) + "\n" ) + sitemap_file.chmod(0o775) + run(["chgrp", group, sitemap_file]) -def build_404(www_root: Path): +def build_404(www_root: Path, group): """Build a nice 404 error page to display in case PDFs are not built yet.""" if not www_root.exists(): logging.info("Skipping 404 page generation (www root does not even exists).") return - shutil.copyfile(HERE / "templates" / "404.html", www_root / "404.html") + not_found_file = www_root / "404.html" + shutil.copyfile(HERE / "templates" / "404.html", not_found_file) + not_found_file.chmod(0o775) + run(["chgrp", group, not_found_file]) def head(text, lines=10): From 704772115fa4414f0c9e96321df4f30bee1b9159 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Wed, 26 Oct 2022 07:40:25 +0200 Subject: [PATCH 015/218] Hello 3.11.0. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 973d766..5795cdb 100755 --- a/build_docs.py +++ b/build_docs.py @@ -205,7 +205,7 @@ def __gt__(self, other): Version( "3.11", branch="origin/3.11", - status="pre-release", + status="stable", sphinx_version="4.5.0", ), Version( From cddba20f24607f219a272e38c0e6aeb8d3136058 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 27 Oct 2022 08:51:23 +0200 Subject: [PATCH 016/218] Better rights for sitemap and 404. --- build_docs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index 5795cdb..b34d6e1 100755 --- a/build_docs.py +++ b/build_docs.py @@ -524,7 +524,7 @@ def build_sitemap(www_root: Path, group): sitemap_file.write( template.render(languages=LANGUAGES, versions=VERSIONS) + "\n" ) - sitemap_file.chmod(0o775) + sitemap_file.chmod(0o664) run(["chgrp", group, sitemap_file]) @@ -535,7 +535,7 @@ def build_404(www_root: Path, group): return not_found_file = www_root / "404.html" shutil.copyfile(HERE / "templates" / "404.html", not_found_file) - not_found_file.chmod(0o775) + not_found_file.chmod(0o664) run(["chgrp", group, not_found_file]) @@ -1023,8 +1023,8 @@ def main(): else: lock.close() - build_sitemap(args.www_root) - build_404(args.www_root) + build_sitemap(args.www_root, args.group) + build_404(args.www_root, args.group) build_robots_txt(args.www_root, args.group, args.skip_cache_invalidation) major_symlinks(args.www_root, args.group) dev_symlink(args.www_root, args.group) From f5f087a381ba65750a6e729bf7eb3e5090cb45a9 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 27 Oct 2022 08:56:14 +0200 Subject: [PATCH 017/218] Use pathlib. --- build_docs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index b34d6e1..a55c827 100755 --- a/build_docs.py +++ b/build_docs.py @@ -520,10 +520,11 @@ def build_sitemap(www_root: Path, group): return with open(HERE / "templates" / "sitemap.xml", encoding="UTF-8") as template_file: template = jinja2.Template(template_file.read()) - with open(www_root / "sitemap.xml", "w", encoding="UTF-8") as sitemap_file: - sitemap_file.write( - template.render(languages=LANGUAGES, versions=VERSIONS) + "\n" - ) + sitemap_file = www_root / "sitemap.xml" + sitemap_file.write_text( + template.render(languages=LANGUAGES, versions=VERSIONS) + "\n", + encoding="UTF-8" + ) sitemap_file.chmod(0o664) run(["chgrp", group, sitemap_file]) From b3c3137efb04c789a60b4ceb65fbb0ad46370f4b Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 27 Oct 2022 09:19:22 +0200 Subject: [PATCH 018/218] Purge when updating symlinks. --- build_docs.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build_docs.py b/build_docs.py index a55c827..361264f 100755 --- a/build_docs.py +++ b/build_docs.py @@ -941,6 +941,7 @@ def symlink(www_root: Path, language: Language, directory: str, name: str, group link.unlink() link.symlink_to(directory) run(["chown", "-h", ":" + group, str(link)]) + purge_path(www_root, link) def major_symlinks(www_root: Path, group): @@ -997,6 +998,13 @@ def proofread_canonicals(www_root: Path, skip_cache_invalidation: bool) -> None: requests.request("PURGE", url) +def purge_path(www_root: Path, path: Path): + to_purge = [str(file.relative_to(www_root)) for file in path.glob("**/*")] + to_purge.append(str(path.relative_to(www_root))) + to_purge.append(str(path.relative_to(www_root)) + "/") + run(["curl", "-XPURGE", f"https://docs.python.org/{{{','.join(to_purge)}}}"]) + + def main(): """Script entry point.""" args = parse_args() From ee7e44ca6ff68bdd264f8e6c04e2d97b1e933e86 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Wed, 7 Dec 2022 22:10:59 +0100 Subject: [PATCH 019/218] Use cpython/Doc/requirements.txt when it's possible. Closes #140. --- build_docs.py | 159 +++++++++++++++++++++----------------------------- 1 file changed, 65 insertions(+), 94 deletions(-) diff --git a/build_docs.py b/build_docs.py index 361264f..9dae35b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -71,10 +71,9 @@ def __init__( self, name, *, + status, branch=None, tag=None, - status, - sphinx_version, sphinxopts=(), ): if status not in self.STATUSES: @@ -88,7 +87,6 @@ def __init__( raise ValueError("Please build a version with at least a branch or a tag.") self.branch_or_tag = branch or tag self.status = status - self.sphinx_version = sphinx_version self.sphinxopts = list(sphinxopts) def __repr__(self): @@ -96,23 +94,29 @@ def __repr__(self): @property def requirements(self): - """Generate the right requirements for this version, pinning breaking - sub-dependencies as needed. + """Generate the right requirements for this version. + + Since CPython 3.8 a Doc/requirements.txt file can be used. + + In case the Doc/requirements.txt is absent or wrong (a + sub-dependency broke), use this function to override it. + + See https://github.com/python/cpython/issues/91294 + See https://github.com/python/cpython/issues/91483 + """ - reqs = [ - "blurb", - "jieba", - f"sphinx=={self.sphinx_version}", + if self.name == "3.5": + return ["jieba", "blurb", "sphinx==1.8.4", "jinja2<3.1", "docutils<=0.17.1"] + if self.name in ("3.7", "3.6", "2.7"): + return ["jieba", "blurb", "sphinx==2.3.1", "jinja2<3.1", "docutils<=0.17.1"] + if self.name == ("3.8", "3.9"): + return ["jieba", "blurb", "sphinx==2.4.4", "jinja2<3.1", "docutils<=0.17.1"] + + return [ + "jieba", # To improve zh search. + "-rrequirements.txt", ] - if version_to_tuple(self.sphinx_version) < (4, 5): - # see https://github.com/python/cpython/issues/91294 - reqs += ["jinja2<3.1"] - if version_to_tuple(self.sphinx_version) < (3, 5, 4): - # see https://github.com/python/cpython/issues/91483 - reqs += ["docutils<=0.17.1"] - return reqs - @property def changefreq(self): """Estimate this version change frequency, for the sitemap.""" @@ -196,60 +200,15 @@ def __gt__(self, other): # # Please keep the list in reverse-order for ease of editing. VERSIONS = [ - Version( - "3.12", - branch="origin/main", - status="in development", - sphinx_version="4.5.0", - ), - Version( - "3.11", - branch="origin/3.11", - status="stable", - sphinx_version="4.5.0", - ), - Version( - "3.10", - branch="origin/3.10", - status="stable", - sphinx_version="3.4.3", - ), - Version( - "3.9", - branch="origin/3.9", - status="security-fixes", - sphinx_version="2.4.4", - ), - Version( - "3.8", - branch="origin/3.8", - status="security-fixes", - sphinx_version="2.4.4", - ), - Version( - "3.7", - branch="origin/3.7", - status="security-fixes", - sphinx_version="2.3.1", - ), - Version( - "3.6", - tag="3.6", - status="EOL", - sphinx_version="2.3.1", - ), - Version( - "3.5", - tag="3.5", - status="EOL", - sphinx_version="1.8.4", - ), - Version( - "2.7", - tag="2.7", - status="EOL", - sphinx_version="2.3.1", - ), + Version("3.12", branch="origin/main", status="in development"), + Version("3.11", branch="origin/3.11", status="stable"), + Version("3.10", branch="origin/3.10", status="stable"), + Version("3.9", branch="origin/3.9", status="security-fixes"), + Version("3.8", branch="origin/3.8", status="security-fixes"), + Version("3.7", branch="origin/3.7", status="security-fixes"), + Version("3.6", tag="3.6", status="EOL"), + Version("3.5", tag="3.5", status="EOL"), + Version("2.7", tag="2.7", status="EOL"), ] XELATEX_DEFAULT = ( @@ -299,13 +258,14 @@ def __gt__(self, other): } -def run(cmd) -> subprocess.CompletedProcess: +def run(cmd, cwd=None) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" cmd = [str(arg) for arg in cmd] cmdstring = shlex.join(cmd) logging.debug("Run: %r", cmdstring) result = subprocess.run( cmd, + cwd=cwd, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, @@ -522,8 +482,7 @@ def build_sitemap(www_root: Path, group): template = jinja2.Template(template_file.read()) sitemap_file = www_root / "sitemap.xml" sitemap_file.write_text( - template.render(languages=LANGUAGES, versions=VERSIONS) + "\n", - encoding="UTF-8" + template.render(languages=LANGUAGES, versions=VERSIONS) + "\n", encoding="UTF-8" ) sitemap_file.chmod(0o664) run(["chgrp", group, sitemap_file]) @@ -680,6 +639,9 @@ class DocBuilder( def run(self): """Build and publish a Python doc, for a language, and a version.""" try: + self.clone_cpython() + if self.language.tag != "en": + self.clone_translation() self.build_venv() self.build() self.copy_build_to_webroot() @@ -697,6 +659,28 @@ def checkout(self) -> Path: """Path to cpython git clone.""" return self.build_root / "cpython" + def clone_translation(self): + locale_repo = f"https://github.com/python/python-docs-{self.language.tag}.git" + locale_clone_dir = ( + self.build_root + / self.version.name + / "locale" + / self.language.iso639_tag + / "LC_MESSAGES" + ) + git_clone( + locale_repo, + locale_clone_dir, + translation_branch(locale_repo, locale_clone_dir, self.version.name), + ) + + def clone_cpython(self): + git_clone( + "https://github.com/python/cpython.git", + self.checkout, + self.version.branch_or_tag, + ) + def build(self): """Build this version/language doc.""" logging.info( @@ -708,15 +692,6 @@ def build(self): sphinxopts.extend(["-q"]) if self.language.tag != "en": locale_dirs = self.build_root / self.version.name / "locale" - locale_clone_dir = locale_dirs / self.language.iso639_tag / "LC_MESSAGES" - locale_repo = ( - f"https://github.com/python/python-docs-{self.language.tag}.git" - ) - git_clone( - locale_repo, - locale_clone_dir, - translation_branch(locale_repo, locale_clone_dir, self.version.name), - ) sphinxopts.extend( ( f"-D locale_dirs={locale_dirs}", @@ -726,11 +701,6 @@ def build(self): ) if self.version.status == "EOL": sphinxopts.append("-D html_context.outdated=1") - git_clone( - "https://github.com/python/cpython.git", - self.checkout, - self.version.branch_or_tag, - ) maketarget = ( "autobuild-" + ( @@ -780,17 +750,18 @@ def build(self): ) def build_venv(self): - """Build a venv for the specific version. - This is used to pin old Sphinx versions to old cpython branches. + """Build a venv for the specific Python version. + + So we can reuse them from builds to builds, while they contain + different Sphinx versions. """ - venv_path = self.build_root / ( - "venv-with-sphinx-" + self.version.sphinx_version - ) + venv_path = self.build_root / ("venv-" + self.version.name) run([sys.executable, "-m", "venv", venv_path]) run( [venv_path / "bin" / "python", "-m", "pip", "install"] + [self.theme] - + self.version.requirements + + self.version.requirements, + cwd=self.checkout / "Doc", ) run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) self.venv = venv_path From 0aa43020bd5f14091499208eb9c386af525d94d5 Mon Sep 17 00:00:00 2001 From: Giuseppe Alaimo <72734028+GiuseppeAlaimo@users.noreply.github.com> Date: Tue, 27 Dec 2022 23:12:49 +0100 Subject: [PATCH 020/218] Add italian build (#139) --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index 9dae35b..245cda2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -255,6 +255,7 @@ def __gt__(self, other): Language("zh-tw", "zh_TW", "Traditional Chinese", True, XELATEX_WITH_CJK), Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), Language("uk", "uk", "Ukrainian", False, XELATEX_DEFAULT), + Language("it", "it", "Italian", False, XELATEX_DEFAULT), } From e5635f82c41a22e95c5e5121294deedb217b12f5 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 27 Dec 2022 23:14:09 +0100 Subject: [PATCH 021/218] Add Turkish build, and keep those languages sorted. --- build_docs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 245cda2..ade3c2b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -248,14 +248,15 @@ def __gt__(self, other): Language("es", "es", "Spanish", True, XELATEX_WITH_FONTSPEC), Language("fr", "fr", "French", True, XELATEX_WITH_FONTSPEC), Language("id", "id", "Indonesian", False, XELATEX_DEFAULT), + Language("it", "it", "Italian", False, XELATEX_DEFAULT), Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT), Language("ko", "ko", "Korean", True, XELATEX_FOR_KOREAN), + Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT), + Language("tr", "tr", "Turkish", False, XELATEX_DEFAULT), + Language("uk", "uk", "Ukrainian", False, XELATEX_DEFAULT), Language("zh-cn", "zh_CN", "Simplified Chinese", True, XELATEX_WITH_CJK), Language("zh-tw", "zh_TW", "Traditional Chinese", True, XELATEX_WITH_CJK), - Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), - Language("uk", "uk", "Ukrainian", False, XELATEX_DEFAULT), - Language("it", "it", "Italian", False, XELATEX_DEFAULT), } From 1a0ad2e6f03af7bf18ab68814f27e88d6b4faee6 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Wed, 28 Dec 2022 10:22:37 +0100 Subject: [PATCH 022/218] Looks like the idea has been rejected: https://github.com/python/psf-salt/pull/197#event-8116052546 --- build_docs_server.py | 169 ------------------------------------------- 1 file changed, 169 deletions(-) delete mode 100644 build_docs_server.py diff --git a/build_docs_server.py b/build_docs_server.py deleted file mode 100644 index 1911a1c..0000000 --- a/build_docs_server.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Github hook server. - -This is a simple HTTP server handling Github Webhooks requests to -build the doc when needed. - -It needs a GH_SECRET environment variable to be able to receive hooks -on `/hook/github`. - -Its logging can be configured by giving a yaml file path to the -`--logging-config` argument. - -By default the loglevel is `DEBUG` on `stderr`, the default config can -be found in the code so one can bootstrap a different config from it. -""" - -from pathlib import Path -import argparse -import asyncio -import logging.config -import os - -from aiohttp import web -from gidgethub import sansio -import yaml - -from build_docs import VERSIONS - - -__version__ = "0.0.1" - -DEFAULT_LOGGING_CONFIG = """ ---- - -version: 1 -disable_existing_loggers: false -formatters: - normal: - format: '%(asctime)s - %(levelname)s - %(message)s' -handlers: - stderr: - class: logging.StreamHandler - stream: ext://sys.stderr - level: DEBUG - formatter: normal -loggers: - build_docs_server: - level: DEBUG - handlers: [stderr] - aiohttp.access: - level: DEBUG - handlers: [stderr] - aiohttp.client: - level: DEBUG - handlers: [stderr] - aiohttp.internal: - level: DEBUG - handlers: [stderr] - aiohttp.server: - level: DEBUG - handlers: [stderr] - aiohttp.web: - level: DEBUG - handlers: [stderr] - aiohttp.websocket: - level: DEBUG - handlers: [stderr] -""" - -logger = logging.getLogger("build_docs_server") - - -async def version(request): - return web.json_response( - { - "name": "docs.python.org Github handler", - "version": __version__, - "source": "https://github.com/python/docsbuild-scripts", - } - ) - - -async def child_waiter(app): - while True: - try: - status = os.waitid(os.P_ALL, 0, os.WNOHANG | os.WEXITED) - logger.debug("Child completed with status %s", str(status)) - except ChildProcessError: - await asyncio.sleep(600) - - -async def start_child_waiter(app): - app["child_waiter"] = asyncio.ensure_future(child_waiter(app)) - - -async def stop_child_waiter(app): - app["child_waiter"].cancel() - - -async def hook(request): - body = await request.read() - event = sansio.Event.from_http( - request.headers, body, secret=os.environ.get("GH_SECRET") - ) - if event.event != "push": - logger.debug("Received a %s event, nothing to do.", event.event) - return web.Response() - touched_files = ( - set(event.data["head_commit"]["added"]) - | set(event.data["head_commit"]["modified"]) - | set(event.data["head_commit"]["removed"]) - ) - if not any("Doc" in touched_file for touched_file in touched_files): - logger.debug("No documentation file modified, ignoring.") - return web.Response() # Nothing to do - branch = event.data["ref"].split("/")[-1] - known_branches = {version.branch for version in VERSION} - if branch not in known_branches: - logger.warning("Ignoring a change in branch %s (unknown branch)", branch) - return web.Response() # Nothing to do - logger.debug("Forking a build for branch %s", branch) - pid = os.fork() - if pid == 0: - os.execl( - "/usr/bin/env", - "/usr/bin/env", - "python", - "build_docs.py", - "--branch", - branch, - ) - else: - return web.Response() - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--path", help="Unix socket to listen for connections.") - parser.add_argument("--port", help="Local port to listen for connections.") - parser.add_argument( - "--logging-config", - help="yml file containing a Python logging dictconfig, see README.md", - ) - return parser.parse_args() - - -def main(): - args = parse_args() - logging.config.dictConfig( - yaml.load( - Path(args.logging_config).read_text() - if args.logging_config - else DEFAULT_LOGGING_CONFIG, - Loader=yaml.SafeLoader, - ) - ) - app = web.Application() - app.on_startup.append(start_child_waiter) - app.on_cleanup.append(stop_child_waiter) - app.add_routes( - [ - web.get("/", version), - web.post("/hooks/github", hook), - ] - ) - web.run_app(app, path=args.path, port=args.port) - - -if __name__ == "__main__": - main() From 5ac59ae41364268d62f0c6eb0f886640b09c8302 Mon Sep 17 00:00:00 2001 From: Ege Akman Date: Thu, 29 Dec 2022 02:01:21 +0300 Subject: [PATCH 023/218] Add Turkish translation to language switcher (#141) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index ade3c2b..94776d1 100755 --- a/build_docs.py +++ b/build_docs.py @@ -253,7 +253,7 @@ def __gt__(self, other): Language("ko", "ko", "Korean", True, XELATEX_FOR_KOREAN), Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT), - Language("tr", "tr", "Turkish", False, XELATEX_DEFAULT), + Language("tr", "tr", "Turkish", True, XELATEX_DEFAULT), Language("uk", "uk", "Ukrainian", False, XELATEX_DEFAULT), Language("zh-cn", "zh_CN", "Simplified Chinese", True, XELATEX_WITH_CJK), Language("zh-tw", "zh_TW", "Traditional Chinese", True, XELATEX_WITH_CJK), From 975cae32539806a19f1beec55a75747bafcaeb2b Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 3 Jan 2023 16:15:34 +0100 Subject: [PATCH 024/218] Allow /ja/ to not build PDF. See #142. --- build_docs.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/build_docs.py b/build_docs.py index 94776d1..190cc56 100755 --- a/build_docs.py +++ b/build_docs.py @@ -21,6 +21,7 @@ from argparse import ArgumentParser from contextlib import suppress +from dataclasses import dataclass import filecmp from itertools import product import json @@ -191,9 +192,16 @@ def __gt__(self, other): return self.as_tuple() > other.as_tuple() -Language = namedtuple( - "Language", ["tag", "iso639_tag", "name", "in_prod", "sphinxopts"] -) + +@dataclass(frozen=True) +class Language: + tag: str + iso639_tag: str + name: str + in_prod: bool + sphinxopts: tuple + html_only: bool = False + # EOL and security-fixes are not automatically built, no need to remove them # from the list, this way we can still rebuild them manually as needed. @@ -249,7 +257,7 @@ def __gt__(self, other): Language("fr", "fr", "French", True, XELATEX_WITH_FONTSPEC), Language("id", "id", "Indonesian", False, XELATEX_DEFAULT), Language("it", "it", "Italian", False, XELATEX_DEFAULT), - Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT), + Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT, html_only=True), # See https://github.com/python/python-docs-ja/issues/35 Language("ko", "ko", "Korean", True, XELATEX_FOR_KOREAN), Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT), @@ -710,7 +718,7 @@ def build(self): if self.version.status in ("in development", "pre-release") else "stable" ) - + ("-html" if self.quick else "") + + ("-html" if self.quick or self.language.html_only else "") ) logging.info("Running make %s", maketarget) python = self.venv / "bin" / "python" From b8f2b31a63325980e61488bcb74f7b96ab4e4a8a Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 3 Jan 2023 18:21:48 +0100 Subject: [PATCH 025/218] Unused since server has been dropped. --- server-requirements.txt | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 server-requirements.txt diff --git a/server-requirements.txt b/server-requirements.txt deleted file mode 100644 index b412767..0000000 --- a/server-requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -aiohttp -gidgethub -pyyaml -sentry-sdk From 4651d465d7ddf03e50a0cc9cdc2a5001a0e85c39 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 3 Jan 2023 18:29:37 +0100 Subject: [PATCH 026/218] Needed by check_versions.py --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 190cc56..73e7162 100755 --- a/build_docs.py +++ b/build_docs.py @@ -193,7 +193,7 @@ def __gt__(self, other): -@dataclass(frozen=True) +@dataclass(frozen=True, order=True) class Language: tag: str iso639_tag: str From 3b692d6f8216f4f0955f2d14f2cbf4cae8177c9f Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 3 Jan 2023 18:30:08 +0100 Subject: [PATCH 027/218] No longer needed since cpython's 0f3b96b368. --- check_versions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/check_versions.py b/check_versions.py index bce051e..eab8c43 100644 --- a/check_versions.py +++ b/check_versions.py @@ -56,7 +56,6 @@ def find_sphinx_in_file(repo: git.Repo, branch, filename): "requirements.txt": "Doc/requirements.txt", "conf.py": "Doc/conf.py", "Makefile": "Doc/Makefile", - "Mac installer": "Mac/BuildScript/build-installer.py", } From f8195b9a3a805591782dbcf67b6fc781c5175c02 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 3 Jan 2023 18:32:04 +0100 Subject: [PATCH 028/218] No longer needed since cpython's 8394500cca --- check_versions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/check_versions.py b/check_versions.py index eab8c43..630eb6e 100644 --- a/check_versions.py +++ b/check_versions.py @@ -55,7 +55,6 @@ def find_sphinx_in_file(repo: git.Repo, branch, filename): "azure": ".azure-pipelines/docs-steps.yml", "requirements.txt": "Doc/requirements.txt", "conf.py": "Doc/conf.py", - "Makefile": "Doc/Makefile", } From 1813f8c3213ed6b3f286b34e6006be36fd8a8297 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Wed, 4 Jan 2023 08:57:44 +0100 Subject: [PATCH 029/218] Other conditions were using self.quick. --- build_docs.py | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index 73e7162..aded2d2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -36,7 +36,7 @@ import sys import time from bisect import bisect_left as bisect -from collections import OrderedDict, namedtuple +from collections import OrderedDict from contextlib import contextmanager from pathlib import Path from string import Template @@ -637,15 +637,32 @@ def setup_logging(log_directory: Path): logging.getLogger().setLevel(logging.DEBUG) -class DocBuilder( - namedtuple( - "DocBuilder", - "version, language, build_root, www_root, quick, group, " - "log_directory, skip_cache_invalidation, theme", - ) -): +@dataclass +class DocBuilder: """Builder for a cpython version and a language.""" + version: Version + language: Language + build_root: Path + www_root: Path + quick: bool + group: str + log_directory: Path + skip_cache_invalidation: bool + theme: Path + + @property + def full_build(self): + """Tell if a full build is needed. + + A full build is slow, it builds pdf, txt, epub, texinfo, and + archives everything. + + A partial build only builds HTML and does not archive, it's + fast. + """ + return not self.quick and not self.language.html_only + def run(self): """Build and publish a Python doc, for a language, and a version.""" try: @@ -718,7 +735,7 @@ def build(self): if self.version.status in ("in development", "pre-release") else "stable" ) - + ("-html" if self.quick or self.language.html_only else "") + + ("" if self.full_build else "-html") ) logging.info("Running make %s", maketarget) python = self.venv / "bin" / "python" @@ -830,11 +847,14 @@ def copy_build_to_webroot(self): ";", ] ) - if self.quick: + if self.full_build: run( [ "rsync", "-a", + "--delete-delay", + "--filter", + "P archives/", str(self.checkout / "Doc" / "build" / "html") + "/", target, ] @@ -844,14 +864,11 @@ def copy_build_to_webroot(self): [ "rsync", "-a", - "--delete-delay", - "--filter", - "P archives/", str(self.checkout / "Doc" / "build" / "html") + "/", target, ] ) - if not self.quick: + if self.full_build: logging.debug("Copying dist files") run( [ @@ -986,7 +1003,7 @@ def purge_path(www_root: Path, path: Path): run(["curl", "-XPURGE", f"https://docs.python.org/{{{','.join(to_purge)}}}"]) -def main(): +def main() -> None: """Script entry point.""" args = parse_args() setup_logging(args.log_directory) From ec33098983688106d9c298ec6da1888deb9dc253 Mon Sep 17 00:00:00 2001 From: take6 <60382512+take6@users.noreply.github.com> Date: Tue, 17 Jan 2023 22:11:36 +0900 Subject: [PATCH 030/218] Fix japanese doc build error (#144) * ad hoc fix for PDF build error for Japanese doc Use kotex package as the error claims that tex failed to process Korean character. * enable PDF build for Japanese doc --- build_docs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index aded2d2..77710ba 100755 --- a/build_docs.py +++ b/build_docs.py @@ -229,6 +229,8 @@ class Language: "-D latex_engine=platex", "-D latex_elements.inputenc=", "-D latex_elements.fontenc=", + # See https://github.com/python/python-docs-ja/issues/35 + r"-D latex_elements.preamble=\\usepackage{kotex}", ) XELATEX_WITH_FONTSPEC = ( @@ -257,7 +259,7 @@ class Language: Language("fr", "fr", "French", True, XELATEX_WITH_FONTSPEC), Language("id", "id", "Indonesian", False, XELATEX_DEFAULT), Language("it", "it", "Italian", False, XELATEX_DEFAULT), - Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT, html_only=True), # See https://github.com/python/python-docs-ja/issues/35 + Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT), Language("ko", "ko", "Korean", True, XELATEX_FOR_KOREAN), Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT), From 3a75c4dcac91e25d6188b750b7beb0546d40eb90 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 19 Jan 2023 09:27:00 +0100 Subject: [PATCH 031/218] Revert "Fix japanese doc build error (#144)" (#146) This reverts commit ec33098983688106d9c298ec6da1888deb9dc253. --- build_docs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 77710ba..aded2d2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -229,8 +229,6 @@ class Language: "-D latex_engine=platex", "-D latex_elements.inputenc=", "-D latex_elements.fontenc=", - # See https://github.com/python/python-docs-ja/issues/35 - r"-D latex_elements.preamble=\\usepackage{kotex}", ) XELATEX_WITH_FONTSPEC = ( @@ -259,7 +257,7 @@ class Language: Language("fr", "fr", "French", True, XELATEX_WITH_FONTSPEC), Language("id", "id", "Indonesian", False, XELATEX_DEFAULT), Language("it", "it", "Italian", False, XELATEX_DEFAULT), - Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT), + Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT, html_only=True), # See https://github.com/python/python-docs-ja/issues/35 Language("ko", "ko", "Korean", True, XELATEX_FOR_KOREAN), Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT), From c49181f1387c202fb8905cd9ca39c6caf569b8b0 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 7 Mar 2023 18:22:07 +0100 Subject: [PATCH 032/218] Let Google discover the canonical we just added. --- templates/robots.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/templates/robots.txt b/templates/robots.txt index c52e054..9af9c5d 100644 --- a/templates/robots.txt +++ b/templates/robots.txt @@ -16,7 +16,3 @@ Disallow: /2.5/ Disallow: /2.6/ Disallow: /2.7/ Disallow: /3.0/ -Disallow: /3.1/ -Disallow: /3.2/ -Disallow: /3.3/ -Disallow: /3.4/ From d5a0a72b373c81f64bbabcbe8b885ba2a409be38 Mon Sep 17 00:00:00 2001 From: Atsuo Ishimoto Date: Sat, 11 Mar 2023 02:35:05 +0900 Subject: [PATCH 033/218] Fix Unicode character error building Japanese PDF documents (#149) * fix Japanese PDF geneeration error * various change for ja/lualatex * Replace U+FFFD with '?' --- build_docs.py | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index aded2d2..c31826a 100755 --- a/build_docs.py +++ b/build_docs.py @@ -225,10 +225,40 @@ class Language: "-D latex_elements.fontenc=", ) -PLATEX_DEFAULT = ( - "-D latex_engine=platex", +LUALATEX_FOR_JP = ( + "-D latex_engine=lualatex", "-D latex_elements.inputenc=", "-D latex_elements.fontenc=", + "-D latex_docclass.manual=ltjsbook", + "-D latex_docclass.howto=ltjsarticle", + + # supress polyglossia warnings + "-D latex_elements.polyglossia=", + "-D latex_elements.fontpkg=", + + # preamble + "-D latex_elements.preamble=" + + # Render non-Japanese letters with luatex + # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b + r"\\usepackage[noto-otf]{luatexja-preset}" + r"\\usepackage{newunicodechar}" + r"\\newunicodechar{^^^^212a}{K}" + + # Workaround for the luatex-ja issue (Thanks to @jfbu) + # https://github.com/sphinx-doc/sphinx/issues/11179#issuecomment-1420715092 + # https://osdn.net/projects/luatex-ja/ticket/47321 + r"\\makeatletter" + r"\\titleformat{\\subsubsection}{\\normalsize\\py@HeaderFamily}" + r"{\\py@TitleColor\\thesubsubsection}{0.5em}{\\py@TitleColor}" + r"\\titleformat{\\paragraph}{\\normalsize\\py@HeaderFamily}" + r"{\\py@TitleColor\\theparagraph}{0.5em}{\\py@TitleColor}" + r"\\titleformat{\\subparagraph}{\\normalsize\\py@HeaderFamily}" + r"{\\py@TitleColor\\thesubparagraph}{0.5em}{\\py@TitleColor}" + r"\\makeatother" + + # subpress warning: (fancyhdr)Make it at least 16.4pt + r"\\setlength{\\footskip}{16.4pt}" ) XELATEX_WITH_FONTSPEC = ( @@ -257,7 +287,7 @@ class Language: Language("fr", "fr", "French", True, XELATEX_WITH_FONTSPEC), Language("id", "id", "Indonesian", False, XELATEX_DEFAULT), Language("it", "it", "Italian", False, XELATEX_DEFAULT), - Language("ja", "ja", "Japanese", True, PLATEX_DEFAULT, html_only=True), # See https://github.com/python/python-docs-ja/issues/35 + Language("ja", "ja", "Japanese", True, LUALATEX_FOR_JP), Language("ko", "ko", "Korean", True, XELATEX_FOR_KOREAN), Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT), @@ -726,6 +756,17 @@ def build(self): "-D gettext_compact=0", ) ) + if self.language.tag == "ja": + # Since luatex doesn't support \ufffd, replace \ufffd with '?'. + # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b + # Luatex already fixed this issue, so we can remove this once Texlive is updated. + # (https://github.com/TeX-Live/luatex/commit/eaa95ce0a141eaf7a02) + subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/?/g " + f"{locale_dirs}/ja/LC_MESSAGES/**/*.po", + shell=True) + subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/?/g " + f"{self.checkout}/Doc/**/*.rst", shell=True) + if self.version.status == "EOL": sphinxopts.append("-D html_context.outdated=1") maketarget = ( From 85b9d5a75866c711d76210f94274b708da744e50 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Tue, 14 Mar 2023 09:30:15 +0100 Subject: [PATCH 034/218] Exit the script process with non-zero status if at least one build was unsuccessful (#150) --- build_docs.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index c31826a..368c242 100755 --- a/build_docs.py +++ b/build_docs.py @@ -48,6 +48,11 @@ HERE = Path(__file__).resolve().parent +try: + from os import EX_OK, EX_SOFTWARE as EX_FAILURE +except ImportError: + EX_OK, EX_FAILURE = 0, 1 + try: import sentry_sdk except ImportError: @@ -693,7 +698,7 @@ def full_build(self): """ return not self.quick and not self.language.html_only - def run(self): + def run(self) -> bool: """Build and publish a Python doc, for a language, and a version.""" try: self.clone_cpython() @@ -710,6 +715,8 @@ def run(self): ) if sentry_sdk: sentry_sdk.capture_exception(err) + return False + return True @property def checkout(self) -> Path: @@ -1044,7 +1051,7 @@ def purge_path(www_root: Path, path: Path): run(["curl", "-XPURGE", f"https://docs.python.org/{{{','.join(to_purge)}}}"]) -def main() -> None: +def main() -> bool: """Script entry point.""" args = parse_args() setup_logging(args.log_directory) @@ -1054,6 +1061,7 @@ def main() -> None: del args.languages del args.branch todo = list(product(versions, languages)) + all_built_successfully = True while todo: version, language = todo.pop() if sentry_sdk: @@ -1063,7 +1071,7 @@ def main() -> None: try: lock = zc.lockfile.LockFile(HERE / "build_docs.lock") builder = DocBuilder(version, language, **vars(args)) - builder.run() + all_built_successfully &= builder.run() except zc.lockfile.LockError: logging.info("Another builder is running... waiting...") time.sleep(10) @@ -1078,6 +1086,9 @@ def main() -> None: dev_symlink(args.www_root, args.group) proofread_canonicals(args.www_root, args.skip_cache_invalidation) + return all_built_successfully + if __name__ == "__main__": - main() + all_built_successfully = main() + sys.exit(EX_OK if all_built_successfully else EX_FAILURE) From 6d97612442255ea57ba118c3a362513e3d5759b4 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 13 Mar 2023 21:25:33 +0100 Subject: [PATCH 035/218] Provide allowed choices for branch CLI argument Set metavar to most recent version --- build_docs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 368c242..2bf92d2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -23,7 +23,7 @@ from contextlib import suppress from dataclasses import dataclass import filecmp -from itertools import product +from itertools import chain, product import json import logging import logging.handlers @@ -595,7 +595,8 @@ def parse_args(): parser.add_argument( "-b", "--branch", - metavar="3.6", + choices=dict.fromkeys(chain(*((v.branch_or_tag, v.name) for v in VERSIONS))), + metavar=Version.current_dev().name, help="Version to build (defaults to all maintained branches).", ) parser.add_argument( From 8881a5f580624adc8686d0f1eb36f6df68563483 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Mon, 27 Mar 2023 14:34:27 +0200 Subject: [PATCH 036/218] [uk] pdf builds are failing. see: https://github.com/python/python-docs-uk/issues/6 --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 2bf92d2..defbe4e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -297,7 +297,7 @@ class Language: Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT), Language("tr", "tr", "Turkish", True, XELATEX_DEFAULT), - Language("uk", "uk", "Ukrainian", False, XELATEX_DEFAULT), + Language("uk", "uk", "Ukrainian", False, XELATEX_DEFAULT, html_only=True), Language("zh-cn", "zh_CN", "Simplified Chinese", True, XELATEX_WITH_CJK), Language("zh-tw", "zh_TW", "Traditional Chinese", True, XELATEX_WITH_CJK), } From 6ef34a21ee094d0217ca3ab806dcfa77e7df0e7c Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Mon, 22 May 2023 22:14:14 +0200 Subject: [PATCH 037/218] Update the list of branches for the creation of the 3.12 branch. --- build_docs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index defbe4e..6060b4f 100755 --- a/build_docs.py +++ b/build_docs.py @@ -213,7 +213,8 @@ class Language: # # Please keep the list in reverse-order for ease of editing. VERSIONS = [ - Version("3.12", branch="origin/main", status="in development"), + Version("3.13", branch="origin/main", status="in development"), + Version("3.12", branch="origin/3.12", status="pre-release"), Version("3.11", branch="origin/3.11", status="stable"), Version("3.10", branch="origin/3.10", status="stable"), Version("3.9", branch="origin/3.9", status="security-fixes"), From a30a1a467b21d37f2fc348e764a3bb27ec1a4f6a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Wed, 24 May 2023 11:15:41 -0400 Subject: [PATCH 038/218] Move 3.10 to security-fix mode. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 6060b4f..25f6426 100755 --- a/build_docs.py +++ b/build_docs.py @@ -216,7 +216,7 @@ class Language: Version("3.13", branch="origin/main", status="in development"), Version("3.12", branch="origin/3.12", status="pre-release"), Version("3.11", branch="origin/3.11", status="stable"), - Version("3.10", branch="origin/3.10", status="stable"), + Version("3.10", branch="origin/3.10", status="security-fixes"), Version("3.9", branch="origin/3.9", status="security-fixes"), Version("3.8", branch="origin/3.8", status="security-fixes"), Version("3.7", branch="origin/3.7", status="security-fixes"), From 4c97d69e02c4810834536bd715246a61ec77d4f7 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 27 Jun 2023 17:46:16 -0400 Subject: [PATCH 039/218] Move 3.7 to end-of-life status --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 25f6426..8f49c3a 100755 --- a/build_docs.py +++ b/build_docs.py @@ -219,7 +219,7 @@ class Language: Version("3.10", branch="origin/3.10", status="security-fixes"), Version("3.9", branch="origin/3.9", status="security-fixes"), Version("3.8", branch="origin/3.8", status="security-fixes"), - Version("3.7", branch="origin/3.7", status="security-fixes"), + Version("3.7", tag="3.7", status="EOL"), Version("3.6", tag="3.6", status="EOL"), Version("3.5", tag="3.5", status="EOL"), Version("2.7", tag="2.7", status="EOL"), From 952c9ee85c676ec58a5126fb0d96a98e47f1f373 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 14 Jul 2023 16:29:39 +0300 Subject: [PATCH 040/218] Replace jQuery with vanilla JavaScript (#160) --- templates/switchers.js | 137 +++++++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 61 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 7a46ea7..29204ae 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -4,7 +4,7 @@ if (!String.prototype.startsWith) { Object.defineProperty(String.prototype, 'startsWith', { value: function(search, rawPos) { - var pos = rawPos > 0 ? rawPos|0 : 0; + const pos = rawPos > 0 ? rawPos|0 : 0; return this.substring(pos, pos + search.length) === search; } }); @@ -12,28 +12,29 @@ // Parses versions in URL segments like: // "3", "dev", "release/2.7" or "3.6rc2" - var version_regexs = [ + const version_regexs = [ '(?:\\d)', '(?:\\d\\.\\d[\\w\\d\\.]*)', '(?:dev)', '(?:release/\\d.\\d[\\x\\d\\.]*)']; - var all_versions = $VERSIONS; - var all_languages = $LANGUAGES; + const all_versions = $VERSIONS; + const all_languages = $LANGUAGES; function quote_attr(str) { return '"' + str.replace('"', '\\"') + '"'; } function build_version_select(release) { - var buf = ['']; + const major_minor = release.split(".").slice(0, 2).join("."); - $.each(all_versions, function(version, title) { - if (version == major_minor) + Object.entries(all_versions).forEach(function([version, title]) { + if (version === major_minor) { buf.push(''); - else + } else { buf.push(''); + } }); buf.push(''); @@ -41,14 +42,14 @@ } function build_language_select(current_language) { - var buf = ['']; - $.each(all_languages, function(language, title) { - if (language == current_language) - buf.push(''); - else + Object.entries(all_languages).forEach(function([language, title]) { + if (language === current_language) { + buf.push(''); + } else { buf.push(''); + } }); if (!(current_language in all_languages)) { // In case we're browsing a language that is not yet in all_languages. @@ -62,29 +63,31 @@ function navigate_to_first_existing(urls) { // Navigate to the first existing URL in urls. - var url = urls.shift(); + const url = urls.shift(); if (urls.length == 0 || url.startsWith("file:///")) { window.location.href = url; return; } - $.ajax({ - url: url, - success: function() { - window.location.href = url; - }, - error: function() { + fetch(url) + .then(function(response) { + if (response.ok) { + window.location.href = url; + } else { + navigate_to_first_existing(urls); + } + }) + .catch(function(error) { navigate_to_first_existing(urls); - } - }); + }); } function on_version_switch() { - var selected_version = $(this).children('option:selected').attr('value') + '/'; - var url = window.location.href; - var current_language = language_segment_from_url(); - var current_version = version_segment_from_url(); - var new_url = url.replace('/' + current_language + current_version, - '/' + current_language + selected_version); + const selected_version = this.options[this.selectedIndex].value + '/'; + const url = window.location.href; + const current_language = language_segment_from_url(); + const current_version = version_segment_from_url(); + const new_url = url.replace('/' + current_language + current_version, + '/' + current_language + selected_version); if (new_url != url) { navigate_to_first_existing([ new_url, @@ -98,13 +101,13 @@ } function on_language_switch() { - var selected_language = $(this).children('option:selected').attr('value') + '/'; - var url = window.location.href; - var current_language = language_segment_from_url(); - var current_version = version_segment_from_url(); - if (selected_language == 'en/') // Special 'default' case for english. + let selected_language = this.options[this.selectedIndex].value + '/'; + const url = window.location.href; + const current_language = language_segment_from_url(); + const current_version = version_segment_from_url(); + if (selected_language == 'en/') // Special 'default' case for English. selected_language = ''; - var new_url = url.replace('/' + current_language + current_version, + let new_url = url.replace('/' + current_language + current_version, '/' + selected_language + current_version); if (new_url != url) { navigate_to_first_existing([ @@ -117,9 +120,9 @@ // Returns the path segment of the language as a string, like 'fr/' // or '' if not found. function language_segment_from_url() { - var path = window.location.pathname; - var language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' - var match = path.match(language_regexp); + const path = window.location.pathname; + const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' + const match = path.match(language_regexp); if (match !== null) return match[1]; return ''; @@ -128,35 +131,36 @@ // Returns the path segment of the version as a string, like '3.6/' // or '' if not found. function version_segment_from_url() { - var path = window.location.pathname; - var language_segment = language_segment_from_url(); - var version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - var version_regexp = language_segment + '(' + version_segment + ')'; - var match = path.match(version_regexp); + const path = window.location.pathname; + const language_segment = language_segment_from_url(); + const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; + const version_regexp = language_segment + '(' + version_segment + ')'; + const match = path.match(version_regexp); if (match !== null) return match[1]; return '' } function create_placeholders_if_missing() { - var version_segment = version_segment_from_url(); - var language_segment = language_segment_from_url(); - var index = "/" + language_segment + version_segment; + const version_segment = version_segment_from_url(); + const language_segment = language_segment_from_url(); + const index = "/" + language_segment + version_segment; - if ($('.version_switcher_placeholder').length) + if (document.querySelectorAll('.version_switcher_placeholder').length > 0) { return; + } - var html = ' \ + const html = ' \ \ Documentation »'; - var probable_places = [ + const probable_places = [ "body>div.related>ul>li:not(.right):contains('Documentation'):first", "body>div.related>ul>li:not(.right):contains('documentation'):first", ]; - for (var i = 0; i < probable_places.length; i++) { - var probable_place = $(probable_places[i]); + for (let i = 0; i < probable_places.length; i++) { + let probable_place = $(probable_places[i]); if (probable_place.length == 1) { probable_place.html(html); document.getElementById('indexlink').href = index; @@ -165,18 +169,29 @@ } } - $(document).ready(function() { - var language_segment = language_segment_from_url(); - var current_language = language_segment.replace(/\/+$/g, '') || 'en'; - var version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); + document.addEventListener('DOMContentLoaded', function() { + const language_segment = language_segment_from_url(); + const current_language = language_segment.replace(/\/+$/g, '') || 'en'; + const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); create_placeholders_if_missing(); - $('.version_switcher_placeholder').html(version_select); - $('.version_switcher_placeholder select').bind('change', on_version_switch); - var language_select = build_language_select(current_language); + let placeholders = document.querySelectorAll('.version_switcher_placeholder'); + placeholders.forEach(function(placeholder) { + placeholder.innerHTML = version_select; - $('.language_switcher_placeholder').html(language_select); - $('.language_switcher_placeholder select').bind('change', on_language_switch); + let selectElement = placeholder.querySelector('select'); + selectElement.addEventListener('change', on_version_switch); + }); + + const language_select = build_language_select(current_language); + + placeholders = document.querySelectorAll('.language_switcher_placeholder'); + placeholders.forEach(function(placeholder) { + placeholder.innerHTML = language_select; + + let selectElement = placeholder.querySelector('select'); + selectElement.addEventListener('change', on_language_switch); + }); }); })(); From 3e0e691507fd1fc12dc466d5a74c806831945eca Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 31 Aug 2023 03:29:13 -0600 Subject: [PATCH 041/218] 'pip install --upgrade' to automatically deploy newer requirements (#161) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 8f49c3a..571fd9d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -835,7 +835,7 @@ def build_venv(self): venv_path = self.build_root / ("venv-" + self.version.name) run([sys.executable, "-m", "venv", venv_path]) run( - [venv_path / "bin" / "python", "-m", "pip", "install"] + [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] + [self.theme] + self.version.requirements, cwd=self.checkout / "Doc", From 3e6bea758d2181e4faf0ceec7a67965acaa1a057 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 2 Oct 2023 21:39:26 +0100 Subject: [PATCH 042/218] Latest and greatest (3.12.0) (#164) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 571fd9d..45fcff6 100755 --- a/build_docs.py +++ b/build_docs.py @@ -214,7 +214,7 @@ class Language: # Please keep the list in reverse-order for ease of editing. VERSIONS = [ Version("3.13", branch="origin/main", status="in development"), - Version("3.12", branch="origin/3.12", status="pre-release"), + Version("3.12", branch="origin/3.12", status="stable"), Version("3.11", branch="origin/3.11", status="stable"), Version("3.10", branch="origin/3.10", status="security-fixes"), Version("3.9", branch="origin/3.9", status="security-fixes"), From 75e84dcf6fb398f5f9ace41f9eefe91e7b0cd523 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 2 Oct 2023 21:40:11 +0100 Subject: [PATCH 043/218] Add EOL versions back to the sitemap (#165) --- templates/robots.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/templates/robots.txt b/templates/robots.txt index 9af9c5d..635bfdc 100644 --- a/templates/robots.txt +++ b/templates/robots.txt @@ -16,3 +16,10 @@ Disallow: /2.5/ Disallow: /2.6/ Disallow: /2.7/ Disallow: /3.0/ +Disallow: /3.1/ +Disallow: /3.2/ +Disallow: /3.3/ +Disallow: /3.4/ +Disallow: /3.5/ +Disallow: /3.6/ +Disallow: /3.7/ From eddbbe2eefa10bebdf7f8e3282bef369873561eb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 5 Oct 2023 10:38:45 +0100 Subject: [PATCH 044/218] Add a link to the download archives (#167) --- templates/404.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/404.html b/templates/404.html index 512faf8..f1490e1 100644 --- a/templates/404.html +++ b/templates/404.html @@ -65,7 +65,9 @@

Navigation

404 — Archive Not Found

The archive you're trying to download has not been built yet.

-

Please try again later.

+

Please try again later or consult the + archives for earlier versions. +

From c8646799dca04dfaff35ae979768a2434b190148 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 5 Oct 2023 03:50:28 -0600 Subject: [PATCH 045/218] Update PEP Index URL to canonical form (#162) * Update to canonical PEP URL * Remove trailing slash Co-authored-by: Ezio Melotti --------- Co-authored-by: Ezio Melotti --- templates/indexsidebar.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/indexsidebar.html b/templates/indexsidebar.html index c7b8177..3a56219 100644 --- a/templates/indexsidebar.html +++ b/templates/indexsidebar.html @@ -21,7 +21,7 @@

{% trans %}Download{% endtrans %}

{% trans %}Other resources{% endtrans %}

    {# XXX: many of these should probably be merged in the main docs #} -
  • {% trans %}PEP Index{% endtrans %}
    • +
  • {% trans %}PEP Index{% endtrans %}
  • {% trans %}Beginner's Guide{% endtrans %}
  • {% trans %}Book List{% endtrans %}
  • {% trans %}Audio/Visual Talks{% endtrans %}
    • From 38f1fd213a8ff0253c6818c9c6ffcf364ee20dbe Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 6 Oct 2023 13:33:37 -0600 Subject: [PATCH 046/218] Bump versions table in README (#156) --- README.md | 63 +++++++++++++++++++++--------------------- tools_requirements.txt | 1 + 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 71156ab..36ccbf2 100644 --- a/README.md +++ b/README.md @@ -23,40 +23,41 @@ of Sphinx we're using where: Sphinx configuration in various branches: - ======== ============= ============= ================== ==================== ============= =============== - version travis azure requirements.txt conf.py Makefile Mac installer - ======== ============= ============= ================== ==================== ============= =============== - 2.7 sphinx~=2.0.1 ø ø needs_sphinx='1.2' ø ø - 3.5 sphinx==1.8.2 ø ø needs_sphinx='1.8' ø ø - 3.6 sphinx==1.8.2 sphinx==1.8.2 ø needs_sphinx='1.2' Sphinx==2.3.1 ø - 3.7 sphinx==1.8.2 sphinx==1.8.2 sphinx==2.3.1 needs_sphinx="1.6.6" ø Sphinx==2.3.1 - 3.8 ø ø sphinx==2.4.4 needs_sphinx='1.8' ø ø - 3.9 ø ø sphinx==2.4.4 needs_sphinx='1.8' ø ø - 3.10 ø ø sphinx==3.4.3 needs_sphinx='3.2' ø ø - 3.11 ø ø sphinx==4.5.0 needs_sphinx='3.2' ø ø - 3.12 ø ø sphinx==4.5.0 needs_sphinx='3.2' ø ø - ======== ============= ============= ================== ==================== ============= =============== + ========= ============= ============= ================== ================== + version travis azure requirements.txt conf.py + ========= ============= ============= ================== ================== + 2.7 sphinx~=2.0.1 ø ø needs_sphinx='1.2' + 3.5 sphinx==1.8.2 ø ø needs_sphinx='1.8' + 3.6 sphinx==1.8.2 sphinx==1.8.2 ø needs_sphinx='1.2' + 3.7 ø ø ø ø + 3.8 ø ø sphinx==2.4.4 needs_sphinx='1.8' + 3.9 ø ø sphinx==2.4.4 needs_sphinx='1.8' + 3.1 ø ø sphinx==3.4.3 needs_sphinx='3.2' + 3.11 ø ø sphinx==4.5.0 needs_sphinx='4.2' + 3.12 ø ø sphinx==4.5.0 needs_sphinx='4.2' + 3.13 ø ø sphinx==6.2.1 needs_sphinx='4.2' + ========= ============= ============= ================== ================== Sphinx build as seen on docs.python.org: - ======== ===== ===== ===== ===== ===== ===== ===== ======= ======= ======= - version en es fr id ja ko pl pt-br zh-cn zh-tw - ======== ===== ===== ===== ===== ===== ===== ===== ======= ======= ======= - 2.7 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 - 3.5 1.8.4 1.8.4 1.8.4 1.8.4 1.8.4 1.8.4 1.8.4 1.8.4 1.8.4 1.8.4 - 3.6 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 - 3.7 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 - 3.8 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 - 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 - 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 - 3.11 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 - 3.12 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 - ======== ===== ===== ===== ===== ===== ===== ===== ======= ======= ======= - - -## The github hook server - -`build_docs_server.py` is a simple HTTP server handling Github Webhooks + ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= + version en es fr id it ja ko pl pt-br tr uk zh-cn zh-tw + ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= + 2.7 ø 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 + 3.5 ø 1.8.4 1.8.4 1.8.4 1.8.4 ø 1.8.4 1.8.4 ø 1.8.4 1.8.4 1.8.4 1.8.4 + 3.6 ø 2.3.1 2.3.1 2.3.1 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 2.3.1 2.3.1 2.3.1 + 3.7 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 + 3.8 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 + 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 + 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 + 3.11 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 + 3.12 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 + 3.13 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 + ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= + +## The GitHub hook server + +`build_docs_server.py` is a simple HTTP server handling GitHub Webhooks requests to build the doc when needed. It only needs `push` events. Its logging can be configured by giving a yaml file path to the diff --git a/tools_requirements.txt b/tools_requirements.txt index cbeb417..be36803 100644 --- a/tools_requirements.txt +++ b/tools_requirements.txt @@ -1,3 +1,4 @@ GitPython httpx tabulate +zc.lockfile From 9604476be3b499910f5a4b0869f2c83245586811 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 10 Oct 2023 10:48:36 +0200 Subject: [PATCH 047/218] Don't queue builds. (#168) It was mandatory for the server to queue builds, but since 1a0ad2e it's no longer needed. Queuing builds has a downside: if the daily builds take more than 24h they starts to infinitly queue. --- build_docs.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index 45fcff6..8e8866f 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1053,10 +1053,8 @@ def purge_path(www_root: Path, path: Path): run(["curl", "-XPURGE", f"https://docs.python.org/{{{','.join(to_purge)}}}"]) -def main() -> bool: - """Script entry point.""" - args = parse_args() - setup_logging(args.log_directory) +def build_docs(args) -> bool: + """Build all docs (each languages and each versions).""" languages_dict = {language.tag: language for language in LANGUAGES} versions = Version.filter(VERSIONS, args.branch) languages = [languages_dict[tag] for tag in args.languages] @@ -1070,17 +1068,8 @@ def main() -> bool: with sentry_sdk.configure_scope() as scope: scope.set_tag("version", version.name) scope.set_tag("language", language.tag) - try: - lock = zc.lockfile.LockFile(HERE / "build_docs.lock") - builder = DocBuilder(version, language, **vars(args)) - all_built_successfully &= builder.run() - except zc.lockfile.LockError: - logging.info("Another builder is running... waiting...") - time.sleep(10) - todo.append((version, language)) - else: - lock.close() - + builder = DocBuilder(version, language, **vars(args)) + all_built_successfully &= builder.run() build_sitemap(args.www_root, args.group) build_404(args.www_root, args.group) build_robots_txt(args.www_root, args.group, args.skip_cache_invalidation) @@ -1091,6 +1080,25 @@ def main() -> bool: return all_built_successfully +def main(): + """Script entry point.""" + args = parse_args() + setup_logging(args.log_directory) + + try: + lock = zc.lockfile.LockFile(HERE / "build_docs.lock") + except zc.lockfile.LockError: + logging.info("Another builder is running... dying...") + return False + + try: + build_docs(args) + finally: + lock.close() + + + + if __name__ == "__main__": all_built_successfully = main() sys.exit(EX_OK if all_built_successfully else EX_FAILURE) From 03e8f0746cb83d3bea2bbd8b2a0e0e1a8afdbc5c Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 17 Oct 2023 22:19:36 +0200 Subject: [PATCH 048/218] Use a config file for the languages and query the devguide for the versions. (#143) --- README.md | 11 -- build_docs.py | 454 +++++++++++++++++++++++------------------------ config.toml | 99 +++++++++++ requirements.txt | 1 + 4 files changed, 327 insertions(+), 238 deletions(-) create mode 100644 config.toml diff --git a/README.md b/README.md index 36ccbf2..3e70fa4 100644 --- a/README.md +++ b/README.md @@ -54,14 +54,3 @@ of Sphinx we're using where: 3.12 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 3.13 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= - -## The GitHub hook server - -`build_docs_server.py` is a simple HTTP server handling GitHub Webhooks -requests to build the doc when needed. It only needs `push` events. - -Its logging can be configured by giving a yaml file path to the -`--logging-config` argument. - -By default the loglevel is `DEBUG` on `stderr`, the default config can -be found in the code so one can bootstrap a different config from it. diff --git a/build_docs.py b/build_docs.py index 8e8866f..efccaca 100755 --- a/build_docs.py +++ b/build_docs.py @@ -2,15 +2,17 @@ """Build the Python docs for various branches and various languages. -Without any arguments builds docs for all active versions configured in the -global VERSIONS list and all languages configured in the LANGUAGES list. +Without any arguments builds docs for all active versions and +languages. + +Languages are stored in `config.toml` while versions are discovered +from the devguide. -q selects "quick build", which means to build only HTML. Translations are fetched from github repositories according to PEP -545. --languages allow select translations, use "--languages" to -build all translations (default) or "--languages en" to skip all -translations (as en is the untranslated version).. +545. `--languages` allows to select translations, like `--languages +en` to just build the english documents. This script was originally created and by Georg Brandl in March 2010. @@ -20,10 +22,9 @@ """ from argparse import ArgumentParser -from contextlib import suppress +from contextlib import suppress, contextmanager from dataclasses import dataclass import filecmp -from itertools import chain, product import json import logging import logging.handlers @@ -34,19 +35,19 @@ import shutil import subprocess import sys -import time from bisect import bisect_left as bisect from collections import OrderedDict -from contextlib import contextmanager from pathlib import Path from string import Template from textwrap import indent +from typing import Iterable +from urllib.parse import urljoin import zc.lockfile import jinja2 import requests +import tomlkit -HERE = Path(__file__).resolve().parent try: from os import EX_OK, EX_SOFTWARE as EX_FAILURE @@ -60,11 +61,7 @@ else: sentry_sdk.init() -if not hasattr(shlex, "join"): - # Add shlex.join if missing (pre 3.8) - shlex.join = lambda split_command: " ".join( - shlex.quote(arg) for arg in split_command - ) +HERE = Path(__file__).resolve().parent @total_ordering @@ -73,27 +70,31 @@ class Version: STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"} + # Those synonyms map branch status vocabulary found in the devguide + # with our vocabulary. + SYNONYMS = { + "feature": "in development", + "bugfix": "stable", + "security": "security-fixes", + "end-of-life": "EOL", + } + def __init__( self, name, *, status, - branch=None, - tag=None, - sphinxopts=(), + branch_or_tag=None, ): + status = self.SYNONYMS.get(status, status) if status not in self.STATUSES: raise ValueError( - f"Version status expected to be in {', '.join(self.STATUSES)}" + "Version status expected to be one of: " + f"{', '.join(self.STATUSES|set(self.SYNONYMS.keys()))}, got {status!r}." ) self.name = name - if branch is not None and tag is not None: - raise ValueError("Please build a version from either a branch or a tag.") - if branch is None and tag is None: - raise ValueError("Please build a version with at least a branch or a tag.") - self.branch_or_tag = branch or tag + self.branch_or_tag = branch_or_tag self.status = status - self.sphinxopts = list(sphinxopts) def __repr__(self): return f"Version({self.name})" @@ -156,14 +157,14 @@ def filter(versions, branch=None): return [v for v in versions if v.status not in ("EOL", "security-fixes")] @staticmethod - def current_stable(): + def current_stable(versions): """Find the current stable cPython version.""" - return max([v for v in VERSIONS if v.status == "stable"], key=Version.as_tuple) + return max((v for v in versions if v.status == "stable"), key=Version.as_tuple) @staticmethod - def current_dev(): + def current_dev(versions): """Find the current de cPython version.""" - return max(VERSIONS, key=Version.as_tuple) + return max(versions, key=Version.as_tuple) @property def picker_label(self): @@ -174,7 +175,7 @@ def picker_label(self): return f"pre ({self.name})" return self.name - def setup_indexsidebar(self, dest_path): + def setup_indexsidebar(self, versions, dest_path): """Build indexsidebar.html for Sphinx.""" with open( HERE / "templates" / "indexsidebar.html", encoding="UTF-8" @@ -185,11 +186,16 @@ def setup_indexsidebar(self, dest_path): sidebar_template.render( current_version=self, versions=sorted( - VERSIONS, key=lambda v: version_to_tuple(v.name), reverse=True + versions, key=lambda v: version_to_tuple(v.name), reverse=True ), ) ) + @classmethod + def from_json(cls, name, values): + """Loads a version from devguide's json representation.""" + return cls(name, status=values["status"], branch_or_tag=values["branch"]) + def __eq__(self, other): return self.name == other.name @@ -197,111 +203,25 @@ def __gt__(self, other): return self.as_tuple() > other.as_tuple() - @dataclass(frozen=True, order=True) class Language: - tag: str iso639_tag: str name: str in_prod: bool sphinxopts: tuple html_only: bool = False + @property + def tag(self): + return self.iso639_tag.replace("_", "-").lower() -# EOL and security-fixes are not automatically built, no need to remove them -# from the list, this way we can still rebuild them manually as needed. -# -# Please keep the list in reverse-order for ease of editing. -VERSIONS = [ - Version("3.13", branch="origin/main", status="in development"), - Version("3.12", branch="origin/3.12", status="stable"), - Version("3.11", branch="origin/3.11", status="stable"), - Version("3.10", branch="origin/3.10", status="security-fixes"), - Version("3.9", branch="origin/3.9", status="security-fixes"), - Version("3.8", branch="origin/3.8", status="security-fixes"), - Version("3.7", tag="3.7", status="EOL"), - Version("3.6", tag="3.6", status="EOL"), - Version("3.5", tag="3.5", status="EOL"), - Version("2.7", tag="2.7", status="EOL"), -] - -XELATEX_DEFAULT = ( - "-D latex_engine=xelatex", - "-D latex_elements.inputenc=", - "-D latex_elements.fontenc=", -) - -LUALATEX_FOR_JP = ( - "-D latex_engine=lualatex", - "-D latex_elements.inputenc=", - "-D latex_elements.fontenc=", - "-D latex_docclass.manual=ltjsbook", - "-D latex_docclass.howto=ltjsarticle", - - # supress polyglossia warnings - "-D latex_elements.polyglossia=", - "-D latex_elements.fontpkg=", - - # preamble - "-D latex_elements.preamble=" - - # Render non-Japanese letters with luatex - # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b - r"\\usepackage[noto-otf]{luatexja-preset}" - r"\\usepackage{newunicodechar}" - r"\\newunicodechar{^^^^212a}{K}" - - # Workaround for the luatex-ja issue (Thanks to @jfbu) - # https://github.com/sphinx-doc/sphinx/issues/11179#issuecomment-1420715092 - # https://osdn.net/projects/luatex-ja/ticket/47321 - r"\\makeatletter" - r"\\titleformat{\\subsubsection}{\\normalsize\\py@HeaderFamily}" - r"{\\py@TitleColor\\thesubsubsection}{0.5em}{\\py@TitleColor}" - r"\\titleformat{\\paragraph}{\\normalsize\\py@HeaderFamily}" - r"{\\py@TitleColor\\theparagraph}{0.5em}{\\py@TitleColor}" - r"\\titleformat{\\subparagraph}{\\normalsize\\py@HeaderFamily}" - r"{\\py@TitleColor\\thesubparagraph}{0.5em}{\\py@TitleColor}" - r"\\makeatother" - - # subpress warning: (fancyhdr)Make it at least 16.4pt - r"\\setlength{\\footskip}{16.4pt}" -) - -XELATEX_WITH_FONTSPEC = ( - "-D latex_engine=xelatex", - "-D latex_elements.inputenc=", - r"-D latex_elements.fontenc=\\usepackage{fontspec}", -) - -XELATEX_FOR_KOREAN = ( - "-D latex_engine=xelatex", - "-D latex_elements.inputenc=", - "-D latex_elements.fontenc=", - r"-D latex_elements.preamble=\\usepackage{kotex}\\setmainhangulfont" - r"{UnBatang}\\setsanshangulfont{UnDotum}\\setmonohangulfont{UnTaza}", -) - -XELATEX_WITH_CJK = ( - "-D latex_engine=xelatex", - "-D latex_elements.inputenc=", - r"-D latex_elements.fontenc=\\usepackage{xeCJK}", -) - -LANGUAGES = { - Language("en", "en", "English", True, XELATEX_DEFAULT), - Language("es", "es", "Spanish", True, XELATEX_WITH_FONTSPEC), - Language("fr", "fr", "French", True, XELATEX_WITH_FONTSPEC), - Language("id", "id", "Indonesian", False, XELATEX_DEFAULT), - Language("it", "it", "Italian", False, XELATEX_DEFAULT), - Language("ja", "ja", "Japanese", True, LUALATEX_FOR_JP), - Language("ko", "ko", "Korean", True, XELATEX_FOR_KOREAN), - Language("pl", "pl", "Polish", False, XELATEX_DEFAULT), - Language("pt-br", "pt_BR", "Brazilian Portuguese", True, XELATEX_DEFAULT), - Language("tr", "tr", "Turkish", True, XELATEX_DEFAULT), - Language("uk", "uk", "Ukrainian", False, XELATEX_DEFAULT, html_only=True), - Language("zh-cn", "zh_CN", "Simplified Chinese", True, XELATEX_WITH_CJK), - Language("zh-tw", "zh_TW", "Traditional Chinese", True, XELATEX_WITH_CJK), -} + @staticmethod + def filter(languages, language_tags=None): + """Filter a sequence of languages according to --languages.""" + if language_tags: + languages_dict = {language.tag: language for language in languages} + return [languages_dict[tag] for tag in language_tags] + return languages def run(cmd, cwd=None) -> subprocess.CompletedProcess: @@ -351,26 +271,45 @@ def traverse(dircmp_result): return changed -def git_clone(repository: str, directory: Path, branch_or_tag=None): - """Clone or update the given repository in the given directory. - Optionally checking out a branch. - """ - logging.info("Updating repository %s in %s", repository, directory) - try: - if not (directory / ".git").is_dir(): - raise AssertionError("Not a git repository.") - run(["git", "-C", directory, "fetch"]) - if branch_or_tag: - run(["git", "-C", directory, "reset", "--hard", branch_or_tag, "--"]) - run(["git", "-C", directory, "clean", "-dfqx"]) - except (subprocess.CalledProcessError, AssertionError): - if directory.exists(): - shutil.rmtree(directory) - logging.info("Cloning %s into %s", repository, directory) - directory.mkdir(mode=0o775, parents=True, exist_ok=True) - run(["git", "clone", repository, directory]) - if branch_or_tag: - run(["git", "-C", directory, "reset", "--hard", branch_or_tag, "--"]) +@dataclass +class Repository: + """Git repository abstraction for our specific needs.""" + + remote: str + directory: Path + + def run(self, *args): + """Run git command in the clone repository.""" + return run(("git", "-C", self.directory) + args) + + def get_ref(self, pattern): + """Return the reference of a given tag or branch.""" + try: + # Maybe it's a branch + return self.run("show-ref", "-s", "origin/" + pattern).stdout.strip() + except subprocess.CalledProcessError: + # Maybe it's a tag + return self.run("show-ref", "-s", "tags/" + pattern).stdout.strip() + + def fetch(self): + self.run("fetch") + + def switch(self, branch_or_tag): + """Reset and cleans the repository to the given branch or tag.""" + self.run("reset", "--hard", self.get_ref(branch_or_tag), "--") + self.run("clean", "-dfqx") + + def clone(self): + """Maybe clone the repository, if not already cloned.""" + if (self.directory / ".git").is_dir(): + return False # Already cloned + logging.info("Cloning %s into %s", self.remote, self.directory) + self.directory.mkdir(mode=0o775, parents=True, exist_ok=True) + run(["git", "clone", self.remote, self.directory]) + return True + + def update(self): + self.clone() or self.fetch() def version_to_tuple(version): @@ -415,20 +354,18 @@ def locate_nearest_version(available_versions, target_version): return tuple_to_version(found) -def translation_branch(locale_repo, locale_clone_dir, needed_version: str): +def translation_branch(repo: Repository, needed_version: str): """Some cpython versions may be untranslated, being either too old or too new. This function looks for remote branches on the given repo, and returns the name of the nearest existing branch. - It could be enhanced to return tags, if needed, just return the - tag as a string (without the `origin/` branch prefix). + It could be enhanced to also search for tags. """ - git_clone(locale_repo, locale_clone_dir) - remote_branches = run(["git", "-C", locale_clone_dir, "branch", "-r"]).stdout + remote_branches = repo.run("branch", "-r").stdout branches = re.findall(r"/([0-9]+\.[0-9]+)$", remote_branches, re.M) - return "origin/" + locate_nearest_version(branches, needed_version) + return locate_nearest_version(branches, needed_version) @contextmanager @@ -448,7 +385,9 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers(html_root: Path): +def setup_switchers( + versions: Iterable[Version], languages: Iterable[Language], html_root: Path +): """Setup cross-links between cpython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher @@ -466,7 +405,7 @@ def setup_switchers(html_root: Path): sorted( [ (language.tag, language.name) - for language in LANGUAGES + for language in languages if language.in_prod ] ) @@ -477,7 +416,7 @@ def setup_switchers(html_root: Path): [ (version.name, version.picker_label) for version in sorted( - VERSIONS, + versions, key=lambda v: version_to_tuple(v.name), reverse=True, ) @@ -501,7 +440,13 @@ def setup_switchers(html_root: Path): ofile.write(line) -def build_robots_txt(www_root: Path, group, skip_cache_invalidation): +def build_robots_txt( + versions: Iterable[Version], + languages: Iterable[Language], + www_root: Path, + group, + skip_cache_invalidation, +): """Disallow crawl of EOL versions in robots.txt.""" if not www_root.exists(): logging.info("Skipping robots.txt generation (www root does not even exists).") @@ -511,15 +456,17 @@ def build_robots_txt(www_root: Path, group, skip_cache_invalidation): template = jinja2.Template(template_file.read()) with open(robots_file, "w", encoding="UTF-8") as robots_txt_file: robots_txt_file.write( - template.render(languages=LANGUAGES, versions=VERSIONS) + "\n" + template.render(languages=languages, versions=versions) + "\n" ) robots_file.chmod(0o775) run(["chgrp", group, robots_file]) if not skip_cache_invalidation: - requests.request("PURGE", "https://docs.python.org/robots.txt") + purge("robots.txt") -def build_sitemap(www_root: Path, group): +def build_sitemap( + versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group +): """Build a sitemap with all live versions and translations.""" if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exists).") @@ -528,7 +475,7 @@ def build_sitemap(www_root: Path, group): template = jinja2.Template(template_file.read()) sitemap_file = www_root / "sitemap.xml" sitemap_file.write_text( - template.render(languages=LANGUAGES, versions=VERSIONS) + "\n", encoding="UTF-8" + template.render(languages=languages, versions=versions) + "\n", encoding="UTF-8" ) sitemap_file.chmod(0o664) run(["chgrp", group, sitemap_file]) @@ -596,8 +543,7 @@ def parse_args(): parser.add_argument( "-b", "--branch", - choices=dict.fromkeys(chain(*((v.branch_or_tag, v.name) for v in VERSIONS))), - metavar=Version.current_dev().name, + metavar="3.12", help="Version to build (defaults to all maintained branches).", ) parser.add_argument( @@ -633,8 +579,9 @@ def parse_args(): parser.add_argument( "--languages", nargs="*", - default={language.tag for language in LANGUAGES}, - help="Language translation, as a PEP 545 language tag like" " 'fr' or 'pt-br'.", + help="Language translation, as a PEP 545 language tag like" + " 'fr' or 'pt-br'. " + "Builds all available languages by default.", metavar="fr", ) parser.add_argument( @@ -679,7 +626,10 @@ class DocBuilder: """Builder for a cpython version and a language.""" version: Version + versions: Iterable[Version] language: Language + languages: Iterable[Language] + cpython_repo: Repository build_root: Path www_root: Path quick: bool @@ -703,7 +653,7 @@ def full_build(self): def run(self) -> bool: """Build and publish a Python doc, for a language, and a version.""" try: - self.clone_cpython() + self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() self.build_venv() @@ -726,6 +676,10 @@ def checkout(self) -> Path: return self.build_root / "cpython" def clone_translation(self): + """Clone the translation repository from github. + + See PEP 545 for repository naming convention. + """ locale_repo = f"https://github.com/python/python-docs-{self.language.tag}.git" locale_clone_dir = ( self.build_root @@ -734,18 +688,9 @@ def clone_translation(self): / self.language.iso639_tag / "LC_MESSAGES" ) - git_clone( - locale_repo, - locale_clone_dir, - translation_branch(locale_repo, locale_clone_dir, self.version.name), - ) - - def clone_cpython(self): - git_clone( - "https://github.com/python/cpython.git", - self.checkout, - self.version.branch_or_tag, - ) + repo = Repository(locale_repo, locale_clone_dir) + repo.update() + repo.switch(translation_branch(repo, self.version.name)) def build(self): """Build this version/language doc.""" @@ -754,7 +699,7 @@ def build(self): self.version.name, self.language.tag, ) - sphinxopts = list(self.language.sphinxopts) + list(self.version.sphinxopts) + sphinxopts = list(self.language.sphinxopts) sphinxopts.extend(["-q"]) if self.language.tag != "en": locale_dirs = self.build_root / self.version.name / "locale" @@ -768,13 +713,19 @@ def build(self): if self.language.tag == "ja": # Since luatex doesn't support \ufffd, replace \ufffd with '?'. # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b - # Luatex already fixed this issue, so we can remove this once Texlive is updated. - # (https://github.com/TeX-Live/luatex/commit/eaa95ce0a141eaf7a02) - subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/?/g " - f"{locale_dirs}/ja/LC_MESSAGES/**/*.po", - shell=True) - subprocess.check_output("sed -i s/\N{REPLACEMENT CHARACTER}/?/g " - f"{self.checkout}/Doc/**/*.rst", shell=True) + # Luatex already fixed this issue, so we can remove this once Texlive + # is updated. + # (https://github.com/TeX-Live/luatex/commit/af5faf1) + subprocess.check_output( + "sed -i s/\N{REPLACEMENT CHARACTER}/?/g " + f"{locale_dirs}/ja/LC_MESSAGES/**/*.po", + shell=True, + ) + subprocess.check_output( + "sed -i s/\N{REPLACEMENT CHARACTER}/?/g " + f"{self.checkout}/Doc/**/*.rst", + shell=True, + ) if self.version.status == "EOL": sphinxopts.append("-D html_context.outdated=1") @@ -785,7 +736,7 @@ def build(self): if self.version.status in ("in development", "pre-release") else "stable" ) - + ("" if self.full_build else "-html") + + ("" if self.full_build else "-html") ) logging.info("Running make %s", maketarget) python = self.venv / "bin" / "python" @@ -801,7 +752,8 @@ def build(self): ] ) self.version.setup_indexsidebar( - self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html" + self.versions, + self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", ) run( [ @@ -819,7 +771,9 @@ def build(self): ) run(["mkdir", "-p", self.log_directory]) run(["chgrp", "-R", self.group, self.log_directory]) - setup_switchers(self.checkout / "Doc" / "build" / "html") + setup_switchers( + self.versions, self.languages, self.checkout / "Doc" / "build" / "html" + ) logging.info( "Build done for version: %s, language: %s", self.version.name, @@ -959,13 +913,9 @@ def copy_build_to_webroot(self): prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout prefixes = prefixes.replace(targets_dir + "/", "") prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix] - to_purge = prefixes[:] + purge(*prefixes) for prefix in prefixes: - to_purge.extend(prefix + p for p in changed) - logging.info("Running CDN purge") - run( - ["curl", "-XPURGE", f"https://docs.python.org/{{{','.join(to_purge)}}}"] - ) + purge(*[prefix + p for p in changed]) logging.info( "Publishing done for version: %s, language: %s", self.version.name, @@ -992,7 +942,9 @@ def symlink(www_root: Path, language: Language, directory: str, name: str, group purge_path(www_root, link) -def major_symlinks(www_root: Path, group): +def major_symlinks( + www_root: Path, group, versions: Iterable[Version], languages: Iterable[Language] +): """Maintains the /2/ and /3/ symlinks for each languages. Like: @@ -1000,13 +952,13 @@ def major_symlinks(www_root: Path, group): - /fr/3/ → /fr/3.9/ - /es/3/ → /es/3.9/ """ - current_stable = Version.current_stable().name - for language in LANGUAGES: + current_stable = Version.current_stable(versions).name + for language in languages: symlink(www_root, language, current_stable, "3", group) symlink(www_root, language, "2.7", "2", group) -def dev_symlink(www_root: Path, group): +def dev_symlink(www_root: Path, group, versions, languages): """Maintains the /dev/ symlinks for each languages. Like: @@ -1014,11 +966,33 @@ def dev_symlink(www_root: Path, group): - /fr/dev/ → /fr/3.11/ - /es/dev/ → /es/3.11/ """ - current_dev = Version.current_dev().name - for language in LANGUAGES: + current_dev = Version.current_dev(versions).name + for language in languages: symlink(www_root, language, current_dev, "dev", group) +def purge(*paths): + """Remove one or many paths from docs.python.org's CDN. + + To be used when a file change, so the CDN fetch the new one. + """ + base = "https://docs.python.org/" + for path in paths: + url = urljoin(base, str(path)) + logging.info("Purging %s from CDN", url) + requests.request("PURGE", url, timeout=30) + + +def purge_path(www_root: Path, path: Path): + """Recursively remove a path from docs.python.org's CDN. + + To be used when a directory change, so the CDN fetch the new one. + """ + purge(*[file.relative_to(www_root) for file in path.glob("**/*")]) + purge(path.relative_to(www_root)) + purge(str(path.relative_to(www_root)) + "/") + + def proofread_canonicals(www_root: Path, skip_cache_invalidation: bool) -> None: """In www_root we check that all canonical links point to existing contents. @@ -1041,40 +1015,69 @@ def proofread_canonicals(www_root: Path, skip_cache_invalidation: bool) -> None: html = html.replace(canonical.group(0), "") file.write_text(html, encoding="UTF-8", errors="surrogateescape") if not skip_cache_invalidation: - url = str(file).replace("/srv/", "https://") - logging.info("Purging %s from CDN", url) - requests.request("PURGE", url) - - -def purge_path(www_root: Path, path: Path): - to_purge = [str(file.relative_to(www_root)) for file in path.glob("**/*")] - to_purge.append(str(path.relative_to(www_root))) - to_purge.append(str(path.relative_to(www_root)) + "/") - run(["curl", "-XPURGE", f"https://docs.python.org/{{{','.join(to_purge)}}}"]) + purge(str(file).replace("/srv/docs.python.org/", "")) + + +def parse_versions_from_devguide(): + releases = requests.get( + "https://raw.githubusercontent.com/" + "python/devguide/main/include/release-cycle.json", + timeout=30, + ).json() + return [Version.from_json(name, release) for name, release in releases.items()] + + +def parse_languages_from_config(): + """Read config.toml to discover languages to build.""" + config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) + languages = [] + defaults = config["defaults"] + for iso639_tag, section in config["languages"].items(): + languages.append( + Language( + iso639_tag, + section["name"], + section.get("in_prod", defaults["in_prod"]), + sphinxopts=section.get("sphinxopts", defaults["sphinxopts"]), + html_only=section.get("html_only", defaults["html_only"]), + ) + ) + return languages def build_docs(args) -> bool: """Build all docs (each languages and each versions).""" - languages_dict = {language.tag: language for language in LANGUAGES} - versions = Version.filter(VERSIONS, args.branch) - languages = [languages_dict[tag] for tag in args.languages] - del args.languages + versions = parse_versions_from_devguide() + languages = parse_languages_from_config() + todo = [ + (version, language) + for version in Version.filter(versions, args.branch) + for language in Language.filter(languages, args.languages) + ] del args.branch - todo = list(product(versions, languages)) + del args.languages all_built_successfully = True + cpython_repo = Repository( + "https://github.com/python/cpython.git", args.build_root / "cpython" + ) + cpython_repo.update() while todo: version, language = todo.pop() if sentry_sdk: with sentry_sdk.configure_scope() as scope: scope.set_tag("version", version.name) scope.set_tag("language", language.tag) - builder = DocBuilder(version, language, **vars(args)) + builder = DocBuilder( + version, versions, language, languages, cpython_repo, **vars(args) + ) all_built_successfully &= builder.run() - build_sitemap(args.www_root, args.group) + build_sitemap(versions, languages, args.www_root, args.group) build_404(args.www_root, args.group) - build_robots_txt(args.www_root, args.group, args.skip_cache_invalidation) - major_symlinks(args.www_root, args.group) - dev_symlink(args.www_root, args.group) + build_robots_txt( + versions, languages, args.www_root, args.group, args.skip_cache_invalidation + ) + major_symlinks(args.www_root, args.group, versions, languages) + dev_symlink(args.www_root, args.group, versions, languages) proofread_canonicals(args.www_root, args.skip_cache_invalidation) return all_built_successfully @@ -1089,16 +1092,13 @@ def main(): lock = zc.lockfile.LockFile(HERE / "build_docs.lock") except zc.lockfile.LockError: logging.info("Another builder is running... dying...") - return False + return EX_FAILURE try: - build_docs(args) + return EX_OK if build_docs(args) else EX_FAILURE finally: lock.close() - - if __name__ == "__main__": - all_built_successfully = main() - sys.exit(EX_OK if all_built_successfully else EX_FAILURE) + sys.exit(main()) diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..c3f4886 --- /dev/null +++ b/config.toml @@ -0,0 +1,99 @@ +[defaults] +# name has no default, it is mandatory. +in_prod = true +html_only = false +sphinxopts = [ + "-D latex_engine=xelatex", + "-D latex_elements.inputenc=", + "-D latex_elements.fontenc=", +] + +[languages.en] +name = "English" + +[languages.es] +name = "Spanish" +sphinxopts = [ + '-D latex_engine=xelatex', + '-D latex_elements.inputenc=', + '-D latex_elements.fontenc=\\usepackage{fontspec}', +] + +[languages.fr] +name = "French" +sphinxopts = [ + '-D latex_engine=xelatex', + '-D latex_elements.inputenc=', + '-D latex_elements.fontenc=\\usepackage{fontspec}', +] + +[languages.id] +name = "Indonesian" +in_prod = false + +[languages.it] +name = "Italian" +in_prod = false + +[languages.ja] +name = "Japanese" +sphinxopts = [ + '-D latex_engine=lualatex', + '-D latex_elements.inputenc=', + '-D latex_elements.fontenc=', + '-D latex_docclass.manual=ltjsbook', + '-D latex_docclass.howto=ltjsarticle', + + # supress polyglossia warnings + '-D latex_elements.polyglossia=', + '-D latex_elements.fontpkg=', + + # preamble + # Render non-Japanese letters with luatex + # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b + # Workaround for the luatex-ja issue (Thanks to @jfbu) + # https://github.com/sphinx-doc/sphinx/issues/11179#issuecomment-1420715092 + # https://osdn.net/projects/luatex-ja/ticket/47321 + # subpress warning: (fancyhdr)Make it at least 16.4pt + '-D latex_elements.preamble=\\usepackage[noto-otf]{luatexja-preset}\\usepackage{newunicodechar}\\newunicodechar{^^^^212a}{K}\\makeatletter\\titleformat{\\subsubsection}{\\normalsize\\py@HeaderFamily}{\\py@TitleColor\\thesubsubsection}{0.5em}{\\py@TitleColor}\\titleformat{\\paragraph}{\\normalsize\\py@HeaderFamily}{\\py@TitleColor\\theparagraph}{0.5em}{\\py@TitleColor}\\titleformat{\\subparagraph}{\\normalsize\\py@HeaderFamily}{\\py@TitleColor\\thesubparagraph}{0.5em}{\\py@TitleColor}\\makeatother\\setlength{\\footskip}{16.4pt}' +] + +[languages.ko] +name = "Korean" +sphinxopts = [ + '-D latex_engine=xelatex', + '-D latex_elements.inputenc=', + '-D latex_elements.fontenc=', + '-D latex_elements.preamble=\\usepackage{kotex}\\setmainhangulfont{UnBatang}\\setsanshangulfont{UnDotum}\\setmonohangulfont{UnTaza}', +] + +[languages.pl] +name = "Polish" +in_prod = false + +[languages.pt_BR] +name = "Brazilian Portuguese" + +[languages.tr] +name = "Turkish" + +[languages.uk] +name = "Ukrainian" +in_prod = false +html_only = true + +[languages.zh_CN] +name = "Simplified Chinese" +sphinxopts = [ + '-D latex_engine=xelatex', + '-D latex_elements.inputenc=', + '-D latex_elements.fontenc=\\usepackage{xeCJK}', +] + +[languages.zh_TW] +name = "Traditional Chinese" +sphinxopts = [ + '-D latex_engine=xelatex', + '-D latex_elements.inputenc=', + '-D latex_elements.fontenc=\\usepackage{xeCJK}', +] diff --git a/requirements.txt b/requirements.txt index 65ae7f2..f51c7d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ jinja2 requests sentry-sdk +tomlkit zc.lockfile From f27dbebe2b4bdc86d7e2fbb6fe7fb442ce58d4a9 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Fri, 20 Oct 2023 14:12:24 +0200 Subject: [PATCH 049/218] Do not rebuild when it's not needed (like there's no updates). (#171) --- build_docs.py | 174 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 120 insertions(+), 54 deletions(-) diff --git a/build_docs.py b/build_docs.py index efccaca..d6eeb6b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -37,9 +37,11 @@ import sys from bisect import bisect_left as bisect from collections import OrderedDict +from datetime import datetime as dt, timezone from pathlib import Path from string import Template from textwrap import indent +from time import perf_counter, sleep from typing import Iterable from urllib.parse import urljoin @@ -246,8 +248,6 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: cmdstring, indent("\n".join(result.stdout.split("\n")[-20:]), " "), ) - else: - logging.debug("Run: %r OK", cmdstring) result.check_returncode() return result @@ -292,7 +292,13 @@ def get_ref(self, pattern): return self.run("show-ref", "-s", "tags/" + pattern).stdout.strip() def fetch(self): - self.run("fetch") + """Try (and retry) to run git fetch.""" + try: + return self.run("fetch") + except subprocess.CalledProcessError as err: + logging.error("'git fetch' failed (%s), retrying...", err.stderr) + sleep(5) + return self.run("fetch") def switch(self, branch_or_tag): """Reset and cleans the repository to the given branch or tag.""" @@ -354,20 +360,6 @@ def locate_nearest_version(available_versions, target_version): return tuple_to_version(found) -def translation_branch(repo: Repository, needed_version: str): - """Some cpython versions may be untranslated, being either too old or - too new. - - This function looks for remote branches on the given repo, and - returns the name of the nearest existing branch. - - It could be enhanced to also search for tags. - """ - remote_branches = repo.run("branch", "-r").stdout - branches = re.findall(r"/([0-9]+\.[0-9]+)$", remote_branches, re.M) - return locate_nearest_version(branches, needed_version) - - @contextmanager def edit(file: Path): """Context manager to edit a file "in place", use it as: @@ -612,11 +604,15 @@ def parse_args(): def setup_logging(log_directory: Path): """Setup logging to stderr if ran by a human, or to a file if ran from a cron.""" if sys.stderr.isatty(): - logging.basicConfig(format="%(levelname)s:%(message)s", stream=sys.stderr) + logging.basicConfig( + format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stderr + ) else: log_directory.mkdir(parents=True, exist_ok=True) handler = logging.handlers.WatchedFileHandler(log_directory / "docsbuild.log") - handler.setFormatter(logging.Formatter("%(levelname)s:%(asctime)s:%(message)s")) + handler.setFormatter( + logging.Formatter("%(asctime)s %(levelname)s: %(message)s") + ) logging.getLogger().addHandler(handler) logging.getLogger().setLevel(logging.DEBUG) @@ -652,19 +648,19 @@ def full_build(self): def run(self) -> bool: """Build and publish a Python doc, for a language, and a version.""" + start_time = perf_counter() + logging.info("Running.") try: self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() - self.build_venv() - self.build() - self.copy_build_to_webroot() + if self.should_rebuild(): + self.build_venv() + self.build() + self.copy_build_to_webroot() + self.save_state(build_duration=perf_counter() - start_time) except Exception as err: - logging.exception( - "Exception while building %s version %s", - self.language.tag, - self.version.name, - ) + logging.exception("Badly handled exception, human, please help.") if sentry_sdk: sentry_sdk.capture_exception(err) return False @@ -676,10 +672,13 @@ def checkout(self) -> Path: return self.build_root / "cpython" def clone_translation(self): - """Clone the translation repository from github. + self.translation_repo.update() + self.translation_repo.switch(self.translation_branch) + + @property + def translation_repo(self): + """See PEP 545 for translations repository naming convention.""" - See PEP 545 for repository naming convention. - """ locale_repo = f"https://github.com/python/python-docs-{self.language.tag}.git" locale_clone_dir = ( self.build_root @@ -688,17 +687,25 @@ def clone_translation(self): / self.language.iso639_tag / "LC_MESSAGES" ) - repo = Repository(locale_repo, locale_clone_dir) - repo.update() - repo.switch(translation_branch(repo, self.version.name)) + return Repository(locale_repo, locale_clone_dir) + + @property + def translation_branch(self): + """Some cpython versions may be untranslated, being either too old or + too new. + + This function looks for remote branches on the given repo, and + returns the name of the nearest existing branch. + + It could be enhanced to also search for tags. + """ + remote_branches = self.translation_repo.run("branch", "-r").stdout + branches = re.findall(r"/([0-9]+\.[0-9]+)$", remote_branches, re.M) + return locate_nearest_version(branches, self.version.name) def build(self): """Build this version/language doc.""" - logging.info( - "Build start for version: %s, language: %s", - self.version.name, - self.language.tag, - ) + logging.info("Build start.") sphinxopts = list(self.language.sphinxopts) sphinxopts.extend(["-q"]) if self.language.tag != "en": @@ -774,11 +781,7 @@ def build(self): setup_switchers( self.versions, self.languages, self.checkout / "Doc" / "build" / "html" ) - logging.info( - "Build done for version: %s, language: %s", - self.version.name, - self.language.tag, - ) + logging.info("Build done.") def build_venv(self): """Build a venv for the specific Python version. @@ -799,11 +802,7 @@ def build_venv(self): def copy_build_to_webroot(self): """Copy a given build to the appropriate webroot with appropriate rights.""" - logging.info( - "Publishing start for version: %s, language: %s", - self.version.name, - self.language.tag, - ) + logging.info("Publishing start.") self.www_root.mkdir(parents=True, exist_ok=True) if self.language.tag == "en": target = self.www_root / self.version.name @@ -873,7 +872,7 @@ def copy_build_to_webroot(self): ] ) if self.full_build: - logging.debug("Copying dist files") + logging.debug("Copying dist files.") run( [ "chown", @@ -916,11 +915,69 @@ def copy_build_to_webroot(self): purge(*prefixes) for prefix in prefixes: purge(*[prefix + p for p in changed]) - logging.info( - "Publishing done for version: %s, language: %s", - self.version.name, - self.language.tag, - ) + logging.info("Publishing done") + + def should_rebuild(self): + state = self.load_state() + if not state: + logging.info("Should rebuild: no previous state found.") + return True + cpython_sha = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() + if self.language.tag != "en": + translation_sha = self.translation_repo.run( + "rev-parse", "HEAD" + ).stdout.strip() + if translation_sha != state["translation_sha"]: + logging.info( + "Should rebuild: new translations (from %s to %s)", + state["translation_sha"], + translation_sha, + ) + return True + if cpython_sha != state["cpython_sha"]: + diff = self.cpython_repo.run( + "diff", "--name-only", state["cpython_sha"], cpython_sha + ).stdout + if "Doc/" in diff: + logging.info( + "Should rebuild: Doc/ has changed (from %s to %s)", + state["cpython_sha"], + cpython_sha, + ) + return True + logging.info("Nothing changed, no rebuild needed.") + return False + + def load_state(self) -> dict: + state_file = self.build_root / "state.toml" + try: + return tomlkit.loads(state_file.read_text(encoding="UTF-8"))[ + f"/{self.language.tag}/{self.version.name}/" + ] + except KeyError: + return {} + + def save_state(self, build_duration: float): + """Save current cpython sha1 and current translation sha1. + + Using this we can deduce if a rebuild is needed or not. + """ + state_file = self.build_root / "state.toml" + try: + states = tomlkit.parse(state_file.read_text(encoding="UTF-8")) + except FileNotFoundError: + states = tomlkit.document() + + state = {} + state["cpython_sha"] = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() + if self.language.tag != "en": + state["translation_sha"] = self.translation_repo.run( + "rev-parse", "HEAD" + ).stdout.strip() + state["last_build"] = dt.now(timezone.utc) + state["last_build_duration"] = build_duration + states[f"/{self.language.tag}/{self.version.name}/"] = state + state_file.write_text(tomlkit.dumps(states), encoding="UTF-8") def symlink(www_root: Path, language: Language, directory: str, name: str, group: str): @@ -1063,6 +1120,11 @@ def build_docs(args) -> bool: cpython_repo.update() while todo: version, language = todo.pop() + logging.root.handlers[0].setFormatter( + logging.Formatter( + f"%(asctime)s %(levelname)s {language.tag}/{version.name}: %(message)s" + ) + ) if sentry_sdk: with sentry_sdk.configure_scope() as scope: scope.set_tag("version", version.name) @@ -1071,6 +1133,10 @@ def build_docs(args) -> bool: version, versions, language, languages, cpython_repo, **vars(args) ) all_built_successfully &= builder.run() + logging.root.handlers[0].setFormatter( + logging.Formatter("%(asctime)s %(levelname)s: %(message)s") + ) + build_sitemap(versions, languages, args.www_root, args.group) build_404(args.www_root, args.group) build_robots_txt( From 1c9faa869f691f85bb5e396ade6babe2086c0801 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Fri, 20 Oct 2023 14:36:19 +0200 Subject: [PATCH 050/218] FIX: Behave properly when state file does not exists yet. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index d6eeb6b..3abd19c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -954,7 +954,7 @@ def load_state(self) -> dict: return tomlkit.loads(state_file.read_text(encoding="UTF-8"))[ f"/{self.language.tag}/{self.version.name}/" ] - except KeyError: + except (KeyError, FileNotFoundError): return {} def save_state(self, build_duration: float): From f0a570aec824c7d775ea9d345a89d1b10d0f11b2 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Mon, 23 Oct 2023 08:01:42 +0200 Subject: [PATCH 051/218] This is too floody for an INFO log. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 3abd19c..eac019a 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1036,7 +1036,7 @@ def purge(*paths): base = "https://docs.python.org/" for path in paths: url = urljoin(base, str(path)) - logging.info("Purging %s from CDN", url) + logging.debug("Purging %s from CDN", url) requests.request("PURGE", url, timeout=30) From 83ebdfcbf2106c9b0d77252004ded620042632be Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Wed, 10 Jan 2024 06:21:40 +0100 Subject: [PATCH 052/218] Fix `--theme` help message typo. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index eac019a..ac532a2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -584,7 +584,7 @@ def parse_args(): parser.add_argument( "--theme", default="python-docs-theme", - help="Python package to use for python-docs-theme: Usefull to test branches:" + help="Python package to use for python-docs-theme: Useful to test branches:" " --theme git+https://github.com/obulat/python-docs-theme@master", ) args = parser.parse_args() From 0aa97a0b279042f4e77cc1e5fb9143806fc560bc Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 9 May 2024 08:17:30 +0200 Subject: [PATCH 053/218] pre-release and prerelease are synonyms. --- build_docs.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/build_docs.py b/build_docs.py index eac019a..b5a1916 100755 --- a/build_docs.py +++ b/build_docs.py @@ -79,15 +79,10 @@ class Version: "bugfix": "stable", "security": "security-fixes", "end-of-life": "EOL", + "prerelease": "pre-release", } - def __init__( - self, - name, - *, - status, - branch_or_tag=None, - ): + def __init__(self, name, *, status, branch_or_tag=None): status = self.SYNONYMS.get(status, status) if status not in self.STATUSES: raise ValueError( From fd8620bb11194b9ec39685fe2aa8f6b827949647 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Sun, 2 Jun 2024 11:08:43 +0200 Subject: [PATCH 054/218] Publishing italian translation. --- config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.toml b/config.toml index c3f4886..3716d7f 100644 --- a/config.toml +++ b/config.toml @@ -33,7 +33,7 @@ in_prod = false [languages.it] name = "Italian" -in_prod = false +in_prod = true [languages.ja] name = "Japanese" From 27b193b4fa931d6e79e1df88591ece9fdb71d014 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 15 Jul 2024 08:29:51 +0200 Subject: [PATCH 055/218] Provide a backup suffix for sed -i (in-place) option when on MacOS (#177) Backup suffix following the -i option is required on macOS. --- build_docs.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/build_docs.py b/build_docs.py index b5a1916..1f9da70 100755 --- a/build_docs.py +++ b/build_docs.py @@ -30,6 +30,7 @@ import logging.handlers from functools import total_ordering from os import readlink +import platform import re import shlex import shutil @@ -745,13 +746,14 @@ def build(self): sphinxbuild = self.venv / "bin" / "sphinx-build" blurb = self.venv / "bin" / "blurb" # Disable cpython switchers, we handle them now: + + def is_mac(): + return platform.system() == 'Darwin' + run( - [ - "sed", - "-i", - "s/ *-A switchers=1//", - self.checkout / "Doc" / "Makefile", - ] + ["sed", "-i"] + + ([""] if is_mac() else []) + + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] ) self.version.setup_indexsidebar( self.versions, From 213530446b083788da9dbf4ca02284b5dc9c5c5c Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Mon, 15 Jul 2024 08:30:49 +0200 Subject: [PATCH 056/218] Pass the skip_cache_invalidation flag to symlinks functions (#178) Follow-up for b3c3137e --- build_docs.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/build_docs.py b/build_docs.py index 1f9da70..212989e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -977,7 +977,7 @@ def save_state(self, build_duration: float): state_file.write_text(tomlkit.dumps(states), encoding="UTF-8") -def symlink(www_root: Path, language: Language, directory: str, name: str, group: str): +def symlink(www_root: Path, language: Language, directory: str, name: str, group: str, skip_cache_invalidation: bool): """Used by major_symlinks and dev_symlink to maintain symlinks.""" if language.tag == "en": # english is rooted on /, no /en/ path = www_root @@ -993,11 +993,12 @@ def symlink(www_root: Path, language: Language, directory: str, name: str, group link.unlink() link.symlink_to(directory) run(["chown", "-h", ":" + group, str(link)]) - purge_path(www_root, link) + if not skip_cache_invalidation: + purge_path(www_root, link) def major_symlinks( - www_root: Path, group, versions: Iterable[Version], languages: Iterable[Language] + www_root: Path, group, versions: Iterable[Version], languages: Iterable[Language], skip_cache_invalidation: bool ): """Maintains the /2/ and /3/ symlinks for each languages. @@ -1008,11 +1009,11 @@ def major_symlinks( """ current_stable = Version.current_stable(versions).name for language in languages: - symlink(www_root, language, current_stable, "3", group) - symlink(www_root, language, "2.7", "2", group) + symlink(www_root, language, current_stable, "3", group, skip_cache_invalidation) + symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation) -def dev_symlink(www_root: Path, group, versions, languages): +def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidation: bool): """Maintains the /dev/ symlinks for each languages. Like: @@ -1022,7 +1023,7 @@ def dev_symlink(www_root: Path, group, versions, languages): """ current_dev = Version.current_dev(versions).name for language in languages: - symlink(www_root, language, current_dev, "dev", group) + symlink(www_root, language, current_dev, "dev", group, skip_cache_invalidation) def purge(*paths): @@ -1139,8 +1140,8 @@ def build_docs(args) -> bool: build_robots_txt( versions, languages, args.www_root, args.group, args.skip_cache_invalidation ) - major_symlinks(args.www_root, args.group, versions, languages) - dev_symlink(args.www_root, args.group, versions, languages) + major_symlinks(args.www_root, args.group, versions, languages, args.skip_cache_invalidation) + dev_symlink(args.www_root, args.group, versions, languages, args.skip_cache_invalidation) proofread_canonicals(args.www_root, args.skip_cache_invalidation) return all_built_successfully From b25088dac8e17f1686580ba75a1adcad4cc0ca7c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 9 Aug 2024 22:29:26 +0100 Subject: [PATCH 057/218] Use ``--upgrade-strategy=eager`` for pip --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index 212989e..429ee45 100755 --- a/build_docs.py +++ b/build_docs.py @@ -790,6 +790,7 @@ def build_venv(self): run([sys.executable, "-m", "venv", venv_path]) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] + + ["--upgrade-strategy=eager"] + [self.theme] + self.version.requirements, cwd=self.checkout / "Doc", From 108fd45c0ae1f0cd5fe836ac45add045d6d03f5d Mon Sep 17 00:00:00 2001 From: egeakman Date: Sun, 11 Aug 2024 18:37:31 +0300 Subject: [PATCH 058/218] migrate sentry from 1.x to 2.x --- build_docs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 212989e..a86d95e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1124,9 +1124,9 @@ def build_docs(args) -> bool: ) ) if sentry_sdk: - with sentry_sdk.configure_scope() as scope: - scope.set_tag("version", version.name) - scope.set_tag("language", language.tag) + scope = sentry_sdk.get_isolation_scope() + scope.set_tag("version", version.name) + scope.set_tag("language", language.tag) builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) From 1babbddb3ac2f956e31b344d7f4d8001cfdd6060 Mon Sep 17 00:00:00 2001 From: egeakman Date: Sun, 11 Aug 2024 19:52:16 +0300 Subject: [PATCH 059/218] ignore build_docs.lock --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a29939..79fca5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /build_root/ /logs/ /www/ +build_docs.lock # Created by https://www.gitignore.io/api/python From 12f773580f470cb9a38a4ba6aa304557ab925d45 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 11 Aug 2024 21:23:28 +0300 Subject: [PATCH 060/218] Speed up purges by re-using a requests session --- build_docs.py | 123 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 91 insertions(+), 32 deletions(-) diff --git a/build_docs.py b/build_docs.py index 212989e..719ee01 100755 --- a/build_docs.py +++ b/build_docs.py @@ -434,7 +434,8 @@ def build_robots_txt( www_root: Path, group, skip_cache_invalidation, -): + session: requests.Session, +) -> None: """Disallow crawl of EOL versions in robots.txt.""" if not www_root.exists(): logging.info("Skipping robots.txt generation (www root does not even exists).") @@ -449,7 +450,7 @@ def build_robots_txt( robots_file.chmod(0o775) run(["chgrp", group, robots_file]) if not skip_cache_invalidation: - purge("robots.txt") + purge(session, "robots.txt") def build_sitemap( @@ -642,7 +643,7 @@ def full_build(self): """ return not self.quick and not self.language.html_only - def run(self) -> bool: + def run(self, session: requests.Session) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() logging.info("Running.") @@ -653,7 +654,7 @@ def run(self) -> bool: if self.should_rebuild(): self.build_venv() self.build() - self.copy_build_to_webroot() + self.copy_build_to_webroot(session) self.save_state(build_duration=perf_counter() - start_time) except Exception as err: logging.exception("Badly handled exception, human, please help.") @@ -797,7 +798,7 @@ def build_venv(self): run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) self.venv = venv_path - def copy_build_to_webroot(self): + def copy_build_to_webroot(self, session: requests.Session) -> None: """Copy a given build to the appropriate webroot with appropriate rights.""" logging.info("Publishing start.") self.www_root.mkdir(parents=True, exist_ok=True) @@ -909,9 +910,9 @@ def copy_build_to_webroot(self): prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout prefixes = prefixes.replace(targets_dir + "/", "") prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix] - purge(*prefixes) + purge(session, *prefixes) for prefix in prefixes: - purge(*[prefix + p for p in changed]) + purge(session, *[prefix + p for p in changed]) logging.info("Publishing done") def should_rebuild(self): @@ -977,7 +978,15 @@ def save_state(self, build_duration: float): state_file.write_text(tomlkit.dumps(states), encoding="UTF-8") -def symlink(www_root: Path, language: Language, directory: str, name: str, group: str, skip_cache_invalidation: bool): +def symlink( + www_root: Path, + language: Language, + directory: str, + name: str, + group: str, + skip_cache_invalidation: bool, + session: requests.Session, +) -> None: """Used by major_symlinks and dev_symlink to maintain symlinks.""" if language.tag == "en": # english is rooted on /, no /en/ path = www_root @@ -994,12 +1003,17 @@ def symlink(www_root: Path, language: Language, directory: str, name: str, group link.symlink_to(directory) run(["chown", "-h", ":" + group, str(link)]) if not skip_cache_invalidation: - purge_path(www_root, link) + purge_path(session, www_root, link) def major_symlinks( - www_root: Path, group, versions: Iterable[Version], languages: Iterable[Language], skip_cache_invalidation: bool -): + www_root: Path, + group: str, + versions: Iterable[Version], + languages: Iterable[Language], + skip_cache_invalidation: bool, + session: requests.Session, +) -> None: """Maintains the /2/ and /3/ symlinks for each languages. Like: @@ -1009,11 +1023,26 @@ def major_symlinks( """ current_stable = Version.current_stable(versions).name for language in languages: - symlink(www_root, language, current_stable, "3", group, skip_cache_invalidation) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation) + symlink( + www_root, + language, + current_stable, + "3", + group, + skip_cache_invalidation, + session, + ) + symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, session) -def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidation: bool): +def dev_symlink( + www_root: Path, + group, + versions, + languages, + skip_cache_invalidation: bool, + session: requests.Session, +) -> None: """Maintains the /dev/ symlinks for each languages. Like: @@ -1023,10 +1052,18 @@ def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidat """ current_dev = Version.current_dev(versions).name for language in languages: - symlink(www_root, language, current_dev, "dev", group, skip_cache_invalidation) + symlink( + www_root, + language, + current_dev, + "dev", + group, + skip_cache_invalidation, + session, + ) -def purge(*paths): +def purge(session: requests.Session, *paths: Path | str) -> None: """Remove one or many paths from docs.python.org's CDN. To be used when a file change, so the CDN fetch the new one. @@ -1035,20 +1072,22 @@ def purge(*paths): for path in paths: url = urljoin(base, str(path)) logging.debug("Purging %s from CDN", url) - requests.request("PURGE", url, timeout=30) + session.request("PURGE", url, timeout=30) -def purge_path(www_root: Path, path: Path): +def purge_path(session: requests.Session, www_root: Path, path: Path) -> None: """Recursively remove a path from docs.python.org's CDN. To be used when a directory change, so the CDN fetch the new one. """ - purge(*[file.relative_to(www_root) for file in path.glob("**/*")]) - purge(path.relative_to(www_root)) - purge(str(path.relative_to(www_root)) + "/") + purge(session, *[file.relative_to(www_root) for file in path.glob("**/*")]) + purge(session, path.relative_to(www_root)) + purge(session, str(path.relative_to(www_root)) + "/") -def proofread_canonicals(www_root: Path, skip_cache_invalidation: bool) -> None: +def proofread_canonicals( + www_root: Path, skip_cache_invalidation: bool, session: requests.Session +) -> None: """In www_root we check that all canonical links point to existing contents. It can happen that a canonical is "broken": @@ -1070,11 +1109,11 @@ def proofread_canonicals(www_root: Path, skip_cache_invalidation: bool) -> None: html = html.replace(canonical.group(0), "") file.write_text(html, encoding="UTF-8", errors="surrogateescape") if not skip_cache_invalidation: - purge(str(file).replace("/srv/docs.python.org/", "")) + purge(session, str(file).replace("/srv/docs.python.org/", "")) -def parse_versions_from_devguide(): - releases = requests.get( +def parse_versions_from_devguide(session: requests.Session) -> list[Version]: + releases = session.get( "https://raw.githubusercontent.com/" "python/devguide/main/include/release-cycle.json", timeout=30, @@ -1101,8 +1140,9 @@ def parse_languages_from_config(): def build_docs(args) -> bool: - """Build all docs (each languages and each versions).""" - versions = parse_versions_from_devguide() + """Build all docs (each language and each version).""" + session = requests.Session() + versions = parse_versions_from_devguide(session) languages = parse_languages_from_config() todo = [ (version, language) @@ -1130,7 +1170,7 @@ def build_docs(args) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - all_built_successfully &= builder.run() + all_built_successfully &= builder.run(session) logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") ) @@ -1138,11 +1178,30 @@ def build_docs(args) -> bool: build_sitemap(versions, languages, args.www_root, args.group) build_404(args.www_root, args.group) build_robots_txt( - versions, languages, args.www_root, args.group, args.skip_cache_invalidation + versions, + languages, + args.www_root, + args.group, + args.skip_cache_invalidation, + session, + ) + major_symlinks( + args.www_root, + args.group, + versions, + languages, + args.skip_cache_invalidation, + session, + ) + dev_symlink( + args.www_root, + args.group, + versions, + languages, + args.skip_cache_invalidation, + session, ) - major_symlinks(args.www_root, args.group, versions, languages, args.skip_cache_invalidation) - dev_symlink(args.www_root, args.group, versions, languages, args.skip_cache_invalidation) - proofread_canonicals(args.www_root, args.skip_cache_invalidation) + proofread_canonicals(args.www_root, args.skip_cache_invalidation, session) return all_built_successfully From 4d05bb0bfe729078bc2f547352bb46934a916740 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 11 Aug 2024 22:12:20 +0300 Subject: [PATCH 061/218] Replace requests with urllib3 --- build_docs.py | 68 ++++++++++++++++++++++++------------------------ requirements.txt | 2 +- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/build_docs.py b/build_docs.py index 719ee01..9b3d592 100755 --- a/build_docs.py +++ b/build_docs.py @@ -46,11 +46,10 @@ from typing import Iterable from urllib.parse import urljoin -import zc.lockfile import jinja2 -import requests import tomlkit - +import urllib3 +import zc.lockfile try: from os import EX_OK, EX_SOFTWARE as EX_FAILURE @@ -434,7 +433,7 @@ def build_robots_txt( www_root: Path, group, skip_cache_invalidation, - session: requests.Session, + http: urllib3.PoolManager, ) -> None: """Disallow crawl of EOL versions in robots.txt.""" if not www_root.exists(): @@ -450,7 +449,7 @@ def build_robots_txt( robots_file.chmod(0o775) run(["chgrp", group, robots_file]) if not skip_cache_invalidation: - purge(session, "robots.txt") + purge(http, "robots.txt") def build_sitemap( @@ -643,7 +642,7 @@ def full_build(self): """ return not self.quick and not self.language.html_only - def run(self, session: requests.Session) -> bool: + def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() logging.info("Running.") @@ -654,7 +653,7 @@ def run(self, session: requests.Session) -> bool: if self.should_rebuild(): self.build_venv() self.build() - self.copy_build_to_webroot(session) + self.copy_build_to_webroot(http) self.save_state(build_duration=perf_counter() - start_time) except Exception as err: logging.exception("Badly handled exception, human, please help.") @@ -798,7 +797,7 @@ def build_venv(self): run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) self.venv = venv_path - def copy_build_to_webroot(self, session: requests.Session) -> None: + def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: """Copy a given build to the appropriate webroot with appropriate rights.""" logging.info("Publishing start.") self.www_root.mkdir(parents=True, exist_ok=True) @@ -910,9 +909,9 @@ def copy_build_to_webroot(self, session: requests.Session) -> None: prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout prefixes = prefixes.replace(targets_dir + "/", "") prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix] - purge(session, *prefixes) + purge(http, *prefixes) for prefix in prefixes: - purge(session, *[prefix + p for p in changed]) + purge(http, *[prefix + p for p in changed]) logging.info("Publishing done") def should_rebuild(self): @@ -985,7 +984,7 @@ def symlink( name: str, group: str, skip_cache_invalidation: bool, - session: requests.Session, + http: urllib3.PoolManager, ) -> None: """Used by major_symlinks and dev_symlink to maintain symlinks.""" if language.tag == "en": # english is rooted on /, no /en/ @@ -1003,7 +1002,7 @@ def symlink( link.symlink_to(directory) run(["chown", "-h", ":" + group, str(link)]) if not skip_cache_invalidation: - purge_path(session, www_root, link) + purge_path(http, www_root, link) def major_symlinks( @@ -1012,7 +1011,7 @@ def major_symlinks( versions: Iterable[Version], languages: Iterable[Language], skip_cache_invalidation: bool, - session: requests.Session, + http: urllib3.PoolManager, ) -> None: """Maintains the /2/ and /3/ symlinks for each languages. @@ -1030,9 +1029,9 @@ def major_symlinks( "3", group, skip_cache_invalidation, - session, + http, ) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, session) + symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) def dev_symlink( @@ -1041,7 +1040,7 @@ def dev_symlink( versions, languages, skip_cache_invalidation: bool, - session: requests.Session, + http: urllib3.PoolManager, ) -> None: """Maintains the /dev/ symlinks for each languages. @@ -1059,11 +1058,11 @@ def dev_symlink( "dev", group, skip_cache_invalidation, - session, + http, ) -def purge(session: requests.Session, *paths: Path | str) -> None: +def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: """Remove one or many paths from docs.python.org's CDN. To be used when a file change, so the CDN fetch the new one. @@ -1072,21 +1071,21 @@ def purge(session: requests.Session, *paths: Path | str) -> None: for path in paths: url = urljoin(base, str(path)) logging.debug("Purging %s from CDN", url) - session.request("PURGE", url, timeout=30) + http.request("PURGE", url, timeout=30) -def purge_path(session: requests.Session, www_root: Path, path: Path) -> None: +def purge_path(http: urllib3.PoolManager, www_root: Path, path: Path) -> None: """Recursively remove a path from docs.python.org's CDN. To be used when a directory change, so the CDN fetch the new one. """ - purge(session, *[file.relative_to(www_root) for file in path.glob("**/*")]) - purge(session, path.relative_to(www_root)) - purge(session, str(path.relative_to(www_root)) + "/") + purge(http, *[file.relative_to(www_root) for file in path.glob("**/*")]) + purge(http, path.relative_to(www_root)) + purge(http, str(path.relative_to(www_root)) + "/") def proofread_canonicals( - www_root: Path, skip_cache_invalidation: bool, session: requests.Session + www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager ) -> None: """In www_root we check that all canonical links point to existing contents. @@ -1109,11 +1108,12 @@ def proofread_canonicals( html = html.replace(canonical.group(0), "") file.write_text(html, encoding="UTF-8", errors="surrogateescape") if not skip_cache_invalidation: - purge(session, str(file).replace("/srv/docs.python.org/", "")) + purge(http, str(file).replace("/srv/docs.python.org/", "")) -def parse_versions_from_devguide(session: requests.Session) -> list[Version]: - releases = session.get( +def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: + releases = http.request( + "GET", "https://raw.githubusercontent.com/" "python/devguide/main/include/release-cycle.json", timeout=30, @@ -1141,8 +1141,8 @@ def parse_languages_from_config(): def build_docs(args) -> bool: """Build all docs (each language and each version).""" - session = requests.Session() - versions = parse_versions_from_devguide(session) + http = urllib3.PoolManager() + versions = parse_versions_from_devguide(http) languages = parse_languages_from_config() todo = [ (version, language) @@ -1170,7 +1170,7 @@ def build_docs(args) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - all_built_successfully &= builder.run(session) + all_built_successfully &= builder.run(http) logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") ) @@ -1183,7 +1183,7 @@ def build_docs(args) -> bool: args.www_root, args.group, args.skip_cache_invalidation, - session, + http, ) major_symlinks( args.www_root, @@ -1191,7 +1191,7 @@ def build_docs(args) -> bool: versions, languages, args.skip_cache_invalidation, - session, + http, ) dev_symlink( args.www_root, @@ -1199,9 +1199,9 @@ def build_docs(args) -> bool: versions, languages, args.skip_cache_invalidation, - session, + http, ) - proofread_canonicals(args.www_root, args.skip_cache_invalidation, session) + proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) return all_built_successfully diff --git a/requirements.txt b/requirements.txt index f51c7d0..cf12434 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ jinja2 -requests sentry-sdk tomlkit +urllib3 zc.lockfile From 81cce71c9e46add68452f77a5a0fe558e13ffb49 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 12 Aug 2024 00:01:51 +0300 Subject: [PATCH 062/218] Install urllib3 >= v2 Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cf12434..4602e89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ jinja2 sentry-sdk tomlkit -urllib3 +urllib3>=2 zc.lockfile From dd53442012ceaf090519ceb9f83c9b5f39b180eb Mon Sep 17 00:00:00 2001 From: egeakman Date: Mon, 12 Aug 2024 13:10:38 +0300 Subject: [PATCH 063/218] require sentry-sdk>=2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f51c7d0..b7e25bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ jinja2 requests -sentry-sdk +sentry-sdk>=2 tomlkit zc.lockfile From 4e7d3d91daef2288f406e9f6d095b3ed041823e1 Mon Sep 17 00:00:00 2001 From: egeakman Date: Mon, 12 Aug 2024 13:48:04 +0300 Subject: [PATCH 064/218] Fix typos --- build_docs.py | 55 +++++++++++++++++++++++------------------------ check_versions.py | 4 ++-- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/build_docs.py b/build_docs.py index 212989e..7cd3737 100755 --- a/build_docs.py +++ b/build_docs.py @@ -10,12 +10,11 @@ -q selects "quick build", which means to build only HTML. -Translations are fetched from github repositories according to PEP -545. `--languages` allows to select translations, like `--languages -en` to just build the english documents. +Translations are fetched from GitHub repositories according to PEP +545. `--languages` allows selecting translations, like `--languages +en` to just build the English documents. -This script was originally created and by Georg Brandl in March -2010. +This script was originally created by Georg Brandl in March 2010. Modified by Benjamin Peterson to do CDN cache invalidation. Modified by Julien Palard to build translations. @@ -69,7 +68,7 @@ @total_ordering class Version: - """Represents a cpython version and its documentation builds dependencies.""" + """Represents a CPython version and its documentation build dependencies.""" STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"} @@ -147,7 +146,7 @@ def filter(versions, branch=None): If *branch* is given, only *versions* matching *branch* are returned. - Else all live version are returned (this mean no EOL and no + Else all live versions are returned (this means no EOL and no security-fixes branches). """ if branch: @@ -156,12 +155,12 @@ def filter(versions, branch=None): @staticmethod def current_stable(versions): - """Find the current stable cPython version.""" + """Find the current stable CPython version.""" return max((v for v in versions if v.status == "stable"), key=Version.as_tuple) @staticmethod def current_dev(versions): - """Find the current de cPython version.""" + """Find the current CPython version in development.""" return max(versions, key=Version.as_tuple) @property @@ -360,7 +359,7 @@ def locate_nearest_version(available_versions, target_version): def edit(file: Path): """Context manager to edit a file "in place", use it as: - with edit("/etc/hosts") as i, o: + with edit("/etc/hosts") as (i, o): for line in i: o.write(line.replace("localhoat", "localhost")) """ @@ -376,7 +375,7 @@ def edit(file: Path): def setup_switchers( versions: Iterable[Version], languages: Iterable[Language], html_root: Path ): - """Setup cross-links between cpython versions: + """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ @@ -437,7 +436,7 @@ def build_robots_txt( ): """Disallow crawl of EOL versions in robots.txt.""" if not www_root.exists(): - logging.info("Skipping robots.txt generation (www root does not even exists).") + logging.info("Skipping robots.txt generation (www root does not even exist).") return robots_file = www_root / "robots.txt" with open(HERE / "templates" / "robots.txt", encoding="UTF-8") as template_file: @@ -457,7 +456,7 @@ def build_sitemap( ): """Build a sitemap with all live versions and translations.""" if not www_root.exists(): - logging.info("Skipping sitemap generation (www root does not even exists).") + logging.info("Skipping sitemap generation (www root does not even exist).") return with open(HERE / "templates" / "sitemap.xml", encoding="UTF-8") as template_file: template = jinja2.Template(template_file.read()) @@ -472,7 +471,7 @@ def build_sitemap( def build_404(www_root: Path, group): """Build a nice 404 error page to display in case PDFs are not built yet.""" if not www_root.exists(): - logging.info("Skipping 404 page generation (www root does not even exists).") + logging.info("Skipping 404 page generation (www root does not even exist).") return not_found_file = www_root / "404.html" shutil.copyfile(HERE / "templates" / "404.html", not_found_file) @@ -550,7 +549,7 @@ def parse_args(): ) parser.add_argument( "--skip-cache-invalidation", - help="Skip fastly cache invalidation.", + help="Skip Fastly cache invalidation.", action="store_true", ) parser.add_argument( @@ -598,7 +597,7 @@ def parse_args(): def setup_logging(log_directory: Path): - """Setup logging to stderr if ran by a human, or to a file if ran from a cron.""" + """Setup logging to stderr if run by a human, or to a file if run from a cron.""" if sys.stderr.isatty(): logging.basicConfig( format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stderr @@ -615,7 +614,7 @@ def setup_logging(log_directory: Path): @dataclass class DocBuilder: - """Builder for a cpython version and a language.""" + """Builder for a CPython version and a language.""" version: Version versions: Iterable[Version] @@ -634,7 +633,7 @@ class DocBuilder: def full_build(self): """Tell if a full build is needed. - A full build is slow, it builds pdf, txt, epub, texinfo, and + A full build is slow; it builds pdf, txt, epub, texinfo, and archives everything. A partial build only builds HTML and does not archive, it's @@ -664,7 +663,7 @@ def run(self) -> bool: @property def checkout(self) -> Path: - """Path to cpython git clone.""" + """Path to CPython git clone.""" return self.build_root / "cpython" def clone_translation(self): @@ -687,7 +686,7 @@ def translation_repo(self): @property def translation_branch(self): - """Some cpython versions may be untranslated, being either too old or + """Some CPython versions may be untranslated, being either too old or too new. This function looks for remote branches on the given repo, and @@ -745,7 +744,7 @@ def build(self): python = self.venv / "bin" / "python" sphinxbuild = self.venv / "bin" / "sphinx-build" blurb = self.venv / "bin" / "blurb" - # Disable cpython switchers, we handle them now: + # Disable CPython switchers, we handle them now: def is_mac(): return platform.system() == 'Darwin' @@ -955,7 +954,7 @@ def load_state(self) -> dict: return {} def save_state(self, build_duration: float): - """Save current cpython sha1 and current translation sha1. + """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. """ @@ -979,7 +978,7 @@ def save_state(self, build_duration: float): def symlink(www_root: Path, language: Language, directory: str, name: str, group: str, skip_cache_invalidation: bool): """Used by major_symlinks and dev_symlink to maintain symlinks.""" - if language.tag == "en": # english is rooted on /, no /en/ + if language.tag == "en": # English is rooted on /, no /en/ path = www_root else: path = www_root / language.tag @@ -1000,7 +999,7 @@ def symlink(www_root: Path, language: Language, directory: str, name: str, group def major_symlinks( www_root: Path, group, versions: Iterable[Version], languages: Iterable[Language], skip_cache_invalidation: bool ): - """Maintains the /2/ and /3/ symlinks for each languages. + """Maintains the /2/ and /3/ symlinks for each language. Like: - /3/ → /3.9/ @@ -1014,7 +1013,7 @@ def major_symlinks( def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidation: bool): - """Maintains the /dev/ symlinks for each languages. + """Maintains the /dev/ symlinks for each language. Like: - /dev/ → /3.11/ @@ -1029,7 +1028,7 @@ def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidat def purge(*paths): """Remove one or many paths from docs.python.org's CDN. - To be used when a file change, so the CDN fetch the new one. + To be used when a file changes, so the CDN fetches the new one. """ base = "https://docs.python.org/" for path in paths: @@ -1041,7 +1040,7 @@ def purge(*paths): def purge_path(www_root: Path, path: Path): """Recursively remove a path from docs.python.org's CDN. - To be used when a directory change, so the CDN fetch the new one. + To be used when a directory changes, so the CDN fetches the new one. """ purge(*[file.relative_to(www_root) for file in path.glob("**/*")]) purge(path.relative_to(www_root)) @@ -1101,7 +1100,7 @@ def parse_languages_from_config(): def build_docs(args) -> bool: - """Build all docs (each languages and each versions).""" + """Build all docs (each language and each version).""" versions = parse_versions_from_devguide() languages = parse_languages_from_config() todo = [ diff --git a/check_versions.py b/check_versions.py index 630eb6e..0ca0770 100644 --- a/check_versions.py +++ b/check_versions.py @@ -20,12 +20,12 @@ def parse_args(): description="""Check the version of our build in different branches Hint: Use with | column -t""" ) - parser.add_argument("cpython_clone", help="Path to a clone of cpython", type=Path) + parser.add_argument("cpython_clone", help="Path to a clone of CPython", type=Path) return parser.parse_args() def remote_by_url(repo: git.Repo, url_pattern: str): - """Find a remote of repo matching the regex url_pattern.""" + """Find a remote in the repo that matches the URL pattern.""" for remote in repo.remotes: for url in remote.urls: if re.search(url_pattern, url): From dd8a562a500e2f41ef378ae1ebf19076115fc0b1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 14 Aug 2024 18:17:21 +0300 Subject: [PATCH 065/218] Show elapsed time for build and publishing --- .github/workflows/lint.yml | 22 ++++++++++++++++++ .github/workflows/test.yml | 39 +++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 47 ++++++++++++++++++++++++++++++++++++++ build_docs.py | 25 +++++++++++++++++--- pyproject.toml | 7 ++++++ tests/test_build_docs.py | 21 +++++++++++++++++ tox.ini | 25 ++++++++++++++++++++ 7 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/test.yml create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml create mode 100644 tests/test_build_docs.py create mode 100644 tox.ini diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..d553e49 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,22 @@ +name: Lint + +on: [push, pull_request, workflow_dispatch] + +env: + FORCE_COLOR: 1 + PIP_DISABLE_PIP_VERSION_CHECK: 1 + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + cache: pip + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..668336d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,39 @@ +name: Test + +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + PIP_DISABLE_PIP_VERSION_CHECK: 1 + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Install uv + uses: hynek/setup-cached-uv@v2 + + - name: Install dependencies + run: | + uv pip install --system -U tox-uv + + - name: Tox tests + run: | + tox -e py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..db47c50 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,47 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: forbid-submodules + - id: requirements-txt-fixer + - id: trailing-whitespace + + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.28.6 + hooks: + - id: check-github-workflows + + - repo: https://github.com/rhysd/actionlint + rev: v1.7.1 + hooks: + - id: actionlint + + - repo: https://github.com/tox-dev/pyproject-fmt + rev: 2.1.3 + hooks: + - id: pyproject-fmt + + - repo: https://github.com/abravalheri/validate-pyproject + rev: v0.18 + hooks: + - id: validate-pyproject + + - repo: https://github.com/tox-dev/tox-ini-fmt + rev: 1.3.1 + hooks: + - id: tox-ini-fmt + + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes + +ci: + autoupdate_schedule: quarterly diff --git a/build_docs.py b/build_docs.py index 212989e..701beb1 100755 --- a/build_docs.py +++ b/build_docs.py @@ -28,7 +28,7 @@ import json import logging import logging.handlers -from functools import total_ordering +from functools import total_ordering, cache from os import readlink import platform import re @@ -702,6 +702,7 @@ def translation_branch(self): def build(self): """Build this version/language doc.""" logging.info("Build start.") + start_time = perf_counter() sphinxopts = list(self.language.sphinxopts) sphinxopts.extend(["-q"]) if self.language.tag != "en": @@ -778,7 +779,7 @@ def is_mac(): setup_switchers( self.versions, self.languages, self.checkout / "Doc" / "build" / "html" ) - logging.info("Build done.") + logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) def build_venv(self): """Build a venv for the specific Python version. @@ -800,6 +801,7 @@ def build_venv(self): def copy_build_to_webroot(self): """Copy a given build to the appropriate webroot with appropriate rights.""" logging.info("Publishing start.") + start_time = perf_counter() self.www_root.mkdir(parents=True, exist_ok=True) if self.language.tag == "en": target = self.www_root / self.version.name @@ -912,7 +914,9 @@ def copy_build_to_webroot(self): purge(*prefixes) for prefix in prefixes: purge(*[prefix + p for p in changed]) - logging.info("Publishing done") + logging.info( + "Publishing done (%s).", format_seconds(perf_counter() - start_time) + ) def should_rebuild(self): state = self.load_state() @@ -1147,6 +1151,21 @@ def build_docs(args) -> bool: return all_built_successfully +@cache +def format_seconds(seconds: float) -> str: + hours, remainder = divmod(seconds, 3600) + minutes, seconds = divmod(remainder, 60) + seconds = round(seconds) + + match (hours, minutes, seconds): + case 0, 0, s: + return f"{s}s" + case 0, m, s: + return f"{m}m {s}s" + case h, m, s: + return f"{h}h {m}m {s}s" + + def main(): """Script entry point.""" args = parse_args() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6c56bf0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.pytest.ini_options] +pythonpath = [ + ".", +] +testpaths = [ + "tests", +] diff --git a/tests/test_build_docs.py b/tests/test_build_docs.py new file mode 100644 index 0000000..975e03c --- /dev/null +++ b/tests/test_build_docs.py @@ -0,0 +1,21 @@ +import pytest + +from build_docs import format_seconds + + +@pytest.mark.parametrize( + "seconds, expected", + [ + (0.4, "0s"), + (0.5, "0s"), + (0.6, "1s"), + (1.5, "2s"), + (30, "30s"), + (60, "1m 0s"), + (185, "3m 5s"), + (454, "7m 34s"), + (7456, "2h 4m 16s"), + ], +) +def test_format_seconds(seconds: int, expected: str) -> None: + assert format_seconds(seconds) == expected diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..40a034d --- /dev/null +++ b/tox.ini @@ -0,0 +1,25 @@ +[tox] +requires = + tox>=4.2 +env_list = + lint + py{313, 312, 311, 310} + +[testenv] +package = wheel +wheel_build_env = .pkg +skip_install = true +deps = + -r requirements.txt + pytest +commands = + {envpython} -m pytest {posargs} + +[testenv:lint] +skip_install = true +deps = + pre-commit +pass_env = + PRE_COMMIT_COLOR +commands = + pre-commit run --all-files --show-diff-on-failure From 3ef52a6613cac9a265c0920c24aa9178b92f3b4c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:20:07 +0300 Subject: [PATCH 066/218] Fix hours and minutes for float input --- build_docs.py | 2 +- tests/test_build_docs.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 701beb1..0e67d7b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1155,7 +1155,7 @@ def build_docs(args) -> bool: def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) - seconds = round(seconds) + hours, minutes, seconds = int(hours), int(minutes), round(seconds) match (hours, minutes, seconds): case 0, 0, s: diff --git a/tests/test_build_docs.py b/tests/test_build_docs.py index 975e03c..4457e95 100644 --- a/tests/test_build_docs.py +++ b/tests/test_build_docs.py @@ -15,7 +15,12 @@ (185, "3m 5s"), (454, "7m 34s"), (7456, "2h 4m 16s"), + (30.1, "30s"), + (60.2, "1m 0s"), + (185.3, "3m 5s"), + (454.4, "7m 34s"), + (7456.5, "2h 4m 16s"), ], ) -def test_format_seconds(seconds: int, expected: str) -> None: +def test_format_seconds(seconds: float, expected: str) -> None: assert format_seconds(seconds) == expected From 6c194eb105f49728cc19d58f3c999eb8dbfc1ebf Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 15 Aug 2024 17:04:34 +0300 Subject: [PATCH 067/218] Remove functools.cache --- build_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0e67d7b..c72c56c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -28,7 +28,7 @@ import json import logging import logging.handlers -from functools import total_ordering, cache +from functools import total_ordering from os import readlink import platform import re @@ -1151,7 +1151,6 @@ def build_docs(args) -> bool: return all_built_successfully -@cache def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) From 56d72d43e5759cc0ed600827b56e81d8310bcaca Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 24 Aug 2024 14:36:09 +0200 Subject: [PATCH 068/218] Update the version tables in the README (#189) * Update README tables * Remove EOL versions from the language table --- README.md | 40 ++++++++-------- build_docs.py | 4 +- check_versions.py | 119 ++++++++++++++++++++++++---------------------- 3 files changed, 85 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 3e70fa4..d5beb39 100644 --- a/README.md +++ b/README.md @@ -23,34 +23,32 @@ of Sphinx we're using where: Sphinx configuration in various branches: - ========= ============= ============= ================== ================== - version travis azure requirements.txt conf.py - ========= ============= ============= ================== ================== - 2.7 sphinx~=2.0.1 ø ø needs_sphinx='1.2' - 3.5 sphinx==1.8.2 ø ø needs_sphinx='1.8' - 3.6 sphinx==1.8.2 sphinx==1.8.2 ø needs_sphinx='1.2' - 3.7 ø ø ø ø - 3.8 ø ø sphinx==2.4.4 needs_sphinx='1.8' - 3.9 ø ø sphinx==2.4.4 needs_sphinx='1.8' - 3.1 ø ø sphinx==3.4.3 needs_sphinx='3.2' - 3.11 ø ø sphinx==4.5.0 needs_sphinx='4.2' - 3.12 ø ø sphinx==4.5.0 needs_sphinx='4.2' - 3.13 ø ø sphinx==6.2.1 needs_sphinx='4.2' - ========= ============= ============= ================== ================== + ========= ============= ================== ==================== + version travis requirements.txt conf.py + ========= ============= ================== ==================== + 2.7 sphinx~=2.0.1 ø needs_sphinx='1.2' + 3.5 sphinx==1.8.2 ø needs_sphinx='1.8' + 3.6 sphinx==1.8.2 ø needs_sphinx='1.2' + 3.7 sphinx==1.8.2 sphinx==2.3.1 needs_sphinx="1.6.6" + 3.8 ø sphinx==2.4.4 needs_sphinx='1.8' + 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' + 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' + 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' + 3.12 ø sphinx~=8.0.0 needs_sphinx='6.2.1' + 3.13 ø sphinx~=8.0.0 needs_sphinx='6.2.1' + 3.14 ø sphinx~=8.0.0 needs_sphinx='6.2.1' + ========= ============= ================== ==================== Sphinx build as seen on docs.python.org: ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= version en es fr id it ja ko pl pt-br tr uk zh-cn zh-tw ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= - 2.7 ø 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 - 3.5 ø 1.8.4 1.8.4 1.8.4 1.8.4 ø 1.8.4 1.8.4 ø 1.8.4 1.8.4 1.8.4 1.8.4 - 3.6 ø 2.3.1 2.3.1 2.3.1 2.3.1 ø 2.3.1 2.3.1 ø 2.3.1 2.3.1 2.3.1 2.3.1 - 3.7 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 3.8 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 - 3.11 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 - 3.12 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 4.5.0 - 3.13 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 6.2.1 + 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 + 3.12 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 + 3.13 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 + 3.14 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= diff --git a/build_docs.py b/build_docs.py index 212989e..93dcac4 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1079,7 +1079,9 @@ def parse_versions_from_devguide(): "python/devguide/main/include/release-cycle.json", timeout=30, ).json() - return [Version.from_json(name, release) for name, release in releases.items()] + versions = [Version.from_json(name, release) for name, release in releases.items()] + versions.sort(key=Version.as_tuple) + return versions def parse_languages_from_config(): diff --git a/check_versions.py b/check_versions.py index 630eb6e..a1900c3 100644 --- a/check_versions.py +++ b/check_versions.py @@ -13,6 +13,8 @@ import build_docs logger = logging.getLogger(__name__) +VERSIONS = build_docs.parse_versions_from_devguide() +LANGUAGES = build_docs.parse_languages_from_config() def parse_args(): @@ -24,102 +26,107 @@ def parse_args(): return parser.parse_args() -def remote_by_url(repo: git.Repo, url_pattern: str): +def find_upstream_remote_name(repo: git.Repo) -> str: """Find a remote of repo matching the regex url_pattern.""" for remote in repo.remotes: for url in remote.urls: - if re.search(url_pattern, url): - return remote + if "github.com/python" in url: + return f"{remote.name}/" def find_sphinx_spec(text: str): if found := re.search( - """sphinx[=<>~]{1,2}[0-9.]{3,}|needs_sphinx = [0-9.'"]*""", text, flags=re.I + """sphinx[=<>~]{1,2}[0-9.]{3,}|needs_sphinx = [0-9.'"]*""", + text, + flags=re.IGNORECASE, ): return found.group(0).replace(" ", "") return "ø" -def find_sphinx_in_file(repo: git.Repo, branch, filename): - upstream = remote_by_url(repo, "github.com.python").name - # Just in case you don't use origin/: - branch = branch.replace("origin/", upstream + "/") - try: - return find_sphinx_spec(repo.git.show(f"{branch}:{filename}")) - except git.exc.GitCommandError: - return "ø" +def find_sphinx_in_files(repo: git.Repo, branch_or_tag, filenames): + upstream = find_upstream_remote_name(repo) + # Just in case you don't use upstream/: + branch_or_tag = branch_or_tag.replace("upstream/", upstream) + specs = [] + for filename in filenames: + try: + blob = repo.git.show(f"{branch_or_tag}:{filename}") + except git.exc.GitCommandError: + specs.append("ø") + else: + specs.append(find_sphinx_spec(blob)) + return specs CONF_FILES = { "travis": ".travis.yml", - "azure": ".azure-pipelines/docs-steps.yml", "requirements.txt": "Doc/requirements.txt", "conf.py": "Doc/conf.py", } -def search_sphinx_versions_in_cpython(repo: git.Repo): - repo.git.fetch("https://github.com/python/cpython") - table = [] - for version in sorted(build_docs.VERSIONS): - table.append( - [ - version.name, - *[ - find_sphinx_in_file(repo, version.branch_or_tag, filename) - for filename in CONF_FILES.values() - ], - ] - ) - print(tabulate(table, headers=["version", *CONF_FILES.keys()], tablefmt="rst")) +def branch_or_tag_for(version): + if version.status == "EOL": + return f"tags/{version.branch_or_tag}" + return f"upstream/{version.branch_or_tag}" -async def get_version_in_prod(language, version): - url = f"https://docs.python.org/{language}/{version}/".replace("/en/", "/") +def search_sphinx_versions_in_cpython(repo: git.Repo): + repo.git.fetch("https://github.com/python/cpython") + filenames = CONF_FILES.values() + table = [ + [ + version.name, + *find_sphinx_in_files(repo, branch_or_tag_for(version), filenames), + ] + for version in VERSIONS + ] + headers = ["version", *CONF_FILES.keys()] + print(tabulate(table, headers=headers, tablefmt="rst", disable_numparse=True)) + + +async def get_version_in_prod(language: str, version: str) -> str: + if language == "en": + url = f"https://docs.python.org/{version}/" + else: + url = f"https://docs.python.org/{language}/{version}/" async with httpx.AsyncClient() as client: try: - response = await client.get(url, timeout=10) + response = await client.get(url, timeout=5) except httpx.ConnectTimeout: return "(timeout)" - text = response.text.encode("ASCII", errors="ignore").decode("ASCII") + # Python 2.6--3.7: sphinx.pocoo.org + # from Python 3.8: www.sphinx-doc.org if created_using := re.search( - r"(?:sphinx.pocoo.org|www.sphinx-doc.org).*?([0-9.]+[0-9])", text, flags=re.M + r"(?:sphinx.pocoo.org|www.sphinx-doc.org).*?([0-9.]+[0-9])", response.text ): return created_using.group(1) return "ø" async def which_sphinx_is_used_in_production(): - table = [] - for version in sorted(build_docs.VERSIONS): - table.append( - [ - version.name, - *await asyncio.gather( - *[ - get_version_in_prod(language.tag, version.name) - for language in build_docs.LANGUAGES - ] - ), - ] - ) - print( - tabulate( - table, - disable_numparse=True, - headers=[ - "version", - *[language.tag for language in sorted(build_docs.LANGUAGES)], - ], - tablefmt="rst", - ) - ) + table = [ + [ + version.name, + *await asyncio.gather( + *[ + get_version_in_prod(language.tag, version.name) + for language in LANGUAGES + ] + ), + ] + for version in VERSIONS + ] + headers = ["version", *[language.tag for language in LANGUAGES]] + print(tabulate(table, headers=headers, tablefmt="rst", disable_numparse=True)) def main(): logging.basicConfig(level=logging.INFO) logging.getLogger("charset_normalizer").setLevel(logging.WARNING) logging.getLogger("asyncio").setLevel(logging.WARNING) + logging.getLogger("httpx").setLevel(logging.WARNING) args = parse_args() repo = git.Repo(args.cpython_clone) print("Sphinx configuration in various branches:", end="\n\n") From 124693f58f4cc22b9a5d40aaab73f1cb26195cb0 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:57:12 +0300 Subject: [PATCH 069/218] Keep build_docs() near main() --- build_docs.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/build_docs.py b/build_docs.py index c72c56c..006b050 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1104,6 +1104,20 @@ def parse_languages_from_config(): return languages +def format_seconds(seconds: float) -> str: + hours, remainder = divmod(seconds, 3600) + minutes, seconds = divmod(remainder, 60) + hours, minutes, seconds = int(hours), int(minutes), round(seconds) + + match (hours, minutes, seconds): + case 0, 0, s: + return f"{s}s" + case 0, m, s: + return f"{m}m {s}s" + case h, m, s: + return f"{h}h {m}m {s}s" + + def build_docs(args) -> bool: """Build all docs (each languages and each versions).""" versions = parse_versions_from_devguide() @@ -1151,20 +1165,6 @@ def build_docs(args) -> bool: return all_built_successfully -def format_seconds(seconds: float) -> str: - hours, remainder = divmod(seconds, 3600) - minutes, seconds = divmod(remainder, 60) - hours, minutes, seconds = int(hours), int(minutes), round(seconds) - - match (hours, minutes, seconds): - case 0, 0, s: - return f"{s}s" - case 0, m, s: - return f"{m}m {s}s" - case h, m, s: - return f"{h}h {m}m {s}s" - - def main(): """Script entry point.""" args = parse_args() From 21d094e2a2e97985ad3690f8066f5f080ec19a16 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 25 Aug 2024 13:01:47 +0300 Subject: [PATCH 070/218] Show elapsed time for full docs build --- build_docs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build_docs.py b/build_docs.py index 006b050..35b794e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1120,6 +1120,8 @@ def format_seconds(seconds: float) -> str: def build_docs(args) -> bool: """Build all docs (each languages and each versions).""" + logging.info("Full build start.") + start_time = perf_counter() versions = parse_versions_from_devguide() languages = parse_languages_from_config() todo = [ @@ -1162,6 +1164,8 @@ def build_docs(args) -> bool: dev_symlink(args.www_root, args.group, versions, languages, args.skip_cache_invalidation) proofread_canonicals(args.www_root, args.skip_cache_invalidation) + logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) + return all_built_successfully From ed42020f66c254dd5bed910b96e5f58602892810 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 26 Aug 2024 08:41:15 +0300 Subject: [PATCH 071/218] Build each docs version with fresh commit --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 93dcac4..b20e489 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1117,7 +1117,6 @@ def build_docs(args) -> bool: cpython_repo = Repository( "https://github.com/python/cpython.git", args.build_root / "cpython" ) - cpython_repo.update() while todo: version, language = todo.pop() logging.root.handlers[0].setFormatter( @@ -1129,6 +1128,7 @@ def build_docs(args) -> bool: with sentry_sdk.configure_scope() as scope: scope.set_tag("version", version.name) scope.set_tag("language", language.tag) + cpython_repo.update() builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) From a21b2bb0840a17a11ce1edf8de4fbc4ed333b46d Mon Sep 17 00:00:00 2001 From: Ege Akman Date: Tue, 10 Sep 2024 13:05:45 -0400 Subject: [PATCH 072/218] Update .gitignore Co-authored-by: Ezio Melotti --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 79fca5f..aaefb08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /build_root/ /logs/ /www/ +# temporary lock file created while building the docs build_docs.lock From f2d2ffb83ce8f3dac961a75d2069033b668b6dc7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:22:21 +0300 Subject: [PATCH 073/218] Update pre-commit --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db47c50..dba593f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.28.6 + rev: 0.29.2 hooks: - id: check-github-workflows @@ -24,17 +24,17 @@ repos: - id: actionlint - repo: https://github.com/tox-dev/pyproject-fmt - rev: 2.1.3 + rev: 2.2.3 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.18 + rev: v0.19 hooks: - id: validate-pyproject - repo: https://github.com/tox-dev/tox-ini-fmt - rev: 1.3.1 + rev: 1.4.0 hooks: - id: tox-ini-fmt From 652c561ac48ef8f3924ad188482058aa7be1ac11 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:44:46 +0300 Subject: [PATCH 074/218] Simplify uv + tox-uv setup --- .github/workflows/test.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 668336d..b2686a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,10 +30,6 @@ jobs: - name: Install uv uses: hynek/setup-cached-uv@v2 - - name: Install dependencies - run: | - uv pip install --system -U tox-uv - - name: Tox tests run: | - tox -e py + uvx --with tox-uv tox -e py From c6a8855c0fba7ba7408fc4b1969ca90e3ce43c1e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:18:54 +0300 Subject: [PATCH 075/218] Using uv not pip --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2686a7..d74e607 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,6 @@ permissions: env: FORCE_COLOR: 1 - PIP_DISABLE_PIP_VERSION_CHECK: 1 jobs: test: From b0921bd556d9b25637019184085e5e13e2830fc7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:42:11 +0300 Subject: [PATCH 076/218] Reformat pyproject.toml --- pyproject.toml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6c56bf0..e85ab2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,3 @@ [tool.pytest.ini_options] -pythonpath = [ - ".", -] -testpaths = [ - "tests", -] +pythonpath = [ "." ] +testpaths = [ "tests" ] From 6360ad789b2c3235086ec4d8e16f5bf7576cbb3b Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:53:02 +0100 Subject: [PATCH 077/218] Improve type hints (use Sequence) --- build_docs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index a3971a5..961c48c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -21,6 +21,7 @@ """ from argparse import ArgumentParser +from collections.abc import Sequence from contextlib import suppress, contextmanager from dataclasses import dataclass import filecmp @@ -372,7 +373,7 @@ def edit(file: Path): def setup_switchers( - versions: Iterable[Version], languages: Iterable[Language], html_root: Path + versions: Sequence[Version], languages: Sequence[Language], html_root: Path ): """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher @@ -617,9 +618,9 @@ class DocBuilder: """Builder for a CPython version and a language.""" version: Version - versions: Iterable[Version] + versions: Sequence[Version] language: Language - languages: Iterable[Language] + languages: Sequence[Language] cpython_repo: Repository build_root: Path www_root: Path @@ -1127,7 +1128,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: return versions -def parse_languages_from_config(): +def parse_languages_from_config() -> list[Language]: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) languages = [] From 2828c2b2f1112b038638a73153c5cbf77a4306f6 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:54:44 +0100 Subject: [PATCH 078/218] Remove needless sorts --- build_docs.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/build_docs.py b/build_docs.py index 961c48c..a77371b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -182,9 +182,7 @@ def setup_indexsidebar(self, versions, dest_path): sidebar_file.write( sidebar_template.render( current_version=self, - versions=sorted( - versions, key=lambda v: version_to_tuple(v.name), reverse=True - ), + versions=versions[::-1], ) ) @@ -339,12 +337,7 @@ def locate_nearest_version(available_versions, target_version): '3.7' """ - available_versions_tuples = sorted( - [ - version_to_tuple(available_version) - for available_version in set(available_versions) - ] - ) + available_versions_tuples = sorted(map(version_to_tuple, set(available_versions))) target_version_tuple = version_to_tuple(target_version) try: found = available_versions_tuples[ @@ -402,11 +395,7 @@ def setup_switchers( OrderedDict( [ (version.name, version.picker_label) - for version in sorted( - versions, - key=lambda v: version_to_tuple(v.name), - reverse=True, - ) + for version in reversed(versions) ] ) ), From 988128e0d6e7f4b35ee968be1a36695f328d898f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:55:10 +0100 Subject: [PATCH 079/218] Run languages in definition order --- build_docs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index a77371b..42e5ad0 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1156,10 +1156,13 @@ def build_docs(args) -> bool: http = urllib3.PoolManager() versions = parse_versions_from_devguide(http) languages = parse_languages_from_config() + # Reverse languages but not versions, because we take version-language + # pairs from the end of the list, effectively reversing it. + # This runs languages in config.toml order and versions newest first. todo = [ (version, language) for version in Version.filter(versions, args.branch) - for language in Language.filter(languages, args.languages) + for language in reversed(Language.filter(languages, args.languages)) ] del args.branch del args.languages From 4eb435ed09d5914e0f24672321f695b6c34c6fcb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:58:24 +0100 Subject: [PATCH 080/218] Remove OrderedDict --- build_docs.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/build_docs.py b/build_docs.py index 42e5ad0..a62efce 100755 --- a/build_docs.py +++ b/build_docs.py @@ -37,7 +37,6 @@ import subprocess import sys from bisect import bisect_left as bisect -from collections import OrderedDict from datetime import datetime as dt, timezone from pathlib import Path from string import Template @@ -372,6 +371,9 @@ def setup_switchers( - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ + languages_map = dict(sorted((l.tag, l.name) for l in languages if l.in_prod)) + versions_map = {v.name: v.picker_label for v in reversed(versions)} + with open( HERE / "templates" / "switchers.js", encoding="UTF-8" ) as switchers_template_file: @@ -380,25 +382,8 @@ def setup_switchers( switchers_path.write_text( template.safe_substitute( { - "LANGUAGES": json.dumps( - OrderedDict( - sorted( - [ - (language.tag, language.name) - for language in languages - if language.in_prod - ] - ) - ) - ), - "VERSIONS": json.dumps( - OrderedDict( - [ - (version.name, version.picker_label) - for version in reversed(versions) - ] - ) - ), + "LANGUAGES": json.dumps(languages_map), + "VERSIONS": json.dumps(versions_map), } ), encoding="UTF-8", From a00bf2bde0569ba9fcb9b21e612e737714f1055b Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:16:00 +0100 Subject: [PATCH 081/218] Use keyword-arguments --- build_docs.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index a62efce..2ebc312 100755 --- a/build_docs.py +++ b/build_docs.py @@ -381,10 +381,8 @@ def setup_switchers( switchers_path = html_root / "_static" / "switchers.js" switchers_path.write_text( template.safe_substitute( - { - "LANGUAGES": json.dumps(languages_map), - "VERSIONS": json.dumps(versions_map), - } + LANGUAGES=json.dumps(languages_map), + VERSIONS=json.dumps(versions_map), ), encoding="UTF-8", ) From 04cc37b17d86a868e7109554ef3b28b061b8594f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:41:44 +0100 Subject: [PATCH 082/218] Add the check-times script (#196) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .pre-commit-config.yaml | 5 +++ build_docs.py | 2 +- check_times.py | 87 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 check_times.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dba593f..c3d2f5f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,6 +13,11 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.7 + hooks: + - id: ruff-format + - repo: https://github.com/python-jsonschema/check-jsonschema rev: 0.29.2 hooks: diff --git a/build_docs.py b/build_docs.py index 2ebc312..8f5f382 100755 --- a/build_docs.py +++ b/build_docs.py @@ -721,7 +721,7 @@ def build(self): # Disable CPython switchers, we handle them now: def is_mac(): - return platform.system() == 'Darwin' + return platform.system() == "Darwin" run( ["sed", "-i"] diff --git a/check_times.py b/check_times.py new file mode 100644 index 0000000..4b6665d --- /dev/null +++ b/check_times.py @@ -0,0 +1,87 @@ +"""Check the frequency of the rebuild loop. + +This must be run in a directory that has the ``docsbuild.log*`` files. +For example: + +.. code-block:: bash + + $ scp "adam@docs.nyc1.psf.io:/var/log/docsbuild/docsbuild.log*" docsbuild-logs + $ python check_times.py +""" + +import datetime as dt +import gzip +from pathlib import Path + +from build_docs import format_seconds + + +def get_lines() -> list[str]: + lines = [] + zipped_logs = list(Path.cwd().glob("docsbuild.log.*.gz")) + zipped_logs.sort(key=lambda p: int(p.name.split(".")[-2]), reverse=True) + for logfile in zipped_logs: + with gzip.open(logfile, "rt", encoding="utf-8") as f: + lines += f.readlines() + with open("docsbuild.log", encoding="utf-8") as f: + lines += f.readlines() + return lines + + +def calc_time(lines: list[str]) -> None: + start = end = language = version = start_timestamp = None + reason = lang_ver = "" + + print("Start | Version | Language | Build | Trigger") + print(":-- | :--: | :--: | --: | :--:") + + for line in lines: + line = line.strip() + + if ": Should rebuild: " in line: + if "no previous state found" in line: + reason = "brand new" + elif "new translations" in line: + reason = "translation" + elif "Doc/ has changed" in line: + reason = "docs" + else: + reason = "" + lang_ver = line.split(" ")[3].removesuffix(":") + + if line.endswith("Build start."): + timestamp = line[:23].replace(",", ".") + language, version = line.split(" ")[3].removesuffix(":").split("/") + start = dt.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") + start_timestamp = f"{line[:16]} UTC" + + if start and ": Build done " in line: + timestamp = line[:23].replace(",", ".") + language, version = line.split(" ")[3].removesuffix(":").split("/") + end = dt.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") + + if start and end: + duration = (end - start).total_seconds() + fmt_duration = format_seconds(duration) + if lang_ver != f"{language}/{version}": + reason = "" + print( + f"{start_timestamp: <20} | {version: <7} | {language: <8} | {fmt_duration :<14} | {reason}" + ) + start = end = start_timestamp = None + + if ": Full build done" in line: + timestamp = f"{line[:16]} UTC" + _, fmt_duration = line.removesuffix(").").split("(") + print( + f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration :<14} | -----------" + ) + + if start and end is None: + print( + f"{start_timestamp: <20} | {version: <7} | {language: <8} | In progress... | {reason}" + ) + + +if __name__ == "__main__": + calc_time(get_lines()) From 8c93ad15bf75ffee72ff22c0f62e7e0727a7bb46 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 23 Sep 2024 21:33:12 +0100 Subject: [PATCH 083/218] Use pathlib more (#197) --- build_docs.py | 80 ++++++++++++++++++++++----------------------------- 1 file changed, 35 insertions(+), 45 deletions(-) diff --git a/build_docs.py b/build_docs.py index 8f5f382..255f45d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -20,6 +20,8 @@ """ +from __future__ import annotations + from argparse import ArgumentParser from collections.abc import Sequence from contextlib import suppress, contextmanager @@ -171,19 +173,15 @@ def picker_label(self): return f"pre ({self.name})" return self.name - def setup_indexsidebar(self, versions, dest_path): + def setup_indexsidebar(self, versions: Sequence[Version], dest_path: Path): """Build indexsidebar.html for Sphinx.""" - with open( - HERE / "templates" / "indexsidebar.html", encoding="UTF-8" - ) as sidebar_template_file: - sidebar_template = jinja2.Template(sidebar_template_file.read()) - with open(dest_path, "w", encoding="UTF-8") as sidebar_file: - sidebar_file.write( - sidebar_template.render( - current_version=self, - versions=versions[::-1], - ) - ) + template_path = HERE / "templates" / "indexsidebar.html" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render( + current_version=self, + versions=versions[::-1], + ) + dest_path.write_text(rendered_template, encoding="UTF-8") @classmethod def from_json(cls, name, values): @@ -374,19 +372,17 @@ def setup_switchers( languages_map = dict(sorted((l.tag, l.name) for l in languages if l.in_prod)) versions_map = {v.name: v.picker_label for v in reversed(versions)} - with open( - HERE / "templates" / "switchers.js", encoding="UTF-8" - ) as switchers_template_file: - template = Template(switchers_template_file.read()) + switchers_template_file = HERE / "templates" / "switchers.js" switchers_path = html_root / "_static" / "switchers.js" - switchers_path.write_text( - template.safe_substitute( - LANGUAGES=json.dumps(languages_map), - VERSIONS=json.dumps(versions_map), - ), - encoding="UTF-8", + + template = Template(switchers_template_file.read_text(encoding="UTF-8")) + rendered_template = template.safe_substitute( + LANGUAGES=json.dumps(languages_map), + VERSIONS=json.dumps(versions_map), ) - for file in Path(html_root).glob("**/*.html"): + switchers_path.write_text(rendered_template, encoding="UTF-8") + + for file in html_root.glob("**/*.html"): depth = len(file.relative_to(html_root).parts) - 1 src = f"{'../' * depth}_static/switchers.js" script = f' \n' @@ -411,15 +407,13 @@ def build_robots_txt( if not www_root.exists(): logging.info("Skipping robots.txt generation (www root does not even exist).") return - robots_file = www_root / "robots.txt" - with open(HERE / "templates" / "robots.txt", encoding="UTF-8") as template_file: - template = jinja2.Template(template_file.read()) - with open(robots_file, "w", encoding="UTF-8") as robots_txt_file: - robots_txt_file.write( - template.render(languages=languages, versions=versions) + "\n" - ) - robots_file.chmod(0o775) - run(["chgrp", group, robots_file]) + template_path = HERE / "templates" / "robots.txt" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render(languages=languages, versions=versions) + robots_path = www_root / "robots.txt" + robots_path.write_text(rendered_template + "\n", encoding="UTF-8") + robots_path.chmod(0o775) + run(["chgrp", group, robots_path]) if not skip_cache_invalidation: purge(http, "robots.txt") @@ -431,14 +425,13 @@ def build_sitemap( if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exist).") return - with open(HERE / "templates" / "sitemap.xml", encoding="UTF-8") as template_file: - template = jinja2.Template(template_file.read()) - sitemap_file = www_root / "sitemap.xml" - sitemap_file.write_text( - template.render(languages=languages, versions=versions) + "\n", encoding="UTF-8" - ) - sitemap_file.chmod(0o664) - run(["chgrp", group, sitemap_file]) + template_path = HERE / "templates" / "sitemap.xml" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render(languages=languages, versions=versions) + sitemap_path = www_root / "sitemap.xml" + sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") + sitemap_path.chmod(0o664) + run(["chgrp", group, sitemap_path]) def build_404(www_root: Path, group): @@ -867,10 +860,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: [ "cp", "-a", - *[ - str(dist) - for dist in (Path(self.checkout) / "Doc" / "dist").glob("*") - ], + *(self.checkout / "Doc" / "dist").glob("*"), target / "archives", ] ) @@ -972,7 +962,7 @@ def symlink( directory_path = path / directory if not directory_path.exists(): return # No touching link, dest doc not built yet. - if link.exists() and readlink(str(link)) == directory: + if link.exists() and readlink(link) == directory: return # Link is already pointing to right doc. if link.exists(): link.unlink() From 50e99b54038464b03a92a39ab026910bb3fb48b1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 25 Sep 2024 07:05:11 +0100 Subject: [PATCH 084/218] Fix directories for suggested use pattern --- check_times.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/check_times.py b/check_times.py index 4b6665d..f9ba49a 100644 --- a/check_times.py +++ b/check_times.py @@ -5,7 +5,8 @@ .. code-block:: bash - $ scp "adam@docs.nyc1.psf.io:/var/log/docsbuild/docsbuild.log*" docsbuild-logs + $ mkdir -p docsbuild-logs + $ scp "adam@docs.nyc1.psf.io:/var/log/docsbuild/docsbuild.log*" docsbuild-logs/ $ python check_times.py """ @@ -15,15 +16,17 @@ from build_docs import format_seconds +LOGS_ROOT = Path('docsbuild-logs').resolve() + def get_lines() -> list[str]: lines = [] - zipped_logs = list(Path.cwd().glob("docsbuild.log.*.gz")) + zipped_logs = list(LOGS_ROOT.glob("docsbuild.log.*.gz")) zipped_logs.sort(key=lambda p: int(p.name.split(".")[-2]), reverse=True) for logfile in zipped_logs: with gzip.open(logfile, "rt", encoding="utf-8") as f: lines += f.readlines() - with open("docsbuild.log", encoding="utf-8") as f: + with open(LOGS_ROOT / "docsbuild.log", encoding="utf-8") as f: lines += f.readlines() return lines From bce1bed404a59e4e1cefe2dbd428294e359c0edf Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 25 Sep 2024 07:11:21 +0100 Subject: [PATCH 085/218] fixup! Fix directories for suggested use pattern --- check_times.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_times.py b/check_times.py index f9ba49a..3b3e53c 100644 --- a/check_times.py +++ b/check_times.py @@ -16,7 +16,7 @@ from build_docs import format_seconds -LOGS_ROOT = Path('docsbuild-logs').resolve() +LOGS_ROOT = Path("docsbuild-logs").resolve() def get_lines() -> list[str]: From b87d94f3ee6690832015079aec7a1ddc0c92ac5e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:06:52 +0100 Subject: [PATCH 086/218] Log updates to ``state.toml`` --- build_docs.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/build_docs.py b/build_docs.py index 255f45d..67bde35 100755 --- a/build_docs.py +++ b/build_docs.py @@ -610,6 +610,7 @@ def full_build(self): def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() + start_timestamp = dt.now(tz=timezone.utc).replace(microsecond=0) logging.info("Running.") try: self.cpython_repo.switch(self.version.branch_or_tag) @@ -619,7 +620,10 @@ def run(self, http: urllib3.PoolManager) -> bool: self.build_venv() self.build() self.copy_build_to_webroot(http) - self.save_state(build_duration=perf_counter() - start_time) + self.save_state( + build_start=start_timestamp, + build_duration=perf_counter() - start_time, + ) except Exception as err: logging.exception("Badly handled exception, human, please help.") if sentry_sdk: @@ -921,7 +925,7 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_duration: float): + def save_state(self, build_start: dt, build_duration: float): """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -932,17 +936,23 @@ def save_state(self, build_duration: float): except FileNotFoundError: states = tomlkit.document() - state = {} - state["cpython_sha"] = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() + key = f"/{self.language.tag}/{self.version.name}/" + state = { + "last_build_start": build_start, + "last_build_duration": round(build_duration, 0), + "cpython_sha": self.cpython_repo.run("rev-parse", "HEAD").stdout.strip(), + } if self.language.tag != "en": state["translation_sha"] = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() - state["last_build"] = dt.now(timezone.utc) - state["last_build_duration"] = build_duration - states[f"/{self.language.tag}/{self.version.name}/"] = state + states[key] = state state_file.write_text(tomlkit.dumps(states), encoding="UTF-8") + tbl = tomlkit.inline_table() + tbl |= state + logging.info("Saved new rebuild state for %s: %s", key, tbl.as_string()) + def symlink( www_root: Path, From 33c527278f953ff60348ab8fb16e4415edaae8f1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 25 Sep 2024 22:13:57 +0100 Subject: [PATCH 087/218] Update build_docs.py Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- build_docs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 67bde35..5e9eb6b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -949,9 +949,9 @@ def save_state(self, build_start: dt, build_duration: float): states[key] = state state_file.write_text(tomlkit.dumps(states), encoding="UTF-8") - tbl = tomlkit.inline_table() - tbl |= state - logging.info("Saved new rebuild state for %s: %s", key, tbl.as_string()) + table = tomlkit.inline_table() + table |= state + logging.info("Saved new rebuild state for %s: %s", key, table.as_string()) def symlink( From 094e52f3cf51e7c97f84b8cf9cac738334108bca Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 26 Sep 2024 17:40:27 +0100 Subject: [PATCH 088/218] Include trigger reason --- build_docs.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/build_docs.py b/build_docs.py index 5e9eb6b..58836e4 100755 --- a/build_docs.py +++ b/build_docs.py @@ -616,13 +616,14 @@ def run(self, http: urllib3.PoolManager) -> bool: self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() - if self.should_rebuild(): + if trigger_reason := self.should_rebuild(): self.build_venv() self.build() self.copy_build_to_webroot(http) self.save_state( build_start=start_timestamp, build_duration=perf_counter() - start_time, + trigger=trigger_reason, ) except Exception as err: logging.exception("Badly handled exception, human, please help.") @@ -889,7 +890,7 @@ def should_rebuild(self): state = self.load_state() if not state: logging.info("Should rebuild: no previous state found.") - return True + return "no previous state" cpython_sha = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() if self.language.tag != "en": translation_sha = self.translation_repo.run( @@ -901,7 +902,7 @@ def should_rebuild(self): state["translation_sha"], translation_sha, ) - return True + return "new translations" if cpython_sha != state["cpython_sha"]: diff = self.cpython_repo.run( "diff", "--name-only", state["cpython_sha"], cpython_sha @@ -912,7 +913,7 @@ def should_rebuild(self): state["cpython_sha"], cpython_sha, ) - return True + return "Doc/ has changed" logging.info("Nothing changed, no rebuild needed.") return False @@ -925,7 +926,7 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_start: dt, build_duration: float): + def save_state(self, build_start: dt, build_duration: float, trigger: str): """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -940,6 +941,7 @@ def save_state(self, build_start: dt, build_duration: float): state = { "last_build_start": build_start, "last_build_duration": round(build_duration, 0), + "triggered_by": trigger, "cpython_sha": self.cpython_repo.run("rev-parse", "HEAD").stdout.strip(), } if self.language.tag != "en": From 5d26f06479cb78912fdbf69a538068aad5f0fd79 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 26 Sep 2024 23:08:04 +0100 Subject: [PATCH 089/218] Add ``--select-output`` (#199) --- .gitignore | 2 ++ build_docs.py | 98 +++++++++++++++++++++++++++------------------------ 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index aaefb08..c257236 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ /www/ # temporary lock file created while building the docs build_docs.lock +build_docs_archives.lock +build_docs_html.lock # Created by https://www.gitignore.io/api/python diff --git a/build_docs.py b/build_docs.py index 58836e4..0805139 100755 --- a/build_docs.py +++ b/build_docs.py @@ -22,7 +22,7 @@ from __future__ import annotations -from argparse import ArgumentParser +from argparse import ArgumentParser, Namespace from collections.abc import Sequence from contextlib import suppress, contextmanager from dataclasses import dataclass @@ -44,7 +44,7 @@ from string import Template from textwrap import indent from time import perf_counter, sleep -from typing import Iterable +from typing import Iterable, Literal from urllib.parse import urljoin import jinja2 @@ -487,11 +487,16 @@ def parse_args(): parser = ArgumentParser( description="Runs a build of the Python docs for various branches." ) + parser.add_argument( + "--select-output", + choices=("no-html", "only-html"), + help="Choose what outputs to build.", + ) parser.add_argument( "-q", "--quick", action="store_true", - help="Make HTML files only (Makefile rules suffixed with -html).", + help="Run a quick build (only HTML files).", ) parser.add_argument( "-b", @@ -589,6 +594,7 @@ class DocBuilder: cpython_repo: Repository build_root: Path www_root: Path + select_output: Literal["no-html", "only-html"] | None quick: bool group: str log_directory: Path @@ -596,16 +602,10 @@ class DocBuilder: theme: Path @property - def full_build(self): - """Tell if a full build is needed. - - A full build is slow; it builds pdf, txt, epub, texinfo, and - archives everything. - - A partial build only builds HTML and does not archive, it's - fast. - """ - return not self.quick and not self.language.html_only + def html_only(self): + return ( + self.select_output == "only-html" or self.quick or self.language.html_only + ) def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" @@ -635,7 +635,7 @@ def run(self, http: urllib3.PoolManager) -> bool: @property def checkout(self) -> Path: """Path to CPython git clone.""" - return self.build_root / "cpython" + return self.build_root / _checkout_name(self.select_output) def clone_translation(self): self.translation_repo.update() @@ -703,15 +703,13 @@ def build(self): if self.version.status == "EOL": sphinxopts.append("-D html_context.outdated=1") - maketarget = ( - "autobuild-" - + ( - "dev" - if self.version.status in ("in development", "pre-release") - else "stable" - ) - + ("" if self.full_build else "-html") - ) + + if self.version.status in ("in development", "pre-release"): + maketarget = "autobuild-dev" + else: + maketarget = "autobuild-stable" + if self.html_only: + maketarget += "-html" logging.info("Running make %s", maketarget) python = self.venv / "bin" / "python" sphinxbuild = self.venv / "bin" / "sphinx-build" @@ -820,28 +818,18 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: ";", ] ) - if self.full_build: - run( - [ - "rsync", - "-a", - "--delete-delay", - "--filter", - "P archives/", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - ] - ) - else: - run( - [ - "rsync", - "-a", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - ] - ) - if self.full_build: + run( + [ + "rsync", + "-a", + "--delete-delay", + "--filter", + "P archives/", + str(self.checkout / "Doc" / "build" / "html") + "/", + target, + ] + ) + if not self.quick: logging.debug("Copying dist files.") run( [ @@ -1153,7 +1141,8 @@ def build_docs(args) -> bool: del args.languages all_built_successfully = True cpython_repo = Repository( - "https://github.com/python/cpython.git", args.build_root / "cpython" + "https://github.com/python/cpython.git", + args.build_root / _checkout_name(args.select_output), ) while todo: version, language = todo.pop() @@ -1208,13 +1197,28 @@ def build_docs(args) -> bool: return all_built_successfully +def _checkout_name(select_output: str | None) -> str: + if select_output is not None: + return f"cpython-{select_output}" + return "cpython" + + def main(): """Script entry point.""" args = parse_args() setup_logging(args.log_directory) + if args.select_output is None: + build_docs_with_lock(args, "build_docs.lock") + elif args.select_output == "no-html": + build_docs_with_lock(args, "build_docs_archives.lock") + elif args.select_output == "only-html": + build_docs_with_lock(args, "build_docs_html.lock") + + +def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: try: - lock = zc.lockfile.LockFile(HERE / "build_docs.lock") + lock = zc.lockfile.LockFile(HERE / lockfile_name) except zc.lockfile.LockError: logging.info("Another builder is running... dying...") return EX_FAILURE From a884721350970f08edfce348da2ba4c7d4ae55bc Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:35:55 +0100 Subject: [PATCH 090/218] Set the logging filename based upon ``--select-output`` --- build_docs.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0805139..17e4f66 100755 --- a/build_docs.py +++ b/build_docs.py @@ -567,18 +567,19 @@ def parse_args(): return args -def setup_logging(log_directory: Path): +def setup_logging(log_directory: Path, select_output: str | None): """Setup logging to stderr if run by a human, or to a file if run from a cron.""" + log_format = "%(asctime)s %(levelname)s: %(message)s" if sys.stderr.isatty(): - logging.basicConfig( - format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stderr - ) + logging.basicConfig(format=log_format, stream=sys.stderr) else: log_directory.mkdir(parents=True, exist_ok=True) - handler = logging.handlers.WatchedFileHandler(log_directory / "docsbuild.log") - handler.setFormatter( - logging.Formatter("%(asctime)s %(levelname)s: %(message)s") - ) + if select_output is None: + filename = log_directory / "docsbuild.log" + else: + filename = log_directory / f"docsbuild-{select_output}.log" + handler = logging.handlers.WatchedFileHandler(filename) + handler.setFormatter(logging.Formatter(log_format)) logging.getLogger().addHandler(handler) logging.getLogger().setLevel(logging.DEBUG) @@ -1206,7 +1207,7 @@ def _checkout_name(select_output: str | None) -> str: def main(): """Script entry point.""" args = parse_args() - setup_logging(args.log_directory) + setup_logging(args.log_directory, args.select_output) if args.select_output is None: build_docs_with_lock(args, "build_docs.lock") From c422c3c9714dcb5170c34c6789482d33e96a5838 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:26:39 +0100 Subject: [PATCH 091/218] Log the output of the sphinx ``make`` command --- build_docs.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0805139..3f18a62 100755 --- a/build_docs.py +++ b/build_docs.py @@ -220,7 +220,7 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" cmd = [str(arg) for arg in cmd] cmdstring = shlex.join(cmd) - logging.debug("Run: %r", cmdstring) + logging.debug("Run: '%s'", cmdstring) result = subprocess.run( cmd, cwd=cwd, @@ -242,6 +242,21 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: return result +def run_with_logging(cmd, cwd=None): + """Like subprocess.check_call, with logging before the command execution.""" + cmd = list(map(str, cmd)) + logging.debug("Run: %s", shlex.join(cmd)) + with subprocess.Popen(cmd, cwd=cwd, encoding="utf-8") as p: + try: + for line in p.stdout: + logging.debug(">>>> %s", line.rstrip()) + except: + p.kill() + raise + if return_code := p.poll(): + raise subprocess.CalledProcessError(return_code, cmd[0]) + + def changed_files(left, right): """Compute a list of different files between left and right, recursively. Resulting paths are relative to left. @@ -728,7 +743,7 @@ def is_mac(): self.versions, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", ) - run( + run_with_logging( [ "make", "-C", From 3bda3f4d0f7d073811e63dc97906af6d2ef16c01 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:32:56 +0100 Subject: [PATCH 092/218] Require tomlkit>=0.13 (#204) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e208755..0cac810 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ jinja2 sentry-sdk>=2 -tomlkit +tomlkit>=0.13 urllib3>=2 zc.lockfile From 3b9d1facdba73b32b1140267bcd85c3b6e345e01 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 28 Sep 2024 14:09:14 +0100 Subject: [PATCH 093/218] Fix ``TypeError: 'NoneType' object is not iterable`` --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 861eace..a6e82ff 100755 --- a/build_docs.py +++ b/build_docs.py @@ -248,7 +248,7 @@ def run_with_logging(cmd, cwd=None): logging.debug("Run: %s", shlex.join(cmd)) with subprocess.Popen(cmd, cwd=cwd, encoding="utf-8") as p: try: - for line in p.stdout: + for line in (p.stdout or ()): logging.debug(">>>> %s", line.rstrip()) except: p.kill() From 5f97ccaab09db6d6d19ab26699834462fd3dc619 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 28 Sep 2024 14:20:24 +0100 Subject: [PATCH 094/218] Fix ``run_with_logging()``: add missing ``subprocess.PIPE`` --- build_docs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index a6e82ff..c5b13f9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -246,7 +246,14 @@ def run_with_logging(cmd, cwd=None): """Like subprocess.check_call, with logging before the command execution.""" cmd = list(map(str, cmd)) logging.debug("Run: %s", shlex.join(cmd)) - with subprocess.Popen(cmd, cwd=cwd, encoding="utf-8") as p: + with subprocess.Popen( + cmd, + cwd=cwd, + stdin=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + encoding="utf-8", + ) as p: try: for line in (p.stdout or ()): logging.debug(">>>> %s", line.rstrip()) From 992d2df9fdd7fc0c73f7b74a8dcebcd5bf26e6c1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 28 Sep 2024 14:22:53 +0100 Subject: [PATCH 095/218] Fix lint failure (ruff format) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index c5b13f9..12c1319 100755 --- a/build_docs.py +++ b/build_docs.py @@ -255,7 +255,7 @@ def run_with_logging(cmd, cwd=None): encoding="utf-8", ) as p: try: - for line in (p.stdout or ()): + for line in p.stdout or (): logging.debug(">>>> %s", line.rstrip()) except: p.kill() From caa4a29fd0d2cbd837e907d2686faf3debeabe73 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:14:36 +0100 Subject: [PATCH 096/218] Only run HTML-specific tasks in builds that output HTML (#207) --- build_docs.py | 112 +++++++++++++++++++++++++++----------------------- 1 file changed, 60 insertions(+), 52 deletions(-) diff --git a/build_docs.py b/build_docs.py index 12c1319..033ef0f 100755 --- a/build_docs.py +++ b/build_docs.py @@ -32,7 +32,6 @@ import logging.handlers from functools import total_ordering from os import readlink -import platform import re import shlex import shutil @@ -630,6 +629,11 @@ def html_only(self): self.select_output == "only-html" or self.quick or self.language.html_only ) + @property + def includes_html(self): + """Does the build we are running include HTML output?""" + return self.select_output != "no-html" + def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() @@ -737,20 +741,18 @@ def build(self): python = self.venv / "bin" / "python" sphinxbuild = self.venv / "bin" / "sphinx-build" blurb = self.venv / "bin" / "blurb" - # Disable CPython switchers, we handle them now: - - def is_mac(): - return platform.system() == "Darwin" - run( - ["sed", "-i"] - + ([""] if is_mac() else []) - + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] - ) - self.version.setup_indexsidebar( - self.versions, - self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", - ) + if self.includes_html: + # Disable CPython switchers, we handle them now: + run( + ["sed", "-i"] + + ([""] if sys.platform == "darwin" else []) + + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] + ) + self.version.setup_indexsidebar( + self.versions, + self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", + ) run_with_logging( [ "make", @@ -767,9 +769,10 @@ def is_mac(): ) run(["mkdir", "-p", self.log_directory]) run(["chgrp", "-R", self.group, self.log_directory]) - setup_switchers( - self.versions, self.languages, self.checkout / "Doc" / "build" / "html" - ) + if self.includes_html: + setup_switchers( + self.versions, self.languages, self.checkout / "Doc" / "build" / "html" + ) logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) def build_venv(self): @@ -817,42 +820,47 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: except subprocess.CalledProcessError as err: logging.warning("Can't change group of %s: %s", target, str(err)) - changed = changed_files(self.checkout / "Doc" / "build" / "html", target) - logging.info("Copying HTML files to %s", target) - run( - [ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "build" / "html/", - ] - ) - run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) - run( - [ - "find", - self.checkout / "Doc" / "build" / "html", - "-type", - "d", - "-exec", - "chmod", - "o+x", - "{}", - ";", - ] - ) - run( - [ - "rsync", - "-a", - "--delete-delay", - "--filter", - "P archives/", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - ] - ) + changed = [] + if self.includes_html: + # Copy built HTML files to webroot (default /srv/docs.python.org) + changed = changed_files(self.checkout / "Doc" / "build" / "html", target) + logging.info("Copying HTML files to %s", target) + run( + [ + "chown", + "-R", + ":" + self.group, + self.checkout / "Doc" / "build" / "html/", + ] + ) + run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) + run( + [ + "find", + self.checkout / "Doc" / "build" / "html", + "-type", + "d", + "-exec", + "chmod", + "o+x", + "{}", + ";", + ] + ) + run( + [ + "rsync", + "-a", + "--delete-delay", + "--filter", + "P archives/", + str(self.checkout / "Doc" / "build" / "html") + "/", + target, + ] + ) + if not self.quick: + # Copy archive files to /archives/ logging.debug("Copying dist files.") run( [ From be14d9554893b106e852cebda5e0149d89445201 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:25:29 +0100 Subject: [PATCH 097/218] Set the ``state.toml`` filename based upon ``--select-output`` (#206) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- build_docs.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 033ef0f..51bfe42 100755 --- a/build_docs.py +++ b/build_docs.py @@ -937,7 +937,10 @@ def should_rebuild(self): return False def load_state(self) -> dict: - state_file = self.build_root / "state.toml" + if self.select_output is not None: + state_file = self.build_root / f"state-{self.select_output}.toml" + else: + state_file = self.build_root / "state.toml" try: return tomlkit.loads(state_file.read_text(encoding="UTF-8"))[ f"/{self.language.tag}/{self.version.name}/" @@ -950,7 +953,10 @@ def save_state(self, build_start: dt, build_duration: float, trigger: str): Using this we can deduce if a rebuild is needed or not. """ - state_file = self.build_root / "state.toml" + if self.select_output is not None: + state_file = self.build_root / f"state-{self.select_output}.toml" + else: + state_file = self.build_root / "state.toml" try: states = tomlkit.parse(state_file.read_text(encoding="UTF-8")) except FileNotFoundError: From caee2be0bf028abe1644c49a9997f5814aa56957 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:26:47 +0100 Subject: [PATCH 098/218] Use saved state information in ``check_times.py`` (#205) --- check_times.py | 84 +++++++++++++++++++++++--------------------------- 1 file changed, 38 insertions(+), 46 deletions(-) diff --git a/check_times.py b/check_times.py index 3b3e53c..07a1af6 100644 --- a/check_times.py +++ b/check_times.py @@ -1,17 +1,17 @@ """Check the frequency of the rebuild loop. -This must be run in a directory that has the ``docsbuild.log*`` files. +This must be run in a directory that has the ``docsbuild*`` log files. For example: .. code-block:: bash $ mkdir -p docsbuild-logs - $ scp "adam@docs.nyc1.psf.io:/var/log/docsbuild/docsbuild.log*" docsbuild-logs/ + $ scp "adam@docs.nyc1.psf.io:/var/log/docsbuild/docsbuild*" docsbuild-logs/ $ python check_times.py """ -import datetime as dt import gzip +import tomllib from pathlib import Path from build_docs import format_seconds @@ -19,21 +19,21 @@ LOGS_ROOT = Path("docsbuild-logs").resolve() -def get_lines() -> list[str]: +def get_lines(filename: str = "docsbuild.log") -> list[str]: lines = [] - zipped_logs = list(LOGS_ROOT.glob("docsbuild.log.*.gz")) + zipped_logs = list(LOGS_ROOT.glob(f"{filename}.*.gz")) zipped_logs.sort(key=lambda p: int(p.name.split(".")[-2]), reverse=True) for logfile in zipped_logs: with gzip.open(logfile, "rt", encoding="utf-8") as f: lines += f.readlines() - with open(LOGS_ROOT / "docsbuild.log", encoding="utf-8") as f: + with open(LOGS_ROOT / filename, encoding="utf-8") as f: lines += f.readlines() return lines def calc_time(lines: list[str]) -> None: - start = end = language = version = start_timestamp = None - reason = lang_ver = "" + in_progress = False + in_progress_line = "" print("Start | Version | Language | Build | Trigger") print(":-- | :--: | :--: | --: | :--:") @@ -41,50 +41,42 @@ def calc_time(lines: list[str]) -> None: for line in lines: line = line.strip() - if ": Should rebuild: " in line: - if "no previous state found" in line: - reason = "brand new" - elif "new translations" in line: - reason = "translation" - elif "Doc/ has changed" in line: - reason = "docs" - else: - reason = "" - lang_ver = line.split(" ")[3].removesuffix(":") - - if line.endswith("Build start."): - timestamp = line[:23].replace(",", ".") - language, version = line.split(" ")[3].removesuffix(":").split("/") - start = dt.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") - start_timestamp = f"{line[:16]} UTC" - - if start and ": Build done " in line: - timestamp = line[:23].replace(",", ".") - language, version = line.split(" ")[3].removesuffix(":").split("/") - end = dt.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f") - - if start and end: - duration = (end - start).total_seconds() - fmt_duration = format_seconds(duration) - if lang_ver != f"{language}/{version}": - reason = "" + if "Saved new rebuild state for" in line: + _, state = line.split("Saved new rebuild state for", 1) + key, state_toml = state.strip().split(": ", 1) + language, version = key.strip("/").split("/", 1) + state_data = tomllib.loads(f"t = {state_toml}")["t"] + start = state_data["last_build_start"] + fmt_duration = format_seconds(state_data["last_build_duration"]) + reason = state_data["triggered_by"] print( - f"{start_timestamp: <20} | {version: <7} | {language: <8} | {fmt_duration :<14} | {reason}" + f"{start:%Y-%m-%d %H:%M UTC} | {version: <7} | {language: <8} | {fmt_duration :<14} | {reason}" ) - start = end = start_timestamp = None - if ": Full build done" in line: - timestamp = f"{line[:16]} UTC" - _, fmt_duration = line.removesuffix(").").split("(") - print( - f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration :<14} | -----------" - ) + if line.endswith("Build start."): + in_progress = True + in_progress_line = line + + if in_progress and ": Build done " in line: + in_progress = False - if start and end is None: + if in_progress: + start_timestamp = f"{in_progress_line[:16]} UTC" + language, version = in_progress_line.split(" ")[3].removesuffix(":").split("/") print( - f"{start_timestamp: <20} | {version: <7} | {language: <8} | In progress... | {reason}" + f"{start_timestamp: <20} | {version: <7} | {language: <8} | In progress... | ..." ) + print() + if __name__ == "__main__": - calc_time(get_lines()) + print("Build times (HTML only)") + print("=======================") + print() + calc_time(get_lines("docsbuild-only-html.log")) + + print("Build times (no HTML)") + print("=====================") + print() + calc_time(get_lines("docsbuild-no-html.log")) From fb5c33d3bae3952e12a45b7f19113352514a231c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:30:52 +0100 Subject: [PATCH 099/218] Skip non-HTML builds for HTML-only languages (#208) --- build_docs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_docs.py b/build_docs.py index 51bfe42..e7b8129 100755 --- a/build_docs.py +++ b/build_docs.py @@ -640,6 +640,9 @@ def run(self, http: urllib3.PoolManager) -> bool: start_timestamp = dt.now(tz=timezone.utc).replace(microsecond=0) logging.info("Running.") try: + if self.language.html_only and not self.includes_html: + logging.info("Skipping non-HTML build (language is HTML-only).") + return True self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() From 38645de81da2d368744400bb78650db3012d6cb9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 2 Oct 2024 22:56:55 +0100 Subject: [PATCH 100/218] Include full build duration (#210) Inadvertently removed. --- check_times.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/check_times.py b/check_times.py index 07a1af6..e561b09 100644 --- a/check_times.py +++ b/check_times.py @@ -60,6 +60,13 @@ def calc_time(lines: list[str]) -> None: if in_progress and ": Build done " in line: in_progress = False + if ": Full build done" in line: + timestamp = f"{line[:16]} UTC" + _, fmt_duration = line.removesuffix(").").split("(") + print( + f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration :<14} | -----------" + ) + if in_progress: start_timestamp = f"{in_progress_line[:16]} UTC" language, version = in_progress_line.split(" ")[3].removesuffix(":").split("/") From a1c7fb8395538eb850ccf018de47c86486f05b3e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 3 Oct 2024 00:12:51 +0100 Subject: [PATCH 101/218] Rebuild on changes to Misc/NEWS.d/ (#211) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index e7b8129..5d3ed4d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -929,7 +929,7 @@ def should_rebuild(self): diff = self.cpython_repo.run( "diff", "--name-only", state["cpython_sha"], cpython_sha ).stdout - if "Doc/" in diff: + if "Doc/" in diff or "Misc/NEWS.d/" in diff: logging.info( "Should rebuild: Doc/ has changed (from %s to %s)", state["cpython_sha"], From 566ca52820fc47b4dd4eb2ba28ecde449a488bec Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 3 Oct 2024 00:13:25 +0100 Subject: [PATCH 102/218] Run ``sphinx-build`` with the default verbosity level (#212) This lets us see which steps are taking more time in sphinx. Quiet mode was originally added in 8d82617d78e25448b379003a811629b7329209b1. --- build_docs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 5d3ed4d..27fee1e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -704,7 +704,6 @@ def build(self): logging.info("Build start.") start_time = perf_counter() sphinxopts = list(self.language.sphinxopts) - sphinxopts.extend(["-q"]) if self.language.tag != "en": locale_dirs = self.build_root / self.version.name / "locale" sphinxopts.extend( From 3a11a361dd38134a9d5c995a1c9a0b415c831e5d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 6 Oct 2024 16:35:38 +0100 Subject: [PATCH 103/218] Harmonise run() and run_with_logging() (#213) --- build_docs.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/build_docs.py b/build_docs.py index 27fee1e..c7122af 100755 --- a/build_docs.py +++ b/build_docs.py @@ -41,7 +41,6 @@ from datetime import datetime as dt, timezone from pathlib import Path from string import Template -from textwrap import indent from time import perf_counter, sleep from typing import Iterable, Literal from urllib.parse import urljoin @@ -217,7 +216,7 @@ def filter(languages, language_tags=None): def run(cmd, cwd=None) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" - cmd = [str(arg) for arg in cmd] + cmd = list(map(str, cmd)) cmdstring = shlex.join(cmd) logging.debug("Run: '%s'", cmdstring) result = subprocess.run( @@ -233,9 +232,9 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: if result.returncode: # Log last 20 lines, those are likely the interesting ones. logging.error( - "Run: %r KO:\n%s", + "Run: '%s' KO:\n%s", cmdstring, - indent("\n".join(result.stdout.split("\n")[-20:]), " "), + "\n".join(f" {line}" for line in result.stdout.split("\n")[-20:]), ) result.check_returncode() return result @@ -244,7 +243,7 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: def run_with_logging(cmd, cwd=None): """Like subprocess.check_call, with logging before the command execution.""" cmd = list(map(str, cmd)) - logging.debug("Run: %s", shlex.join(cmd)) + logging.debug("Run: '%s'", shlex.join(cmd)) with subprocess.Popen( cmd, cwd=cwd, @@ -255,7 +254,7 @@ def run_with_logging(cmd, cwd=None): ) as p: try: for line in p.stdout or (): - logging.debug(">>>> %s", line.rstrip()) + logging.debug(">>>> %s", line.rstrip()) except: p.kill() raise From dcb1aa07594eb6600cb40fb7536351caa8ac3403 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:57:33 +0100 Subject: [PATCH 104/218] Add 3.8 to robots.txt (#214) --- templates/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/robots.txt b/templates/robots.txt index 635bfdc..a7de4a6 100644 --- a/templates/robots.txt +++ b/templates/robots.txt @@ -23,3 +23,4 @@ Disallow: /3.4/ Disallow: /3.5/ Disallow: /3.6/ Disallow: /3.7/ +Disallow: /3.8/ From 65c2c305b0d6863f871bb954814cf061ca5855ec Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 9 Oct 2024 21:43:37 +0100 Subject: [PATCH 105/218] Simplify copying robots.txt (#215) --- build_docs.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/build_docs.py b/build_docs.py index c7122af..6541070 100755 --- a/build_docs.py +++ b/build_docs.py @@ -415,23 +415,19 @@ def setup_switchers( ofile.write(line) -def build_robots_txt( - versions: Iterable[Version], - languages: Iterable[Language], +def copy_robots_txt( www_root: Path, group, skip_cache_invalidation, http: urllib3.PoolManager, ) -> None: - """Disallow crawl of EOL versions in robots.txt.""" + """Copy robots.txt to www_root.""" if not www_root.exists(): - logging.info("Skipping robots.txt generation (www root does not even exist).") + logging.info("Skipping copying robots.txt (www root does not even exist).") return template_path = HERE / "templates" / "robots.txt" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render(languages=languages, versions=versions) robots_path = www_root / "robots.txt" - robots_path.write_text(rendered_template + "\n", encoding="UTF-8") + shutil.copyfile(template_path, robots_path) robots_path.chmod(0o775) run(["chgrp", group, robots_path]) if not skip_cache_invalidation: @@ -1204,9 +1200,7 @@ def build_docs(args) -> bool: build_sitemap(versions, languages, args.www_root, args.group) build_404(args.www_root, args.group) - build_robots_txt( - versions, - languages, + copy_robots_txt( args.www_root, args.group, args.skip_cache_invalidation, From 743a3a35fb04dd8b67ea8b0fdfc4fd47e876e774 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 10 Oct 2024 20:56:15 +0100 Subject: [PATCH 106/218] Simplify `Version.requirements` (#218) --- build_docs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 6541070..ffae5be 100755 --- a/build_docs.py +++ b/build_docs.py @@ -110,10 +110,8 @@ def requirements(self): """ if self.name == "3.5": return ["jieba", "blurb", "sphinx==1.8.4", "jinja2<3.1", "docutils<=0.17.1"] - if self.name in ("3.7", "3.6", "2.7"): + if self.name in {"3.7", "3.6", "2.7"}: return ["jieba", "blurb", "sphinx==2.3.1", "jinja2<3.1", "docutils<=0.17.1"] - if self.name == ("3.8", "3.9"): - return ["jieba", "blurb", "sphinx==2.4.4", "jinja2<3.1", "docutils<=0.17.1"] return [ "jieba", # To improve zh search. From 134f57cda62cbfb70340daaf80d27cd0c9472f22 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:00:15 +0100 Subject: [PATCH 107/218] Use PyStemmer (#217) PyStemmer exposes bindings to libstemmer_c, the core Snowball library written in C. This can improve performance of word stemming. --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index ffae5be..a744a42 100755 --- a/build_docs.py +++ b/build_docs.py @@ -115,6 +115,7 @@ def requirements(self): return [ "jieba", # To improve zh search. + "PyStemmer~=2.2.0", # To improve performance for word stemming. "-rrequirements.txt", ] From 5ac8ae9ae00951246c746616434f3e20ccac8d9a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 10 Oct 2024 23:20:29 +0100 Subject: [PATCH 108/218] Add an HTML-only (English) build variant (#219) --- build_docs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index a744a42..207250b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -504,7 +504,7 @@ def parse_args(): ) parser.add_argument( "--select-output", - choices=("no-html", "only-html"), + choices=("no-html", "only-html", "only-html-en"), help="Choose what outputs to build.", ) parser.add_argument( @@ -610,7 +610,7 @@ class DocBuilder: cpython_repo: Repository build_root: Path www_root: Path - select_output: Literal["no-html", "only-html"] | None + select_output: Literal["no-html", "only-html", "only-html-en"] | None quick: bool group: str log_directory: Path @@ -620,7 +620,9 @@ class DocBuilder: @property def html_only(self): return ( - self.select_output == "only-html" or self.quick or self.language.html_only + self.select_output in {"only-html", "only-html-en"} + or self.quick + or self.language.html_only ) @property @@ -1245,6 +1247,8 @@ def main(): build_docs_with_lock(args, "build_docs_archives.lock") elif args.select_output == "only-html": build_docs_with_lock(args, "build_docs_html.lock") + elif args.select_output == "only-html-en": + build_docs_with_lock(args, "build_docs_html_en.lock") def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: From 2406204b5e24743309a2257de4e66ffda9056975 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Oct 2024 03:43:46 +0100 Subject: [PATCH 109/218] Add English HTML-only to check_times --- check_times.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/check_times.py b/check_times.py index e561b09..9310cd4 100644 --- a/check_times.py +++ b/check_times.py @@ -78,6 +78,11 @@ def calc_time(lines: list[str]) -> None: if __name__ == "__main__": + print("Build times (HTML only; English)") + print("=======================") + print() + calc_time(get_lines("docsbuild-only-html-en.log")) + print("Build times (HTML only)") print("=======================") print() From e6c58e56af21837e6874f34457b3db1ca93ba3dd Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Oct 2024 03:49:26 +0100 Subject: [PATCH 110/218] Update versions in README.md --- README.md | 13 ++++++------- check_versions.py | 4 +++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d5beb39..5176cd0 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ of Sphinx we're using where: 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' - 3.12 ø sphinx~=8.0.0 needs_sphinx='6.2.1' - 3.13 ø sphinx~=8.0.0 needs_sphinx='6.2.1' - 3.14 ø sphinx~=8.0.0 needs_sphinx='6.2.1' + 3.12 ø sphinx~=8.1.0 needs_sphinx='6.2.1' + 3.13 ø sphinx~=8.1.0 needs_sphinx='6.2.1' + 3.14 ø sphinx~=8.1.0 needs_sphinx='6.2.1' ========= ============= ================== ==================== Sphinx build as seen on docs.python.org: @@ -44,11 +44,10 @@ of Sphinx we're using where: ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= version en es fr id it ja ko pl pt-br tr uk zh-cn zh-tw ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= - 3.8 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 - 3.12 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 - 3.13 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 - 3.14 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 8.0.2 + 3.12 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 + 3.13 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 + 3.14 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= diff --git a/check_versions.py b/check_versions.py index 63ffab5..70cade9 100644 --- a/check_versions.py +++ b/check_versions.py @@ -7,13 +7,15 @@ import re import httpx +import urllib3 from tabulate import tabulate import git import build_docs logger = logging.getLogger(__name__) -VERSIONS = build_docs.parse_versions_from_devguide() +http = urllib3.PoolManager() +VERSIONS = build_docs.parse_versions_from_devguide(http) LANGUAGES = build_docs.parse_languages_from_config() From cfae93d691b94533af11dcd38bb98f2ecae00926 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 20 Oct 2024 01:36:49 +0100 Subject: [PATCH 111/218] Update versions in README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5176cd0..056f73d 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ of Sphinx we're using where: 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' - 3.12 ø sphinx~=8.1.0 needs_sphinx='6.2.1' - 3.13 ø sphinx~=8.1.0 needs_sphinx='6.2.1' - 3.14 ø sphinx~=8.1.0 needs_sphinx='6.2.1' + 3.12 ø sphinx~=8.1.0 needs_sphinx='7.2.6' + 3.13 ø sphinx~=8.1.0 needs_sphinx='7.2.6' + 3.14 ø sphinx~=8.1.0 needs_sphinx='7.2.6' ========= ============= ================== ==================== Sphinx build as seen on docs.python.org: From a97ad6839868ba9781dc14fc1e0f5aa73fe63db8 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 20 Oct 2024 01:41:18 +0100 Subject: [PATCH 112/218] Update versions in README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 056f73d..767014c 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ of Sphinx we're using where: 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 - 3.12 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 - 3.13 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 - 3.14 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 8.1.0 + 3.12 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 + 3.13 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 + 3.14 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= From 949821476e5adc509873c7efba17872e97917b50 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:47:26 +0100 Subject: [PATCH 113/218] Add logging in post-build tasks (#221) --- build_docs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build_docs.py b/build_docs.py index 207250b..a7b4758 100755 --- a/build_docs.py +++ b/build_docs.py @@ -424,6 +424,7 @@ def copy_robots_txt( if not www_root.exists(): logging.info("Skipping copying robots.txt (www root does not even exist).") return + logging.info("Copying robots.txt...") template_path = HERE / "templates" / "robots.txt" robots_path = www_root / "robots.txt" shutil.copyfile(template_path, robots_path) @@ -440,6 +441,7 @@ def build_sitemap( if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exist).") return + logging.info("Starting sitemap generation...") template_path = HERE / "templates" / "sitemap.xml" template = jinja2.Template(template_path.read_text(encoding="UTF-8")) rendered_template = template.render(languages=languages, versions=versions) @@ -454,6 +456,7 @@ def build_404(www_root: Path, group): if not www_root.exists(): logging.info("Skipping 404 page generation (www root does not even exist).") return + logging.info("Copying 404 page...") not_found_file = www_root / "404.html" shutil.copyfile(HERE / "templates" / "404.html", not_found_file) not_found_file.chmod(0o664) @@ -1022,6 +1025,7 @@ def major_symlinks( - /fr/3/ → /fr/3.9/ - /es/3/ → /es/3.9/ """ + logging.info("Creating major version symlinks...") current_stable = Version.current_stable(versions).name for language in languages: symlink( @@ -1051,6 +1055,7 @@ def dev_symlink( - /fr/dev/ → /fr/3.11/ - /es/dev/ → /es/3.11/ """ + logging.info("Creating development version symlinks...") current_dev = Version.current_dev(versions).name for language in languages: symlink( @@ -1096,6 +1101,7 @@ def proofread_canonicals( - /3.11/whatsnew/3.11.html typically would link to /3/whatsnew/3.11.html, which may not exist yet. """ + logging.info("Checking canonical links...") canonical_re = re.compile( """""" ) From bbf1112ac03e2045dfbcd76d04c99c5765fd35db Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:27:39 +0100 Subject: [PATCH 114/218] Purge using Surrogate-Key headers (#220) --- build_docs.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/build_docs.py b/build_docs.py index a7b4758..fefcb15 100755 --- a/build_docs.py +++ b/build_docs.py @@ -31,7 +31,7 @@ import logging import logging.handlers from functools import total_ordering -from os import readlink +from os import getenv, readlink import re import shlex import shutil @@ -895,13 +895,8 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: logging.info("%s files changed", len(changed)) if changed and not self.skip_cache_invalidation: - targets_dir = str(self.www_root) - prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout - prefixes = prefixes.replace(targets_dir + "/", "") - prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix] - purge(http, *prefixes) - for prefix in prefixes: - purge(http, *[prefix + p for p in changed]) + surrogate_key = f"{self.language.tag}/{self.version.name}" + purge_surrogate_key(http, surrogate_key) logging.info( "Publishing done (%s).", format_seconds(perf_counter() - start_time) ) @@ -1007,7 +1002,8 @@ def symlink( link.symlink_to(directory) run(["chown", "-h", ":" + group, str(link)]) if not skip_cache_invalidation: - purge_path(http, www_root, link) + surrogate_key = f"{language.tag}/{name}" + purge_surrogate_key(http, surrogate_key) def major_symlinks( @@ -1081,14 +1077,25 @@ def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: http.request("PURGE", url, timeout=30) -def purge_path(http: urllib3.PoolManager, www_root: Path, path: Path) -> None: - """Recursively remove a path from docs.python.org's CDN. +def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: + """Remove paths from docs.python.org's CDN. + All paths matching the given 'Surrogate-Key' will be removed. + This is set by the Nginx server for every language-version pair. To be used when a directory changes, so the CDN fetches the new one. + + https://www.fastly.com/documentation/reference/api/purging/#purge-tag """ - purge(http, *[file.relative_to(www_root) for file in path.glob("**/*")]) - purge(http, path.relative_to(www_root)) - purge(http, str(path.relative_to(www_root)) + "/") + service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") + fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") + + logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) + http.request( + "POST", + f"https://api.fastly.com/service/{service_id}/purge/{surrogate_key}", + headers={"Fastly-Key": fastly_key}, + timeout=30, + ) def proofread_canonicals( From d708c717b4cbd22d12adb125d5354e7e11c21343 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 24 Oct 2024 19:19:14 +0100 Subject: [PATCH 115/218] Always purge symlinks --- build_docs.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/build_docs.py b/build_docs.py index fefcb15..9f26bf9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -995,12 +995,14 @@ def symlink( directory_path = path / directory if not directory_path.exists(): return # No touching link, dest doc not built yet. - if link.exists() and readlink(link) == directory: - return # Link is already pointing to right doc. - if link.exists(): - link.unlink() - link.symlink_to(directory) - run(["chown", "-h", ":" + group, str(link)]) + + link_exists = link.exists() + if not link_exists or readlink(link) != directory: + # Link does not exist or points to the wrong target. + if link_exists: + link.unlink() + link.symlink_to(directory) + run(["chown", "-h", f":{group}", str(link)]) if not skip_cache_invalidation: surrogate_key = f"{language.tag}/{name}" purge_surrogate_key(http, surrogate_key) From e0c1fed030ce4e1d92426f2e45e615b3be621ac6 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 24 Oct 2024 20:15:07 +0100 Subject: [PATCH 116/218] Simplify Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- build_docs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 9f26bf9..658ba63 100755 --- a/build_docs.py +++ b/build_docs.py @@ -996,10 +996,9 @@ def symlink( if not directory_path.exists(): return # No touching link, dest doc not built yet. - link_exists = link.exists() - if not link_exists or readlink(link) != directory: + if not link.exists() or readlink(link) != directory: # Link does not exist or points to the wrong target. - if link_exists: + if link.exists(): link.unlink() link.symlink_to(directory) run(["chown", "-h", f":{group}", str(link)]) From 36f8c1ebf61f5d3c970566e2da0a273ff29ca3d2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:29:32 +0000 Subject: [PATCH 117/218] Further simplify symlink removal (#224) Co-authored-by: Ezio Melotti --- build_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 658ba63..6596a25 100755 --- a/build_docs.py +++ b/build_docs.py @@ -998,8 +998,7 @@ def symlink( if not link.exists() or readlink(link) != directory: # Link does not exist or points to the wrong target. - if link.exists(): - link.unlink() + link.unlink(missing_ok=True) link.symlink_to(directory) run(["chown", "-h", f":{group}", str(link)]) if not skip_cache_invalidation: From 1807c70aff5a050077ede04fe65c7689110bf5f1 Mon Sep 17 00:00:00 2001 From: Kerim Kabirov Date: Mon, 28 Oct 2024 16:31:05 +0100 Subject: [PATCH 118/218] Add context labels to the version switcher (#223) --- templates/switchers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 29204ae..999ca10 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -26,7 +26,7 @@ } function build_version_select(release) { - let buf = ['']; const major_minor = release.split(".").slice(0, 2).join("."); Object.entries(all_versions).forEach(function([version, title]) { @@ -42,7 +42,7 @@ } function build_language_select(current_language) { - let buf = ['']; Object.entries(all_languages).forEach(function([language, title]) { if (language === current_language) { From 7e274eb87a36acf9ca35e3f1671ee4eb2d87d9f3 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:47:52 +0000 Subject: [PATCH 119/218] Remove String.startsWith polyfill --- templates/switchers.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 999ca10..ad31a03 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,15 +1,6 @@ (function() { 'use strict'; - if (!String.prototype.startsWith) { - Object.defineProperty(String.prototype, 'startsWith', { - value: function(search, rawPos) { - const pos = rawPos > 0 ? rawPos|0 : 0; - return this.substring(pos, pos + search.length) === search; - } - }); - } - // Parses versions in URL segments like: // "3", "dev", "release/2.7" or "3.6rc2" const version_regexs = [ From 27e2dc7fbf61c48bd58c79152b5a0480c2d02ef1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:53:09 +0000 Subject: [PATCH 120/218] Remove create_placeholders_if_missing() The placeholders have been in the theme since v2021.5 --- templates/switchers.js | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index ad31a03..ffedb7d 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -132,41 +132,11 @@ return '' } - function create_placeholders_if_missing() { - const version_segment = version_segment_from_url(); - const language_segment = language_segment_from_url(); - const index = "/" + language_segment + version_segment; - - if (document.querySelectorAll('.version_switcher_placeholder').length > 0) { - return; - } - - const html = ' \ - \ -Documentation »'; - - const probable_places = [ - "body>div.related>ul>li:not(.right):contains('Documentation'):first", - "body>div.related>ul>li:not(.right):contains('documentation'):first", - ]; - - for (let i = 0; i < probable_places.length; i++) { - let probable_place = $(probable_places[i]); - if (probable_place.length == 1) { - probable_place.html(html); - document.getElementById('indexlink').href = index; - return; - } - } - } - document.addEventListener('DOMContentLoaded', function() { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - create_placeholders_if_missing(); - let placeholders = document.querySelectorAll('.version_switcher_placeholder'); placeholders.forEach(function(placeholder) { placeholder.innerHTML = version_select; From 75594be5806eb6ed91aecac46a9a4036f734dc02 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:54:10 +0000 Subject: [PATCH 121/218] Use exact comparison operators --- templates/switchers.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index ffedb7d..664db12 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -55,7 +55,7 @@ function navigate_to_first_existing(urls) { // Navigate to the first existing URL in urls. const url = urls.shift(); - if (urls.length == 0 || url.startsWith("file:///")) { + if (urls.length === 0 || url.startsWith("file:///")) { window.location.href = url; return; } @@ -79,7 +79,7 @@ const current_version = version_segment_from_url(); const new_url = url.replace('/' + current_language + current_version, '/' + current_language + selected_version); - if (new_url != url) { + if (new_url !== url) { navigate_to_first_existing([ new_url, url.replace('/' + current_language + current_version, @@ -96,11 +96,11 @@ const url = window.location.href; const current_language = language_segment_from_url(); const current_version = version_segment_from_url(); - if (selected_language == 'en/') // Special 'default' case for English. + if (selected_language === 'en/') // Special 'default' case for English. selected_language = ''; let new_url = url.replace('/' + current_language + current_version, '/' + selected_language + current_version); - if (new_url != url) { + if (new_url !== url) { navigate_to_first_existing([ new_url, '/' From 8a71e4273a287deb62e8ed84cc9d837a1a13a0f4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:56:33 +0000 Subject: [PATCH 122/218] Remove an unneeded variable --- templates/switchers.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 664db12..70997e1 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -135,10 +135,9 @@ document.addEventListener('DOMContentLoaded', function() { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - let placeholders = document.querySelectorAll('.version_switcher_placeholder'); - placeholders.forEach(function(placeholder) { + const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); + document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { placeholder.innerHTML = version_select; let selectElement = placeholder.querySelector('select'); @@ -146,9 +145,7 @@ }); const language_select = build_language_select(current_language); - - placeholders = document.querySelectorAll('.language_switcher_placeholder'); - placeholders.forEach(function(placeholder) { + document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { placeholder.innerHTML = language_select; let selectElement = placeholder.querySelector('select'); From ce8b4742aa957612ce6734b78bdf6e242a6bd43c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:57:55 +0000 Subject: [PATCH 123/218] Remove anonymous function (IIFE) --- templates/switchers.js | 284 ++++++++++++++++++++--------------------- 1 file changed, 141 insertions(+), 143 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 70997e1..d436d5e 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,155 +1,153 @@ -(function() { - 'use strict'; - - // Parses versions in URL segments like: - // "3", "dev", "release/2.7" or "3.6rc2" - const version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; - - const all_versions = $VERSIONS; - const all_languages = $LANGUAGES; - - function quote_attr(str) { - return '"' + str.replace('"', '\\"') + '"'; - } +'use strict'; + +// Parses versions in URL segments like: +// "3", "dev", "release/2.7" or "3.6rc2" +const version_regexs = [ + '(?:\\d)', + '(?:\\d\\.\\d[\\w\\d\\.]*)', + '(?:dev)', + '(?:release/\\d.\\d[\\x\\d\\.]*)']; + +const all_versions = $VERSIONS; +const all_languages = $LANGUAGES; + +function quote_attr(str) { + return '"' + str.replace('"', '\\"') + '"'; +} + +function build_version_select(release) { + let buf = ['']; - const major_minor = release.split(".").slice(0, 2).join("."); + buf.push(''); + return buf.join(''); +} - Object.entries(all_versions).forEach(function([version, title]) { - if (version === major_minor) { - buf.push(''); - } else { - buf.push(''); - } - }); +function build_language_select(current_language) { + let buf = [''); - return buf.join(''); + Object.entries(all_languages).forEach(function([language, title]) { + if (language === current_language) { + buf.push(''); + } else { + buf.push(''); + } + }); + if (!(current_language in all_languages)) { + // In case we're browsing a language that is not yet in all_languages. + buf.push(''); + all_languages[current_language] = current_language; } - - function build_language_select(current_language) { - let buf = [''); + return buf.join(''); +} + +function navigate_to_first_existing(urls) { + // Navigate to the first existing URL in urls. + const url = urls.shift(); + if (urls.length === 0 || url.startsWith("file:///")) { + window.location.href = url; + return; + } + fetch(url) + .then(function(response) { + if (response.ok) { + window.location.href = url; } else { - buf.push(''); + navigate_to_first_existing(urls); } + }) + .catch(function(error) { + navigate_to_first_existing(urls); }); - if (!(current_language in all_languages)) { - // In case we're browsing a language that is not yet in all_languages. - buf.push(''); - all_languages[current_language] = current_language; - } - buf.push(''); - return buf.join(''); - } - - function navigate_to_first_existing(urls) { - // Navigate to the first existing URL in urls. - const url = urls.shift(); - if (urls.length === 0 || url.startsWith("file:///")) { - window.location.href = url; - return; - } - fetch(url) - .then(function(response) { - if (response.ok) { - window.location.href = url; - } else { - navigate_to_first_existing(urls); - } - }) - .catch(function(error) { - navigate_to_first_existing(urls); - }); - } - - function on_version_switch() { - const selected_version = this.options[this.selectedIndex].value + '/'; - const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); - const new_url = url.replace('/' + current_language + current_version, - '/' + current_language + selected_version); - if (new_url !== url) { - navigate_to_first_existing([ - new_url, - url.replace('/' + current_language + current_version, - '/' + selected_version), - '/' + current_language + selected_version, - '/' + selected_version, - '/' - ]); - } - } - - function on_language_switch() { - let selected_language = this.options[this.selectedIndex].value + '/'; - const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); - if (selected_language === 'en/') // Special 'default' case for English. - selected_language = ''; - let new_url = url.replace('/' + current_language + current_version, - '/' + selected_language + current_version); - if (new_url !== url) { - navigate_to_first_existing([ - new_url, - '/' - ]); - } - } - - // Returns the path segment of the language as a string, like 'fr/' - // or '' if not found. - function language_segment_from_url() { - const path = window.location.pathname; - const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' - const match = path.match(language_regexp); - if (match !== null) - return match[1]; - return ''; +} + +function on_version_switch() { + const selected_version = this.options[this.selectedIndex].value + '/'; + const url = window.location.href; + const current_language = language_segment_from_url(); + const current_version = version_segment_from_url(); + const new_url = url.replace('/' + current_language + current_version, + '/' + current_language + selected_version); + if (new_url !== url) { + navigate_to_first_existing([ + new_url, + url.replace('/' + current_language + current_version, + '/' + selected_version), + '/' + current_language + selected_version, + '/' + selected_version, + '/' + ]); } - - // Returns the path segment of the version as a string, like '3.6/' - // or '' if not found. - function version_segment_from_url() { - const path = window.location.pathname; - const language_segment = language_segment_from_url(); - const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - const version_regexp = language_segment + '(' + version_segment + ')'; - const match = path.match(version_regexp); - if (match !== null) - return match[1]; - return '' +} + +function on_language_switch() { + let selected_language = this.options[this.selectedIndex].value + '/'; + const url = window.location.href; + const current_language = language_segment_from_url(); + const current_version = version_segment_from_url(); + if (selected_language === 'en/') // Special 'default' case for English. + selected_language = ''; + let new_url = url.replace('/' + current_language + current_version, + '/' + selected_language + current_version); + if (new_url !== url) { + navigate_to_first_existing([ + new_url, + '/' + ]); } +} + +// Returns the path segment of the language as a string, like 'fr/' +// or '' if not found. +function language_segment_from_url() { + const path = window.location.pathname; + const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' + const match = path.match(language_regexp); + if (match !== null) + return match[1]; + return ''; +} + +// Returns the path segment of the version as a string, like '3.6/' +// or '' if not found. +function version_segment_from_url() { + const path = window.location.pathname; + const language_segment = language_segment_from_url(); + const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; + const version_regexp = language_segment + '(' + version_segment + ')'; + const match = path.match(version_regexp); + if (match !== null) + return match[1]; + return '' +} + +document.addEventListener('DOMContentLoaded', function() { + const language_segment = language_segment_from_url(); + const current_language = language_segment.replace(/\/+$/g, '') || 'en'; + + const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); + document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { + placeholder.innerHTML = version_select; + + let selectElement = placeholder.querySelector('select'); + selectElement.addEventListener('change', on_version_switch); + }); - document.addEventListener('DOMContentLoaded', function() { - const language_segment = language_segment_from_url(); - const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - - const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = version_select; - - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_version_switch); - }); - - const language_select = build_language_select(current_language); - document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = language_select; + const language_select = build_language_select(current_language); + document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { + placeholder.innerHTML = language_select; - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_language_switch); - }); + let selectElement = placeholder.querySelector('select'); + selectElement.addEventListener('change', on_language_switch); }); -})(); +}); From cdcd52305ee0b7b6d88f2cf859ae5d2f37e2ddd3 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:59:28 +0000 Subject: [PATCH 124/218] Name the initialisation function --- templates/switchers.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index d436d5e..8bdf3d5 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -130,8 +130,7 @@ function version_segment_from_url() { return match[1]; return '' } - -document.addEventListener('DOMContentLoaded', function() { +const _initialise_switchers = () => { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; @@ -150,4 +149,10 @@ document.addEventListener('DOMContentLoaded', function() { let selectElement = placeholder.querySelector('select'); selectElement.addEventListener('change', on_language_switch); }); -}); +}; + +if (document.readyState !== 'loading') { + _initialise_switchers(); +} else { + document.addEventListener('DOMContentLoaded', _initialise_switchers); +} From 84a3f9c5d8794fa20e738ddf195d2f6c0abb36f7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:09:38 +0000 Subject: [PATCH 125/218] Construct DOM nodes instead of parsing HTML --- templates/switchers.js | 90 ++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 8bdf3d5..094c2ce 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -15,41 +15,45 @@ function quote_attr(str) { return '"' + str.replace('"', '\\"') + '"'; } -function build_version_select(release) { - let buf = [''); - return buf.join(''); -} + select.add(option); + } -function build_language_select(current_language) { - let buf = [''); - return buf.join(''); -} + + const select = document.createElement('select'); + select.className = 'language-select'; + + for (const [language, title] in all_languages) { + const option = document.createElement('option'); + option.value = language; + option.text = title; + if (language === current_language) option.selected = true; + select.add(option); + } + + return select; +}; function navigate_to_first_existing(urls) { // Navigate to the first existing URL in urls. @@ -134,21 +138,23 @@ const _initialise_switchers = () => { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = version_select; - - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_version_switch); - }); - - const language_select = build_language_select(current_language); - document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = language_select; + const version_select = _create_version_select(DOCUMENTATION_OPTIONS.VERSION); + document + .querySelectorAll('.version_switcher_placeholder') + .forEach((placeholder) => { + const s = version_select.cloneNode(true); + s.addEventListener('change', on_version_switch); + placeholder.append(s); + }); - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_language_switch); - }); + const language_select = _create_language_select(current_language); + document + .querySelectorAll('.language_switcher_placeholder') + .forEach((placeholder) => { + const s = language_select.cloneNode(true); + s.addEventListener('change', on_language_switch); + placeholder.append(s); + }); }; if (document.readyState !== 'loading') { From 2a5adf99c1af5d8b2f1874c696651ca72c2a0518 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:18:22 +0000 Subject: [PATCH 126/218] Use arrow functions --- templates/switchers.js | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 094c2ce..b5b46ba 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -55,7 +55,7 @@ const _create_language_select = (current_language) => { return select; }; -function navigate_to_first_existing(urls) { +const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. const url = urls.shift(); if (urls.length === 0 || url.startsWith("file:///")) { @@ -63,19 +63,20 @@ function navigate_to_first_existing(urls) { return; } fetch(url) - .then(function(response) { + .then((response) => { if (response.ok) { window.location.href = url; } else { navigate_to_first_existing(urls); } }) - .catch(function(error) { + .catch((err) => { + void err; navigate_to_first_existing(urls); }); -} +}; -function on_version_switch() { +const _on_version_switch = () => { const selected_version = this.options[this.selectedIndex].value + '/'; const url = window.location.href; const current_language = language_segment_from_url(); @@ -83,7 +84,7 @@ function on_version_switch() { const new_url = url.replace('/' + current_language + current_version, '/' + current_language + selected_version); if (new_url !== url) { - navigate_to_first_existing([ + _navigate_to_first_existing([ new_url, url.replace('/' + current_language + current_version, '/' + selected_version), @@ -92,9 +93,9 @@ function on_version_switch() { '/' ]); } -} +}; -function on_language_switch() { +const _on_language_switch = () => { let selected_language = this.options[this.selectedIndex].value + '/'; const url = window.location.href; const current_language = language_segment_from_url(); @@ -104,12 +105,12 @@ function on_language_switch() { let new_url = url.replace('/' + current_language + current_version, '/' + selected_language + current_version); if (new_url !== url) { - navigate_to_first_existing([ + _navigate_to_first_existing([ new_url, '/' ]); } -} +}; // Returns the path segment of the language as a string, like 'fr/' // or '' if not found. @@ -143,7 +144,7 @@ const _initialise_switchers = () => { .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { const s = version_select.cloneNode(true); - s.addEventListener('change', on_version_switch); + s.addEventListener('change', _on_version_switch); placeholder.append(s); }); @@ -152,7 +153,7 @@ const _initialise_switchers = () => { .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { const s = language_select.cloneNode(true); - s.addEventListener('change', on_language_switch); + s.addEventListener('change', _on_language_switch); placeholder.append(s); }); }; From b4b36c0983a8693cdfdd743268630678c737b158 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:18:37 +0000 Subject: [PATCH 127/218] Remove unused quote_attr() function --- templates/switchers.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index b5b46ba..d70fc8f 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -11,10 +11,6 @@ const version_regexs = [ const all_versions = $VERSIONS; const all_languages = $LANGUAGES; -function quote_attr(str) { - return '"' + str.replace('"', '\\"') + '"'; -} - const _create_version_select = (release) => { const major_minor = release.split('.').slice(0, 2).join('.'); const select = document.createElement('select'); From 2cac9f627ffc927ade8c09b3b0eec45181ff7c46 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:25:08 +0000 Subject: [PATCH 128/218] Run prettier --- templates/switchers.js | 44 +++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index d70fc8f..1b7a71d 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -6,7 +6,8 @@ const version_regexs = [ '(?:\\d)', '(?:\\d\\.\\d[\\w\\d\\.]*)', '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; + '(?:release/\\d.\\d[\\x\\d\\.]*)', +]; const all_versions = $VERSIONS; const all_languages = $LANGUAGES; @@ -54,7 +55,7 @@ const _create_language_select = (current_language) => { const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. const url = urls.shift(); - if (urls.length === 0 || url.startsWith("file:///")) { + if (urls.length === 0 || url.startsWith('file:///')) { window.location.href = url; return; } @@ -77,16 +78,20 @@ const _on_version_switch = () => { const url = window.location.href; const current_language = language_segment_from_url(); const current_version = version_segment_from_url(); - const new_url = url.replace('/' + current_language + current_version, - '/' + current_language + selected_version); + const new_url = url.replace( + '/' + current_language + current_version, + '/' + current_language + selected_version, + ); if (new_url !== url) { _navigate_to_first_existing([ new_url, - url.replace('/' + current_language + current_version, - '/' + selected_version), + url.replace( + '/' + current_language + current_version, + '/' + selected_version, + ), '/' + current_language + selected_version, '/' + selected_version, - '/' + '/', ]); } }; @@ -96,15 +101,15 @@ const _on_language_switch = () => { const url = window.location.href; const current_language = language_segment_from_url(); const current_version = version_segment_from_url(); - if (selected_language === 'en/') // Special 'default' case for English. + if (selected_language === 'en/') + // Special 'default' case for English. selected_language = ''; - let new_url = url.replace('/' + current_language + current_version, - '/' + selected_language + current_version); + let new_url = url.replace( + '/' + current_language + current_version, + '/' + selected_language + current_version, + ); if (new_url !== url) { - _navigate_to_first_existing([ - new_url, - '/' - ]); + _navigate_to_first_existing([new_url, '/']); } }; @@ -112,10 +117,10 @@ const _on_language_switch = () => { // or '' if not found. function language_segment_from_url() { const path = window.location.pathname; - const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' + const language_regexp = + '/((?:' + Object.keys(all_languages).join('|') + ')/)'; const match = path.match(language_regexp); - if (match !== null) - return match[1]; + if (match !== null) return match[1]; return ''; } @@ -127,9 +132,8 @@ function version_segment_from_url() { const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; const version_regexp = language_segment + '(' + version_segment + ')'; const match = path.match(version_regexp); - if (match !== null) - return match[1]; - return '' + if (match !== null) return match[1]; + return ''; } const _initialise_switchers = () => { const language_segment = language_segment_from_url(); From 7b5e7730954e7b788195ecddb12f8f7056b92675 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:29:30 +0000 Subject: [PATCH 129/218] Make the regular expression for version parts a constant --- templates/switchers.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 1b7a71d..6b12e32 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,13 +1,14 @@ 'use strict'; // Parses versions in URL segments like: -// "3", "dev", "release/2.7" or "3.6rc2" -const version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)', -]; +const _VERSION_PATTERN = ( + '((?:' + + '(?:\\d)' // e.g. "3" + +'|(?:\\d\\.\\d[\\w\\d\\.]*)' // e.g. "3.6rc2" + +'|(?:dev)' // e.g. "dev" + +'|(?:release/\\d.\\d[\\x\\d\\.]*)'// e.g. "release/2.7" + + ')/)' +); const all_versions = $VERSIONS; const all_languages = $LANGUAGES; @@ -128,9 +129,7 @@ function language_segment_from_url() { // or '' if not found. function version_segment_from_url() { const path = window.location.pathname; - const language_segment = language_segment_from_url(); - const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - const version_regexp = language_segment + '(' + version_segment + ')'; + const version_regexp = language_segment_from_url() + _VERSION_PATTERN; const match = path.match(version_regexp); if (match !== null) return match[1]; return ''; From ab3b07962977ace9fea0c8c4034d53cb129872d0 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:05:32 +0000 Subject: [PATCH 130/218] Use constants from DOCUMENTATION_OPTIONS where possible --- templates/switchers.js | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 6b12e32..f98de05 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,5 +1,18 @@ 'use strict'; +const _CURRENT_VERSION = DOCUMENTATION_OPTIONS.VERSION; +const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; +const _CURRENT_PREFIX = (() => { + // Sphinx 7.2+ defines the content root data attribute in the HTML element. + const _CONTENT_ROOT = document.documentElement.dataset.content_root; + if (_CONTENT_ROOT !== undefined) { + return new URL(_CONTENT_ROOT, window.location).pathname; + } + // Fallback for older versions of Sphinx (used in Python 3.10 and older). + const _NUM_PREFIX_PARTS = _CURRENT_LANGUAGE === 'en' ? 2 : 3; + return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; +})(); + // Parses versions in URL segments like: const _VERSION_PATTERN = ( '((?:' @@ -77,20 +90,15 @@ const _navigate_to_first_existing = (urls) => { const _on_version_switch = () => { const selected_version = this.options[this.selectedIndex].value + '/'; const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); const new_url = url.replace( - '/' + current_language + current_version, - '/' + current_language + selected_version, + _CURRENT_PREFIX, + '/' + _CURRENT_LANGUAGE + selected_version, ); if (new_url !== url) { _navigate_to_first_existing([ new_url, - url.replace( - '/' + current_language + current_version, - '/' + selected_version, - ), - '/' + current_language + selected_version, + url.replace(_CURRENT_PREFIX, '/' + selected_version), + '/' + _CURRENT_LANGUAGE + selected_version, '/' + selected_version, '/', ]); @@ -100,14 +108,12 @@ const _on_version_switch = () => { const _on_language_switch = () => { let selected_language = this.options[this.selectedIndex].value + '/'; const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); if (selected_language === 'en/') // Special 'default' case for English. selected_language = ''; let new_url = url.replace( - '/' + current_language + current_version, - '/' + selected_language + current_version, + _CURRENT_PREFIX, + '/' + selected_language + _CURRENT_VERSION, ); if (new_url !== url) { _navigate_to_first_existing([new_url, '/']); @@ -135,10 +141,7 @@ function version_segment_from_url() { return ''; } const _initialise_switchers = () => { - const language_segment = language_segment_from_url(); - const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - - const version_select = _create_version_select(DOCUMENTATION_OPTIONS.VERSION); + const version_select = _create_version_select(_CURRENT_VERSION); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { @@ -147,7 +150,7 @@ const _initialise_switchers = () => { placeholder.append(s); }); - const language_select = _create_language_select(current_language); + const language_select = _create_language_select(_CURRENT_LANGUAGE); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { From a5936462d9811cd843ddb9337208272d996a098d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:06:10 +0000 Subject: [PATCH 131/218] Remove now-unused segment_from_url() functions --- templates/switchers.js | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index f98de05..600c955 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -13,16 +13,6 @@ const _CURRENT_PREFIX = (() => { return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; })(); -// Parses versions in URL segments like: -const _VERSION_PATTERN = ( - '((?:' - + '(?:\\d)' // e.g. "3" - +'|(?:\\d\\.\\d[\\w\\d\\.]*)' // e.g. "3.6rc2" - +'|(?:dev)' // e.g. "dev" - +'|(?:release/\\d.\\d[\\x\\d\\.]*)'// e.g. "release/2.7" - + ')/)' -); - const all_versions = $VERSIONS; const all_languages = $LANGUAGES; @@ -120,26 +110,6 @@ const _on_language_switch = () => { } }; -// Returns the path segment of the language as a string, like 'fr/' -// or '' if not found. -function language_segment_from_url() { - const path = window.location.pathname; - const language_regexp = - '/((?:' + Object.keys(all_languages).join('|') + ')/)'; - const match = path.match(language_regexp); - if (match !== null) return match[1]; - return ''; -} - -// Returns the path segment of the version as a string, like '3.6/' -// or '' if not found. -function version_segment_from_url() { - const path = window.location.pathname; - const version_regexp = language_segment_from_url() + _VERSION_PATTERN; - const match = path.match(version_regexp); - if (match !== null) return match[1]; - return ''; -} const _initialise_switchers = () => { const version_select = _create_version_select(_CURRENT_VERSION); document From b62ea7a325fb2dc53350f07be9c0129de1f11c6a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:08:12 +0000 Subject: [PATCH 132/218] Use the event argument of select element callback functions --- templates/switchers.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 600c955..dbd4809 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -77,8 +77,8 @@ const _navigate_to_first_existing = (urls) => { }); }; -const _on_version_switch = () => { - const selected_version = this.options[this.selectedIndex].value + '/'; +const _on_version_switch = (event) => { + const selected_version = event.target.value + '/'; const url = window.location.href; const new_url = url.replace( _CURRENT_PREFIX, @@ -95,8 +95,8 @@ const _on_version_switch = () => { } }; -const _on_language_switch = () => { - let selected_language = this.options[this.selectedIndex].value + '/'; +const _on_language_switch = (event) => { + let selected_language = event.target.value + '/'; const url = window.location.href; if (selected_language === 'en/') // Special 'default' case for English. From e5b53bbf6d545d0d95cff2b9f74c1f602efb2511 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:43:21 +0000 Subject: [PATCH 133/218] Improve logic for the callback functions --- templates/switchers.js | 90 ++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index dbd4809..ababf04 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -58,55 +58,67 @@ const _create_language_select = (current_language) => { const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. - const url = urls.shift(); - if (urls.length === 0 || url.startsWith('file:///')) { - window.location.href = url; - return; + for (const url of urls) { + if (url.startsWith('file:///')) { + window.location.href = url; + return; + } + fetch(url) + .then((response) => { + if (response.ok) { + window.location.href = url; + return url; + } + }) + .catch((err) => { + console.error(`Error when fetching '${url}'!`); + console.error(err); + }); } - fetch(url) - .then((response) => { - if (response.ok) { - window.location.href = url; - } else { - navigate_to_first_existing(urls); - } - }) - .catch((err) => { - void err; - navigate_to_first_existing(urls); - }); + + // if all else fails, redirect to the d.p.o root + window.location.href = '/'; + return '/'; }; const _on_version_switch = (event) => { - const selected_version = event.target.value + '/'; - const url = window.location.href; - const new_url = url.replace( - _CURRENT_PREFIX, - '/' + _CURRENT_LANGUAGE + selected_version, - ); - if (new_url !== url) { + const selected_version = event.target.value; + // English has no language prefix. + const new_prefix_en = `/${selected_version}/`; + const new_prefix = + _CURRENT_LANGUAGE === 'en' + ? new_prefix_en + : `/${_CURRENT_LANGUAGE}/${selected_version}/`; + if (_CURRENT_PREFIX !== new_prefix) { + // Try the following pages in order: + // 1. The current page in the current language with the new version + // 2. The current page in English with the new version + // 3. The documentation home in the current language with the new version + // 4. The documentation home in English with the new version _navigate_to_first_existing([ - new_url, - url.replace(_CURRENT_PREFIX, '/' + selected_version), - '/' + _CURRENT_LANGUAGE + selected_version, - '/' + selected_version, - '/', + window.location.href.replace(_CURRENT_PREFIX, new_prefix), + window.location.href.replace(_CURRENT_PREFIX, new_prefix_en), + new_prefix, + new_prefix_en, ]); } }; const _on_language_switch = (event) => { - let selected_language = event.target.value + '/'; - const url = window.location.href; - if (selected_language === 'en/') - // Special 'default' case for English. - selected_language = ''; - let new_url = url.replace( - _CURRENT_PREFIX, - '/' + selected_language + _CURRENT_VERSION, - ); - if (new_url !== url) { - _navigate_to_first_existing([new_url, '/']); + const selected_language = event.target.value; + // English has no language prefix. + const new_prefix = + selected_language === 'en' + ? `/${_CURRENT_VERSION}/` + : `/${selected_language}/${_CURRENT_VERSION}/`; + if (_CURRENT_PREFIX !== new_prefix) { + // Try the following pages in order: + // 1. The current page in the new language with the current version + // 2. The documentation home in the new language with the current version + _navigate_to_first_existing([ + window.location.href.replace(_CURRENT_PREFIX, new_prefix), + new_prefix, + ]); } }; From 1179ee9d3b62ee38f978ffa6b97e311b2775d230 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:49:33 +0000 Subject: [PATCH 134/218] Iterate over arrays with for...of --- templates/switchers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index ababf04..af73ced 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -21,7 +21,7 @@ const _create_version_select = (release) => { const select = document.createElement('select'); select.className = 'version-select'; - for (const [version, title] in all_versions) { + for (const [version, title] of Object.entries(all_versions)) { const option = document.createElement('option'); option.value = version; if (version === major_minor) { @@ -45,7 +45,7 @@ const _create_language_select = (current_language) => { const select = document.createElement('select'); select.className = 'language-select'; - for (const [language, title] in all_languages) { + for (const [language, title] of Object.entries(all_languages)) { const option = document.createElement('option'); option.value = language; option.text = title; From d1b7ad8450b1ef58b307c61882c0336c60ef1276 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:52:54 +0000 Subject: [PATCH 135/218] Fix _CURRENT_VERSION for pre-releases --- templates/switchers.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/switchers.js b/templates/switchers.js index af73ced..afa26f4 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,6 +1,7 @@ 'use strict'; -const _CURRENT_VERSION = DOCUMENTATION_OPTIONS.VERSION; +const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ''; +const _CURRENT_VERSION = _CURRENT_RELEASE.split('.', 2).join('.'); const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; const _CURRENT_PREFIX = (() => { // Sphinx 7.2+ defines the content root data attribute in the HTML element. From 7efb6bf363a2f22338d0f9732e39895fa28abbe7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:00:10 +0000 Subject: [PATCH 136/218] Use _CURRENT_RELEASE in _create_version_select() --- templates/switchers.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index afa26f4..774f04e 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -17,16 +17,15 @@ const _CURRENT_PREFIX = (() => { const all_versions = $VERSIONS; const all_languages = $LANGUAGES; -const _create_version_select = (release) => { - const major_minor = release.split('.').slice(0, 2).join('.'); +const _create_version_select = () => { const select = document.createElement('select'); select.className = 'version-select'; for (const [version, title] of Object.entries(all_versions)) { const option = document.createElement('option'); option.value = version; - if (version === major_minor) { - option.text = release; + if (version === _CURRENT_VERSION) { + option.text = _CURRENT_RELEASE; option.selected = true; } else { option.text = title; @@ -124,7 +123,7 @@ const _on_language_switch = (event) => { }; const _initialise_switchers = () => { - const version_select = _create_version_select(_CURRENT_VERSION); + const version_select = _create_version_select(); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { From 359c393d92dbec5f383b974652f19259edf9fdbb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:02:14 +0000 Subject: [PATCH 137/218] Use _CURRENT_LANGUAGE in _create_language_select() --- templates/switchers.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 774f04e..740cc12 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -36,10 +36,10 @@ const _create_version_select = () => { return select; }; -const _create_language_select = (current_language) => { - if (!(current_language in all_languages)) { +const _create_language_select = () => { + if (!(_CURRENT_LANGUAGE in all_languages)) { // In case we are browsing a language that is not yet in all_languages. - all_languages[current_language] = current_language; + all_languages[_CURRENT_LANGUAGE] = _CURRENT_LANGUAGE; } const select = document.createElement('select'); @@ -49,7 +49,7 @@ const _create_language_select = (current_language) => { const option = document.createElement('option'); option.value = language; option.text = title; - if (language === current_language) option.selected = true; + if (language === _CURRENT_LANGUAGE) option.selected = true; select.add(option); } @@ -132,7 +132,7 @@ const _initialise_switchers = () => { placeholder.append(s); }); - const language_select = _create_language_select(_CURRENT_LANGUAGE); + const language_select = _create_language_select(); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { From e55b26c44c6410d2720d8d4b0b73d5140a29721e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:05:57 +0000 Subject: [PATCH 138/218] Add better handling for file URIs --- templates/switchers.js | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 740cc12..025daed 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,9 +1,14 @@ 'use strict'; +// File URIs must begin with either one or three forward slashes +const _is_file_uri = (uri) => uri.startsWith('file:/'); + +const _IS_LOCAL = _is_file_uri(window.location.href); const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ''; const _CURRENT_VERSION = _CURRENT_RELEASE.split('.', 2).join('.'); const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; const _CURRENT_PREFIX = (() => { + if (_IS_LOCAL) return null; // Sphinx 7.2+ defines the content root data attribute in the HTML element. const _CONTENT_ROOT = document.documentElement.dataset.content_root; if (_CONTENT_ROOT !== undefined) { @@ -20,6 +25,10 @@ const all_languages = $LANGUAGES; const _create_version_select = () => { const select = document.createElement('select'); select.className = 'version-select'; + if (_IS_LOCAL) { + select.disabled = true; + select.title = 'Version switching is disabled in local builds'; + } for (const [version, title] of Object.entries(all_versions)) { const option = document.createElement('option'); @@ -44,6 +53,10 @@ const _create_language_select = () => { const select = document.createElement('select'); select.className = 'language-select'; + if (_IS_LOCAL) { + select.disabled = true; + select.title = 'Language switching is disabled in local builds'; + } for (const [language, title] of Object.entries(all_languages)) { const option = document.createElement('option'); @@ -59,10 +72,6 @@ const _create_language_select = () => { const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { - if (url.startsWith('file:///')) { - window.location.href = url; - return; - } fetch(url) .then((response) => { if (response.ok) { @@ -82,6 +91,8 @@ const _navigate_to_first_existing = (urls) => { }; const _on_version_switch = (event) => { + if (_IS_LOCAL) return; + const selected_version = event.target.value; // English has no language prefix. const new_prefix_en = `/${selected_version}/`; @@ -105,6 +116,8 @@ const _on_version_switch = (event) => { }; const _on_language_switch = (event) => { + if (_IS_LOCAL) return; + const selected_language = event.target.value; // English has no language prefix. const new_prefix = From 4c495ccf8bb0eb512a4b7acba7b6d682babdf91c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:35:03 +0000 Subject: [PATCH 139/218] Remove placeholder classes when initialisation is complete --- templates/switchers.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 025daed..c6dd30e 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -143,6 +143,7 @@ const _initialise_switchers = () => { const s = version_select.cloneNode(true); s.addEventListener('change', _on_version_switch); placeholder.append(s); + placeholder.classList.remove('version_switcher_placeholder'); }); const language_select = _create_language_select(); @@ -152,6 +153,7 @@ const _initialise_switchers = () => { const s = language_select.cloneNode(true); s.addEventListener('change', _on_language_switch); placeholder.append(s); + placeholder.classList.remove('language_switcher_placeholder'); }); }; From 3031085ac20893336bd9f317b66d97db0ce21af1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:46:44 +0000 Subject: [PATCH 140/218] Use a Map for versions and languages --- templates/switchers.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index c6dd30e..2e4fd76 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -19,10 +19,10 @@ const _CURRENT_PREFIX = (() => { return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; })(); -const all_versions = $VERSIONS; -const all_languages = $LANGUAGES; +const _ALL_VERSIONS = new Map(Object.entries($VERSIONS)); +const _ALL_LANGUAGES = new Map(Object.entries($LANGUAGES)); -const _create_version_select = () => { +const _create_version_select = (versions) => { const select = document.createElement('select'); select.className = 'version-select'; if (_IS_LOCAL) { @@ -30,7 +30,7 @@ const _create_version_select = () => { select.title = 'Version switching is disabled in local builds'; } - for (const [version, title] of Object.entries(all_versions)) { + for (const [version, title] of versions) { const option = document.createElement('option'); option.value = version; if (version === _CURRENT_VERSION) { @@ -45,10 +45,10 @@ const _create_version_select = () => { return select; }; -const _create_language_select = () => { - if (!(_CURRENT_LANGUAGE in all_languages)) { - // In case we are browsing a language that is not yet in all_languages. - all_languages[_CURRENT_LANGUAGE] = _CURRENT_LANGUAGE; +const _create_language_select = (languages) => { + if (!languages.has(_CURRENT_LANGUAGE)) { + // In case we are browsing a language that is not yet in languages. + languages.set(_CURRENT_LANGUAGE, _CURRENT_LANGUAGE); } const select = document.createElement('select'); @@ -58,7 +58,7 @@ const _create_language_select = () => { select.title = 'Language switching is disabled in local builds'; } - for (const [language, title] of Object.entries(all_languages)) { + for (const [language, title] of languages) { const option = document.createElement('option'); option.value = language; option.text = title; @@ -136,7 +136,10 @@ const _on_language_switch = (event) => { }; const _initialise_switchers = () => { - const version_select = _create_version_select(); + const versions = _ALL_VERSIONS; + const languages = _ALL_LANGUAGES; + + const version_select = _create_version_select(versions); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { @@ -146,7 +149,7 @@ const _initialise_switchers = () => { placeholder.classList.remove('version_switcher_placeholder'); }); - const language_select = _create_language_select(); + const language_select = _create_language_select(languages); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { From 8cb67060a4fd9e4508b006df3052d91c2d545a3a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:51:02 +0000 Subject: [PATCH 141/218] Add JSDoc comments --- templates/switchers.js | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 2e4fd76..cd4cbc2 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -22,6 +22,11 @@ const _CURRENT_PREFIX = (() => { const _ALL_VERSIONS = new Map(Object.entries($VERSIONS)); const _ALL_LANGUAGES = new Map(Object.entries($LANGUAGES)); +/** + * @param {Map} versions + * @returns {HTMLSelectElement} + * @private + */ const _create_version_select = (versions) => { const select = document.createElement('select'); select.className = 'version-select'; @@ -45,6 +50,11 @@ const _create_version_select = (versions) => { return select; }; +/** + * @param {Map} languages + * @returns {HTMLSelectElement} + * @private + */ const _create_language_select = (languages) => { if (!languages.has(_CURRENT_LANGUAGE)) { // In case we are browsing a language that is not yet in languages. @@ -69,6 +79,11 @@ const _create_language_select = (languages) => { return select; }; +/** + * Change the current page to the first existing URL in the list. + * @param {Array} urls + * @private + */ const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { @@ -90,6 +105,12 @@ const _navigate_to_first_existing = (urls) => { return '/'; }; +/** + * Callback for the version switcher. + * @param {Event} event + * @returns {void} + * @private + */ const _on_version_switch = (event) => { if (_IS_LOCAL) return; @@ -115,6 +136,12 @@ const _on_version_switch = (event) => { } }; +/** + * Callback for the language switcher. + * @param {Event} event + * @returns {void} + * @private + */ const _on_language_switch = (event) => { if (_IS_LOCAL) return; @@ -135,6 +162,11 @@ const _on_language_switch = (event) => { } }; +/** + * Initialisation function for the version and language switchers. + * @returns {void} + * @private + */ const _initialise_switchers = () => { const versions = _ALL_VERSIONS; const languages = _ALL_LANGUAGES; From 58c9634bcad42f8114e8665955d73dcbfbfab02e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 29 Oct 2024 22:22:46 +0000 Subject: [PATCH 142/218] Convert promises to async-await (#227) --- templates/switchers.js | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index cd4cbc2..b479b3c 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -84,20 +84,18 @@ const _create_language_select = (languages) => { * @param {Array} urls * @private */ -const _navigate_to_first_existing = (urls) => { +const _navigate_to_first_existing = async (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { - fetch(url) - .then((response) => { - if (response.ok) { - window.location.href = url; - return url; - } - }) - .catch((err) => { - console.error(`Error when fetching '${url}'!`); - console.error(err); - }); + try { + const response = await fetch(url, { method: 'HEAD' }); + if (response.ok) { + window.location.href = url; + return url; + } + } catch (err) { + console.error(`Error when fetching '${url}': ${err}`); + } } // if all else fails, redirect to the d.p.o root @@ -111,7 +109,7 @@ const _navigate_to_first_existing = (urls) => { * @returns {void} * @private */ -const _on_version_switch = (event) => { +const _on_version_switch = async (event) => { if (_IS_LOCAL) return; const selected_version = event.target.value; @@ -127,7 +125,7 @@ const _on_version_switch = (event) => { // 2. The current page in English with the new version // 3. The documentation home in the current language with the new version // 4. The documentation home in English with the new version - _navigate_to_first_existing([ + await _navigate_to_first_existing([ window.location.href.replace(_CURRENT_PREFIX, new_prefix), window.location.href.replace(_CURRENT_PREFIX, new_prefix_en), new_prefix, @@ -142,7 +140,7 @@ const _on_version_switch = (event) => { * @returns {void} * @private */ -const _on_language_switch = (event) => { +const _on_language_switch = async (event) => { if (_IS_LOCAL) return; const selected_language = event.target.value; @@ -155,7 +153,7 @@ const _on_language_switch = (event) => { // Try the following pages in order: // 1. The current page in the new language with the current version // 2. The documentation home in the new language with the current version - _navigate_to_first_existing([ + await _navigate_to_first_existing([ window.location.href.replace(_CURRENT_PREFIX, new_prefix), new_prefix, ]); From c1fc85cfe4aeb2e99cc09096e28b4d8ca76e3db2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 29 Oct 2024 22:32:11 +0000 Subject: [PATCH 143/218] Always create new select nodes (#228) --- templates/switchers.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index b479b3c..dd28044 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -169,21 +169,19 @@ const _initialise_switchers = () => { const versions = _ALL_VERSIONS; const languages = _ALL_LANGUAGES; - const version_select = _create_version_select(versions); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { - const s = version_select.cloneNode(true); + const s = _create_version_select(versions); s.addEventListener('change', _on_version_switch); placeholder.append(s); placeholder.classList.remove('version_switcher_placeholder'); }); - const language_select = _create_language_select(languages); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { - const s = language_select.cloneNode(true); + const s = _create_language_select(languages); s.addEventListener('change', _on_language_switch); placeholder.append(s); placeholder.classList.remove('language_switcher_placeholder'); From eff0dd956eb3e58cb294100aea1bf0bf12a53ba9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:36:29 +0000 Subject: [PATCH 144/218] Run prettier in GitHub Actions (#232) --- .github/workflows/lint.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d553e49..26f2cc1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ permissions: contents: read jobs: - lint: + pre-commit: runs-on: ubuntu-latest steps: @@ -20,3 +20,14 @@ jobs: python-version: "3.x" cache: pip - uses: pre-commit/action@v3.0.1 + + prettier: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Lint with Prettier + run: npx prettier templates/switchers.js --check --single-quote From 49641c17b23dac6ecf3fefbe4483dca530dfd19f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:42:25 +0000 Subject: [PATCH 145/218] Create switchers maps from arrays of pairs (#231) --- build_docs.py | 8 ++++---- templates/switchers.js | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build_docs.py b/build_docs.py index 6596a25..f14cc2b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -388,16 +388,16 @@ def setup_switchers( - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - languages_map = dict(sorted((l.tag, l.name) for l in languages if l.in_prod)) - versions_map = {v.name: v.picker_label for v in reversed(versions)} + language_pairs = sorted((l.tag, l.name) for l in languages if l.in_prod) + version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] switchers_template_file = HERE / "templates" / "switchers.js" switchers_path = html_root / "_static" / "switchers.js" template = Template(switchers_template_file.read_text(encoding="UTF-8")) rendered_template = template.safe_substitute( - LANGUAGES=json.dumps(languages_map), - VERSIONS=json.dumps(versions_map), + LANGUAGES=json.dumps(language_pairs), + VERSIONS=json.dumps(version_pairs), ) switchers_path.write_text(rendered_template, encoding="UTF-8") diff --git a/templates/switchers.js b/templates/switchers.js index dd28044..d1a4768 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -19,8 +19,8 @@ const _CURRENT_PREFIX = (() => { return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; })(); -const _ALL_VERSIONS = new Map(Object.entries($VERSIONS)); -const _ALL_LANGUAGES = new Map(Object.entries($LANGUAGES)); +const _ALL_VERSIONS = new Map($VERSIONS); +const _ALL_LANGUAGES = new Map($LANGUAGES); /** * @param {Map} versions From 91d0994a794d255c767e42873ffee6502c88b91a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:45:36 +0000 Subject: [PATCH 146/218] Keep switcher placeholder classes --- templates/switchers.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index d1a4768..3eefa71 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -175,7 +175,6 @@ const _initialise_switchers = () => { const s = _create_version_select(versions); s.addEventListener('change', _on_version_switch); placeholder.append(s); - placeholder.classList.remove('version_switcher_placeholder'); }); document @@ -184,7 +183,6 @@ const _initialise_switchers = () => { const s = _create_language_select(languages); s.addEventListener('change', _on_language_switch); placeholder.append(s); - placeholder.classList.remove('language_switcher_placeholder'); }); }; From cc4f5531c333c0beaaa4e89e528787e877db6c43 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 31 Oct 2024 18:13:40 +0200 Subject: [PATCH 147/218] Run prettier via pre-commit (#234) --- .github/workflows/lint.yml | 13 +--------- .pre-commit-config.yaml | 20 ++++++++++----- templates/switchers.js | 52 +++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 45 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 26f2cc1..d553e49 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ permissions: contents: read jobs: - pre-commit: + lint: runs-on: ubuntu-latest steps: @@ -20,14 +20,3 @@ jobs: python-version: "3.x" cache: pip - uses: pre-commit/action@v3.0.1 - - prettier: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "22" - - name: Lint with Prettier - run: npx prettier templates/switchers.js --check --single-quote diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c3d2f5f..60a928c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -14,35 +14,41 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.7 + rev: v0.7.1 hooks: - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.2 + rev: 0.29.4 hooks: - id: check-github-workflows - repo: https://github.com/rhysd/actionlint - rev: v1.7.1 + rev: v1.7.3 hooks: - id: actionlint - repo: https://github.com/tox-dev/pyproject-fmt - rev: 2.2.3 + rev: v2.5.0 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.19 + rev: v0.22 hooks: - id: validate-pyproject - repo: https://github.com/tox-dev/tox-ini-fmt - rev: 1.4.0 + rev: 1.4.1 hooks: - id: tox-ini-fmt + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.3.3 + hooks: + - id: prettier + files: templates/switchers.js + - repo: meta hooks: - id: check-hooks-apply diff --git a/templates/switchers.js b/templates/switchers.js index 3eefa71..44c71bb 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,12 +1,12 @@ -'use strict'; +"use strict"; // File URIs must begin with either one or three forward slashes -const _is_file_uri = (uri) => uri.startsWith('file:/'); +const _is_file_uri = (uri) => uri.startsWith("file:/"); const _IS_LOCAL = _is_file_uri(window.location.href); -const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ''; -const _CURRENT_VERSION = _CURRENT_RELEASE.split('.', 2).join('.'); -const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; +const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ""; +const _CURRENT_VERSION = _CURRENT_RELEASE.split(".", 2).join("."); +const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || "en"; const _CURRENT_PREFIX = (() => { if (_IS_LOCAL) return null; // Sphinx 7.2+ defines the content root data attribute in the HTML element. @@ -15,8 +15,8 @@ const _CURRENT_PREFIX = (() => { return new URL(_CONTENT_ROOT, window.location).pathname; } // Fallback for older versions of Sphinx (used in Python 3.10 and older). - const _NUM_PREFIX_PARTS = _CURRENT_LANGUAGE === 'en' ? 2 : 3; - return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; + const _NUM_PREFIX_PARTS = _CURRENT_LANGUAGE === "en" ? 2 : 3; + return window.location.pathname.split("/", _NUM_PREFIX_PARTS).join("/") + "/"; })(); const _ALL_VERSIONS = new Map($VERSIONS); @@ -28,15 +28,15 @@ const _ALL_LANGUAGES = new Map($LANGUAGES); * @private */ const _create_version_select = (versions) => { - const select = document.createElement('select'); - select.className = 'version-select'; + const select = document.createElement("select"); + select.className = "version-select"; if (_IS_LOCAL) { select.disabled = true; - select.title = 'Version switching is disabled in local builds'; + select.title = "Version switching is disabled in local builds"; } for (const [version, title] of versions) { - const option = document.createElement('option'); + const option = document.createElement("option"); option.value = version; if (version === _CURRENT_VERSION) { option.text = _CURRENT_RELEASE; @@ -61,15 +61,15 @@ const _create_language_select = (languages) => { languages.set(_CURRENT_LANGUAGE, _CURRENT_LANGUAGE); } - const select = document.createElement('select'); - select.className = 'language-select'; + const select = document.createElement("select"); + select.className = "language-select"; if (_IS_LOCAL) { select.disabled = true; - select.title = 'Language switching is disabled in local builds'; + select.title = "Language switching is disabled in local builds"; } for (const [language, title] of languages) { - const option = document.createElement('option'); + const option = document.createElement("option"); option.value = language; option.text = title; if (language === _CURRENT_LANGUAGE) option.selected = true; @@ -88,7 +88,7 @@ const _navigate_to_first_existing = async (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { try { - const response = await fetch(url, { method: 'HEAD' }); + const response = await fetch(url, { method: "HEAD" }); if (response.ok) { window.location.href = url; return url; @@ -99,8 +99,8 @@ const _navigate_to_first_existing = async (urls) => { } // if all else fails, redirect to the d.p.o root - window.location.href = '/'; - return '/'; + window.location.href = "/"; + return "/"; }; /** @@ -116,7 +116,7 @@ const _on_version_switch = async (event) => { // English has no language prefix. const new_prefix_en = `/${selected_version}/`; const new_prefix = - _CURRENT_LANGUAGE === 'en' + _CURRENT_LANGUAGE === "en" ? new_prefix_en : `/${_CURRENT_LANGUAGE}/${selected_version}/`; if (_CURRENT_PREFIX !== new_prefix) { @@ -146,7 +146,7 @@ const _on_language_switch = async (event) => { const selected_language = event.target.value; // English has no language prefix. const new_prefix = - selected_language === 'en' + selected_language === "en" ? `/${_CURRENT_VERSION}/` : `/${selected_language}/${_CURRENT_VERSION}/`; if (_CURRENT_PREFIX !== new_prefix) { @@ -170,24 +170,24 @@ const _initialise_switchers = () => { const languages = _ALL_LANGUAGES; document - .querySelectorAll('.version_switcher_placeholder') + .querySelectorAll(".version_switcher_placeholder") .forEach((placeholder) => { const s = _create_version_select(versions); - s.addEventListener('change', _on_version_switch); + s.addEventListener("change", _on_version_switch); placeholder.append(s); }); document - .querySelectorAll('.language_switcher_placeholder') + .querySelectorAll(".language_switcher_placeholder") .forEach((placeholder) => { const s = _create_language_select(languages); - s.addEventListener('change', _on_language_switch); + s.addEventListener("change", _on_language_switch); placeholder.append(s); }); }; -if (document.readyState !== 'loading') { +if (document.readyState !== "loading") { _initialise_switchers(); } else { - document.addEventListener('DOMContentLoaded', _initialise_switchers); + document.addEventListener("DOMContentLoaded", _initialise_switchers); } From b2b548353b50a3316ba098b89382fe6b8f8474a9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 31 Oct 2024 23:04:00 +0000 Subject: [PATCH 148/218] Update JSDoc comments in switchers.js --- templates/switchers.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 44c71bb..7a4fb63 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -104,7 +104,7 @@ const _navigate_to_first_existing = async (urls) => { }; /** - * Callback for the version switcher. + * Navigate to the selected version. * @param {Event} event * @returns {void} * @private @@ -135,7 +135,7 @@ const _on_version_switch = async (event) => { }; /** - * Callback for the language switcher. + * Navigate to the selected language. * @param {Event} event * @returns {void} * @private @@ -161,7 +161,7 @@ const _on_language_switch = async (event) => { }; /** - * Initialisation function for the version and language switchers. + * Set up the version and language switchers. * @returns {void} * @private */ From b5f5a52faf28ab13990ebbc577c1a3c206343850 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:15:46 +0200 Subject: [PATCH 149/218] Add instructions on manually rebuilding a branch --- README.md | 4 ++-- RELEASING.md | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 RELEASING.md diff --git a/README.md b/README.md index 767014c..b9881e7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This repository contains scripts for automatically building the Python documentation on [docs.python.org](https://docs.python.org). -# How to test it? +## How to test it? The following command should build all maintained versions and translations in `./www`, beware it can take a few hours: @@ -15,7 +15,7 @@ If you don't need to build all translations of all branches, add `--language en --branch main`. -# Check current version +## Check current version Install `tools_requirements.txt` then run `python check_versions.py ../cpython/` (pointing to a real CPython clone) to see which version diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..abe72cd --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,16 @@ +# Manually rebuild a branch + +Docs for [feature and bugfix branches](https://devguide.python.org/versions/) are +automatically built from a cron. + +Manual rebuilds are needed for new security releases, +and to add the end-of-life banner for newly end-of-life branches. + +To manually rebuild a branch, for example 3.11: + +```shell +ssh docs.nyc1.psf.io +sudo su --shell=/bin/bash docsbuild +screen -DUR # Rejoin screen session if it exists, otherwise create a new one +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 +``` From 10064ee59d8c6f2fbad2391055d746f1d5deb991 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 10 Dec 2024 23:10:01 +0200 Subject: [PATCH 150/218] Move to README --- README.md | 17 +++++++++++++++++ RELEASING.md | 16 ---------------- 2 files changed, 17 insertions(+), 16 deletions(-) delete mode 100644 RELEASING.md diff --git a/README.md b/README.md index b9881e7..1b76bcd 100644 --- a/README.md +++ b/README.md @@ -51,3 +51,20 @@ of Sphinx we're using where: 3.13 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 3.14 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= + +## Manually rebuild a branch + +Docs for [feature and bugfix branches](https://devguide.python.org/versions/) are +automatically built from a cron. + +Manual rebuilds are needed for new security releases, +and to add the end-of-life banner for newly end-of-life branches. + +To manually rebuild a branch, for example 3.11: + +```shell +ssh docs.nyc1.psf.io +sudo su --shell=/bin/bash docsbuild +screen -DUR # Rejoin screen session if it exists, otherwise create a new one +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 +``` diff --git a/RELEASING.md b/RELEASING.md deleted file mode 100644 index abe72cd..0000000 --- a/RELEASING.md +++ /dev/null @@ -1,16 +0,0 @@ -# Manually rebuild a branch - -Docs for [feature and bugfix branches](https://devguide.python.org/versions/) are -automatically built from a cron. - -Manual rebuilds are needed for new security releases, -and to add the end-of-life banner for newly end-of-life branches. - -To manually rebuild a branch, for example 3.11: - -```shell -ssh docs.nyc1.psf.io -sudo su --shell=/bin/bash docsbuild -screen -DUR # Rejoin screen session if it exists, otherwise create a new one -/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 -``` From e4a8aff9772738a63d0945042777d18c3d926930 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 28 Dec 2024 12:30:57 +0000 Subject: [PATCH 151/218] Enable the Polish translation in the language switcher (#237) --- config.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/config.toml b/config.toml index 3716d7f..3679c73 100644 --- a/config.toml +++ b/config.toml @@ -69,7 +69,6 @@ sphinxopts = [ [languages.pl] name = "Polish" -in_prod = false [languages.pt_BR] name = "Brazilian Portuguese" From b3f238f367d70c4f8cab4d3163d1a5eecc771c2f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 17 Jan 2025 12:51:16 +0000 Subject: [PATCH 152/218] Enable translation_progress_classes (#239) --- build_docs.py | 26 ++++++++++++++------------ config.toml | 1 - 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/build_docs.py b/build_docs.py index f14cc2b..b5cf717 100755 --- a/build_docs.py +++ b/build_docs.py @@ -192,7 +192,7 @@ def __gt__(self, other): return self.as_tuple() > other.as_tuple() -@dataclass(frozen=True, order=True) +@dataclass(order=True, frozen=True, kw_only=True) class Language: iso639_tag: str name: str @@ -710,6 +710,7 @@ def build(self): f"-D locale_dirs={locale_dirs}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", + "-D translation_progress_classes=1", ) ) if self.language.tag == "ja": @@ -1141,19 +1142,20 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: def parse_languages_from_config() -> list[Language]: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) - languages = [] defaults = config["defaults"] - for iso639_tag, section in config["languages"].items(): - languages.append( - Language( - iso639_tag, - section["name"], - section.get("in_prod", defaults["in_prod"]), - sphinxopts=section.get("sphinxopts", defaults["sphinxopts"]), - html_only=section.get("html_only", defaults["html_only"]), - ) + default_in_prod = defaults.get("in_prod", True) + default_sphinxopts = defaults.get("sphinxopts", []) + default_html_only = defaults.get("html_only", False) + return [ + Language( + iso639_tag=iso639_tag, + name=section["name"], + in_prod=section.get("in_prod", default_in_prod), + sphinxopts=section.get("sphinxopts", default_sphinxopts), + html_only=section.get("html_only", default_html_only), ) - return languages + for iso639_tag, section in config["languages"].items() + ] def format_seconds(seconds: float) -> str: diff --git a/config.toml b/config.toml index 3679c73..b0994ad 100644 --- a/config.toml +++ b/config.toml @@ -33,7 +33,6 @@ in_prod = false [languages.it] name = "Italian" -in_prod = true [languages.ja] name = "Japanese" From be699c482265555fda16dc8dccf1bcef25d23fb8 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 18 Jan 2025 15:38:54 +0000 Subject: [PATCH 153/218] Disable translation_progress_classes Sphinx does not properly convert command-line overrides to Boolean values, so ``-D translation_progress_classes=1`` fails with: Sphinx parallel build error: sphinx.errors.ConfigError: translation_progress_classes must be True, False, "translated" or "untranslated" --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index b5cf717..40384b8 100755 --- a/build_docs.py +++ b/build_docs.py @@ -710,7 +710,7 @@ def build(self): f"-D locale_dirs={locale_dirs}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", - "-D translation_progress_classes=1", + # "-D translation_progress_classes=1", ) ) if self.language.tag == "ja": From 4e7299dfd3719b08d42a162c0bbb9181218b5596 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 18 Jan 2025 18:47:45 +0000 Subject: [PATCH 154/218] Identify the language correctly for older versions (#240) --- templates/switchers.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/switchers.js b/templates/switchers.js index 7a4fb63..774366f 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -6,7 +6,13 @@ const _is_file_uri = (uri) => uri.startsWith("file:/"); const _IS_LOCAL = _is_file_uri(window.location.href); const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ""; const _CURRENT_VERSION = _CURRENT_RELEASE.split(".", 2).join("."); -const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || "en"; +const _CURRENT_LANGUAGE = (() => { + const _LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || "en"; + // Python 2.7 and 3.5--3.10 use ``LANGUAGE: 'None'`` for English + // in ``documentation_options.js``. + if (_LANGUAGE === "none") return "en"; + return _LANGUAGE; +})(); const _CURRENT_PREFIX = (() => { if (_IS_LOCAL) return null; // Sphinx 7.2+ defines the content root data attribute in the HTML element. From 2efa5bce54a2d09573e62a580047b5c7419bbedc Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 20 Jan 2025 23:35:16 +0000 Subject: [PATCH 155/218] Install matplotlib for opengraph preview images (#242) --- build_docs.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 40384b8..a6b08bb 100755 --- a/build_docs.py +++ b/build_docs.py @@ -745,6 +745,10 @@ def build(self): blurb = self.venv / "bin" / "blurb" if self.includes_html: + # Define a tag to enable opengraph socialcards previews + # (used in Doc/conf.py and requires matplotlib) + sphinxopts.append("-t create-social-cards") + # Disable CPython switchers, we handle them now: run( ["sed", "-i"] @@ -783,13 +787,17 @@ def build_venv(self): So we can reuse them from builds to builds, while they contain different Sphinx versions. """ + requirements = [self.theme] + self.version.requirements + if self.includes_html: + # opengraph previews + requirements.append("matplotlib>=3") + venv_path = self.build_root / ("venv-" + self.version.name) run([sys.executable, "-m", "venv", venv_path]) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] + ["--upgrade-strategy=eager"] - + [self.theme] - + self.version.requirements, + + requirements, cwd=self.checkout / "Doc", ) run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) From 6aa487d97044d111c27deae12baa85755351e4fb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 19 Feb 2025 03:23:46 +0000 Subject: [PATCH 156/218] Include languages' translated names in the switcher (#245) Co-authored-by: Ezio Melotti Co-authored-by: Rafael Fontenelle Co-authored-by: W. H. Wang --- build_docs.py | 11 ++++++++++- config.toml | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index a6b08bb..8b7f3f9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -196,6 +196,7 @@ def __gt__(self, other): class Language: iso639_tag: str name: str + translated_name: str in_prod: bool sphinxopts: tuple html_only: bool = False @@ -204,6 +205,12 @@ class Language: def tag(self): return self.iso639_tag.replace("_", "-").lower() + @property + def switcher_label(self): + if self.translated_name: + return f"{self.name} | {self.translated_name}" + return self.name + @staticmethod def filter(languages, language_tags=None): """Filter a sequence of languages according to --languages.""" @@ -388,7 +395,7 @@ def setup_switchers( - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - language_pairs = sorted((l.tag, l.name) for l in languages if l.in_prod) + language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] switchers_template_file = HERE / "templates" / "switchers.js" @@ -1151,6 +1158,7 @@ def parse_languages_from_config() -> list[Language]: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) defaults = config["defaults"] + default_translated_name = defaults.get("translated_name", "") default_in_prod = defaults.get("in_prod", True) default_sphinxopts = defaults.get("sphinxopts", []) default_html_only = defaults.get("html_only", False) @@ -1158,6 +1166,7 @@ def parse_languages_from_config() -> list[Language]: Language( iso639_tag=iso639_tag, name=section["name"], + translated_name=section.get("translated_name", default_translated_name), in_prod=section.get("in_prod", default_in_prod), sphinxopts=section.get("sphinxopts", default_sphinxopts), html_only=section.get("html_only", default_html_only), diff --git a/config.toml b/config.toml index b0994ad..489c774 100644 --- a/config.toml +++ b/config.toml @@ -1,5 +1,12 @@ +# name: the English name for the language. +# translated_name: the 'local' name for the language. +# in_prod: If true, include in the language switcher. +# html_only: If true, only create HTML files. +# sphinxopts: Extra options to pass to SPHINXOPTS in the Makefile. + [defaults] # name has no default, it is mandatory. +translated_name = "" in_prod = true html_only = false sphinxopts = [ @@ -13,6 +20,7 @@ name = "English" [languages.es] name = "Spanish" +translated_name = "español" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -21,6 +29,7 @@ sphinxopts = [ [languages.fr] name = "French" +translated_name = "français" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -29,13 +38,16 @@ sphinxopts = [ [languages.id] name = "Indonesian" +translated_name = "Indonesia" in_prod = false [languages.it] name = "Italian" +translated_name = "italiano" [languages.ja] name = "Japanese" +translated_name = "日本語" sphinxopts = [ '-D latex_engine=lualatex', '-D latex_elements.inputenc=', @@ -59,6 +71,7 @@ sphinxopts = [ [languages.ko] name = "Korean" +translated_name = "한국어" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -68,20 +81,25 @@ sphinxopts = [ [languages.pl] name = "Polish" +translated_name = "polski" [languages.pt_BR] name = "Brazilian Portuguese" +translated_name = "Português brasileiro" [languages.tr] name = "Turkish" +translated_name = "Türkçe" [languages.uk] name = "Ukrainian" +translated_name = "українська" in_prod = false html_only = true [languages.zh_CN] name = "Simplified Chinese" +translated_name = "简体中文" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -90,6 +108,7 @@ sphinxopts = [ [languages.zh_TW] name = "Traditional Chinese" +translated_name = "繁體中文" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', From b66ca348eeea7d97eed0617cee091162320a8704 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 19 Feb 2025 12:53:56 +0000 Subject: [PATCH 157/218] Re-enable translation_progress_classes We now use Sphinx 8.2, which should properly convert command-line overrides to Boolean values. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 8b7f3f9..ca57ca5 100755 --- a/build_docs.py +++ b/build_docs.py @@ -717,7 +717,7 @@ def build(self): f"-D locale_dirs={locale_dirs}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", - # "-D translation_progress_classes=1", + "-D translation_progress_classes=1", ) ) if self.language.tag == "ja": From 273cd3c5ab4242c37117a15938e439f289b4c995 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:21:34 +0200 Subject: [PATCH 158/218] Test Python 3.14 & remove Python 3.10-3.12 (#251) --- .github/workflows/test.yml | 2 +- build_docs.py | 8 ++++---- tox.ini | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d74e607..ebff180 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.13", "3.14"] os: [ubuntu-latest] steps: diff --git a/build_docs.py b/build_docs.py index ca57ca5..fb51cf9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -23,7 +23,7 @@ from __future__ import annotations from argparse import ArgumentParser, Namespace -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from contextlib import suppress, contextmanager from dataclasses import dataclass import filecmp @@ -42,7 +42,7 @@ from pathlib import Path from string import Template from time import perf_counter, sleep -from typing import Iterable, Literal +from typing import Literal from urllib.parse import urljoin import jinja2 @@ -479,7 +479,7 @@ def version_info(): """Handler for --version.""" try: platex_version = head( - subprocess.check_output(["platex", "--version"], universal_newlines=True), + subprocess.check_output(["platex", "--version"], text=True), lines=3, ) except FileNotFoundError: @@ -487,7 +487,7 @@ def version_info(): try: xelatex_version = head( - subprocess.check_output(["xelatex", "--version"], universal_newlines=True), + subprocess.check_output(["xelatex", "--version"], text=True), lines=2, ) except FileNotFoundError: diff --git a/tox.ini b/tox.ini index 40a034d..56c6420 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ requires = tox>=4.2 env_list = lint - py{313, 312, 311, 310} + py{314, 313} [testenv] package = wheel From 6902d1c2302411dc560655f7948bd2dee6dcf264 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 20 Feb 2025 15:15:09 +0200 Subject: [PATCH 159/218] Update pre-commit (#249) --- .github/workflows/lint.yml | 7 ++++--- .github/workflows/test.yml | 5 +++-- .pre-commit-config.yaml | 17 +++++++++++------ build_docs.py | 5 ++--- check_times.py | 4 ++-- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d553e49..cd9d07e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,19 +2,20 @@ name: Lint on: [push, pull_request, workflow_dispatch] +permissions: {} + env: FORCE_COLOR: 1 PIP_DISABLE_PIP_VERSION_CHECK: 1 -permissions: - contents: read - jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ebff180..2976bae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,7 @@ name: Test on: [push, pull_request, workflow_dispatch] -permissions: - contents: read +permissions: {} env: FORCE_COLOR: 1 @@ -19,6 +18,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60a928c..44948ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,37 +14,42 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.9.6 hooks: - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.4 + rev: 0.31.1 hooks: - id: check-github-workflows - repo: https://github.com/rhysd/actionlint - rev: v1.7.3 + rev: v1.7.7 hooks: - id: actionlint + - repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v1.3.1 + hooks: + - id: zizmor + - repo: https://github.com/tox-dev/pyproject-fmt rev: v2.5.0 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.22 + rev: v0.23 hooks: - id: validate-pyproject - repo: https://github.com/tox-dev/tox-ini-fmt - rev: 1.4.1 + rev: 1.5.0 hooks: - id: tox-ini-fmt - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.3.3 + rev: v3.5.1 hooks: - id: prettier files: templates/switchers.js diff --git a/build_docs.py b/build_docs.py index fb51cf9..7da3520 100755 --- a/build_docs.py +++ b/build_docs.py @@ -86,7 +86,7 @@ def __init__(self, name, *, status, branch_or_tag=None): if status not in self.STATUSES: raise ValueError( "Version status expected to be one of: " - f"{', '.join(self.STATUSES|set(self.SYNONYMS.keys()))}, got {status!r}." + f"{', '.join(self.STATUSES | set(self.SYNONYMS.keys()))}, got {status!r}." ) self.name = name self.branch_or_tag = branch_or_tag @@ -732,8 +732,7 @@ def build(self): shell=True, ) subprocess.check_output( - "sed -i s/\N{REPLACEMENT CHARACTER}/?/g " - f"{self.checkout}/Doc/**/*.rst", + f"sed -i s/\N{REPLACEMENT CHARACTER}/?/g {self.checkout}/Doc/**/*.rst", shell=True, ) diff --git a/check_times.py b/check_times.py index 9310cd4..2b3d2f9 100644 --- a/check_times.py +++ b/check_times.py @@ -50,7 +50,7 @@ def calc_time(lines: list[str]) -> None: fmt_duration = format_seconds(state_data["last_build_duration"]) reason = state_data["triggered_by"] print( - f"{start:%Y-%m-%d %H:%M UTC} | {version: <7} | {language: <8} | {fmt_duration :<14} | {reason}" + f"{start:%Y-%m-%d %H:%M UTC} | {version: <7} | {language: <8} | {fmt_duration:<14} | {reason}" ) if line.endswith("Build start."): @@ -64,7 +64,7 @@ def calc_time(lines: list[str]) -> None: timestamp = f"{line[:16]} UTC" _, fmt_duration = line.removesuffix(").").split("(") print( - f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration :<14} | -----------" + f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration:<14} | -----------" ) if in_progress: From 7583b1744007e74cf9aafce98fa59677a75ce28d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 5 Apr 2025 16:29:14 +0100 Subject: [PATCH 160/218] Define ``ogp_site_url`` for social media cards (#252) --- build_docs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 7da3520..23fd6c0 100755 --- a/build_docs.py +++ b/build_docs.py @@ -751,9 +751,15 @@ def build(self): blurb = self.venv / "bin" / "blurb" if self.includes_html: + site_url = self.version.url + if self.language.tag != "en": + site_url += f"{self.language.tag}/" # Define a tag to enable opengraph socialcards previews # (used in Doc/conf.py and requires matplotlib) - sphinxopts.append("-t create-social-cards") + sphinxopts += ( + "-t create-social-cards", + f"-D ogp_site_url={site_url}", + ) # Disable CPython switchers, we handle them now: run( From 667cd01ec2b694fdecae6a3416f16774d51d390a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 01:09:44 +0100 Subject: [PATCH 161/218] Sort imports --- build_docs.py | 12 ++++++------ check_versions.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build_docs.py b/build_docs.py index 23fd6c0..1ab2fdd 100755 --- a/build_docs.py +++ b/build_docs.py @@ -22,23 +22,23 @@ from __future__ import annotations -from argparse import ArgumentParser, Namespace -from collections.abc import Iterable, Sequence -from contextlib import suppress, contextmanager -from dataclasses import dataclass import filecmp import json import logging import logging.handlers -from functools import total_ordering -from os import getenv, readlink import re import shlex import shutil import subprocess import sys +from argparse import ArgumentParser, Namespace from bisect import bisect_left as bisect +from collections.abc import Iterable, Sequence +from contextlib import contextmanager, suppress +from dataclasses import dataclass from datetime import datetime as dt, timezone +from functools import total_ordering +from os import getenv, readlink from pathlib import Path from string import Template from time import perf_counter, sleep diff --git a/check_versions.py b/check_versions.py index 70cade9..343c85a 100644 --- a/check_versions.py +++ b/check_versions.py @@ -1,15 +1,15 @@ #!/usr/bin/env python -from pathlib import Path import argparse import asyncio import logging import re +from pathlib import Path +import git import httpx import urllib3 from tabulate import tabulate -import git import build_docs From 62903155bd36c7ce09e8136b5d804552f4ba477c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 01:50:33 +0100 Subject: [PATCH 162/218] Move ``parse_args()`` and ``setup_logging()`` after ``main()`` --- build_docs.py | 206 +++++++++++++++++++++++++------------------------- 1 file changed, 103 insertions(+), 103 deletions(-) diff --git a/build_docs.py b/build_docs.py index 1ab2fdd..3da84df 100755 --- a/build_docs.py +++ b/build_docs.py @@ -506,109 +506,6 @@ def version_info(): ) -def parse_args(): - """Parse command-line arguments.""" - - parser = ArgumentParser( - description="Runs a build of the Python docs for various branches." - ) - parser.add_argument( - "--select-output", - choices=("no-html", "only-html", "only-html-en"), - help="Choose what outputs to build.", - ) - parser.add_argument( - "-q", - "--quick", - action="store_true", - help="Run a quick build (only HTML files).", - ) - parser.add_argument( - "-b", - "--branch", - metavar="3.12", - help="Version to build (defaults to all maintained branches).", - ) - parser.add_argument( - "-r", - "--build-root", - type=Path, - help="Path to a directory containing a checkout per branch.", - default=Path("/srv/docsbuild"), - ) - parser.add_argument( - "-w", - "--www-root", - type=Path, - help="Path where generated files will be copied.", - default=Path("/srv/docs.python.org"), - ) - parser.add_argument( - "--skip-cache-invalidation", - help="Skip Fastly cache invalidation.", - action="store_true", - ) - parser.add_argument( - "--group", - help="Group files on targets and www-root file should get.", - default="docs", - ) - parser.add_argument( - "--log-directory", - type=Path, - help="Directory used to store logs.", - default=Path("/var/log/docsbuild/"), - ) - parser.add_argument( - "--languages", - nargs="*", - help="Language translation, as a PEP 545 language tag like" - " 'fr' or 'pt-br'. " - "Builds all available languages by default.", - metavar="fr", - ) - parser.add_argument( - "--version", - action="store_true", - help="Get build_docs and dependencies version info", - ) - parser.add_argument( - "--theme", - default="python-docs-theme", - help="Python package to use for python-docs-theme: Useful to test branches:" - " --theme git+https://github.com/obulat/python-docs-theme@master", - ) - args = parser.parse_args() - if args.version: - version_info() - sys.exit(0) - del args.version - if args.log_directory: - args.log_directory = args.log_directory.resolve() - if args.build_root: - args.build_root = args.build_root.resolve() - if args.www_root: - args.www_root = args.www_root.resolve() - return args - - -def setup_logging(log_directory: Path, select_output: str | None): - """Setup logging to stderr if run by a human, or to a file if run from a cron.""" - log_format = "%(asctime)s %(levelname)s: %(message)s" - if sys.stderr.isatty(): - logging.basicConfig(format=log_format, stream=sys.stderr) - else: - log_directory.mkdir(parents=True, exist_ok=True) - if select_output is None: - filename = log_directory / "docsbuild.log" - else: - filename = log_directory / f"docsbuild-{select_output}.log" - handler = logging.handlers.WatchedFileHandler(filename) - handler.setFormatter(logging.Formatter(log_format)) - logging.getLogger().addHandler(handler) - logging.getLogger().setLevel(logging.DEBUG) - - @dataclass class DocBuilder: """Builder for a CPython version and a language.""" @@ -1288,6 +1185,109 @@ def main(): build_docs_with_lock(args, "build_docs_html_en.lock") +def parse_args(): + """Parse command-line arguments.""" + + parser = ArgumentParser( + description="Runs a build of the Python docs for various branches." + ) + parser.add_argument( + "--select-output", + choices=("no-html", "only-html", "only-html-en"), + help="Choose what outputs to build.", + ) + parser.add_argument( + "-q", + "--quick", + action="store_true", + help="Run a quick build (only HTML files).", + ) + parser.add_argument( + "-b", + "--branch", + metavar="3.12", + help="Version to build (defaults to all maintained branches).", + ) + parser.add_argument( + "-r", + "--build-root", + type=Path, + help="Path to a directory containing a checkout per branch.", + default=Path("/srv/docsbuild"), + ) + parser.add_argument( + "-w", + "--www-root", + type=Path, + help="Path where generated files will be copied.", + default=Path("/srv/docs.python.org"), + ) + parser.add_argument( + "--skip-cache-invalidation", + help="Skip Fastly cache invalidation.", + action="store_true", + ) + parser.add_argument( + "--group", + help="Group files on targets and www-root file should get.", + default="docs", + ) + parser.add_argument( + "--log-directory", + type=Path, + help="Directory used to store logs.", + default=Path("/var/log/docsbuild/"), + ) + parser.add_argument( + "--languages", + nargs="*", + help="Language translation, as a PEP 545 language tag like" + " 'fr' or 'pt-br'. " + "Builds all available languages by default.", + metavar="fr", + ) + parser.add_argument( + "--version", + action="store_true", + help="Get build_docs and dependencies version info", + ) + parser.add_argument( + "--theme", + default="python-docs-theme", + help="Python package to use for python-docs-theme: Useful to test branches:" + " --theme git+https://github.com/obulat/python-docs-theme@master", + ) + args = parser.parse_args() + if args.version: + version_info() + sys.exit(0) + del args.version + if args.log_directory: + args.log_directory = args.log_directory.resolve() + if args.build_root: + args.build_root = args.build_root.resolve() + if args.www_root: + args.www_root = args.www_root.resolve() + return args + + +def setup_logging(log_directory: Path, select_output: str | None): + """Setup logging to stderr if run by a human, or to a file if run from a cron.""" + log_format = "%(asctime)s %(levelname)s: %(message)s" + if sys.stderr.isatty(): + logging.basicConfig(format=log_format, stream=sys.stderr) + else: + log_directory.mkdir(parents=True, exist_ok=True) + if select_output is None: + filename = log_directory / "docsbuild.log" + else: + filename = log_directory / f"docsbuild-{select_output}.log" + handler = logging.handlers.WatchedFileHandler(filename) + handler.setFormatter(logging.Formatter(log_format)) + logging.getLogger().addHandler(handler) + logging.getLogger().setLevel(logging.DEBUG) + + def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: try: lock = zc.lockfile.LockFile(HERE / lockfile_name) From 8d0b9c3237cc34e9ebc0480cb85bd3345f8da3a9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 01:52:49 +0100 Subject: [PATCH 163/218] Move ``build_docs()`` after ``build_docs_with_lock()`` --- build_docs.py | 608 +++++++++++++++++++++++++------------------------- 1 file changed, 304 insertions(+), 304 deletions(-) diff --git a/build_docs.py b/build_docs.py index 3da84df..fa0ce44 100755 --- a/build_docs.py +++ b/build_docs.py @@ -421,55 +421,6 @@ def setup_switchers( ofile.write(line) -def copy_robots_txt( - www_root: Path, - group, - skip_cache_invalidation, - http: urllib3.PoolManager, -) -> None: - """Copy robots.txt to www_root.""" - if not www_root.exists(): - logging.info("Skipping copying robots.txt (www root does not even exist).") - return - logging.info("Copying robots.txt...") - template_path = HERE / "templates" / "robots.txt" - robots_path = www_root / "robots.txt" - shutil.copyfile(template_path, robots_path) - robots_path.chmod(0o775) - run(["chgrp", group, robots_path]) - if not skip_cache_invalidation: - purge(http, "robots.txt") - - -def build_sitemap( - versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group -): - """Build a sitemap with all live versions and translations.""" - if not www_root.exists(): - logging.info("Skipping sitemap generation (www root does not even exist).") - return - logging.info("Starting sitemap generation...") - template_path = HERE / "templates" / "sitemap.xml" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render(languages=languages, versions=versions) - sitemap_path = www_root / "sitemap.xml" - sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") - sitemap_path.chmod(0o664) - run(["chgrp", group, sitemap_path]) - - -def build_404(www_root: Path, group): - """Build a nice 404 error page to display in case PDFs are not built yet.""" - if not www_root.exists(): - logging.info("Skipping 404 page generation (www root does not even exist).") - return - logging.info("Copying 404 page...") - not_found_file = www_root / "404.html" - shutil.copyfile(HERE / "templates" / "404.html", not_found_file) - not_found_file.chmod(0o664) - run(["chgrp", group, not_found_file]) - - def head(text, lines=10): """Return the first *lines* lines from the given text.""" return "\n".join(text.split("\n")[:lines]) @@ -895,188 +846,6 @@ def save_state(self, build_start: dt, build_duration: float, trigger: str): logging.info("Saved new rebuild state for %s: %s", key, table.as_string()) -def symlink( - www_root: Path, - language: Language, - directory: str, - name: str, - group: str, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Used by major_symlinks and dev_symlink to maintain symlinks.""" - if language.tag == "en": # English is rooted on /, no /en/ - path = www_root - else: - path = www_root / language.tag - link = path / name - directory_path = path / directory - if not directory_path.exists(): - return # No touching link, dest doc not built yet. - - if not link.exists() or readlink(link) != directory: - # Link does not exist or points to the wrong target. - link.unlink(missing_ok=True) - link.symlink_to(directory) - run(["chown", "-h", f":{group}", str(link)]) - if not skip_cache_invalidation: - surrogate_key = f"{language.tag}/{name}" - purge_surrogate_key(http, surrogate_key) - - -def major_symlinks( - www_root: Path, - group: str, - versions: Iterable[Version], - languages: Iterable[Language], - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /2/ and /3/ symlinks for each language. - - Like: - - /3/ → /3.9/ - - /fr/3/ → /fr/3.9/ - - /es/3/ → /es/3.9/ - """ - logging.info("Creating major version symlinks...") - current_stable = Version.current_stable(versions).name - for language in languages: - symlink( - www_root, - language, - current_stable, - "3", - group, - skip_cache_invalidation, - http, - ) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) - - -def dev_symlink( - www_root: Path, - group, - versions, - languages, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /dev/ symlinks for each language. - - Like: - - /dev/ → /3.11/ - - /fr/dev/ → /fr/3.11/ - - /es/dev/ → /es/3.11/ - """ - logging.info("Creating development version symlinks...") - current_dev = Version.current_dev(versions).name - for language in languages: - symlink( - www_root, - language, - current_dev, - "dev", - group, - skip_cache_invalidation, - http, - ) - - -def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: - """Remove one or many paths from docs.python.org's CDN. - - To be used when a file changes, so the CDN fetches the new one. - """ - base = "https://docs.python.org/" - for path in paths: - url = urljoin(base, str(path)) - logging.debug("Purging %s from CDN", url) - http.request("PURGE", url, timeout=30) - - -def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: - """Remove paths from docs.python.org's CDN. - - All paths matching the given 'Surrogate-Key' will be removed. - This is set by the Nginx server for every language-version pair. - To be used when a directory changes, so the CDN fetches the new one. - - https://www.fastly.com/documentation/reference/api/purging/#purge-tag - """ - service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") - fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") - - logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) - http.request( - "POST", - f"https://api.fastly.com/service/{service_id}/purge/{surrogate_key}", - headers={"Fastly-Key": fastly_key}, - timeout=30, - ) - - -def proofread_canonicals( - www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager -) -> None: - """In www_root we check that all canonical links point to existing contents. - - It can happen that a canonical is "broken": - - - /3.11/whatsnew/3.11.html typically would link to - /3/whatsnew/3.11.html, which may not exist yet. - """ - logging.info("Checking canonical links...") - canonical_re = re.compile( - """""" - ) - for file in www_root.glob("**/*.html"): - html = file.read_text(encoding="UTF-8", errors="surrogateescape") - canonical = canonical_re.search(html) - if not canonical: - continue - target = canonical.group(1) - if not (www_root / target).exists(): - logging.info("Removing broken canonical from %s to %s", file, target) - html = html.replace(canonical.group(0), "") - file.write_text(html, encoding="UTF-8", errors="surrogateescape") - if not skip_cache_invalidation: - purge(http, str(file).replace("/srv/docs.python.org/", "")) - - -def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: - releases = http.request( - "GET", - "https://raw.githubusercontent.com/" - "python/devguide/main/include/release-cycle.json", - timeout=30, - ).json() - versions = [Version.from_json(name, release) for name, release in releases.items()] - versions.sort(key=Version.as_tuple) - return versions - - -def parse_languages_from_config() -> list[Language]: - """Read config.toml to discover languages to build.""" - config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) - defaults = config["defaults"] - default_translated_name = defaults.get("translated_name", "") - default_in_prod = defaults.get("in_prod", True) - default_sphinxopts = defaults.get("sphinxopts", []) - default_html_only = defaults.get("html_only", False) - return [ - Language( - iso639_tag=iso639_tag, - name=section["name"], - translated_name=section.get("translated_name", default_translated_name), - in_prod=section.get("in_prod", default_in_prod), - sphinxopts=section.get("sphinxopts", default_sphinxopts), - html_only=section.get("html_only", default_html_only), - ) - for iso639_tag, section in config["languages"].items() - ] - - def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) @@ -1091,79 +860,6 @@ def format_seconds(seconds: float) -> str: return f"{h}h {m}m {s}s" -def build_docs(args) -> bool: - """Build all docs (each language and each version).""" - logging.info("Full build start.") - start_time = perf_counter() - http = urllib3.PoolManager() - versions = parse_versions_from_devguide(http) - languages = parse_languages_from_config() - # Reverse languages but not versions, because we take version-language - # pairs from the end of the list, effectively reversing it. - # This runs languages in config.toml order and versions newest first. - todo = [ - (version, language) - for version in Version.filter(versions, args.branch) - for language in reversed(Language.filter(languages, args.languages)) - ] - del args.branch - del args.languages - all_built_successfully = True - cpython_repo = Repository( - "https://github.com/python/cpython.git", - args.build_root / _checkout_name(args.select_output), - ) - while todo: - version, language = todo.pop() - logging.root.handlers[0].setFormatter( - logging.Formatter( - f"%(asctime)s %(levelname)s {language.tag}/{version.name}: %(message)s" - ) - ) - if sentry_sdk: - scope = sentry_sdk.get_isolation_scope() - scope.set_tag("version", version.name) - scope.set_tag("language", language.tag) - cpython_repo.update() - builder = DocBuilder( - version, versions, language, languages, cpython_repo, **vars(args) - ) - all_built_successfully &= builder.run(http) - logging.root.handlers[0].setFormatter( - logging.Formatter("%(asctime)s %(levelname)s: %(message)s") - ) - - build_sitemap(versions, languages, args.www_root, args.group) - build_404(args.www_root, args.group) - copy_robots_txt( - args.www_root, - args.group, - args.skip_cache_invalidation, - http, - ) - major_symlinks( - args.www_root, - args.group, - versions, - languages, - args.skip_cache_invalidation, - http, - ) - dev_symlink( - args.www_root, - args.group, - versions, - languages, - args.skip_cache_invalidation, - http, - ) - proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) - - logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - - return all_built_successfully - - def _checkout_name(select_output: str | None) -> str: if select_output is not None: return f"cpython-{select_output}" @@ -1301,5 +997,309 @@ def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: lock.close() +def build_docs(args) -> bool: + """Build all docs (each language and each version).""" + logging.info("Full build start.") + start_time = perf_counter() + http = urllib3.PoolManager() + versions = parse_versions_from_devguide(http) + languages = parse_languages_from_config() + # Reverse languages but not versions, because we take version-language + # pairs from the end of the list, effectively reversing it. + # This runs languages in config.toml order and versions newest first. + todo = [ + (version, language) + for version in Version.filter(versions, args.branch) + for language in reversed(Language.filter(languages, args.languages)) + ] + del args.branch + del args.languages + all_built_successfully = True + cpython_repo = Repository( + "https://github.com/python/cpython.git", + args.build_root / _checkout_name(args.select_output), + ) + while todo: + version, language = todo.pop() + logging.root.handlers[0].setFormatter( + logging.Formatter( + f"%(asctime)s %(levelname)s {language.tag}/{version.name}: %(message)s" + ) + ) + if sentry_sdk: + scope = sentry_sdk.get_isolation_scope() + scope.set_tag("version", version.name) + scope.set_tag("language", language.tag) + cpython_repo.update() + builder = DocBuilder( + version, versions, language, languages, cpython_repo, **vars(args) + ) + all_built_successfully &= builder.run(http) + logging.root.handlers[0].setFormatter( + logging.Formatter("%(asctime)s %(levelname)s: %(message)s") + ) + + build_sitemap(versions, languages, args.www_root, args.group) + build_404(args.www_root, args.group) + copy_robots_txt( + args.www_root, + args.group, + args.skip_cache_invalidation, + http, + ) + major_symlinks( + args.www_root, + args.group, + versions, + languages, + args.skip_cache_invalidation, + http, + ) + dev_symlink( + args.www_root, + args.group, + versions, + languages, + args.skip_cache_invalidation, + http, + ) + proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) + + logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) + + return all_built_successfully + + +def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: + releases = http.request( + "GET", + "https://raw.githubusercontent.com/" + "python/devguide/main/include/release-cycle.json", + timeout=30, + ).json() + versions = [Version.from_json(name, release) for name, release in releases.items()] + versions.sort(key=Version.as_tuple) + return versions + + +def parse_languages_from_config() -> list[Language]: + """Read config.toml to discover languages to build.""" + config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) + defaults = config["defaults"] + default_translated_name = defaults.get("translated_name", "") + default_in_prod = defaults.get("in_prod", True) + default_sphinxopts = defaults.get("sphinxopts", []) + default_html_only = defaults.get("html_only", False) + return [ + Language( + iso639_tag=iso639_tag, + name=section["name"], + translated_name=section.get("translated_name", default_translated_name), + in_prod=section.get("in_prod", default_in_prod), + sphinxopts=section.get("sphinxopts", default_sphinxopts), + html_only=section.get("html_only", default_html_only), + ) + for iso639_tag, section in config["languages"].items() + ] + + +def build_sitemap( + versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group +): + """Build a sitemap with all live versions and translations.""" + if not www_root.exists(): + logging.info("Skipping sitemap generation (www root does not even exist).") + return + logging.info("Starting sitemap generation...") + template_path = HERE / "templates" / "sitemap.xml" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render(languages=languages, versions=versions) + sitemap_path = www_root / "sitemap.xml" + sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") + sitemap_path.chmod(0o664) + run(["chgrp", group, sitemap_path]) + + +def build_404(www_root: Path, group): + """Build a nice 404 error page to display in case PDFs are not built yet.""" + if not www_root.exists(): + logging.info("Skipping 404 page generation (www root does not even exist).") + return + logging.info("Copying 404 page...") + not_found_file = www_root / "404.html" + shutil.copyfile(HERE / "templates" / "404.html", not_found_file) + not_found_file.chmod(0o664) + run(["chgrp", group, not_found_file]) + + +def copy_robots_txt( + www_root: Path, + group, + skip_cache_invalidation, + http: urllib3.PoolManager, +) -> None: + """Copy robots.txt to www_root.""" + if not www_root.exists(): + logging.info("Skipping copying robots.txt (www root does not even exist).") + return + logging.info("Copying robots.txt...") + template_path = HERE / "templates" / "robots.txt" + robots_path = www_root / "robots.txt" + shutil.copyfile(template_path, robots_path) + robots_path.chmod(0o775) + run(["chgrp", group, robots_path]) + if not skip_cache_invalidation: + purge(http, "robots.txt") + + +def major_symlinks( + www_root: Path, + group: str, + versions: Iterable[Version], + languages: Iterable[Language], + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Maintains the /2/ and /3/ symlinks for each language. + + Like: + - /3/ → /3.9/ + - /fr/3/ → /fr/3.9/ + - /es/3/ → /es/3.9/ + """ + logging.info("Creating major version symlinks...") + current_stable = Version.current_stable(versions).name + for language in languages: + symlink( + www_root, + language, + current_stable, + "3", + group, + skip_cache_invalidation, + http, + ) + symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) + + +def dev_symlink( + www_root: Path, + group, + versions, + languages, + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Maintains the /dev/ symlinks for each language. + + Like: + - /dev/ → /3.11/ + - /fr/dev/ → /fr/3.11/ + - /es/dev/ → /es/3.11/ + """ + logging.info("Creating development version symlinks...") + current_dev = Version.current_dev(versions).name + for language in languages: + symlink( + www_root, + language, + current_dev, + "dev", + group, + skip_cache_invalidation, + http, + ) + + +def symlink( + www_root: Path, + language: Language, + directory: str, + name: str, + group: str, + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Used by major_symlinks and dev_symlink to maintain symlinks.""" + if language.tag == "en": # English is rooted on /, no /en/ + path = www_root + else: + path = www_root / language.tag + link = path / name + directory_path = path / directory + if not directory_path.exists(): + return # No touching link, dest doc not built yet. + + if not link.exists() or readlink(link) != directory: + # Link does not exist or points to the wrong target. + link.unlink(missing_ok=True) + link.symlink_to(directory) + run(["chown", "-h", f":{group}", str(link)]) + if not skip_cache_invalidation: + surrogate_key = f"{language.tag}/{name}" + purge_surrogate_key(http, surrogate_key) + + +def proofread_canonicals( + www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager +) -> None: + """In www_root we check that all canonical links point to existing contents. + + It can happen that a canonical is "broken": + + - /3.11/whatsnew/3.11.html typically would link to + /3/whatsnew/3.11.html, which may not exist yet. + """ + logging.info("Checking canonical links...") + canonical_re = re.compile( + """""" + ) + for file in www_root.glob("**/*.html"): + html = file.read_text(encoding="UTF-8", errors="surrogateescape") + canonical = canonical_re.search(html) + if not canonical: + continue + target = canonical.group(1) + if not (www_root / target).exists(): + logging.info("Removing broken canonical from %s to %s", file, target) + html = html.replace(canonical.group(0), "") + file.write_text(html, encoding="UTF-8", errors="surrogateescape") + if not skip_cache_invalidation: + purge(http, str(file).replace("/srv/docs.python.org/", "")) + + +def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: + """Remove one or many paths from docs.python.org's CDN. + + To be used when a file changes, so the CDN fetches the new one. + """ + base = "https://docs.python.org/" + for path in paths: + url = urljoin(base, str(path)) + logging.debug("Purging %s from CDN", url) + http.request("PURGE", url, timeout=30) + + +def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: + """Remove paths from docs.python.org's CDN. + + All paths matching the given 'Surrogate-Key' will be removed. + This is set by the Nginx server for every language-version pair. + To be used when a directory changes, so the CDN fetches the new one. + + https://www.fastly.com/documentation/reference/api/purging/#purge-tag + """ + service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") + fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") + + logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) + http.request( + "POST", + f"https://api.fastly.com/service/{service_id}/purge/{surrogate_key}", + headers={"Fastly-Key": fastly_key}, + timeout=30, + ) + + if __name__ == "__main__": sys.exit(main()) From e096701ea6bf82281dc1c73936bf8ca2d8b39bee Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:17:57 +0100 Subject: [PATCH 164/218] Improve type annotations for versions and languages (#253) --- build_docs.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index fa0ce44..21b3e6a 100755 --- a/build_docs.py +++ b/build_docs.py @@ -33,7 +33,6 @@ import sys from argparse import ArgumentParser, Namespace from bisect import bisect_left as bisect -from collections.abc import Iterable, Sequence from contextlib import contextmanager, suppress from dataclasses import dataclass from datetime import datetime as dt, timezone @@ -42,7 +41,6 @@ from pathlib import Path from string import Template from time import perf_counter, sleep -from typing import Literal from urllib.parse import urljoin import jinja2 @@ -50,6 +48,14 @@ import urllib3 import zc.lockfile +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Sequence + from typing import Literal, TypeAlias + + Versions: TypeAlias = Sequence["Version"] + Languages: TypeAlias = Sequence["Language"] + try: from os import EX_OK, EX_SOFTWARE as EX_FAILURE except ImportError: @@ -170,7 +176,7 @@ def picker_label(self): return f"pre ({self.name})" return self.name - def setup_indexsidebar(self, versions: Sequence[Version], dest_path: Path): + def setup_indexsidebar(self, versions: Versions, dest_path: Path): """Build indexsidebar.html for Sphinx.""" template_path = HERE / "templates" / "indexsidebar.html" template = jinja2.Template(template_path.read_text(encoding="UTF-8")) @@ -388,9 +394,7 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers( - versions: Sequence[Version], languages: Sequence[Language], html_root: Path -): +def setup_switchers(versions: Versions, languages: Languages, html_root: Path): """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher @@ -462,9 +466,9 @@ class DocBuilder: """Builder for a CPython version and a language.""" version: Version - versions: Sequence[Version] + versions: Versions language: Language - languages: Sequence[Language] + languages: Languages cpython_repo: Repository build_root: Path www_root: Path @@ -1070,7 +1074,7 @@ def build_docs(args) -> bool: return all_built_successfully -def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: +def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: releases = http.request( "GET", "https://raw.githubusercontent.com/" @@ -1082,7 +1086,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: return versions -def parse_languages_from_config() -> list[Language]: +def parse_languages_from_config() -> Languages: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) defaults = config["defaults"] @@ -1103,9 +1107,7 @@ def parse_languages_from_config() -> list[Language]: ] -def build_sitemap( - versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group -): +def build_sitemap(versions: Versions, languages: Languages, www_root: Path, group): """Build a sitemap with all live versions and translations.""" if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exist).") @@ -1155,8 +1157,8 @@ def copy_robots_txt( def major_symlinks( www_root: Path, group: str, - versions: Iterable[Version], - languages: Iterable[Language], + versions: Versions, + languages: Languages, skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: From 044aba7d5a27748d6d28f043446b3334fe76dd68 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:36:45 +0100 Subject: [PATCH 165/218] Create a dataclass for Versions (#254) --- build_docs.py | 123 ++++++++++++++++++++++++++++---------------------- 1 file changed, 69 insertions(+), 54 deletions(-) diff --git a/build_docs.py b/build_docs.py index 21b3e6a..3c8d435 100755 --- a/build_docs.py +++ b/build_docs.py @@ -50,10 +50,9 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Iterator, Sequence from typing import Literal, TypeAlias - Versions: TypeAlias = Sequence["Version"] Languages: TypeAlias = Sequence["Language"] try: @@ -71,6 +70,57 @@ HERE = Path(__file__).resolve().parent +@dataclass(frozen=True, slots=True) +class Versions: + _seq: Sequence[Version] + + def __iter__(self) -> Iterator[Version]: + return iter(self._seq) + + def __reversed__(self) -> Iterator[Version]: + return reversed(self._seq) + + @classmethod + def from_json(cls, data) -> Versions: + versions = sorted( + [Version.from_json(name, release) for name, release in data.items()], + key=Version.as_tuple, + ) + return cls(versions) + + def filter(self, branch: str = "") -> Sequence[Version]: + """Filter the given versions. + + If *branch* is given, only *versions* matching *branch* are returned. + + Else all live versions are returned (this means no EOL and no + security-fixes branches). + """ + if branch: + return [v for v in self if branch in (v.name, v.branch_or_tag)] + return [v for v in self if v.status not in {"EOL", "security-fixes"}] + + @property + def current_stable(self) -> Version: + """Find the current stable CPython version.""" + return max((v for v in self if v.status == "stable"), key=Version.as_tuple) + + @property + def current_dev(self) -> Version: + """Find the current CPython version in development.""" + return max(self, key=Version.as_tuple) + + def setup_indexsidebar(self, current: Version, dest_path: Path) -> None: + """Build indexsidebar.html for Sphinx.""" + template_path = HERE / "templates" / "indexsidebar.html" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render( + current_version=current, + versions=list(reversed(self)), + ) + dest_path.write_text(rendered_template, encoding="UTF-8") + + @total_ordering class Version: """Represents a CPython version and its documentation build dependencies.""" @@ -101,6 +151,17 @@ def __init__(self, name, *, status, branch_or_tag=None): def __repr__(self): return f"Version({self.name})" + def __eq__(self, other): + return self.name == other.name + + def __gt__(self, other): + return self.as_tuple() > other.as_tuple() + + @classmethod + def from_json(cls, name, values): + """Loads a version from devguide's json representation.""" + return cls(name, status=values["status"], branch_or_tag=values["branch"]) + @property def requirements(self): """Generate the right requirements for this version. @@ -144,29 +205,6 @@ def title(self): """The title of this version's doc, for the sidebar.""" return f"Python {self.name} ({self.status})" - @staticmethod - def filter(versions, branch=None): - """Filter the given versions. - - If *branch* is given, only *versions* matching *branch* are returned. - - Else all live versions are returned (this means no EOL and no - security-fixes branches). - """ - if branch: - return [v for v in versions if branch in (v.name, v.branch_or_tag)] - return [v for v in versions if v.status not in ("EOL", "security-fixes")] - - @staticmethod - def current_stable(versions): - """Find the current stable CPython version.""" - return max((v for v in versions if v.status == "stable"), key=Version.as_tuple) - - @staticmethod - def current_dev(versions): - """Find the current CPython version in development.""" - return max(versions, key=Version.as_tuple) - @property def picker_label(self): """Forge the label of a version picker.""" @@ -176,27 +214,6 @@ def picker_label(self): return f"pre ({self.name})" return self.name - def setup_indexsidebar(self, versions: Versions, dest_path: Path): - """Build indexsidebar.html for Sphinx.""" - template_path = HERE / "templates" / "indexsidebar.html" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render( - current_version=self, - versions=versions[::-1], - ) - dest_path.write_text(rendered_template, encoding="UTF-8") - - @classmethod - def from_json(cls, name, values): - """Loads a version from devguide's json representation.""" - return cls(name, status=values["status"], branch_or_tag=values["branch"]) - - def __eq__(self, other): - return self.name == other.name - - def __gt__(self, other): - return self.as_tuple() > other.as_tuple() - @dataclass(order=True, frozen=True, kw_only=True) class Language: @@ -619,8 +636,8 @@ def build(self): + ([""] if sys.platform == "darwin" else []) + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] ) - self.version.setup_indexsidebar( - self.versions, + self.versions.setup_indexsidebar( + self.version, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", ) run_with_logging( @@ -1013,7 +1030,7 @@ def build_docs(args) -> bool: # This runs languages in config.toml order and versions newest first. todo = [ (version, language) - for version in Version.filter(versions, args.branch) + for version in versions.filter(args.branch) for language in reversed(Language.filter(languages, args.languages)) ] del args.branch @@ -1081,9 +1098,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: "python/devguide/main/include/release-cycle.json", timeout=30, ).json() - versions = [Version.from_json(name, release) for name, release in releases.items()] - versions.sort(key=Version.as_tuple) - return versions + return Versions.from_json(releases) def parse_languages_from_config() -> Languages: @@ -1170,7 +1185,7 @@ def major_symlinks( - /es/3/ → /es/3.9/ """ logging.info("Creating major version symlinks...") - current_stable = Version.current_stable(versions).name + current_stable = versions.current_stable.name for language in languages: symlink( www_root, @@ -1200,7 +1215,7 @@ def dev_symlink( - /es/dev/ → /es/3.11/ """ logging.info("Creating development version symlinks...") - current_dev = Version.current_dev(versions).name + current_dev = versions.current_dev.name for language in languages: symlink( www_root, From a8070868f67096d5c05122c29d408ac4c1f1083d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:51:46 +0100 Subject: [PATCH 166/218] Create a dataclass for Languages (#255) --- build_docs.py | 70 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/build_docs.py b/build_docs.py index 3c8d435..ebb5a94 100755 --- a/build_docs.py +++ b/build_docs.py @@ -51,9 +51,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: from collections.abc import Iterator, Sequence - from typing import Literal, TypeAlias - - Languages: TypeAlias = Sequence["Language"] + from typing import Literal try: from os import EX_OK, EX_SOFTWARE as EX_FAILURE @@ -215,13 +213,50 @@ def picker_label(self): return self.name +@dataclass(frozen=True, slots=True) +class Languages: + _seq: Sequence[Language] + + def __iter__(self) -> Iterator[Language]: + return iter(self._seq) + + def __reversed__(self) -> Iterator[Language]: + return reversed(self._seq) + + @classmethod + def from_json(cls, defaults, languages) -> Languages: + default_translated_name = defaults.get("translated_name", "") + default_in_prod = defaults.get("in_prod", True) + default_sphinxopts = defaults.get("sphinxopts", []) + default_html_only = defaults.get("html_only", False) + langs = [ + Language( + iso639_tag=iso639_tag, + name=section["name"], + translated_name=section.get("translated_name", default_translated_name), + in_prod=section.get("in_prod", default_in_prod), + sphinxopts=section.get("sphinxopts", default_sphinxopts), + html_only=section.get("html_only", default_html_only), + ) + for iso639_tag, section in languages.items() + ] + return cls(langs) + + def filter(self, language_tags: Sequence[str] = ()) -> Sequence[Language]: + """Filter a sequence of languages according to --languages.""" + if language_tags: + language_tags = frozenset(language_tags) + return [l for l in self if l.tag in language_tags] + return list(self) + + @dataclass(order=True, frozen=True, kw_only=True) class Language: iso639_tag: str name: str translated_name: str in_prod: bool - sphinxopts: tuple + sphinxopts: Sequence[str] html_only: bool = False @property @@ -234,14 +269,6 @@ def switcher_label(self): return f"{self.name} | {self.translated_name}" return self.name - @staticmethod - def filter(languages, language_tags=None): - """Filter a sequence of languages according to --languages.""" - if language_tags: - languages_dict = {language.tag: language for language in languages} - return [languages_dict[tag] for tag in language_tags] - return languages - def run(cmd, cwd=None) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" @@ -1031,7 +1058,7 @@ def build_docs(args) -> bool: todo = [ (version, language) for version in versions.filter(args.branch) - for language in reversed(Language.filter(languages, args.languages)) + for language in reversed(languages.filter(args.languages)) ] del args.branch del args.languages @@ -1104,22 +1131,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: def parse_languages_from_config() -> Languages: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) - defaults = config["defaults"] - default_translated_name = defaults.get("translated_name", "") - default_in_prod = defaults.get("in_prod", True) - default_sphinxopts = defaults.get("sphinxopts", []) - default_html_only = defaults.get("html_only", False) - return [ - Language( - iso639_tag=iso639_tag, - name=section["name"], - translated_name=section.get("translated_name", default_translated_name), - in_prod=section.get("in_prod", default_in_prod), - sphinxopts=section.get("sphinxopts", default_sphinxopts), - html_only=section.get("html_only", default_html_only), - ) - for iso639_tag, section in config["languages"].items() - ] + return Languages.from_json(config["defaults"], config["languages"]) def build_sitemap(versions: Versions, languages: Languages, www_root: Path, group): From 1a8a586d37840fa5ed43aed225fd9dcb6e8ca07f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:59:27 +0100 Subject: [PATCH 167/218] Convert some relative imports to absolute (#256) --- build_docs.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/build_docs.py b/build_docs.py index ebb5a94..a19f2f8 100755 --- a/build_docs.py +++ b/build_docs.py @@ -22,22 +22,22 @@ from __future__ import annotations +import argparse +import dataclasses +import datetime as dt import filecmp import json import logging import logging.handlers +import os import re import shlex import shutil import subprocess import sys -from argparse import ArgumentParser, Namespace from bisect import bisect_left as bisect from contextlib import contextmanager, suppress -from dataclasses import dataclass -from datetime import datetime as dt, timezone from functools import total_ordering -from os import getenv, readlink from pathlib import Path from string import Template from time import perf_counter, sleep @@ -68,7 +68,7 @@ HERE = Path(__file__).resolve().parent -@dataclass(frozen=True, slots=True) +@dataclasses.dataclass(frozen=True, slots=True) class Versions: _seq: Sequence[Version] @@ -213,7 +213,7 @@ def picker_label(self): return self.name -@dataclass(frozen=True, slots=True) +@dataclasses.dataclass(frozen=True, slots=True) class Languages: _seq: Sequence[Language] @@ -250,7 +250,7 @@ def filter(self, language_tags: Sequence[str] = ()) -> Sequence[Language]: return list(self) -@dataclass(order=True, frozen=True, kw_only=True) +@dataclasses.dataclass(order=True, frozen=True, kw_only=True) class Language: iso639_tag: str name: str @@ -337,7 +337,7 @@ def traverse(dircmp_result): return changed -@dataclass +@dataclasses.dataclass class Repository: """Git repository abstraction for our specific needs.""" @@ -505,7 +505,7 @@ def version_info(): ) -@dataclass +@dataclasses.dataclass class DocBuilder: """Builder for a CPython version and a language.""" @@ -539,7 +539,7 @@ def includes_html(self): def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() - start_timestamp = dt.now(tz=timezone.utc).replace(microsecond=0) + start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) logging.info("Running.") try: if self.language.html_only and not self.includes_html: @@ -861,7 +861,7 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_start: dt, build_duration: float, trigger: str): + def save_state(self, build_start: dt.datetime, build_duration: float, trigger: str): """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -932,8 +932,9 @@ def main(): def parse_args(): """Parse command-line arguments.""" - parser = ArgumentParser( - description="Runs a build of the Python docs for various branches." + parser = argparse.ArgumentParser( + description="Runs a build of the Python docs for various branches.", + allow_abbrev=False, ) parser.add_argument( "--select-output", @@ -1032,7 +1033,7 @@ def setup_logging(log_directory: Path, select_output: str | None): logging.getLogger().setLevel(logging.DEBUG) -def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: +def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: try: lock = zc.lockfile.LockFile(HERE / lockfile_name) except zc.lockfile.LockError: @@ -1045,7 +1046,7 @@ def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: lock.close() -def build_docs(args) -> bool: +def build_docs(args: argparse.Namespace) -> bool: """Build all docs (each language and each version).""" logging.info("Full build start.") start_time = perf_counter() @@ -1259,7 +1260,7 @@ def symlink( if not directory_path.exists(): return # No touching link, dest doc not built yet. - if not link.exists() or readlink(link) != directory: + if not link.exists() or os.readlink(link) != directory: # Link does not exist or points to the wrong target. link.unlink(missing_ok=True) link.symlink_to(directory) @@ -1318,8 +1319,8 @@ def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: https://www.fastly.com/documentation/reference/api/purging/#purge-tag """ - service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") - fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") + service_id = os.environ.get("FASTLY_SERVICE_ID", "__UNSET__") + fastly_key = os.environ.get("FASTLY_TOKEN", "__UNSET__") logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) http.request( From a6c458d9ffaa135ec4a317012c21e9eae04814eb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 03:25:25 +0100 Subject: [PATCH 168/218] Only recreate a symlink when the relevant version has changed (#257) --- build_docs.py | 109 +++++++++++++++++++++----------------------------- 1 file changed, 45 insertions(+), 64 deletions(-) diff --git a/build_docs.py b/build_docs.py index a19f2f8..d03d687 100755 --- a/build_docs.py +++ b/build_docs.py @@ -50,7 +50,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Iterator, Sequence + from collections.abc import Iterator, Sequence, Set from typing import Literal try: @@ -1063,7 +1063,9 @@ def build_docs(args: argparse.Namespace) -> bool: ] del args.branch del args.languages - all_built_successfully = True + + build_succeeded = set() + build_failed = set() cpython_repo = Repository( "https://github.com/python/cpython.git", args.build_root / _checkout_name(args.select_output), @@ -1083,7 +1085,12 @@ def build_docs(args: argparse.Namespace) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - all_built_successfully &= builder.run(http) + built_successfully = builder.run(http) + if built_successfully: + build_succeeded.add((version.name, language.tag)) + else: + build_failed.add((version.name, language.tag)) + logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") ) @@ -1096,19 +1103,12 @@ def build_docs(args: argparse.Namespace) -> bool: args.skip_cache_invalidation, http, ) - major_symlinks( - args.www_root, - args.group, - versions, - languages, - args.skip_cache_invalidation, - http, - ) - dev_symlink( + make_symlinks( args.www_root, args.group, versions, languages, + build_succeeded, args.skip_cache_invalidation, http, ) @@ -1116,7 +1116,7 @@ def build_docs(args: argparse.Namespace) -> bool: logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - return all_built_successfully + return len(build_failed) == 0 def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: @@ -1182,68 +1182,46 @@ def copy_robots_txt( purge(http, "robots.txt") -def major_symlinks( +def make_symlinks( www_root: Path, group: str, versions: Versions, languages: Languages, + successful_builds: Set[tuple[str, str]], skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: - """Maintains the /2/ and /3/ symlinks for each language. + """Maintains the /2/, /3/, and /dev/ symlinks for each language. Like: - - /3/ → /3.9/ - - /fr/3/ → /fr/3.9/ - - /es/3/ → /es/3.9/ + - /2/ → /2.7/ + - /3/ → /3.12/ + - /dev/ → /3.14/ + - /fr/3/ → /fr/3.12/ + - /es/dev/ → /es/3.14/ """ - logging.info("Creating major version symlinks...") - current_stable = versions.current_stable.name - for language in languages: - symlink( - www_root, - language, - current_stable, - "3", - group, - skip_cache_invalidation, - http, - ) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) - - -def dev_symlink( - www_root: Path, - group, - versions, - languages, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /dev/ symlinks for each language. - - Like: - - /dev/ → /3.11/ - - /fr/dev/ → /fr/3.11/ - - /es/dev/ → /es/3.11/ - """ - logging.info("Creating development version symlinks...") - current_dev = versions.current_dev.name - for language in languages: - symlink( - www_root, - language, - current_dev, - "dev", - group, - skip_cache_invalidation, - http, - ) + logging.info("Creating major and development version symlinks...") + for symlink_name, symlink_target in ( + ("3", versions.current_stable.name), + ("2", "2.7"), + ("dev", versions.current_dev.name), + ): + for language in languages: + if (symlink_target, language.tag) in successful_builds: + symlink( + www_root, + language.tag, + symlink_target, + symlink_name, + group, + skip_cache_invalidation, + http, + ) def symlink( www_root: Path, - language: Language, + language_tag: str, directory: str, name: str, group: str, @@ -1251,10 +1229,13 @@ def symlink( http: urllib3.PoolManager, ) -> None: """Used by major_symlinks and dev_symlink to maintain symlinks.""" - if language.tag == "en": # English is rooted on /, no /en/ + msg = "Creating symlink from %s to %s" + if language_tag == "en": # English is rooted on /, no /en/ path = www_root + logging.debug(msg, name, directory) else: - path = www_root / language.tag + path = www_root / language_tag + logging.debug(msg, f"{language_tag}/{name}", f"{language_tag}/{directory}") link = path / name directory_path = path / directory if not directory_path.exists(): @@ -1266,7 +1247,7 @@ def symlink( link.symlink_to(directory) run(["chown", "-h", f":{group}", str(link)]) if not skip_cache_invalidation: - surrogate_key = f"{language.tag}/{name}" + surrogate_key = f"{language_tag}/{name}" purge_surrogate_key(http, surrogate_key) From 3d068f403607669b9408115dce9918a9b915dc62 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 05:41:28 +0100 Subject: [PATCH 169/218] Adjust symlink log message (#259) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index d03d687..b6a58a9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1229,7 +1229,7 @@ def symlink( http: urllib3.PoolManager, ) -> None: """Used by major_symlinks and dev_symlink to maintain symlinks.""" - msg = "Creating symlink from %s to %s" + msg = "Creating symlink from /%s/ to /%s/" if language_tag == "en": # English is rooted on /, no /en/ path = www_root logging.debug(msg, name, directory) From 0ef872fd7e7b00e6e1193a73ebaf37d7eff7b444 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 05:43:29 +0100 Subject: [PATCH 170/218] Skip CDN requests when secrets are unset (#260) --- build_docs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index b6a58a9..4d7bf8d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1300,8 +1300,13 @@ def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: https://www.fastly.com/documentation/reference/api/purging/#purge-tag """ - service_id = os.environ.get("FASTLY_SERVICE_ID", "__UNSET__") - fastly_key = os.environ.get("FASTLY_TOKEN", "__UNSET__") + unset = "__UNSET__" + service_id = os.environ.get("FASTLY_SERVICE_ID", unset) + fastly_key = os.environ.get("FASTLY_TOKEN", unset) + + if service_id == unset or fastly_key == unset: + logging.info("CDN secrets not set, skipping Surrogate-Key purge") + return logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) http.request( From 6d537969095c21f34406ff954535020b690f0d3d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 06:00:37 +0100 Subject: [PATCH 171/218] Add ``--language`` as a synonym of ``--languages`` --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 4d7bf8d..8f5d763 100755 --- a/build_docs.py +++ b/build_docs.py @@ -984,7 +984,7 @@ def parse_args(): default=Path("/var/log/docsbuild/"), ) parser.add_argument( - "--languages", + "--languages", "--language", nargs="*", help="Language translation, as a PEP 545 language tag like" " 'fr' or 'pt-br'. " From 77a363be7d4e4574bb950e02cb504434bdb2e830 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 06:29:32 +0100 Subject: [PATCH 172/218] Add Ruff configuration file (#262) --- .pre-commit-config.yaml | 2 +- .ruff.toml | 7 ++ build_docs.py | 143 ++++++++++++++++++---------------------- check_versions.py | 10 ++- 4 files changed, 76 insertions(+), 86 deletions(-) create mode 100644 .ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 44948ef..4ad4370 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.6 + rev: v0.11.5 hooks: - id: ruff-format diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..e837d03 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,7 @@ +target-version = "py313" # Pin Ruff to Python 3.13 +line-length = 88 +output-format = "full" + +[format] +preview = true +docstring-code-format = true diff --git a/build_docs.py b/build_docs.py index 8f5d763..0ca807e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -608,14 +608,12 @@ def build(self): sphinxopts = list(self.language.sphinxopts) if self.language.tag != "en": locale_dirs = self.build_root / self.version.name / "locale" - sphinxopts.extend( - ( - f"-D locale_dirs={locale_dirs}", - f"-D language={self.language.iso639_tag}", - "-D gettext_compact=0", - "-D translation_progress_classes=1", - ) - ) + sphinxopts.extend(( + f"-D locale_dirs={locale_dirs}", + f"-D language={self.language.iso639_tag}", + "-D gettext_compact=0", + "-D translation_progress_classes=1", + )) if self.language.tag == "ja": # Since luatex doesn't support \ufffd, replace \ufffd with '?'. # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b @@ -667,20 +665,18 @@ def build(self): self.version, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", ) - run_with_logging( - [ - "make", - "-C", - self.checkout / "Doc", - "PYTHON=" + str(python), - "SPHINXBUILD=" + str(sphinxbuild), - "BLURB=" + str(blurb), - "VENVDIR=" + str(self.venv), - "SPHINXOPTS=" + " ".join(sphinxopts), - "SPHINXERRORHANDLING=", - maketarget, - ] - ) + run_with_logging([ + "make", + "-C", + self.checkout / "Doc", + "PYTHON=" + str(python), + "SPHINXBUILD=" + str(sphinxbuild), + "BLURB=" + str(blurb), + "VENVDIR=" + str(self.venv), + "SPHINXOPTS=" + " ".join(sphinxopts), + "SPHINXERRORHANDLING=", + maketarget, + ]) run(["mkdir", "-p", self.log_directory]) run(["chgrp", "-R", self.group, self.log_directory]) if self.includes_html: @@ -743,69 +739,57 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: # Copy built HTML files to webroot (default /srv/docs.python.org) changed = changed_files(self.checkout / "Doc" / "build" / "html", target) logging.info("Copying HTML files to %s", target) - run( - [ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "build" / "html/", - ] - ) + run([ + "chown", + "-R", + ":" + self.group, + self.checkout / "Doc" / "build" / "html/", + ]) run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) - run( - [ - "find", - self.checkout / "Doc" / "build" / "html", - "-type", - "d", - "-exec", - "chmod", - "o+x", - "{}", - ";", - ] - ) - run( - [ - "rsync", - "-a", - "--delete-delay", - "--filter", - "P archives/", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - ] - ) + run([ + "find", + self.checkout / "Doc" / "build" / "html", + "-type", + "d", + "-exec", + "chmod", + "o+x", + "{}", + ";", + ]) + run([ + "rsync", + "-a", + "--delete-delay", + "--filter", + "P archives/", + str(self.checkout / "Doc" / "build" / "html") + "/", + target, + ]) if not self.quick: # Copy archive files to /archives/ logging.debug("Copying dist files.") - run( - [ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "dist", - ] - ) - run( - [ - "chmod", - "-R", - "o+r", - self.checkout / "Doc" / "dist", - ] - ) + run([ + "chown", + "-R", + ":" + self.group, + self.checkout / "Doc" / "dist", + ]) + run([ + "chmod", + "-R", + "o+r", + self.checkout / "Doc" / "dist", + ]) run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) run(["chown", ":" + self.group, target / "archives"]) - run( - [ - "cp", - "-a", - *(self.checkout / "Doc" / "dist").glob("*"), - target / "archives", - ] - ) + run([ + "cp", + "-a", + *(self.checkout / "Doc" / "dist").glob("*"), + target / "archives", + ]) changed.append("archives/") for file in (target / "archives").iterdir(): changed.append("archives/" + file.name) @@ -984,7 +968,8 @@ def parse_args(): default=Path("/var/log/docsbuild/"), ) parser.add_argument( - "--languages", "--language", + "--languages", + "--language", nargs="*", help="Language translation, as a PEP 545 language tag like" " 'fr' or 'pt-br'. " diff --git a/check_versions.py b/check_versions.py index 343c85a..1a1016f 100644 --- a/check_versions.py +++ b/check_versions.py @@ -111,12 +111,10 @@ async def which_sphinx_is_used_in_production(): table = [ [ version.name, - *await asyncio.gather( - *[ - get_version_in_prod(language.tag, version.name) - for language in LANGUAGES - ] - ), + *await asyncio.gather(*[ + get_version_in_prod(language.tag, version.name) + for language in LANGUAGES + ]), ] for version in VERSIONS ] From a6a666d9f5007351803eb83b4525864d314732f9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 07:17:01 +0100 Subject: [PATCH 173/218] Account for skipped builds in ``build_docs()`` (#261) --- build_docs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0ca807e..08ce611 100755 --- a/build_docs.py +++ b/build_docs.py @@ -536,7 +536,7 @@ def includes_html(self): """Does the build we are running include HTML output?""" return self.select_output != "no-html" - def run(self, http: urllib3.PoolManager) -> bool: + def run(self, http: urllib3.PoolManager) -> bool | None: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) @@ -544,7 +544,7 @@ def run(self, http: urllib3.PoolManager) -> bool: try: if self.language.html_only and not self.includes_html: logging.info("Skipping non-HTML build (language is HTML-only).") - return True + return None # skipped self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() @@ -557,6 +557,8 @@ def run(self, http: urllib3.PoolManager) -> bool: build_duration=perf_counter() - start_time, trigger=trigger_reason, ) + else: + return None # skipped except Exception as err: logging.exception("Badly handled exception, human, please help.") if sentry_sdk: @@ -1073,7 +1075,7 @@ def build_docs(args: argparse.Namespace) -> bool: built_successfully = builder.run(http) if built_successfully: build_succeeded.add((version.name, language.tag)) - else: + elif built_successfully is not None: build_failed.add((version.name, language.tag)) logging.root.handlers[0].setFormatter( From e80b7296bdff2d7034d397bdd73b3852e8e1c4c1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 14:10:00 +0100 Subject: [PATCH 174/218] Improve performance for ``proofread_canonicals()`` (#258) --- build_docs.py | 51 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index 08ce611..63b561c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -23,6 +23,7 @@ from __future__ import annotations import argparse +import concurrent.futures import dataclasses import datetime as dt import filecmp @@ -1249,21 +1250,41 @@ def proofread_canonicals( /3/whatsnew/3.11.html, which may not exist yet. """ logging.info("Checking canonical links...") - canonical_re = re.compile( - """""" - ) - for file in www_root.glob("**/*.html"): - html = file.read_text(encoding="UTF-8", errors="surrogateescape") - canonical = canonical_re.search(html) - if not canonical: - continue - target = canonical.group(1) - if not (www_root / target).exists(): - logging.info("Removing broken canonical from %s to %s", file, target) - html = html.replace(canonical.group(0), "") - file.write_text(html, encoding="UTF-8", errors="surrogateescape") - if not skip_cache_invalidation: - purge(http, str(file).replace("/srv/docs.python.org/", "")) + worker_count = (os.cpu_count() or 1) + 2 + with concurrent.futures.ThreadPoolExecutor(worker_count) as executor: + futures = { + executor.submit(_check_canonical_rel, file, www_root) + for file in www_root.glob("**/*.html") + } + paths_to_purge = { + res.relative_to(www_root) # strip the leading /srv/docs.python.org + for fut in concurrent.futures.as_completed(futures) + if (res := fut.result()) is not None + } + if not skip_cache_invalidation: + purge(http, *paths_to_purge) + + +def _check_canonical_rel(file: Path, www_root: Path): + # Check for a canonical relation link in the HTML. + # If one exists, ensure that the target exists + # or otherwise remove the canonical link element. + prefix = b'' + pfx_len = len(prefix) + sfx_len = len(suffix) + html = file.read_bytes() + try: + start = html.index(prefix) + end = html.index(suffix, start + pfx_len) + except ValueError: + return None + target = html[start + pfx_len : end].decode(errors="surrogateescape") + if (www_root / target).exists(): + return None + logging.info("Removing broken canonical from %s to %s", file, target) + file.write_bytes(html[:start] + html[end + sfx_len :]) + return file def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: From 946b6bc68b239931579e56f43f45f6e15d3e60c4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:19:21 +0100 Subject: [PATCH 175/218] Account for recent removal of self-closing tags (#264) --- build_docs.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/build_docs.py b/build_docs.py index 63b561c..ff9eb59 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1265,25 +1265,26 @@ def proofread_canonicals( purge(http, *paths_to_purge) +# Python 3.12 onwards doesn't use self-closing tags for +_canonical_re = re.compile( + b"""""" +) + + def _check_canonical_rel(file: Path, www_root: Path): # Check for a canonical relation link in the HTML. # If one exists, ensure that the target exists # or otherwise remove the canonical link element. - prefix = b'' - pfx_len = len(prefix) - sfx_len = len(suffix) html = file.read_bytes() - try: - start = html.index(prefix) - end = html.index(suffix, start + pfx_len) - except ValueError: + canonical = _canonical_re.search(html) + if canonical is None: return None - target = html[start + pfx_len : end].decode(errors="surrogateescape") + target = canonical[1].decode(encoding="UTF-8", errors="surrogateescape") if (www_root / target).exists(): return None logging.info("Removing broken canonical from %s to %s", file, target) - file.write_bytes(html[:start] + html[end + sfx_len :]) + start, end = canonical.span() + file.write_bytes(html[:start] + html[end:]) return file From 2113fd7daaed924663c986384048defcab68b342 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 11 Apr 2025 17:46:23 +0300 Subject: [PATCH 176/218] Allow passing multiple branches to build via CLI (#235) --- .coveragerc | 7 ++++ .github/workflows/test.yml | 7 ++++ README.md | 7 +++- build_docs.py | 19 +++++----- tests/test_build_docs_versions.py | 62 +++++++++++++++++++++++++++++++ tox.ini | 13 ++++++- 6 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 .coveragerc create mode 100644 tests/test_build_docs_versions.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..0f12707 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +# .coveragerc to control coverage.py + +[report] +# Regexes for lines to exclude from consideration +exclude_also = + # Don't complain if non-runnable code isn't run: + if __name__ == .__main__.: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2976bae..e272e76 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,3 +33,10 @@ jobs: - name: Tox tests run: | uvx --with tox-uv tox -e py + + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + flags: ${{ matrix.os }} + name: ${{ matrix.os }} Python ${{ matrix.python-version }} + token: ${{ secrets.CODECOV_ORG_TOKEN }} diff --git a/README.md b/README.md index 1b76bcd..d78bdb7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ +# docsbuild-scripts + +[![GitHub Actions status](https://github.com/python/docsbuild-scripts/actions/workflows/test.yml/badge.svg)](https://github.com/python/docsbuild-scripts/actions/workflows/test.yml) +[![Codecov](https://codecov.io/gh/python/docsbuild-scripts/branch/main/graph/badge.svg)](https://codecov.io/gh/python/docsbuild-scripts) + This repository contains scripts for automatically building the Python documentation on [docs.python.org](https://docs.python.org). @@ -12,7 +17,7 @@ python3 ./build_docs.py --quick --build-root ./build_root --www-root ./www --log ``` If you don't need to build all translations of all branches, add -`--language en --branch main`. +`--languages en --branches main`. ## Check current version diff --git a/build_docs.py b/build_docs.py index ff9eb59..d4578eb 100755 --- a/build_docs.py +++ b/build_docs.py @@ -87,16 +87,17 @@ def from_json(cls, data) -> Versions: ) return cls(versions) - def filter(self, branch: str = "") -> Sequence[Version]: + def filter(self, branches: Sequence[str] = ()) -> Sequence[Version]: """Filter the given versions. - If *branch* is given, only *versions* matching *branch* are returned. + If *branches* is given, only *versions* matching *branches* are returned. Else all live versions are returned (this means no EOL and no security-fixes branches). """ - if branch: - return [v for v in self if branch in (v.name, v.branch_or_tag)] + if branches: + branches = frozenset(branches) + return [v for v in self if {v.name, v.branch_or_tag} & branches] return [v for v in self if v.status not in {"EOL", "security-fixes"}] @property @@ -936,9 +937,10 @@ def parse_args(): ) parser.add_argument( "-b", - "--branch", + "--branches", + nargs="*", metavar="3.12", - help="Version to build (defaults to all maintained branches).", + help="Versions to build (defaults to all maintained branches).", ) parser.add_argument( "-r", @@ -972,7 +974,6 @@ def parse_args(): ) parser.add_argument( "--languages", - "--language", nargs="*", help="Language translation, as a PEP 545 language tag like" " 'fr' or 'pt-br'. " @@ -1046,10 +1047,10 @@ def build_docs(args: argparse.Namespace) -> bool: # This runs languages in config.toml order and versions newest first. todo = [ (version, language) - for version in versions.filter(args.branch) + for version in versions.filter(args.branches) for language in reversed(languages.filter(args.languages)) ] - del args.branch + del args.branches del args.languages build_succeeded = set() diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py new file mode 100644 index 0000000..5ed9bcb --- /dev/null +++ b/tests/test_build_docs_versions.py @@ -0,0 +1,62 @@ +from build_docs import Versions, Version + + +def test_filter_default() -> None: + # Arrange + versions = Versions([ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + Version("3.11", status="security"), + Version("3.10", status="security"), + Version("3.9", status="security"), + ]) + + # Act + filtered = versions.filter() + + # Assert + assert filtered == [ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + ] + + +def test_filter_one() -> None: + # Arrange + versions = Versions([ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + Version("3.11", status="security"), + Version("3.10", status="security"), + Version("3.9", status="security"), + ]) + + # Act + filtered = versions.filter(["3.13"]) + + # Assert + assert filtered == [Version("3.13", status="security")] + + +def test_filter_multiple() -> None: + # Arrange + versions = Versions([ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + Version("3.11", status="security"), + Version("3.10", status="security"), + Version("3.9", status="security"), + ]) + + # Act + filtered = versions.filter(["3.13", "3.14"]) + + # Assert + assert filtered == [ + Version("3.14", status="feature"), + Version("3.13", status="security"), + ] diff --git a/tox.ini b/tox.ini index 56c6420..12efcdf 100644 --- a/tox.ini +++ b/tox.ini @@ -12,8 +12,19 @@ skip_install = true deps = -r requirements.txt pytest + pytest-cov +pass_env = + FORCE_COLOR +set_env = + COVERAGE_CORE = sysmon commands = - {envpython} -m pytest {posargs} + {envpython} -m pytest \ + --cov . \ + --cov tests \ + --cov-report html \ + --cov-report term \ + --cov-report xml \ + {posargs} [testenv:lint] skip_install = true From bd0e222e4e0844f2790d61e5d66e0243b97073c8 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Fri, 11 Apr 2025 14:49:11 -0400 Subject: [PATCH 177/218] Move environment variables to a configuration file (#269) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- build_docs.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 48 insertions(+) diff --git a/build_docs.py b/build_docs.py index d4578eb..22fc5bd 100755 --- a/build_docs.py +++ b/build_docs.py @@ -5,6 +5,26 @@ Without any arguments builds docs for all active versions and languages. +Environment variables for: + +- `SENTRY_DSN` (Error reporting) +- `FASTLY_SERVICE_ID` / `FASTLY_TOKEN` (CDN purges) +- `PYTHON_DOCS_ENABLE_ANALYTICS` (Enable Plausible for online docs) + +are read from the site configuration path for your platform +(/etc/xdg/docsbuild-scripts on linux) if available, +and can be overriden by writing a file to the user config dir +for your platform ($HOME/.config/docsbuild-scripts on linux). +The contents of the file is parsed as toml: + +```toml +[env] +SENTRY_DSN = "https://0a0a0a0a0a0a0a0a0a0a0a@sentry.io/69420" +FASTLY_SERVICE_ID = "deadbeefdeadbeefdead" +FASTLY_TOKEN = "secureme!" +PYTHON_DOCS_ENABLE_ANALYTICS = "1" +``` + Languages are stored in `config.toml` while versions are discovered from the devguide. @@ -48,6 +68,7 @@ import tomlkit import urllib3 import zc.lockfile +from platformdirs import user_config_path, site_config_path TYPE_CHECKING = False if TYPE_CHECKING: @@ -906,6 +927,7 @@ def main(): """Script entry point.""" args = parse_args() setup_logging(args.log_directory, args.select_output) + load_environment_variables() if args.select_output is None: build_docs_with_lock(args, "build_docs.lock") @@ -1022,6 +1044,31 @@ def setup_logging(log_directory: Path, select_output: str | None): logging.getLogger().setLevel(logging.DEBUG) +def load_environment_variables() -> None: + _user_config_path = user_config_path("docsbuild-scripts") + _site_config_path = site_config_path("docsbuild-scripts") + if _user_config_path.is_file(): + ENV_CONF_FILE = _user_config_path + elif _site_config_path.is_file(): + ENV_CONF_FILE = _site_config_path + else: + logging.info( + "No environment variables configured. " + f"Configure in {_site_config_path} or {_user_config_path}." + ) + return + + logging.info(f"Reading environment variables from {ENV_CONF_FILE}.") + if ENV_CONF_FILE == _site_config_path: + logging.info(f"You can override settings in {_user_config_path}.") + elif _site_config_path.is_file(): + logging.info(f"Overriding {_site_config_path}.") + with open(ENV_CONF_FILE, "r") as f: + for key, value in tomlkit.parse(f.read()).get("env", {}).items(): + logging.debug(f"Setting {key} in environment.") + os.environ[key] = value + + def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: try: lock = zc.lockfile.LockFile(HERE / lockfile_name) diff --git a/requirements.txt b/requirements.txt index 0cac810..535b36a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ jinja2 +platformdirs sentry-sdk>=2 tomlkit>=0.13 urllib3>=2 From 10e20e45c8aaa473b667ec9e2bb4f3a277685488 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:25:54 +0100 Subject: [PATCH 178/218] Require imghdr for building Python 3.8-3.10 (#267) --- build_docs.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/build_docs.py b/build_docs.py index 22fc5bd..1b790c6 100755 --- a/build_docs.py +++ b/build_docs.py @@ -196,23 +196,30 @@ def requirements(self): See https://github.com/python/cpython/issues/91483 """ - if self.name == "3.5": - return ["jieba", "blurb", "sphinx==1.8.4", "jinja2<3.1", "docutils<=0.17.1"] - if self.name in {"3.7", "3.6", "2.7"}: - return ["jieba", "blurb", "sphinx==2.3.1", "jinja2<3.1", "docutils<=0.17.1"] - - return [ + dependencies = [ "jieba", # To improve zh search. "PyStemmer~=2.2.0", # To improve performance for word stemming. "-rrequirements.txt", ] + if self.as_tuple() >= (3, 11): + return dependencies + if self.as_tuple() >= (3, 8): + # Restore the imghdr module for Python 3.8-3.10. + return dependencies + ["standard-imghdr"] + + # Requirements/constraints for Python 3.7 and older, pre-requirements.txt + reqs = ["jieba", "blurb", "jinja2<3.1", "docutils<=0.17.1", "standard-imghdr"] + if self.name in {"3.7", "3.6", "2.7"}: + return reqs + ["sphinx==2.3.1"] + if self.name == "3.5": + return reqs + ["sphinx==1.8.4"] @property def changefreq(self): """Estimate this version change frequency, for the sitemap.""" return {"EOL": "never", "security-fixes": "yearly"}.get(self.status, "daily") - def as_tuple(self): + def as_tuple(self) -> tuple[int, ...]: """This version name as tuple, for easy comparisons.""" return version_to_tuple(self.name) @@ -407,7 +414,7 @@ def update(self): self.clone() or self.fetch() -def version_to_tuple(version): +def version_to_tuple(version) -> tuple[int, ...]: """Transform a version string to a tuple, for easy comparisons.""" return tuple(int(part) for part in version.split(".")) From 776f413c7e174017d59aa38ff9f75dc135f749e1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:30:16 +0100 Subject: [PATCH 179/218] Add ``--force`` to always rebuild (#268) --- README.md | 2 +- build_docs.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d78bdb7..2c84353 100644 --- a/README.md +++ b/README.md @@ -71,5 +71,5 @@ To manually rebuild a branch, for example 3.11: ssh docs.nyc1.psf.io sudo su --shell=/bin/bash docsbuild screen -DUR # Rejoin screen session if it exists, otherwise create a new one -/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --force --branch 3.11 ``` diff --git a/build_docs.py b/build_docs.py index 1b790c6..9727a85 100755 --- a/build_docs.py +++ b/build_docs.py @@ -566,7 +566,7 @@ def includes_html(self): """Does the build we are running include HTML output?""" return self.select_output != "no-html" - def run(self, http: urllib3.PoolManager) -> bool | None: + def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) @@ -578,7 +578,7 @@ def run(self, http: urllib3.PoolManager) -> bool | None: self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() - if trigger_reason := self.should_rebuild(): + if trigger_reason := self.should_rebuild(force_build): self.build_venv() self.build() self.copy_build_to_webroot(http) @@ -834,7 +834,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: "Publishing done (%s).", format_seconds(perf_counter() - start_time) ) - def should_rebuild(self): + def should_rebuild(self, force: bool): state = self.load_state() if not state: logging.info("Should rebuild: no previous state found.") @@ -862,6 +862,9 @@ def should_rebuild(self): cpython_sha, ) return "Doc/ has changed" + if force: + logging.info("Should rebuild: forced.") + return "forced" logging.info("Nothing changed, no rebuild needed.") return False @@ -985,6 +988,12 @@ def parse_args(): help="Path where generated files will be copied.", default=Path("/srv/docs.python.org"), ) + parser.add_argument( + "--force", + action="store_true", + help="Always build the chosen languages and versions, " + "regardless of existing state.", + ) parser.add_argument( "--skip-cache-invalidation", help="Skip Fastly cache invalidation.", @@ -1128,7 +1137,7 @@ def build_docs(args: argparse.Namespace) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - built_successfully = builder.run(http) + built_successfully = builder.run(http, force_build=args.force) if built_successfully: build_succeeded.add((version.name, language.tag)) elif built_successfully is not None: From 82e2a41997ce7bb02b113d429b894119ac2c8437 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:38:18 +0100 Subject: [PATCH 180/218] Delete ``force`` from ``args`` namespace (#270) --- build_docs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 9727a85..309d786 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1115,6 +1115,8 @@ def build_docs(args: argparse.Namespace) -> bool: ] del args.branches del args.languages + force_build = args.force + del args.force build_succeeded = set() build_failed = set() @@ -1137,7 +1139,7 @@ def build_docs(args: argparse.Namespace) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - built_successfully = builder.run(http, force_build=args.force) + built_successfully = builder.run(http, force_build=force_build) if built_successfully: build_succeeded.add((version.name, language.tag)) elif built_successfully is not None: From 4a5d2ca799ae7a7b9240cde7a987b8c63da070b1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 12 Apr 2025 02:14:44 +0300 Subject: [PATCH 181/218] Test for GNU sed instead of macOS (#263) --- build_docs.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 309d786..97eee6d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -687,10 +687,25 @@ def build(self): f"-D ogp_site_url={site_url}", ) + def is_gnu_sed() -> bool: + """Check if we are using GNU sed.""" + try: + subprocess.run( + ["sed", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + return True + except subprocess.CalledProcessError: + return False + except FileNotFoundError: + return False + # Disable CPython switchers, we handle them now: run( ["sed", "-i"] - + ([""] if sys.platform == "darwin" else []) + + ([] if is_gnu_sed() else [""]) + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] ) self.versions.setup_indexsidebar( From c341248dbc162b2ab9c527d95e8de0a9b6958285 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 00:19:50 +0100 Subject: [PATCH 182/218] Ensure that `Doc/dist` exists (#271) --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index 97eee6d..c0a954e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -817,6 +817,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: if not self.quick: # Copy archive files to /archives/ logging.debug("Copying dist files.") + (self.checkout / "Doc" / "dist").mkdir(exist_ok=True) run([ "chown", "-R", From 7a9bca9945f149486a366d4050f658f3f34dc81b Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 01:14:16 +0100 Subject: [PATCH 183/218] Remove use of the ``sed`` command (#272) --- build_docs.py | 42 ++++++------------------------------------ 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/build_docs.py b/build_docs.py index c0a954e..ac4f351 100755 --- a/build_docs.py +++ b/build_docs.py @@ -646,21 +646,6 @@ def build(self): "-D gettext_compact=0", "-D translation_progress_classes=1", )) - if self.language.tag == "ja": - # Since luatex doesn't support \ufffd, replace \ufffd with '?'. - # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b - # Luatex already fixed this issue, so we can remove this once Texlive - # is updated. - # (https://github.com/TeX-Live/luatex/commit/af5faf1) - subprocess.check_output( - "sed -i s/\N{REPLACEMENT CHARACTER}/?/g " - f"{locale_dirs}/ja/LC_MESSAGES/**/*.po", - shell=True, - ) - subprocess.check_output( - f"sed -i s/\N{REPLACEMENT CHARACTER}/?/g {self.checkout}/Doc/**/*.rst", - shell=True, - ) if self.version.status == "EOL": sphinxopts.append("-D html_context.outdated=1") @@ -687,27 +672,12 @@ def build(self): f"-D ogp_site_url={site_url}", ) - def is_gnu_sed() -> bool: - """Check if we are using GNU sed.""" - try: - subprocess.run( - ["sed", "--version"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=True, - ) - return True - except subprocess.CalledProcessError: - return False - except FileNotFoundError: - return False - - # Disable CPython switchers, we handle them now: - run( - ["sed", "-i"] - + ([] if is_gnu_sed() else [""]) - + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] - ) + if self.version.as_tuple() < (3, 8): + # Disable CPython switchers, we handle them now: + text = (self.checkout / "Doc" / "Makefile").read_text(encoding="utf-8") + text = text.replace(" -A switchers=1", "") + (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") + self.versions.setup_indexsidebar( self.version, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", From feef5f169a1a2155097d3b44eb649f34405b0653 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 01:38:36 +0100 Subject: [PATCH 184/218] Only copy archive files if the dist directory exists (#273) --- build_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index ac4f351..654c945 100755 --- a/build_docs.py +++ b/build_docs.py @@ -784,10 +784,9 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: target, ]) - if not self.quick: + if not self.quick and (self.checkout / "Doc" / "dist").is_dir(): # Copy archive files to /archives/ logging.debug("Copying dist files.") - (self.checkout / "Doc" / "dist").mkdir(exist_ok=True) run([ "chown", "-R", From 76192128301d3b553b9b6f287b880d454c245606 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 03:05:54 +0100 Subject: [PATCH 185/218] Add more type hints (#274) --- build_docs.py | 109 ++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 48 deletions(-) diff --git a/build_docs.py b/build_docs.py index 654c945..0eb900e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -72,7 +72,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Iterator, Sequence, Set + from collections.abc import Collection, Iterator, Sequence, Set from typing import Literal try: @@ -101,7 +101,7 @@ def __reversed__(self) -> Iterator[Version]: return reversed(self._seq) @classmethod - def from_json(cls, data) -> Versions: + def from_json(cls, data: dict) -> Versions: versions = sorted( [Version.from_json(name, release) for name, release in data.items()], key=Version.as_tuple, @@ -158,7 +158,9 @@ class Version: "prerelease": "pre-release", } - def __init__(self, name, *, status, branch_or_tag=None): + def __init__( + self, name: str, *, status: str, branch_or_tag: str | None = None + ) -> None: status = self.SYNONYMS.get(status, status) if status not in self.STATUSES: raise ValueError( @@ -169,22 +171,22 @@ def __init__(self, name, *, status, branch_or_tag=None): self.branch_or_tag = branch_or_tag self.status = status - def __repr__(self): + def __repr__(self) -> str: return f"Version({self.name})" - def __eq__(self, other): + def __eq__(self, other: Version) -> bool: return self.name == other.name - def __gt__(self, other): + def __gt__(self, other: Version) -> bool: return self.as_tuple() > other.as_tuple() @classmethod - def from_json(cls, name, values): + def from_json(cls, name: str, values: dict) -> Version: """Loads a version from devguide's json representation.""" return cls(name, status=values["status"], branch_or_tag=values["branch"]) @property - def requirements(self): + def requirements(self) -> list[str]: """Generate the right requirements for this version. Since CPython 3.8 a Doc/requirements.txt file can be used. @@ -213,9 +215,10 @@ def requirements(self): return reqs + ["sphinx==2.3.1"] if self.name == "3.5": return reqs + ["sphinx==1.8.4"] + raise ValueError("unreachable") @property - def changefreq(self): + def changefreq(self) -> str: """Estimate this version change frequency, for the sitemap.""" return {"EOL": "never", "security-fixes": "yearly"}.get(self.status, "daily") @@ -224,17 +227,17 @@ def as_tuple(self) -> tuple[int, ...]: return version_to_tuple(self.name) @property - def url(self): + def url(self) -> str: """The doc URL of this version in production.""" return f"https://docs.python.org/{self.name}/" @property - def title(self): + def title(self) -> str: """The title of this version's doc, for the sidebar.""" return f"Python {self.name} ({self.status})" @property - def picker_label(self): + def picker_label(self) -> str: """Forge the label of a version picker.""" if self.status == "in development": return f"dev ({self.name})" @@ -254,7 +257,7 @@ def __reversed__(self) -> Iterator[Language]: return reversed(self._seq) @classmethod - def from_json(cls, defaults, languages) -> Languages: + def from_json(cls, defaults: dict, languages: dict) -> Languages: default_translated_name = defaults.get("translated_name", "") default_in_prod = defaults.get("in_prod", True) default_sphinxopts = defaults.get("sphinxopts", []) @@ -290,17 +293,19 @@ class Language: html_only: bool = False @property - def tag(self): + def tag(self) -> str: return self.iso639_tag.replace("_", "-").lower() @property - def switcher_label(self): + def switcher_label(self) -> str: if self.translated_name: return f"{self.name} | {self.translated_name}" return self.name -def run(cmd, cwd=None) -> subprocess.CompletedProcess: +def run( + cmd: Sequence[str | Path], cwd: Path | None = None +) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" cmd = list(map(str, cmd)) cmdstring = shlex.join(cmd) @@ -326,7 +331,7 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: return result -def run_with_logging(cmd, cwd=None): +def run_with_logging(cmd: Sequence[str | Path], cwd: Path | None = None) -> None: """Like subprocess.check_call, with logging before the command execution.""" cmd = list(map(str, cmd)) logging.debug("Run: '%s'", shlex.join(cmd)) @@ -348,13 +353,13 @@ def run_with_logging(cmd, cwd=None): raise subprocess.CalledProcessError(return_code, cmd[0]) -def changed_files(left, right): +def changed_files(left: Path, right: Path) -> list[str]: """Compute a list of different files between left and right, recursively. Resulting paths are relative to left. """ changed = [] - def traverse(dircmp_result): + def traverse(dircmp_result: filecmp.dircmp) -> None: base = Path(dircmp_result.left).relative_to(left) for file in dircmp_result.diff_files: changed.append(str(base / file)) @@ -374,11 +379,11 @@ class Repository: remote: str directory: Path - def run(self, *args): + def run(self, *args: str) -> subprocess.CompletedProcess: """Run git command in the clone repository.""" return run(("git", "-C", self.directory) + args) - def get_ref(self, pattern): + def get_ref(self, pattern: str) -> str: """Return the reference of a given tag or branch.""" try: # Maybe it's a branch @@ -387,7 +392,7 @@ def get_ref(self, pattern): # Maybe it's a tag return self.run("show-ref", "-s", "tags/" + pattern).stdout.strip() - def fetch(self): + def fetch(self) -> subprocess.CompletedProcess: """Try (and retry) to run git fetch.""" try: return self.run("fetch") @@ -396,12 +401,12 @@ def fetch(self): sleep(5) return self.run("fetch") - def switch(self, branch_or_tag): + def switch(self, branch_or_tag: str) -> None: """Reset and cleans the repository to the given branch or tag.""" self.run("reset", "--hard", self.get_ref(branch_or_tag), "--") self.run("clean", "-dfqx") - def clone(self): + def clone(self) -> bool: """Maybe clone the repository, if not already cloned.""" if (self.directory / ".git").is_dir(): return False # Already cloned @@ -410,21 +415,23 @@ def clone(self): run(["git", "clone", self.remote, self.directory]) return True - def update(self): + def update(self) -> None: self.clone() or self.fetch() -def version_to_tuple(version) -> tuple[int, ...]: +def version_to_tuple(version: str) -> tuple[int, ...]: """Transform a version string to a tuple, for easy comparisons.""" return tuple(int(part) for part in version.split(".")) -def tuple_to_version(version_tuple): +def tuple_to_version(version_tuple: tuple[int, ...]) -> str: """Reverse version_to_tuple.""" return ".".join(str(part) for part in version_tuple) -def locate_nearest_version(available_versions, target_version): +def locate_nearest_version( + available_versions: Collection[str], target_version: str +) -> str: """Look for the nearest version of target_version in available_versions. Versions are to be given as tuples, like (3, 7) for 3.7. @@ -468,7 +475,7 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers(versions: Versions, languages: Languages, html_root: Path): +def setup_switchers(versions: Versions, languages: Languages, html_root: Path) -> None: """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher @@ -499,12 +506,12 @@ def setup_switchers(versions: Versions, languages: Languages, html_root: Path): ofile.write(line) -def head(text, lines=10): +def head(text: str, lines: int = 10) -> str: """Return the first *lines* lines from the given text.""" return "\n".join(text.split("\n")[:lines]) -def version_info(): +def version_info() -> None: """Handler for --version.""" try: platex_version = head( @@ -554,7 +561,7 @@ class DocBuilder: theme: Path @property - def html_only(self): + def html_only(self) -> bool: return ( self.select_output in {"only-html", "only-html-en"} or self.quick @@ -562,7 +569,7 @@ def html_only(self): ) @property - def includes_html(self): + def includes_html(self) -> bool: """Does the build we are running include HTML output?""" return self.select_output != "no-html" @@ -601,12 +608,12 @@ def checkout(self) -> Path: """Path to CPython git clone.""" return self.build_root / _checkout_name(self.select_output) - def clone_translation(self): + def clone_translation(self) -> None: self.translation_repo.update() self.translation_repo.switch(self.translation_branch) @property - def translation_repo(self): + def translation_repo(self) -> Repository: """See PEP 545 for translations repository naming convention.""" locale_repo = f"https://github.com/python/python-docs-{self.language.tag}.git" @@ -620,7 +627,7 @@ def translation_repo(self): return Repository(locale_repo, locale_clone_dir) @property - def translation_branch(self): + def translation_branch(self) -> str: """Some CPython versions may be untranslated, being either too old or too new. @@ -633,7 +640,7 @@ def translation_branch(self): branches = re.findall(r"/([0-9]+\.[0-9]+)$", remote_branches, re.M) return locate_nearest_version(branches, self.version.name) - def build(self): + def build(self) -> None: """Build this version/language doc.""" logging.info("Build start.") start_time = perf_counter() @@ -702,7 +709,7 @@ def build(self): ) logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) - def build_venv(self): + def build_venv(self) -> None: """Build a venv for the specific Python version. So we can reuse them from builds to builds, while they contain @@ -819,7 +826,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: "Publishing done (%s).", format_seconds(perf_counter() - start_time) ) - def should_rebuild(self, force: bool): + def should_rebuild(self, force: bool) -> str | Literal[False]: state = self.load_state() if not state: logging.info("Should rebuild: no previous state found.") @@ -865,7 +872,9 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_start: dt.datetime, build_duration: float, trigger: str): + def save_state( + self, build_start: dt.datetime, build_duration: float, trigger: str + ) -> None: """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -911,6 +920,8 @@ def format_seconds(seconds: float) -> str: case h, m, s: return f"{h}h {m}m {s}s" + raise ValueError("unreachable") + def _checkout_name(select_output: str | None) -> str: if select_output is not None: @@ -918,7 +929,7 @@ def _checkout_name(select_output: str | None) -> str: return "cpython" -def main(): +def main() -> None: """Script entry point.""" args = parse_args() setup_logging(args.log_directory, args.select_output) @@ -934,7 +945,7 @@ def main(): build_docs_with_lock(args, "build_docs_html_en.lock") -def parse_args(): +def parse_args() -> argparse.Namespace: """Parse command-line arguments.""" parser = argparse.ArgumentParser( @@ -1028,7 +1039,7 @@ def parse_args(): return args -def setup_logging(log_directory: Path, select_output: str | None): +def setup_logging(log_directory: Path, select_output: str | None) -> None: """Setup logging to stderr if run by a human, or to a file if run from a cron.""" log_format = "%(asctime)s %(levelname)s: %(message)s" if sys.stderr.isatty(): @@ -1174,7 +1185,9 @@ def parse_languages_from_config() -> Languages: return Languages.from_json(config["defaults"], config["languages"]) -def build_sitemap(versions: Versions, languages: Languages, www_root: Path, group): +def build_sitemap( + versions: Versions, languages: Languages, www_root: Path, group: str +) -> None: """Build a sitemap with all live versions and translations.""" if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exist).") @@ -1189,7 +1202,7 @@ def build_sitemap(versions: Versions, languages: Languages, www_root: Path, grou run(["chgrp", group, sitemap_path]) -def build_404(www_root: Path, group): +def build_404(www_root: Path, group: str) -> None: """Build a nice 404 error page to display in case PDFs are not built yet.""" if not www_root.exists(): logging.info("Skipping 404 page generation (www root does not even exist).") @@ -1203,8 +1216,8 @@ def build_404(www_root: Path, group): def copy_robots_txt( www_root: Path, - group, - skip_cache_invalidation, + group: str, + skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: """Copy robots.txt to www_root.""" @@ -1322,7 +1335,7 @@ def proofread_canonicals( ) -def _check_canonical_rel(file: Path, www_root: Path): +def _check_canonical_rel(file: Path, www_root: Path) -> Path | None: # Check for a canonical relation link in the HTML. # If one exists, ensure that the target exists # or otherwise remove the canonical link element. From 2fee30f3edc36216b0e329a93703d038b960d446 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:05:11 +0100 Subject: [PATCH 186/218] Refresh versions table in README (#276) --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2c84353..8f914c6 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,9 @@ of Sphinx we're using where: 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' - 3.12 ø sphinx~=8.1.0 needs_sphinx='7.2.6' - 3.13 ø sphinx~=8.1.0 needs_sphinx='7.2.6' - 3.14 ø sphinx~=8.1.0 needs_sphinx='7.2.6' + 3.12 ø sphinx~=8.2.0 needs_sphinx='8.2.0' + 3.13 ø sphinx~=8.2.0 needs_sphinx='8.2.0' + 3.14 ø sphinx~=8.2.0 needs_sphinx='8.2.0' ========= ============= ================== ==================== Sphinx build as seen on docs.python.org: @@ -52,9 +52,9 @@ of Sphinx we're using where: 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 - 3.12 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 - 3.13 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 - 3.14 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 + 3.12 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 + 3.13 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 + 3.14 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= ## Manually rebuild a branch From 57136bc88571e8115b30f94ca9adaebb2cd7cd38 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:06:16 +0100 Subject: [PATCH 187/218] Use Ruff linting (#277) --- .pre-commit-config.yaml | 1 + .ruff.toml | 22 ++++++++++++++++ build_docs.py | 42 ++++++++++++++++--------------- tests/test_build_docs.py | 2 +- tests/test_build_docs_versions.py | 2 +- 5 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4ad4370..c0eb053 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.11.5 hooks: + - id: ruff - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema diff --git a/.ruff.toml b/.ruff.toml index e837d03..6862f11 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -5,3 +5,25 @@ output-format = "full" [format] preview = true docstring-code-format = true + +[lint] +preview = true +select = [ + "C4", # flake8-comprehensions + "B", # flake8-bugbear + "E", # pycodestyle + "F", # pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "I", # isort + "N", # pep8-naming + "PERF", # perflint + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "TC", # flake8-type-checking + "UP", # pyupgrade + "W", # pycodestyle +] +ignore = [ + "E501", # Ignore line length errors (we use auto-formatting) +] diff --git a/build_docs.py b/build_docs.py index 0eb900e..7329bbe 100755 --- a/build_docs.py +++ b/build_docs.py @@ -65,10 +65,10 @@ from urllib.parse import urljoin import jinja2 +import platformdirs import tomlkit import urllib3 import zc.lockfile -from platformdirs import user_config_path, site_config_path TYPE_CHECKING = False if TYPE_CHECKING: @@ -76,7 +76,8 @@ from typing import Literal try: - from os import EX_OK, EX_SOFTWARE as EX_FAILURE + from os import EX_OK + from os import EX_SOFTWARE as EX_FAILURE except ImportError: EX_OK, EX_FAILURE = 0, 1 @@ -279,7 +280,7 @@ def filter(self, language_tags: Sequence[str] = ()) -> Sequence[Language]: """Filter a sequence of languages according to --languages.""" if language_tags: language_tags = frozenset(language_tags) - return [l for l in self if l.tag in language_tags] + return [l for l in self if l.tag in language_tags] # NoQA: E741 return list(self) @@ -480,7 +481,7 @@ def setup_switchers(versions: Versions, languages: Languages, html_root: Path) - - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) + language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] switchers_template_file = HERE / "templates" / "switchers.js" @@ -1057,28 +1058,29 @@ def setup_logging(log_directory: Path, select_output: str | None) -> None: def load_environment_variables() -> None: - _user_config_path = user_config_path("docsbuild-scripts") - _site_config_path = site_config_path("docsbuild-scripts") - if _user_config_path.is_file(): - ENV_CONF_FILE = _user_config_path - elif _site_config_path.is_file(): - ENV_CONF_FILE = _site_config_path + dbs_user_config = platformdirs.user_config_path("docsbuild-scripts") + dbs_site_config = platformdirs.site_config_path("docsbuild-scripts") + if dbs_user_config.is_file(): + env_conf_file = dbs_user_config + elif dbs_site_config.is_file(): + env_conf_file = dbs_site_config else: logging.info( "No environment variables configured. " - f"Configure in {_site_config_path} or {_user_config_path}." + f"Configure in {dbs_site_config} or {dbs_user_config}." ) return - logging.info(f"Reading environment variables from {ENV_CONF_FILE}.") - if ENV_CONF_FILE == _site_config_path: - logging.info(f"You can override settings in {_user_config_path}.") - elif _site_config_path.is_file(): - logging.info(f"Overriding {_site_config_path}.") - with open(ENV_CONF_FILE, "r") as f: - for key, value in tomlkit.parse(f.read()).get("env", {}).items(): - logging.debug(f"Setting {key} in environment.") - os.environ[key] = value + logging.info(f"Reading environment variables from {env_conf_file}.") + if env_conf_file == dbs_site_config: + logging.info(f"You can override settings in {dbs_user_config}.") + elif dbs_site_config.is_file(): + logging.info(f"Overriding {dbs_site_config}.") + + env_config = env_conf_file.read_text(encoding="utf-8") + for key, value in tomlkit.parse(env_config).get("env", {}).items(): + logging.debug(f"Setting {key} in environment.") + os.environ[key] = value def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: diff --git a/tests/test_build_docs.py b/tests/test_build_docs.py index 4457e95..028da90 100644 --- a/tests/test_build_docs.py +++ b/tests/test_build_docs.py @@ -4,7 +4,7 @@ @pytest.mark.parametrize( - "seconds, expected", + ("seconds", "expected"), [ (0.4, "0s"), (0.5, "0s"), diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py index 5ed9bcb..662838e 100644 --- a/tests/test_build_docs_versions.py +++ b/tests/test_build_docs_versions.py @@ -1,4 +1,4 @@ -from build_docs import Versions, Version +from build_docs import Version, Versions def test_filter_default() -> None: From 3deb0c3248f9016419868ed25a87d617ad786d04 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:10:16 +0100 Subject: [PATCH 188/218] Use strict mode for ``flake8-type-checking`` --- .ruff.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ruff.toml b/.ruff.toml index 6862f11..52976fe 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -27,3 +27,7 @@ select = [ ignore = [ "E501", # Ignore line length errors (we use auto-formatting) ] + +[lint.flake8-type-checking] +exempt-modules = [] +strict = true From 8c8b000889e51c078045a61b822ced8e6f226609 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:25:36 +0100 Subject: [PATCH 189/218] Enable flake8-logging-format in Ruff (#278) --- .ruff.toml | 1 + build_docs.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 52976fe..47cbf74 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -15,6 +15,7 @@ select = [ "F", # pyflakes "FA", # flake8-future-annotations "FLY", # flynt + "G", # flake8-logging-format "I", # isort "N", # pep8-naming "PERF", # perflint diff --git a/build_docs.py b/build_docs.py index 7329bbe..a204560 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1066,20 +1066,21 @@ def load_environment_variables() -> None: env_conf_file = dbs_site_config else: logging.info( - "No environment variables configured. " - f"Configure in {dbs_site_config} or {dbs_user_config}." + "No environment variables configured. Configure in %s or %s.", + dbs_site_config, + dbs_user_config, ) return - logging.info(f"Reading environment variables from {env_conf_file}.") + logging.info("Reading environment variables from %s.", env_conf_file) if env_conf_file == dbs_site_config: - logging.info(f"You can override settings in {dbs_user_config}.") + logging.info("You can override settings in %s.", dbs_user_config) elif dbs_site_config.is_file(): - logging.info(f"Overriding {dbs_site_config}.") + logging.info("Overriding %s.", dbs_site_config) env_config = env_conf_file.read_text(encoding="utf-8") for key, value in tomlkit.parse(env_config).get("env", {}).items(): - logging.debug(f"Setting {key} in environment.") + logging.debug("Setting %s in environment.", key) os.environ[key] = value From 88404600ea006ee14ea1bdbe716c925acba67bbb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 15:46:32 +0100 Subject: [PATCH 190/218] Add a simple integration test (#275) --- .github/workflows/test.yml | 87 ++++++++++++++++++++++++++------------ build_docs.py | 2 +- 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e272e76..d7ebf7f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,9 @@ name: Test -on: [push, pull_request, workflow_dispatch] +on: + push: + pull_request: + workflow_dispatch: permissions: {} @@ -8,35 +11,63 @@ env: FORCE_COLOR: 1 jobs: - test: - runs-on: ${{ matrix.os }} + integration: + name: Integration test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Set up requirements + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Build documentation + run: > + python ./build_docs.py + --quick + --build-root ./build_root + --www-root ./www + --log-directory ./logs + --group "$(id -g)" + --skip-cache-invalidation + --languages en + --branches 3.14 + + unit: + name: Unit tests + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.13", "3.14"] - os: [ubuntu-latest] + python-version: + - "3.13" + - "3.14" steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - - - name: Install uv - uses: hynek/setup-cached-uv@v2 - - - name: Tox tests - run: | - uvx --with tox-uv tox -e py - - - name: Upload coverage - uses: codecov/codecov-action@v5 - with: - flags: ${{ matrix.os }} - name: ${{ matrix.os }} Python ${{ matrix.python-version }} - token: ${{ secrets.CODECOV_ORG_TOKEN }} + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Install uv + uses: hynek/setup-cached-uv@v2 + + - name: Tox tests + run: uvx --with tox-uv tox -e py + + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + name: Python ${{ matrix.python-version }} + token: ${{ secrets.CODECOV_ORG_TOKEN }} diff --git a/build_docs.py b/build_docs.py index a204560..2301918 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1043,7 +1043,7 @@ def parse_args() -> argparse.Namespace: def setup_logging(log_directory: Path, select_output: str | None) -> None: """Setup logging to stderr if run by a human, or to a file if run from a cron.""" log_format = "%(asctime)s %(levelname)s: %(message)s" - if sys.stderr.isatty(): + if sys.stderr.isatty() or "CI" in os.environ: logging.basicConfig(format=log_format, stream=sys.stderr) else: log_directory.mkdir(parents=True, exist_ok=True) From 5cfd94e84ead25ce6baa3d859a53c30ea37eac7e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 15:51:56 +0100 Subject: [PATCH 191/218] Convert ``Version`` to a dataclass (#279) --- build_docs.py | 53 +++++++++++++------------------ tests/test_build_docs_versions.py | 48 ++++++++++++++-------------- 2 files changed, 46 insertions(+), 55 deletions(-) diff --git a/build_docs.py b/build_docs.py index 2301918..1efdabf 100755 --- a/build_docs.py +++ b/build_docs.py @@ -58,7 +58,6 @@ import sys from bisect import bisect_left as bisect from contextlib import contextmanager, suppress -from functools import total_ordering from pathlib import Path from string import Template from time import perf_counter, sleep @@ -103,11 +102,23 @@ def __reversed__(self) -> Iterator[Version]: @classmethod def from_json(cls, data: dict) -> Versions: - versions = sorted( - [Version.from_json(name, release) for name, release in data.items()], - key=Version.as_tuple, - ) - return cls(versions) + """Load versions from the devguide's JSON representation.""" + permitted = ", ".join(sorted(Version.STATUSES | Version.SYNONYMS.keys())) + + versions = [] + for name, release in data.items(): + branch = release["branch"] + status = release["status"] + status = Version.SYNONYMS.get(status, status) + if status not in Version.STATUSES: + msg = ( + f"Saw invalid version status {status!r}, " + f"expected to be one of {permitted}." + ) + raise ValueError(msg) + versions.append(Version(name=name, status=status, branch_or_tag=branch)) + + return cls(sorted(versions, key=Version.as_tuple)) def filter(self, branches: Sequence[str] = ()) -> Sequence[Version]: """Filter the given versions. @@ -143,10 +154,14 @@ def setup_indexsidebar(self, current: Version, dest_path: Path) -> None: dest_path.write_text(rendered_template, encoding="UTF-8") -@total_ordering +@dataclasses.dataclass(frozen=True, kw_only=True, slots=True) class Version: """Represents a CPython version and its documentation build dependencies.""" + name: str + status: Literal["EOL", "security-fixes", "stable", "pre-release", "in development"] + branch_or_tag: str + STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"} # Those synonyms map branch status vocabulary found in the devguide @@ -159,33 +174,9 @@ class Version: "prerelease": "pre-release", } - def __init__( - self, name: str, *, status: str, branch_or_tag: str | None = None - ) -> None: - status = self.SYNONYMS.get(status, status) - if status not in self.STATUSES: - raise ValueError( - "Version status expected to be one of: " - f"{', '.join(self.STATUSES | set(self.SYNONYMS.keys()))}, got {status!r}." - ) - self.name = name - self.branch_or_tag = branch_or_tag - self.status = status - - def __repr__(self) -> str: - return f"Version({self.name})" - def __eq__(self, other: Version) -> bool: return self.name == other.name - def __gt__(self, other: Version) -> bool: - return self.as_tuple() > other.as_tuple() - - @classmethod - def from_json(cls, name: str, values: dict) -> Version: - """Loads a version from devguide's json representation.""" - return cls(name, status=values["status"], branch_or_tag=values["branch"]) - @property def requirements(self) -> list[str]: """Generate the right requirements for this version. diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py index 662838e..42b5392 100644 --- a/tests/test_build_docs_versions.py +++ b/tests/test_build_docs_versions.py @@ -4,12 +4,12 @@ def test_filter_default() -> None: # Arrange versions = Versions([ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), - Version("3.11", status="security"), - Version("3.10", status="security"), - Version("3.9", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.11", status="security-fixes", branch_or_tag=""), + Version(name="3.10", status="security-fixes", branch_or_tag=""), + Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) # Act @@ -17,39 +17,39 @@ def test_filter_default() -> None: # Assert assert filtered == [ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), ] def test_filter_one() -> None: # Arrange versions = Versions([ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), - Version("3.11", status="security"), - Version("3.10", status="security"), - Version("3.9", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.11", status="security-fixes", branch_or_tag=""), + Version(name="3.10", status="security-fixes", branch_or_tag=""), + Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) # Act filtered = versions.filter(["3.13"]) # Assert - assert filtered == [Version("3.13", status="security")] + assert filtered == [Version(name="3.13", status="security-fixes", branch_or_tag="")] def test_filter_multiple() -> None: # Arrange versions = Versions([ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), - Version("3.11", status="security"), - Version("3.10", status="security"), - Version("3.9", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.11", status="security-fixes", branch_or_tag=""), + Version(name="3.10", status="security-fixes", branch_or_tag=""), + Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) # Act @@ -57,6 +57,6 @@ def test_filter_multiple() -> None: # Assert assert filtered == [ - Version("3.14", status="feature"), - Version("3.13", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="security-fixes", branch_or_tag=""), ] From 9f3437707aa698ecc560a5d911cfbad9803541e7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 19:00:31 +0100 Subject: [PATCH 192/218] Extract ``switchers.js`` rendering into a new function. (#280) Previously, identical content for ``switchers.js`` was rendered for each version-language pair. This commit pre-renders the content of the file, then writing it to disk for each version-language pair. --- build_docs.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index 1efdabf..08f457e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -467,23 +467,13 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers(versions: Versions, languages: Languages, html_root: Path) -> None: +def setup_switchers(script_content: bytes, html_root: Path) -> None: """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 - version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] - - switchers_template_file = HERE / "templates" / "switchers.js" switchers_path = html_root / "_static" / "switchers.js" - - template = Template(switchers_template_file.read_text(encoding="UTF-8")) - rendered_template = template.safe_substitute( - LANGUAGES=json.dumps(language_pairs), - VERSIONS=json.dumps(version_pairs), - ) - switchers_path.write_text(rendered_template, encoding="UTF-8") + switchers_path.write_text(script_content, encoding="UTF-8") for file in html_root.glob("**/*.html"): depth = len(file.relative_to(html_root).parts) - 1 @@ -541,8 +531,8 @@ class DocBuilder: version: Version versions: Versions language: Language - languages: Languages cpython_repo: Repository + switchers_content: bytes build_root: Path www_root: Path select_output: Literal["no-html", "only-html", "only-html-en"] | None @@ -697,7 +687,7 @@ def build(self) -> None: run(["chgrp", "-R", self.group, self.log_directory]) if self.includes_html: setup_switchers( - self.versions, self.languages, self.checkout / "Doc" / "build" / "html" + self.switchers_content, self.checkout / "Doc" / "build" / "html" ) logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) @@ -1108,6 +1098,8 @@ def build_docs(args: argparse.Namespace) -> bool: force_build = args.force del args.force + switchers_content = render_switchers(versions, languages) + build_succeeded = set() build_failed = set() cpython_repo = Repository( @@ -1127,7 +1119,12 @@ def build_docs(args: argparse.Namespace) -> bool: scope.set_tag("language", language.tag) cpython_repo.update() builder = DocBuilder( - version, versions, language, languages, cpython_repo, **vars(args) + version, + versions, + language, + cpython_repo, + switchers_content, + **vars(args), ) built_successfully = builder.run(http, force_build=force_build) if built_successfully: @@ -1179,6 +1176,19 @@ def parse_languages_from_config() -> Languages: return Languages.from_json(config["defaults"], config["languages"]) +def render_switchers(versions: Versions, languages: Languages) -> bytes: + language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 + version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] + + switchers_template_file = HERE / "templates" / "switchers.js" + template = Template(switchers_template_file.read_text(encoding="UTF-8")) + rendered_template = template.safe_substitute( + LANGUAGES=json.dumps(language_pairs), + VERSIONS=json.dumps(version_pairs), + ) + return rendered_template.encode("UTF-8") + + def build_sitemap( versions: Versions, languages: Languages, www_root: Path, group: str ) -> None: From 638d5e6634869c769ac49ac655c92c47d9f4a662 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 19:02:28 +0100 Subject: [PATCH 193/218] Correct the type annotation for ``DocBuilder.theme`` (#281) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 08f457e..07c98ae 100755 --- a/build_docs.py +++ b/build_docs.py @@ -540,7 +540,7 @@ class DocBuilder: group: str log_directory: Path skip_cache_invalidation: bool - theme: Path + theme: str @property def html_only(self) -> bool: From f30fec8a7a811e7f325cadd705f7db4578e2d7a4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 22:07:23 +0100 Subject: [PATCH 194/218] Skip checking canonical links if nothing was built (#282) --- build_docs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index 07c98ae..faa8648 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1101,7 +1101,7 @@ def build_docs(args: argparse.Namespace) -> bool: switchers_content = render_switchers(versions, languages) build_succeeded = set() - build_failed = set() + any_build_failed = False cpython_repo = Repository( "https://github.com/python/cpython.git", args.build_root / _checkout_name(args.select_output), @@ -1130,7 +1130,7 @@ def build_docs(args: argparse.Namespace) -> bool: if built_successfully: build_succeeded.add((version.name, language.tag)) elif built_successfully is not None: - build_failed.add((version.name, language.tag)) + any_build_failed = True logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") @@ -1153,11 +1153,13 @@ def build_docs(args: argparse.Namespace) -> bool: args.skip_cache_invalidation, http, ) - proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) + if build_succeeded: + # Only check canonicals if at least one version was built. + proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - return len(build_failed) == 0 + return any_build_failed def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: From 982c0f8b71f6fa10a8378eb3e3a60ed50fea4e5a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 13 Apr 2025 11:32:47 +0100 Subject: [PATCH 195/218] Fix TypeError in ``setup_switchers()`` (#284) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index faa8648..fef379e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -473,7 +473,7 @@ def setup_switchers(script_content: bytes, html_root: Path) -> None: - Cross-link various versions in a version switcher """ switchers_path = html_root / "_static" / "switchers.js" - switchers_path.write_text(script_content, encoding="UTF-8") + switchers_path.write_bytes(script_content) for file in html_root.glob("**/*.html"): depth = len(file.relative_to(html_root).parts) - 1 From cc5e153b4afc6bbc7dd5aac99ee8f396d9b9b7a4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 13 Apr 2025 13:50:33 +0100 Subject: [PATCH 196/218] Ensure exit status is returned by ``build_docs.py`` (#286) --- build_docs.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/build_docs.py b/build_docs.py index fef379e..e40259c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -911,20 +911,21 @@ def _checkout_name(select_output: str | None) -> str: return "cpython" -def main() -> None: +def main() -> int: """Script entry point.""" args = parse_args() setup_logging(args.log_directory, args.select_output) load_environment_variables() if args.select_output is None: - build_docs_with_lock(args, "build_docs.lock") - elif args.select_output == "no-html": - build_docs_with_lock(args, "build_docs_archives.lock") - elif args.select_output == "only-html": - build_docs_with_lock(args, "build_docs_html.lock") - elif args.select_output == "only-html-en": - build_docs_with_lock(args, "build_docs_html_en.lock") + return build_docs_with_lock(args, "build_docs.lock") + if args.select_output == "no-html": + return build_docs_with_lock(args, "build_docs_archives.lock") + if args.select_output == "only-html": + return build_docs_with_lock(args, "build_docs_html.lock") + if args.select_output == "only-html-en": + return build_docs_with_lock(args, "build_docs_html_en.lock") + return EX_FAILURE def parse_args() -> argparse.Namespace: @@ -1073,12 +1074,12 @@ def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: return EX_FAILURE try: - return EX_OK if build_docs(args) else EX_FAILURE + return build_docs(args) finally: lock.close() -def build_docs(args: argparse.Namespace) -> bool: +def build_docs(args: argparse.Namespace) -> int: """Build all docs (each language and each version).""" logging.info("Full build start.") start_time = perf_counter() @@ -1159,7 +1160,7 @@ def build_docs(args: argparse.Namespace) -> bool: logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - return any_build_failed + return EX_FAILURE if any_build_failed else EX_OK def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: @@ -1397,4 +1398,4 @@ def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) From 1e856ea3a38944e9743c11110d67599c3159cb07 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 13 Apr 2025 19:08:49 +0300 Subject: [PATCH 197/218] Run unit tests on three operating systems (#285) Co-authored-by: Ezio Melotti --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7ebf7f..8a9324f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,13 +41,12 @@ jobs: unit: name: Unit tests - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: - - "3.13" - - "3.14" + python-version: ["3.13", "3.14"] + os: [windows-latest, macos-latest, ubuntu-latest] steps: - uses: actions/checkout@v4 @@ -69,5 +68,6 @@ jobs: - name: Upload coverage uses: codecov/codecov-action@v5 with: + flags: ${{ matrix.os }} name: Python ${{ matrix.python-version }} token: ${{ secrets.CODECOV_ORG_TOKEN }} From 9f45c4e38487289db9dacaea600ecec2720b1507 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:44:32 +0300 Subject: [PATCH 198/218] Test the ``Version`` class (#287) --- tests/test_build_docs_version.py | 88 ++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/test_build_docs_version.py diff --git a/tests/test_build_docs_version.py b/tests/test_build_docs_version.py new file mode 100644 index 0000000..bd5e644 --- /dev/null +++ b/tests/test_build_docs_version.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import pytest + +from build_docs import Version + + +def test_equality() -> None: + # Arrange + version1 = Version(name="3.13", status="stable", branch_or_tag="3.13") + version2 = Version(name="3.13", status="stable", branch_or_tag="3.13") + + # Act / Assert + assert version1 == version2 + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("3.13", "-rrequirements.txt"), + ("3.10", "standard-imghdr"), + ("3.7", "sphinx==2.3.1"), + ("3.5", "sphinx==1.8.4"), + ], +) +def test_requirements(name: str, expected: str) -> None: + # Arrange + version = Version(name=name, status="stable", branch_or_tag="") + + # Act / Assert + assert expected in version.requirements + + +def test_requirements_error() -> None: + # Arrange + version = Version(name="2.8", status="ex-release", branch_or_tag="") + + # Act / Assert + with pytest.raises(ValueError, match="unreachable"): + _ = version.requirements + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + ("EOL", "never"), + ("security-fixes", "yearly"), + ("stable", "daily"), + ], +) +def test_changefreq(status: str, expected: str) -> None: + # Arrange + version = Version(name="3.13", status=status, branch_or_tag="") + + # Act / Assert + assert version.changefreq == expected + + +def test_url() -> None: + # Arrange + version = Version(name="3.13", status="stable", branch_or_tag="") + + # Act / Assert + assert version.url == "https://docs.python.org/3.13/" + + +def test_title() -> None: + # Arrange + version = Version(name="3.14", status="in development", branch_or_tag="") + + # Act / Assert + assert version.title == "Python 3.14 (in development)" + + +@pytest.mark.parametrize( + ("name", "status", "expected"), + [ + ("3.15", "in development", "dev (3.15)"), + ("3.14", "pre-release", "pre (3.14)"), + ("3.13", "stable", "3.13"), + ], +) +def test_picker_label(name: str, status: str, expected: str) -> None: + # Arrange + version = Version(name=name, status=status, branch_or_tag="") + + # Act / Assert + assert version.picker_label == expected From 9bcde435e586cca62c694678b0966b4b84d93047 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:45:44 +0100 Subject: [PATCH 199/218] Simplify generation of ``indexsidebar.html`` (#283) Remove the double-rendering of ``indexsidebar.html``. For end-of-life versions, change the sidebar to contain a link to the current stable version. For non-EOL versions, overwrite the new ``_docs_by_version.html`` file with the list of links. --- build_docs.py | 40 +++++++++++++++++++-------------- templates/_docs_by_version.html | 11 +++++++++ templates/indexsidebar.html | 23 +++++-------------- 3 files changed, 39 insertions(+), 35 deletions(-) create mode 100644 templates/_docs_by_version.html diff --git a/build_docs.py b/build_docs.py index e40259c..9b498fe 100755 --- a/build_docs.py +++ b/build_docs.py @@ -143,16 +143,6 @@ def current_dev(self) -> Version: """Find the current CPython version in development.""" return max(self, key=Version.as_tuple) - def setup_indexsidebar(self, current: Version, dest_path: Path) -> None: - """Build indexsidebar.html for Sphinx.""" - template_path = HERE / "templates" / "indexsidebar.html" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render( - current_version=current, - versions=list(reversed(self)), - ) - dest_path.write_text(rendered_template, encoding="UTF-8") - @dataclasses.dataclass(frozen=True, kw_only=True, slots=True) class Version: @@ -529,9 +519,9 @@ class DocBuilder: """Builder for a CPython version and a language.""" version: Version - versions: Versions language: Language cpython_repo: Repository + docs_by_version_content: bytes switchers_content: bytes build_root: Path www_root: Path @@ -667,10 +657,7 @@ def build(self) -> None: text = text.replace(" -A switchers=1", "") (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") - self.versions.setup_indexsidebar( - self.version, - self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", - ) + self.setup_indexsidebar() run_with_logging([ "make", "-C", @@ -713,6 +700,18 @@ def build_venv(self) -> None: run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) self.venv = venv_path + def setup_indexsidebar(self) -> None: + """Copy indexsidebar.html for Sphinx.""" + tmpl_src = HERE / "templates" + tmpl_dst = self.checkout / "Doc" / "tools" / "templates" + dbv_path = tmpl_dst / "_docs_by_version.html" + + shutil.copy(tmpl_src / "indexsidebar.html", tmpl_dst / "indexsidebar.html") + if self.version.status != "EOL": + dbv_path.write_bytes(self.docs_by_version_content) + else: + shutil.copy(tmpl_src / "_docs_by_version.html", dbv_path) + def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: """Copy a given build to the appropriate webroot with appropriate rights.""" logging.info("Publishing start.") @@ -1099,6 +1098,7 @@ def build_docs(args: argparse.Namespace) -> int: force_build = args.force del args.force + docs_by_version_content = render_docs_by_version(versions).encode() switchers_content = render_switchers(versions, languages) build_succeeded = set() @@ -1118,12 +1118,12 @@ def build_docs(args: argparse.Namespace) -> int: scope = sentry_sdk.get_isolation_scope() scope.set_tag("version", version.name) scope.set_tag("language", language.tag) - cpython_repo.update() + cpython_repo.update() builder = DocBuilder( version, - versions, language, cpython_repo, + docs_by_version_content, switchers_content, **vars(args), ) @@ -1179,6 +1179,12 @@ def parse_languages_from_config() -> Languages: return Languages.from_json(config["defaults"], config["languages"]) +def render_docs_by_version(versions: Versions) -> str: + """Generate content for _docs_by_version.html.""" + links = [f'
  • {v.title}
    • ' for v in reversed(versions)] + return "\n".join(links) + + def render_switchers(versions: Versions, languages: Languages) -> bytes: language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] diff --git a/templates/_docs_by_version.html b/templates/_docs_by_version.html new file mode 100644 index 0000000..1a84cfb --- /dev/null +++ b/templates/_docs_by_version.html @@ -0,0 +1,11 @@ +{# +This file is only used in indexsidebar.html, where it is included in the docs +by version list. For non-end-of-life branches, build_docs.py overwrites this +list with the full list of versions. + +Keep the following two files synchronised: +* cpython/Doc/tools/templates/_docs_by_version.html +* docsbuild-scripts/templates/_docs_by_version.html +#} +
  • {% trans %}Stable{% endtrans %}
    • +
  • {% trans %}In development{% endtrans %}
    • diff --git a/templates/indexsidebar.html b/templates/indexsidebar.html index 3a56219..eea29e2 100644 --- a/templates/indexsidebar.html +++ b/templates/indexsidebar.html @@ -1,30 +1,17 @@ -{# -Beware, this file is rendered twice via Jinja2: -- First by build_docs.py, given 'current_version' and 'versions'. -- A 2nd time by Sphinx. -#} - -{% raw %}

    {% trans %}Download{% endtrans %}

    {% trans %}Download these documents{% endtrans %}

    -{% endraw %} -{% if current_version.status != "EOL" %} -{% raw %}

    {% trans %}Docs by version{% endtrans %}

    {% endraw %} +

    {% trans %}Docs by version{% endtrans %}

    -{% endif %} -{% raw %}

    {% trans %}Other resources{% endtrans %}

    -{% endraw %} From efd1e17c947f426af9301ec0387f2e56bf3f86bf Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:45:53 +0100 Subject: [PATCH 200/218] Use ``venv.create()`` to create virtual environments (#289) --- build_docs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 9b498fe..733473e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -56,6 +56,7 @@ import shutil import subprocess import sys +import venv from bisect import bisect_left as bisect from contextlib import contextmanager, suppress from pathlib import Path @@ -690,7 +691,7 @@ def build_venv(self) -> None: requirements.append("matplotlib>=3") venv_path = self.build_root / ("venv-" + self.version.name) - run([sys.executable, "-m", "venv", venv_path]) + venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] + ["--upgrade-strategy=eager"] From 776e64996819c11cec1a4154342630f3e7ea4c1f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:56:15 +0100 Subject: [PATCH 201/218] Create a function to perform ``chgrp`` operations (#290) --- build_docs.py | 70 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/build_docs.py b/build_docs.py index 733473e..2f164f8 100755 --- a/build_docs.py +++ b/build_docs.py @@ -671,8 +671,8 @@ def build(self) -> None: "SPHINXERRORHANDLING=", maketarget, ]) - run(["mkdir", "-p", self.log_directory]) - run(["chgrp", "-R", self.group, self.log_directory]) + self.log_directory.mkdir(parents=True, exist_ok=True) + chgrp(self.log_directory, group=self.group, recursive=True) if self.includes_html: setup_switchers( self.switchers_content, self.checkout / "Doc" / "build" / "html" @@ -723,10 +723,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: else: language_dir = self.www_root / self.language.tag language_dir.mkdir(parents=True, exist_ok=True) - try: - run(["chgrp", "-R", self.group, language_dir]) - except subprocess.CalledProcessError as err: - logging.warning("Can't change group of %s: %s", language_dir, str(err)) + chgrp(language_dir, group=self.group, recursive=True) language_dir.chmod(0o775) target = language_dir / self.version.name @@ -735,22 +732,18 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: target.chmod(0o775) except PermissionError as err: logging.warning("Can't change mod of %s: %s", target, str(err)) - try: - run(["chgrp", "-R", self.group, target]) - except subprocess.CalledProcessError as err: - logging.warning("Can't change group of %s: %s", target, str(err)) + chgrp(target, group=self.group, recursive=True) changed = [] if self.includes_html: # Copy built HTML files to webroot (default /srv/docs.python.org) changed = changed_files(self.checkout / "Doc" / "build" / "html", target) logging.info("Copying HTML files to %s", target) - run([ - "chown", - "-R", - ":" + self.group, + chgrp( self.checkout / "Doc" / "build" / "html/", - ]) + group=self.group, + recursive=True, + ) run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) run([ "find", @@ -776,12 +769,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: if not self.quick and (self.checkout / "Doc" / "dist").is_dir(): # Copy archive files to /archives/ logging.debug("Copying dist files.") - run([ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "dist", - ]) + chgrp(self.checkout / "Doc" / "dist", group=self.group, recursive=True) run([ "chmod", "-R", @@ -789,7 +777,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: self.checkout / "Doc" / "dist", ]) run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) - run(["chown", ":" + self.group, target / "archives"]) + chgrp(target / "archives", group=self.group) run([ "cp", "-a", @@ -889,6 +877,36 @@ def save_state( logging.info("Saved new rebuild state for %s: %s", key, table.as_string()) +def chgrp( + path: Path, + /, + group: int | str | None, + *, + recursive: bool = False, + follow_symlinks: bool = True, +) -> None: + if sys.platform == "win32": + return + + from grp import getgrnam + + try: + try: + group_id = int(group) + except ValueError: + group_id = getgrnam(group)[2] + except (LookupError, TypeError, ValueError): + return + + try: + os.chown(path, -1, group_id, follow_symlinks=follow_symlinks) + if recursive: + for p in path.rglob("*"): + os.chown(p, -1, group_id, follow_symlinks=follow_symlinks) + except OSError as err: + logging.warning("Can't change group of %s: %s", path, str(err)) + + def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) @@ -1213,7 +1231,7 @@ def build_sitemap( sitemap_path = www_root / "sitemap.xml" sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") sitemap_path.chmod(0o664) - run(["chgrp", group, sitemap_path]) + chgrp(sitemap_path, group=group) def build_404(www_root: Path, group: str) -> None: @@ -1225,7 +1243,7 @@ def build_404(www_root: Path, group: str) -> None: not_found_file = www_root / "404.html" shutil.copyfile(HERE / "templates" / "404.html", not_found_file) not_found_file.chmod(0o664) - run(["chgrp", group, not_found_file]) + chgrp(not_found_file, group=group) def copy_robots_txt( @@ -1243,7 +1261,7 @@ def copy_robots_txt( robots_path = www_root / "robots.txt" shutil.copyfile(template_path, robots_path) robots_path.chmod(0o775) - run(["chgrp", group, robots_path]) + chgrp(robots_path, group=group) if not skip_cache_invalidation: purge(http, "robots.txt") @@ -1311,7 +1329,7 @@ def symlink( # Link does not exist or points to the wrong target. link.unlink(missing_ok=True) link.symlink_to(directory) - run(["chown", "-h", f":{group}", str(link)]) + chgrp(link, group=group, follow_symlinks=False) if not skip_cache_invalidation: surrogate_key = f"{language_tag}/{name}" purge_surrogate_key(http, surrogate_key) From 6164fac89acf0e3890d70bdd60320cd9b5de77e2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 21:29:27 +0100 Subject: [PATCH 202/218] Create a function for ``chmod`` operations (#291) --- build_docs.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/build_docs.py b/build_docs.py index 2f164f8..d1afdf2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -54,6 +54,7 @@ import re import shlex import shutil +import stat import subprocess import sys import venv @@ -744,18 +745,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: group=self.group, recursive=True, ) - run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) - run([ - "find", - self.checkout / "Doc" / "build" / "html", - "-type", - "d", - "-exec", - "chmod", - "o+x", - "{}", - ";", - ]) + chmod_make_readable(self.checkout / "Doc" / "build" / "html") run([ "rsync", "-a", @@ -770,12 +760,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: # Copy archive files to /archives/ logging.debug("Copying dist files.") chgrp(self.checkout / "Doc" / "dist", group=self.group, recursive=True) - run([ - "chmod", - "-R", - "o+r", - self.checkout / "Doc" / "dist", - ]) + chmod_make_readable(self.checkout / "Doc" / "dist") run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) chgrp(target / "archives", group=self.group) run([ @@ -907,6 +892,18 @@ def chgrp( logging.warning("Can't change group of %s: %s", path, str(err)) +def chmod_make_readable(path: Path, /, mode: int = stat.S_IROTH) -> None: + if not path.is_dir(): + raise ValueError + + path.chmod(path.stat().st_mode | stat.S_IROTH | stat.S_IXOTH) # o+rx + for p in path.rglob("*"): + if p.is_dir(): + p.chmod(p.stat().st_mode | stat.S_IROTH | stat.S_IXOTH) # o+rx + else: + p.chmod(p.stat().st_mode | stat.S_IROTH) # o+r + + def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) From e1dcdfcaaac76fbb6a4608e169bef9d2e5f6f5a2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 22:53:25 +0100 Subject: [PATCH 203/218] Remove subprocess calls for copying archive files (#292) --- .github/workflows/test.yml | 6 ++++++ build_docs.py | 27 ++++++++++++++------------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a9324f..850fec7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,12 @@ jobs: --languages en --branches 3.14 + - name: Upload documentation + uses: actions/upload-artifact@v4 + with: + name: www-root + path: ./www + unit: name: Unit tests runs-on: ${{ matrix.os }} diff --git a/build_docs.py b/build_docs.py index d1afdf2..83bc26c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -756,22 +756,23 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: target, ]) - if not self.quick and (self.checkout / "Doc" / "dist").is_dir(): + dist_dir = self.checkout / "Doc" / "dist" + if dist_dir.is_dir(): # Copy archive files to /archives/ logging.debug("Copying dist files.") - chgrp(self.checkout / "Doc" / "dist", group=self.group, recursive=True) - chmod_make_readable(self.checkout / "Doc" / "dist") - run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) - chgrp(target / "archives", group=self.group) - run([ - "cp", - "-a", - *(self.checkout / "Doc" / "dist").glob("*"), - target / "archives", - ]) + chgrp(dist_dir, group=self.group, recursive=True) + chmod_make_readable(dist_dir) + archives_dir = target / "archives" + archives_dir.mkdir(parents=True, exist_ok=True) + archives_dir.chmod( + archives_dir.stat().st_mode | stat.S_IROTH | stat.S_IXOTH + ) + chgrp(archives_dir, group=self.group) + for dist_file in dist_dir.iterdir(): + shutil.copy2(dist_file, archives_dir / dist_file.name) changed.append("archives/") - for file in (target / "archives").iterdir(): - changed.append("archives/" + file.name) + for file in archives_dir.iterdir(): + changed.append(f"archives/{file.name}") logging.info("%s files changed", len(changed)) if changed and not self.skip_cache_invalidation: From d6f84292abef7150bdd0a9467d0fca08df7188f5 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 22:58:51 +0100 Subject: [PATCH 204/218] Set explicit artefact retention time --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 850fec7..b64e8a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,7 @@ jobs: with: name: www-root path: ./www + retention-days: 2 unit: name: Unit tests From 783e70445fa48a8d3d7268e75eb45b2926b04c05 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 23:22:12 +0100 Subject: [PATCH 205/218] Convert ``changed`` to an integer count --- build_docs.py | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/build_docs.py b/build_docs.py index 83bc26c..d20aea4 100755 --- a/build_docs.py +++ b/build_docs.py @@ -337,23 +337,15 @@ def run_with_logging(cmd: Sequence[str | Path], cwd: Path | None = None) -> None raise subprocess.CalledProcessError(return_code, cmd[0]) -def changed_files(left: Path, right: Path) -> list[str]: - """Compute a list of different files between left and right, recursively. - Resulting paths are relative to left. - """ - changed = [] +def changed_files(left: Path, right: Path) -> int: + """Compute the number of different files in the two directory trees.""" - def traverse(dircmp_result: filecmp.dircmp) -> None: - base = Path(dircmp_result.left).relative_to(left) - for file in dircmp_result.diff_files: - changed.append(str(base / file)) - if file == "index.html": - changed.append(str(base) + "/") - for dircomp in dircmp_result.subdirs.values(): - traverse(dircomp) + def traverse(dircmp_result: filecmp.dircmp) -> int: + changed = len(dircmp_result.diff_files) + changed += sum(map(traverse, dircmp_result.subdirs.values())) + return changed - traverse(filecmp.dircmp(left, right)) - return changed + return traverse(filecmp.dircmp(left, right)) @dataclasses.dataclass @@ -735,10 +727,10 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: logging.warning("Can't change mod of %s: %s", target, str(err)) chgrp(target, group=self.group, recursive=True) - changed = [] + changed = 0 if self.includes_html: # Copy built HTML files to webroot (default /srv/docs.python.org) - changed = changed_files(self.checkout / "Doc" / "build" / "html", target) + changed += changed_files(self.checkout / "Doc" / "build" / "html", target) logging.info("Copying HTML files to %s", target) chgrp( self.checkout / "Doc" / "build" / "html/", @@ -768,13 +760,12 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: archives_dir.stat().st_mode | stat.S_IROTH | stat.S_IXOTH ) chgrp(archives_dir, group=self.group) + changed += 1 for dist_file in dist_dir.iterdir(): shutil.copy2(dist_file, archives_dir / dist_file.name) - changed.append("archives/") - for file in archives_dir.iterdir(): - changed.append(f"archives/{file.name}") + changed += 1 - logging.info("%s files changed", len(changed)) + logging.info("%s files changed", changed) if changed and not self.skip_cache_invalidation: surrogate_key = f"{self.language.tag}/{self.version.name}" purge_surrogate_key(http, surrogate_key) From 9082a8eed43f3c2ec4f8101176d3932b4949ecde Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 23:32:48 +0100 Subject: [PATCH 206/218] Use f-strings instead of string concatenation --- build_docs.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build_docs.py b/build_docs.py index d20aea4..847bfe2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -363,10 +363,10 @@ def get_ref(self, pattern: str) -> str: """Return the reference of a given tag or branch.""" try: # Maybe it's a branch - return self.run("show-ref", "-s", "origin/" + pattern).stdout.strip() + return self.run("show-ref", "-s", f"origin/{pattern}").stdout.strip() except subprocess.CalledProcessError: # Maybe it's a tag - return self.run("show-ref", "-s", "tags/" + pattern).stdout.strip() + return self.run("show-ref", "-s", f"tags/{pattern}").stdout.strip() def fetch(self) -> subprocess.CompletedProcess: """Try (and retry) to run git fetch.""" @@ -656,11 +656,11 @@ def build(self) -> None: "make", "-C", self.checkout / "Doc", - "PYTHON=" + str(python), - "SPHINXBUILD=" + str(sphinxbuild), - "BLURB=" + str(blurb), - "VENVDIR=" + str(self.venv), - "SPHINXOPTS=" + " ".join(sphinxopts), + f"PYTHON={python}", + f"SPHINXBUILD={sphinxbuild}", + f"BLURB={blurb}", + f"VENVDIR={self.venv}", + f"SPHINXOPTS={' '.join(sphinxopts)}", "SPHINXERRORHANDLING=", maketarget, ]) @@ -683,7 +683,7 @@ def build_venv(self) -> None: # opengraph previews requirements.append("matplotlib>=3") - venv_path = self.build_root / ("venv-" + self.version.name) + venv_path = self.build_root / f"venv-{self.version.name}" venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] From fe84a0bb4c442bcdb32154768c160468b1e90d6e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 23:41:47 +0100 Subject: [PATCH 207/218] Use tuples for subprocess argument lists --- build_docs.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/build_docs.py b/build_docs.py index 847bfe2..d31ebdd 100755 --- a/build_docs.py +++ b/build_docs.py @@ -388,7 +388,7 @@ def clone(self) -> bool: return False # Already cloned logging.info("Cloning %s into %s", self.remote, self.directory) self.directory.mkdir(mode=0o775, parents=True, exist_ok=True) - run(["git", "clone", self.remote, self.directory]) + run(("git", "clone", self.remote, self.directory)) return True def update(self) -> None: @@ -481,7 +481,7 @@ def version_info() -> None: """Handler for --version.""" try: platex_version = head( - subprocess.check_output(["platex", "--version"], text=True), + subprocess.check_output(("platex", "--version"), text=True), lines=3, ) except FileNotFoundError: @@ -489,7 +489,7 @@ def version_info() -> None: try: xelatex_version = head( - subprocess.check_output(["xelatex", "--version"], text=True), + subprocess.check_output(("xelatex", "--version"), text=True), lines=2, ) except FileNotFoundError: @@ -652,7 +652,7 @@ def build(self) -> None: (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") self.setup_indexsidebar() - run_with_logging([ + run_with_logging(( "make", "-C", self.checkout / "Doc", @@ -663,7 +663,7 @@ def build(self) -> None: f"SPHINXOPTS={' '.join(sphinxopts)}", "SPHINXERRORHANDLING=", maketarget, - ]) + )) self.log_directory.mkdir(parents=True, exist_ok=True) chgrp(self.log_directory, group=self.group, recursive=True) if self.includes_html: @@ -678,7 +678,7 @@ def build_venv(self) -> None: So we can reuse them from builds to builds, while they contain different Sphinx versions. """ - requirements = [self.theme] + self.version.requirements + requirements = list(self.version.requirements) if self.includes_html: # opengraph previews requirements.append("matplotlib>=3") @@ -686,12 +686,19 @@ def build_venv(self) -> None: venv_path = self.build_root / f"venv-{self.version.name}" venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( - [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] - + ["--upgrade-strategy=eager"] - + requirements, + ( + venv_path / "bin" / "python", + "-m", + "pip", + "install", + "--upgrade", + "--upgrade-strategy=eager", + self.theme, + *requirements, + ), cwd=self.checkout / "Doc", ) - run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) + run((venv_path / "bin" / "python", "-m", "pip", "freeze", "--all")) self.venv = venv_path def setup_indexsidebar(self) -> None: @@ -738,7 +745,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: recursive=True, ) chmod_make_readable(self.checkout / "Doc" / "build" / "html") - run([ + run(( "rsync", "-a", "--delete-delay", @@ -746,7 +753,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: "P archives/", str(self.checkout / "Doc" / "build" / "html") + "/", target, - ]) + )) dist_dir = self.checkout / "Doc" / "dist" if dist_dir.is_dir(): From d1f0418ea5aadb14a670151eb16ce0e2ba4d6cda Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 00:02:39 +0100 Subject: [PATCH 208/218] Add more helper properties in the ``Language`` class (#293) --- build_docs.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/build_docs.py b/build_docs.py index d31ebdd..bd94ba1 100755 --- a/build_docs.py +++ b/build_docs.py @@ -280,6 +280,14 @@ class Language: def tag(self) -> str: return self.iso639_tag.replace("_", "-").lower() + @property + def is_translation(self) -> bool: + return self.tag != "en" + + @property + def locale_repo_url(self) -> str: + return f"https://github.com/python/python-docs-{self.tag}.git" + @property def switcher_label(self) -> str: if self.translated_name: @@ -549,7 +557,7 @@ def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: logging.info("Skipping non-HTML build (language is HTML-only).") return None # skipped self.cpython_repo.switch(self.version.branch_or_tag) - if self.language.tag != "en": + if self.language.is_translation: self.clone_translation() if trigger_reason := self.should_rebuild(force_build): self.build_venv() @@ -569,6 +577,10 @@ def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: return False return True + @property + def locale_dir(self) -> Path: + return self.build_root / self.version.name / "locale" + @property def checkout(self) -> Path: """Path to CPython git clone.""" @@ -582,15 +594,8 @@ def clone_translation(self) -> None: def translation_repo(self) -> Repository: """See PEP 545 for translations repository naming convention.""" - locale_repo = f"https://github.com/python/python-docs-{self.language.tag}.git" - locale_clone_dir = ( - self.build_root - / self.version.name - / "locale" - / self.language.iso639_tag - / "LC_MESSAGES" - ) - return Repository(locale_repo, locale_clone_dir) + locale_clone_dir = self.locale_dir / self.language.iso639_tag / "LC_MESSAGES" + return Repository(self.language.locale_repo_url, locale_clone_dir) @property def translation_branch(self) -> str: @@ -611,10 +616,9 @@ def build(self) -> None: logging.info("Build start.") start_time = perf_counter() sphinxopts = list(self.language.sphinxopts) - if self.language.tag != "en": - locale_dirs = self.build_root / self.version.name / "locale" + if self.language.is_translation: sphinxopts.extend(( - f"-D locale_dirs={locale_dirs}", + f"-D locale_dirs={self.locale_dir}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", "-D translation_progress_classes=1", @@ -636,7 +640,7 @@ def build(self) -> None: if self.includes_html: site_url = self.version.url - if self.language.tag != "en": + if self.language.is_translation: site_url += f"{self.language.tag}/" # Define a tag to enable opengraph socialcards previews # (used in Doc/conf.py and requires matplotlib) @@ -718,7 +722,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: logging.info("Publishing start.") start_time = perf_counter() self.www_root.mkdir(parents=True, exist_ok=True) - if self.language.tag == "en": + if not self.language.is_translation: target = self.www_root / self.version.name else: language_dir = self.www_root / self.language.tag @@ -786,7 +790,7 @@ def should_rebuild(self, force: bool) -> str | Literal[False]: logging.info("Should rebuild: no previous state found.") return "no previous state" cpython_sha = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() - if self.language.tag != "en": + if self.language.is_translation: translation_sha = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() @@ -849,7 +853,7 @@ def save_state( "triggered_by": trigger, "cpython_sha": self.cpython_repo.run("rev-parse", "HEAD").stdout.strip(), } - if self.language.tag != "en": + if self.language.is_translation: state["translation_sha"] = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() From c64294658cf6ceffd6c169a95646ee14ef148b3f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 00:53:19 +0100 Subject: [PATCH 209/218] Constrain blurb to version 1.1 or older (#294) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index bd94ba1..d95b408 100755 --- a/build_docs.py +++ b/build_docs.py @@ -194,7 +194,7 @@ def requirements(self) -> list[str]: return dependencies + ["standard-imghdr"] # Requirements/constraints for Python 3.7 and older, pre-requirements.txt - reqs = ["jieba", "blurb", "jinja2<3.1", "docutils<=0.17.1", "standard-imghdr"] + reqs = ["jieba", "blurb<1.2", "jinja2<3.1", "docutils<0.18", "standard-imghdr"] if self.name in {"3.7", "3.6", "2.7"}: return reqs + ["sphinx==2.3.1"] if self.name == "3.5": From d53c8a6e4d64571d4902523d8d42ed4af5cb2907 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 01:10:42 +0100 Subject: [PATCH 210/218] Add constraints for sphinxcontrib dependencies (#295) --- build_docs.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index d95b408..e1ca9a3 100755 --- a/build_docs.py +++ b/build_docs.py @@ -183,9 +183,9 @@ def requirements(self) -> list[str]: """ dependencies = [ + "-rrequirements.txt", "jieba", # To improve zh search. "PyStemmer~=2.2.0", # To improve performance for word stemming. - "-rrequirements.txt", ] if self.as_tuple() >= (3, 11): return dependencies @@ -194,7 +194,19 @@ def requirements(self) -> list[str]: return dependencies + ["standard-imghdr"] # Requirements/constraints for Python 3.7 and older, pre-requirements.txt - reqs = ["jieba", "blurb<1.2", "jinja2<3.1", "docutils<0.18", "standard-imghdr"] + reqs = [ + "blurb<1.2", + "docutils<=0.17.1", + "jieba", + "jinja2<3.1", + "sphinxcontrib-applehelp<=1.0.2", + "sphinxcontrib-devhelp<=1.0.2", + "sphinxcontrib-htmlhelp<=2.0", + "sphinxcontrib-jsmath<=1.0.1", + "sphinxcontrib-qthelp<=1.0.3", + "sphinxcontrib-serializinghtml<=1.1.5", + "standard-imghdr", + ] if self.name in {"3.7", "3.6", "2.7"}: return reqs + ["sphinx==2.3.1"] if self.name == "3.5": From 3254c48f3cdab522896cc8d5d9d654dde11f3614 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 01:13:23 +0100 Subject: [PATCH 211/218] Constrain alabaster to version 0.7.12 or older --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index e1ca9a3..c257e2f 100755 --- a/build_docs.py +++ b/build_docs.py @@ -195,6 +195,7 @@ def requirements(self) -> list[str]: # Requirements/constraints for Python 3.7 and older, pre-requirements.txt reqs = [ + "alabaster<0.7.12", "blurb<1.2", "docutils<=0.17.1", "jieba", From 9a365d64d329aa3c286691dccd7ae475fc425829 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:24:10 +0100 Subject: [PATCH 212/218] Constrain python-docs-theme to version 2023.3.1 or older --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index c257e2f..6895618 100755 --- a/build_docs.py +++ b/build_docs.py @@ -200,6 +200,7 @@ def requirements(self) -> list[str]: "docutils<=0.17.1", "jieba", "jinja2<3.1", + "python-docs-theme<=2023.3.1", "sphinxcontrib-applehelp<=1.0.2", "sphinxcontrib-devhelp<=1.0.2", "sphinxcontrib-htmlhelp<=2.0", From c215a786f60a9e24311b038155f5646e548287e2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:25:43 +0100 Subject: [PATCH 213/218] Restore _create_placeholders_if_missing() (#296) --- templates/switchers.js | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 774366f..324fd65 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -28,6 +28,30 @@ const _CURRENT_PREFIX = (() => { const _ALL_VERSIONS = new Map($VERSIONS); const _ALL_LANGUAGES = new Map($LANGUAGES); +/** + * Required for Python 3.7 and earlier. + * @returns {void} + * @private + */ +const _create_placeholders_if_missing = () => { + if (document.querySelectorAll(".version_switcher_placeholder").length) return; + + const items = document.querySelectorAll("body>div.related>ul>li:not(.right)"); + for (const item of items) { + if (item.innerText.toLowerCase().includes("documentation")) { + const container = document.createElement("li"); + container.className = "switchers"; + for (const placeholder_name of ["language", "version"]) { + const placeholder = document.createElement("div"); + placeholder.className = `${placeholder_name}_switcher_placeholder`; + container.appendChild(placeholder); + } + item.parentElement.insertBefore(container, item); + return; + } + } +}; + /** * @param {Map} versions * @returns {HTMLSelectElement} @@ -175,6 +199,8 @@ const _initialise_switchers = () => { const versions = _ALL_VERSIONS; const languages = _ALL_LANGUAGES; + _create_placeholders_if_missing(); + document .querySelectorAll(".version_switcher_placeholder") .forEach((placeholder) => { From 65735049bc9de117d9493a7677259cecc1f4de22 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:32:19 +0100 Subject: [PATCH 214/218] Require pipes for building Python 3.5 (#297) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 6895618..c75f096 100755 --- a/build_docs.py +++ b/build_docs.py @@ -212,7 +212,7 @@ def requirements(self) -> list[str]: if self.name in {"3.7", "3.6", "2.7"}: return reqs + ["sphinx==2.3.1"] if self.name == "3.5": - return reqs + ["sphinx==1.8.4"] + return reqs + ["sphinx==1.8.4", "standard-pipes"] raise ValueError("unreachable") @property From c1be80ec82c318076b7c10b2e854674ca7d4ea62 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:49:00 +0100 Subject: [PATCH 215/218] Add inline styles in _create_placeholders_if_missing() --- templates/switchers.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 324fd65..e54a278 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -41,9 +41,12 @@ const _create_placeholders_if_missing = () => { if (item.innerText.toLowerCase().includes("documentation")) { const container = document.createElement("li"); container.className = "switchers"; + container.style.display = "inline-flex"; for (const placeholder_name of ["language", "version"]) { const placeholder = document.createElement("div"); placeholder.className = `${placeholder_name}_switcher_placeholder`; + placeholder.style.marginRight = "5px"; + placeholder.style.paddingLeft = "5px"; container.appendChild(placeholder); } item.parentElement.insertBefore(container, item); From 84a749ebd9f7957baee6121f79aa86f930456218 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 18 Apr 2025 22:27:20 +0300 Subject: [PATCH 216/218] Add more tests for `Versions` class (#288) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- .coveragerc | 1 + .pre-commit-config.yaml | 1 + tests/test_build_docs_versions.py | 104 +++++++++++++++++++++++------- 3 files changed, 82 insertions(+), 24 deletions(-) diff --git a/.coveragerc b/.coveragerc index 0f12707..f970781 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,3 +5,4 @@ exclude_also = # Don't complain if non-runnable code isn't run: if __name__ == .__main__.: + if TYPE_CHECKING: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c0eb053..869a979 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,7 @@ repos: rev: v0.11.5 hooks: - id: ruff + args: [--fix] - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py index 42b5392..1d8f6dc 100644 --- a/tests/test_build_docs_versions.py +++ b/tests/test_build_docs_versions.py @@ -1,9 +1,13 @@ +from __future__ import annotations + +import pytest + from build_docs import Version, Versions -def test_filter_default() -> None: - # Arrange - versions = Versions([ +@pytest.fixture +def versions() -> Versions: + return Versions([ Version(name="3.14", status="in development", branch_or_tag=""), Version(name="3.13", status="stable", branch_or_tag=""), Version(name="3.12", status="stable", branch_or_tag=""), @@ -12,28 +16,90 @@ def test_filter_default() -> None: Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) + +def test_reversed(versions: Versions) -> None: # Act - filtered = versions.filter() + output = list(reversed(versions)) # Assert - assert filtered == [ - Version(name="3.14", status="in development", branch_or_tag=""), + assert output[0].name == "3.9" + assert output[-1].name == "3.14" + + +def test_from_json() -> None: + # Arrange + json_data = { + "3.14": { + "branch": "main", + "pep": 745, + "status": "feature", + "first_release": "2025-10-01", + "end_of_life": "2030-10", + "release_manager": "Hugo van Kemenade", + }, + "3.13": { + "branch": "3.13", + "pep": 719, + "status": "bugfix", + "first_release": "2024-10-07", + "end_of_life": "2029-10", + "release_manager": "Thomas Wouters", + }, + } + + # Act + versions = list(Versions.from_json(json_data)) + + # Assert + assert versions == [ Version(name="3.13", status="stable", branch_or_tag=""), - Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.14", status="in development", branch_or_tag=""), ] -def test_filter_one() -> None: +def test_from_json_error() -> None: # Arrange - versions = Versions([ + json_data = {"2.8": {"branch": "2.8", "pep": 404, "status": "ex-release"}} + + # Act / Assert + with pytest.raises( + ValueError, + match="Saw invalid version status 'ex-release', expected to be one of", + ): + Versions.from_json(json_data) + + +def test_current_stable(versions) -> None: + # Act + current_stable = versions.current_stable + + # Assert + assert current_stable.name == "3.13" + assert current_stable.status == "stable" + + +def test_current_dev(versions) -> None: + # Act + current_dev = versions.current_dev + + # Assert + assert current_dev.name == "3.14" + assert current_dev.status == "in development" + + +def test_filter_default(versions) -> None: + # Act + filtered = versions.filter() + + # Assert + assert filtered == [ Version(name="3.14", status="in development", branch_or_tag=""), Version(name="3.13", status="stable", branch_or_tag=""), Version(name="3.12", status="stable", branch_or_tag=""), - Version(name="3.11", status="security-fixes", branch_or_tag=""), - Version(name="3.10", status="security-fixes", branch_or_tag=""), - Version(name="3.9", status="security-fixes", branch_or_tag=""), - ]) + ] + +def test_filter_one(versions) -> None: # Act filtered = versions.filter(["3.13"]) @@ -41,17 +107,7 @@ def test_filter_one() -> None: assert filtered == [Version(name="3.13", status="security-fixes", branch_or_tag="")] -def test_filter_multiple() -> None: - # Arrange - versions = Versions([ - Version(name="3.14", status="in development", branch_or_tag=""), - Version(name="3.13", status="stable", branch_or_tag=""), - Version(name="3.12", status="stable", branch_or_tag=""), - Version(name="3.11", status="security-fixes", branch_or_tag=""), - Version(name="3.10", status="security-fixes", branch_or_tag=""), - Version(name="3.9", status="security-fixes", branch_or_tag=""), - ]) - +def test_filter_multiple(versions) -> None: # Act filtered = versions.filter(["3.13", "3.14"]) From 0e83b79b2d2488ec9226f28c93e215b5217e6730 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 22 May 2025 14:35:36 +0200 Subject: [PATCH 217/218] =?UTF-8?q?Hello=20=E0=A6=AC=E0=A6=BE=E0=A6=82?= =?UTF-8?q?=E0=A6=B2=E0=A6=BE=20(#301)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config.toml b/config.toml index 489c774..f25052a 100644 --- a/config.toml +++ b/config.toml @@ -36,6 +36,11 @@ sphinxopts = [ '-D latex_elements.fontenc=\\usepackage{fontspec}', ] +[languages.bn_IN] +name = "Bengali" +translated_name = "বাংলা" +in_prod = false + [languages.id] name = "Indonesian" translated_name = "Indonesia" From 15f94c0b7caec8b12f4bab3227967fd859ff6bad Mon Sep 17 00:00:00 2001 From: Octavian Mustafa Date: Mon, 16 Jun 2025 21:44:24 +0300 Subject: [PATCH 218/218] Start building Romanian translations (#302) --- config.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config.toml b/config.toml index f25052a..d628ca0 100644 --- a/config.toml +++ b/config.toml @@ -92,6 +92,11 @@ translated_name = "polski" name = "Brazilian Portuguese" translated_name = "Português brasileiro" +[languages.ro] +name = "Romanian" +translated_name = "Românește" +in_prod = false + [languages.tr] name = "Turkish" translated_name = "Türkçe"