diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..f970781 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,8 @@ +# .coveragerc to control coverage.py + +[report] +# Regexes for lines to exclude from consideration +exclude_also = + # Don't complain if non-runnable code isn't run: + if __name__ == .__main__.: + if TYPE_CHECKING: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d553e49..cd9d07e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,19 +2,20 @@ name: Lint on: [push, pull_request, workflow_dispatch] +permissions: {} + env: FORCE_COLOR: 1 PIP_DISABLE_PIP_VERSION_CHECK: 1 -permissions: - contents: read - jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d74e607..b64e8a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,34 +1,80 @@ name: Test -on: [push, pull_request, workflow_dispatch] +on: + push: + pull_request: + workflow_dispatch: -permissions: - contents: read +permissions: {} env: FORCE_COLOR: 1 jobs: - test: + integration: + name: Integration test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Set up requirements + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Build documentation + run: > + python ./build_docs.py + --quick + --build-root ./build_root + --www-root ./www + --log-directory ./logs + --group "$(id -g)" + --skip-cache-invalidation + --languages en + --branches 3.14 + + - name: Upload documentation + uses: actions/upload-artifact@v4 + with: + name: www-root + path: ./www + retention-days: 2 + + unit: + name: Unit tests runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] - os: [ubuntu-latest] + python-version: ["3.13", "3.14"] + os: [windows-latest, macos-latest, ubuntu-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true + - name: Install uv + uses: hynek/setup-cached-uv@v2 - - name: Install uv - uses: hynek/setup-cached-uv@v2 + - name: Tox tests + run: uvx --with tox-uv tox -e py - - name: Tox tests - run: | - uvx --with tox-uv tox -e py + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + flags: ${{ matrix.os }} + name: Python ${{ matrix.python-version }} + token: ${{ secrets.CODECOV_ORG_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c3d2f5f..869a979 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -14,35 +14,48 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.7 + rev: v0.11.5 hooks: + - id: ruff + args: [--fix] - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.2 + rev: 0.31.1 hooks: - id: check-github-workflows - repo: https://github.com/rhysd/actionlint - rev: v1.7.1 + rev: v1.7.7 hooks: - id: actionlint + - repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v1.3.1 + hooks: + - id: zizmor + - repo: https://github.com/tox-dev/pyproject-fmt - rev: 2.2.3 + rev: v2.5.0 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.19 + rev: v0.23 hooks: - id: validate-pyproject - repo: https://github.com/tox-dev/tox-ini-fmt - rev: 1.4.0 + rev: 1.5.0 hooks: - id: tox-ini-fmt + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.5.1 + hooks: + - id: prettier + files: templates/switchers.js + - repo: meta hooks: - id: check-hooks-apply diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..47cbf74 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,34 @@ +target-version = "py313" # Pin Ruff to Python 3.13 +line-length = 88 +output-format = "full" + +[format] +preview = true +docstring-code-format = true + +[lint] +preview = true +select = [ + "C4", # flake8-comprehensions + "B", # flake8-bugbear + "E", # pycodestyle + "F", # pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "G", # flake8-logging-format + "I", # isort + "N", # pep8-naming + "PERF", # perflint + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "TC", # flake8-type-checking + "UP", # pyupgrade + "W", # pycodestyle +] +ignore = [ + "E501", # Ignore line length errors (we use auto-formatting) +] + +[lint.flake8-type-checking] +exempt-modules = [] +strict = true diff --git a/README.md b/README.md index 767014c..8f914c6 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,13 @@ +# docsbuild-scripts + +[![GitHub Actions status](https://github.com/python/docsbuild-scripts/actions/workflows/test.yml/badge.svg)](https://github.com/python/docsbuild-scripts/actions/workflows/test.yml) +[![Codecov](https://codecov.io/gh/python/docsbuild-scripts/branch/main/graph/badge.svg)](https://codecov.io/gh/python/docsbuild-scripts) + This repository contains scripts for automatically building the Python documentation on [docs.python.org](https://docs.python.org). -# How to test it? +## How to test it? The following command should build all maintained versions and translations in `./www`, beware it can take a few hours: @@ -12,10 +17,10 @@ python3 ./build_docs.py --quick --build-root ./build_root --www-root ./www --log ``` If you don't need to build all translations of all branches, add -`--language en --branch main`. +`--languages en --branches main`. -# Check current version +## Check current version Install `tools_requirements.txt` then run `python check_versions.py ../cpython/` (pointing to a real CPython clone) to see which version @@ -34,9 +39,9 @@ of Sphinx we're using where: 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' - 3.12 ø sphinx~=8.1.0 needs_sphinx='7.2.6' - 3.13 ø sphinx~=8.1.0 needs_sphinx='7.2.6' - 3.14 ø sphinx~=8.1.0 needs_sphinx='7.2.6' + 3.12 ø sphinx~=8.2.0 needs_sphinx='8.2.0' + 3.13 ø sphinx~=8.2.0 needs_sphinx='8.2.0' + 3.14 ø sphinx~=8.2.0 needs_sphinx='8.2.0' ========= ============= ================== ==================== Sphinx build as seen on docs.python.org: @@ -47,7 +52,24 @@ of Sphinx we're using where: 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 - 3.12 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 - 3.13 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 - 3.14 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 + 3.12 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 + 3.13 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 + 3.14 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= + +## Manually rebuild a branch + +Docs for [feature and bugfix branches](https://devguide.python.org/versions/) are +automatically built from a cron. + +Manual rebuilds are needed for new security releases, +and to add the end-of-life banner for newly end-of-life branches. + +To manually rebuild a branch, for example 3.11: + +```shell +ssh docs.nyc1.psf.io +sudo su --shell=/bin/bash docsbuild +screen -DUR # Rejoin screen session if it exists, otherwise create a new one +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --force --branch 3.11 +``` diff --git a/build_docs.py b/build_docs.py index 658ba63..c75f096 100755 --- a/build_docs.py +++ b/build_docs.py @@ -5,6 +5,26 @@ Without any arguments builds docs for all active versions and languages. +Environment variables for: + +- `SENTRY_DSN` (Error reporting) +- `FASTLY_SERVICE_ID` / `FASTLY_TOKEN` (CDN purges) +- `PYTHON_DOCS_ENABLE_ANALYTICS` (Enable Plausible for online docs) + +are read from the site configuration path for your platform +(/etc/xdg/docsbuild-scripts on linux) if available, +and can be overriden by writing a file to the user config dir +for your platform ($HOME/.config/docsbuild-scripts on linux). +The contents of the file is parsed as toml: + +```toml +[env] +SENTRY_DSN = "https://0a0a0a0a0a0a0a0a0a0a0a@sentry.io/69420" +FASTLY_SERVICE_ID = "deadbeefdeadbeefdead" +FASTLY_TOKEN = "secureme!" +PYTHON_DOCS_ENABLE_ANALYTICS = "1" +``` + Languages are stored in `config.toml` while versions are discovered from the devguide. @@ -22,36 +42,43 @@ from __future__ import annotations -from argparse import ArgumentParser, Namespace -from collections.abc import Sequence -from contextlib import suppress, contextmanager -from dataclasses import dataclass +import argparse +import concurrent.futures +import dataclasses +import datetime as dt import filecmp import json import logging import logging.handlers -from functools import total_ordering -from os import getenv, readlink +import os import re import shlex import shutil +import stat import subprocess import sys +import venv from bisect import bisect_left as bisect -from datetime import datetime as dt, timezone +from contextlib import contextmanager, suppress from pathlib import Path from string import Template from time import perf_counter, sleep -from typing import Iterable, Literal from urllib.parse import urljoin import jinja2 +import platformdirs import tomlkit import urllib3 import zc.lockfile +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Collection, Iterator, Sequence, Set + from typing import Literal + try: - from os import EX_OK, EX_SOFTWARE as EX_FAILURE + from os import EX_OK + from os import EX_SOFTWARE as EX_FAILURE except ImportError: EX_OK, EX_FAILURE = 0, 1 @@ -65,10 +92,68 @@ HERE = Path(__file__).resolve().parent -@total_ordering +@dataclasses.dataclass(frozen=True, slots=True) +class Versions: + _seq: Sequence[Version] + + def __iter__(self) -> Iterator[Version]: + return iter(self._seq) + + def __reversed__(self) -> Iterator[Version]: + return reversed(self._seq) + + @classmethod + def from_json(cls, data: dict) -> Versions: + """Load versions from the devguide's JSON representation.""" + permitted = ", ".join(sorted(Version.STATUSES | Version.SYNONYMS.keys())) + + versions = [] + for name, release in data.items(): + branch = release["branch"] + status = release["status"] + status = Version.SYNONYMS.get(status, status) + if status not in Version.STATUSES: + msg = ( + f"Saw invalid version status {status!r}, " + f"expected to be one of {permitted}." + ) + raise ValueError(msg) + versions.append(Version(name=name, status=status, branch_or_tag=branch)) + + return cls(sorted(versions, key=Version.as_tuple)) + + def filter(self, branches: Sequence[str] = ()) -> Sequence[Version]: + """Filter the given versions. + + If *branches* is given, only *versions* matching *branches* are returned. + + Else all live versions are returned (this means no EOL and no + security-fixes branches). + """ + if branches: + branches = frozenset(branches) + return [v for v in self if {v.name, v.branch_or_tag} & branches] + return [v for v in self if v.status not in {"EOL", "security-fixes"}] + + @property + def current_stable(self) -> Version: + """Find the current stable CPython version.""" + return max((v for v in self if v.status == "stable"), key=Version.as_tuple) + + @property + def current_dev(self) -> Version: + """Find the current CPython version in development.""" + return max(self, key=Version.as_tuple) + + +@dataclasses.dataclass(frozen=True, kw_only=True, slots=True) class Version: """Represents a CPython version and its documentation build dependencies.""" + name: str + status: Literal["EOL", "security-fixes", "stable", "pre-release", "in development"] + branch_or_tag: str + STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"} # Those synonyms map branch status vocabulary found in the devguide @@ -81,22 +166,11 @@ class Version: "prerelease": "pre-release", } - def __init__(self, name, *, status, branch_or_tag=None): - status = self.SYNONYMS.get(status, status) - if status not in self.STATUSES: - raise ValueError( - "Version status expected to be one of: " - f"{', '.join(self.STATUSES|set(self.SYNONYMS.keys()))}, got {status!r}." - ) - self.name = name - self.branch_or_tag = branch_or_tag - self.status = status - - def __repr__(self): - return f"Version({self.name})" + def __eq__(self, other: Version) -> bool: + return self.name == other.name @property - def requirements(self): + def requirements(self) -> list[str]: """Generate the right requirements for this version. Since CPython 3.8 a Doc/requirements.txt file can be used. @@ -108,61 +182,60 @@ def requirements(self): See https://github.com/python/cpython/issues/91483 """ - if self.name == "3.5": - return ["jieba", "blurb", "sphinx==1.8.4", "jinja2<3.1", "docutils<=0.17.1"] - if self.name in {"3.7", "3.6", "2.7"}: - return ["jieba", "blurb", "sphinx==2.3.1", "jinja2<3.1", "docutils<=0.17.1"] - - return [ + dependencies = [ + "-rrequirements.txt", "jieba", # To improve zh search. "PyStemmer~=2.2.0", # To improve performance for word stemming. - "-rrequirements.txt", ] + if self.as_tuple() >= (3, 11): + return dependencies + if self.as_tuple() >= (3, 8): + # Restore the imghdr module for Python 3.8-3.10. + return dependencies + ["standard-imghdr"] + + # Requirements/constraints for Python 3.7 and older, pre-requirements.txt + reqs = [ + "alabaster<0.7.12", + "blurb<1.2", + "docutils<=0.17.1", + "jieba", + "jinja2<3.1", + "python-docs-theme<=2023.3.1", + "sphinxcontrib-applehelp<=1.0.2", + "sphinxcontrib-devhelp<=1.0.2", + "sphinxcontrib-htmlhelp<=2.0", + "sphinxcontrib-jsmath<=1.0.1", + "sphinxcontrib-qthelp<=1.0.3", + "sphinxcontrib-serializinghtml<=1.1.5", + "standard-imghdr", + ] + if self.name in {"3.7", "3.6", "2.7"}: + return reqs + ["sphinx==2.3.1"] + if self.name == "3.5": + return reqs + ["sphinx==1.8.4", "standard-pipes"] + raise ValueError("unreachable") @property - def changefreq(self): + def changefreq(self) -> str: """Estimate this version change frequency, for the sitemap.""" return {"EOL": "never", "security-fixes": "yearly"}.get(self.status, "daily") - def as_tuple(self): + def as_tuple(self) -> tuple[int, ...]: """This version name as tuple, for easy comparisons.""" return version_to_tuple(self.name) @property - def url(self): + def url(self) -> str: """The doc URL of this version in production.""" return f"https://docs.python.org/{self.name}/" @property - def title(self): + def title(self) -> str: """The title of this version's doc, for the sidebar.""" return f"Python {self.name} ({self.status})" - @staticmethod - def filter(versions, branch=None): - """Filter the given versions. - - If *branch* is given, only *versions* matching *branch* are returned. - - Else all live versions are returned (this means no EOL and no - security-fixes branches). - """ - if branch: - return [v for v in versions if branch in (v.name, v.branch_or_tag)] - return [v for v in versions if v.status not in ("EOL", "security-fixes")] - - @staticmethod - def current_stable(versions): - """Find the current stable CPython version.""" - return max((v for v in versions if v.status == "stable"), key=Version.as_tuple) - - @staticmethod - def current_dev(versions): - """Find the current CPython version in development.""" - return max(versions, key=Version.as_tuple) - @property - def picker_label(self): + def picker_label(self) -> str: """Forge the label of a version picker.""" if self.status == "in development": return f"dev ({self.name})" @@ -170,50 +243,75 @@ def picker_label(self): return f"pre ({self.name})" return self.name - def setup_indexsidebar(self, versions: Sequence[Version], dest_path: Path): - """Build indexsidebar.html for Sphinx.""" - template_path = HERE / "templates" / "indexsidebar.html" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render( - current_version=self, - versions=versions[::-1], - ) - dest_path.write_text(rendered_template, encoding="UTF-8") - @classmethod - def from_json(cls, name, values): - """Loads a version from devguide's json representation.""" - return cls(name, status=values["status"], branch_or_tag=values["branch"]) +@dataclasses.dataclass(frozen=True, slots=True) +class Languages: + _seq: Sequence[Language] - def __eq__(self, other): - return self.name == other.name + def __iter__(self) -> Iterator[Language]: + return iter(self._seq) + + def __reversed__(self) -> Iterator[Language]: + return reversed(self._seq) + + @classmethod + def from_json(cls, defaults: dict, languages: dict) -> Languages: + default_translated_name = defaults.get("translated_name", "") + default_in_prod = defaults.get("in_prod", True) + default_sphinxopts = defaults.get("sphinxopts", []) + default_html_only = defaults.get("html_only", False) + langs = [ + Language( + iso639_tag=iso639_tag, + name=section["name"], + translated_name=section.get("translated_name", default_translated_name), + in_prod=section.get("in_prod", default_in_prod), + sphinxopts=section.get("sphinxopts", default_sphinxopts), + html_only=section.get("html_only", default_html_only), + ) + for iso639_tag, section in languages.items() + ] + return cls(langs) - def __gt__(self, other): - return self.as_tuple() > other.as_tuple() + def filter(self, language_tags: Sequence[str] = ()) -> Sequence[Language]: + """Filter a sequence of languages according to --languages.""" + if language_tags: + language_tags = frozenset(language_tags) + return [l for l in self if l.tag in language_tags] # NoQA: E741 + return list(self) -@dataclass(frozen=True, order=True) +@dataclasses.dataclass(order=True, frozen=True, kw_only=True) class Language: iso639_tag: str name: str + translated_name: str in_prod: bool - sphinxopts: tuple + sphinxopts: Sequence[str] html_only: bool = False @property - def tag(self): + def tag(self) -> str: return self.iso639_tag.replace("_", "-").lower() - @staticmethod - def filter(languages, language_tags=None): - """Filter a sequence of languages according to --languages.""" - if language_tags: - languages_dict = {language.tag: language for language in languages} - return [languages_dict[tag] for tag in language_tags] - return languages + @property + def is_translation(self) -> bool: + return self.tag != "en" + + @property + def locale_repo_url(self) -> str: + return f"https://github.com/python/python-docs-{self.tag}.git" + + @property + def switcher_label(self) -> str: + if self.translated_name: + return f"{self.name} | {self.translated_name}" + return self.name -def run(cmd, cwd=None) -> subprocess.CompletedProcess: +def run( + cmd: Sequence[str | Path], cwd: Path | None = None +) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" cmd = list(map(str, cmd)) cmdstring = shlex.join(cmd) @@ -239,7 +337,7 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: return result -def run_with_logging(cmd, cwd=None): +def run_with_logging(cmd: Sequence[str | Path], cwd: Path | None = None) -> None: """Like subprocess.check_call, with logging before the command execution.""" cmd = list(map(str, cmd)) logging.debug("Run: '%s'", shlex.join(cmd)) @@ -261,46 +359,38 @@ def run_with_logging(cmd, cwd=None): raise subprocess.CalledProcessError(return_code, cmd[0]) -def changed_files(left, right): - """Compute a list of different files between left and right, recursively. - Resulting paths are relative to left. - """ - changed = [] +def changed_files(left: Path, right: Path) -> int: + """Compute the number of different files in the two directory trees.""" - def traverse(dircmp_result): - base = Path(dircmp_result.left).relative_to(left) - for file in dircmp_result.diff_files: - changed.append(str(base / file)) - if file == "index.html": - changed.append(str(base) + "/") - for dircomp in dircmp_result.subdirs.values(): - traverse(dircomp) + def traverse(dircmp_result: filecmp.dircmp) -> int: + changed = len(dircmp_result.diff_files) + changed += sum(map(traverse, dircmp_result.subdirs.values())) + return changed - traverse(filecmp.dircmp(left, right)) - return changed + return traverse(filecmp.dircmp(left, right)) -@dataclass +@dataclasses.dataclass class Repository: """Git repository abstraction for our specific needs.""" remote: str directory: Path - def run(self, *args): + def run(self, *args: str) -> subprocess.CompletedProcess: """Run git command in the clone repository.""" return run(("git", "-C", self.directory) + args) - def get_ref(self, pattern): + def get_ref(self, pattern: str) -> str: """Return the reference of a given tag or branch.""" try: # Maybe it's a branch - return self.run("show-ref", "-s", "origin/" + pattern).stdout.strip() + return self.run("show-ref", "-s", f"origin/{pattern}").stdout.strip() except subprocess.CalledProcessError: # Maybe it's a tag - return self.run("show-ref", "-s", "tags/" + pattern).stdout.strip() + return self.run("show-ref", "-s", f"tags/{pattern}").stdout.strip() - def fetch(self): + def fetch(self) -> subprocess.CompletedProcess: """Try (and retry) to run git fetch.""" try: return self.run("fetch") @@ -309,35 +399,37 @@ def fetch(self): sleep(5) return self.run("fetch") - def switch(self, branch_or_tag): + def switch(self, branch_or_tag: str) -> None: """Reset and cleans the repository to the given branch or tag.""" self.run("reset", "--hard", self.get_ref(branch_or_tag), "--") self.run("clean", "-dfqx") - def clone(self): + def clone(self) -> bool: """Maybe clone the repository, if not already cloned.""" if (self.directory / ".git").is_dir(): return False # Already cloned logging.info("Cloning %s into %s", self.remote, self.directory) self.directory.mkdir(mode=0o775, parents=True, exist_ok=True) - run(["git", "clone", self.remote, self.directory]) + run(("git", "clone", self.remote, self.directory)) return True - def update(self): + def update(self) -> None: self.clone() or self.fetch() -def version_to_tuple(version): +def version_to_tuple(version: str) -> tuple[int, ...]: """Transform a version string to a tuple, for easy comparisons.""" return tuple(int(part) for part in version.split(".")) -def tuple_to_version(version_tuple): +def tuple_to_version(version_tuple: tuple[int, ...]) -> str: """Reverse version_to_tuple.""" return ".".join(str(part) for part in version_tuple) -def locate_nearest_version(available_versions, target_version): +def locate_nearest_version( + available_versions: Collection[str], target_version: str +) -> str: """Look for the nearest version of target_version in available_versions. Versions are to be given as tuples, like (3, 7) for 3.7. @@ -381,25 +473,13 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers( - versions: Sequence[Version], languages: Sequence[Language], html_root: Path -): +def setup_switchers(script_content: bytes, html_root: Path) -> None: """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - languages_map = dict(sorted((l.tag, l.name) for l in languages if l.in_prod)) - versions_map = {v.name: v.picker_label for v in reversed(versions)} - - switchers_template_file = HERE / "templates" / "switchers.js" switchers_path = html_root / "_static" / "switchers.js" - - template = Template(switchers_template_file.read_text(encoding="UTF-8")) - rendered_template = template.safe_substitute( - LANGUAGES=json.dumps(languages_map), - VERSIONS=json.dumps(versions_map), - ) - switchers_path.write_text(rendered_template, encoding="UTF-8") + switchers_path.write_bytes(script_content) for file in html_root.glob("**/*.html"): depth = len(file.relative_to(html_root).parts) - 1 @@ -414,65 +494,16 @@ def setup_switchers( ofile.write(line) -def copy_robots_txt( - www_root: Path, - group, - skip_cache_invalidation, - http: urllib3.PoolManager, -) -> None: - """Copy robots.txt to www_root.""" - if not www_root.exists(): - logging.info("Skipping copying robots.txt (www root does not even exist).") - return - logging.info("Copying robots.txt...") - template_path = HERE / "templates" / "robots.txt" - robots_path = www_root / "robots.txt" - shutil.copyfile(template_path, robots_path) - robots_path.chmod(0o775) - run(["chgrp", group, robots_path]) - if not skip_cache_invalidation: - purge(http, "robots.txt") - - -def build_sitemap( - versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group -): - """Build a sitemap with all live versions and translations.""" - if not www_root.exists(): - logging.info("Skipping sitemap generation (www root does not even exist).") - return - logging.info("Starting sitemap generation...") - template_path = HERE / "templates" / "sitemap.xml" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render(languages=languages, versions=versions) - sitemap_path = www_root / "sitemap.xml" - sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") - sitemap_path.chmod(0o664) - run(["chgrp", group, sitemap_path]) - - -def build_404(www_root: Path, group): - """Build a nice 404 error page to display in case PDFs are not built yet.""" - if not www_root.exists(): - logging.info("Skipping 404 page generation (www root does not even exist).") - return - logging.info("Copying 404 page...") - not_found_file = www_root / "404.html" - shutil.copyfile(HERE / "templates" / "404.html", not_found_file) - not_found_file.chmod(0o664) - run(["chgrp", group, not_found_file]) - - -def head(text, lines=10): +def head(text: str, lines: int = 10) -> str: """Return the first *lines* lines from the given text.""" return "\n".join(text.split("\n")[:lines]) -def version_info(): +def version_info() -> None: """Handler for --version.""" try: platex_version = head( - subprocess.check_output(["platex", "--version"], universal_newlines=True), + subprocess.check_output(("platex", "--version"), text=True), lines=3, ) except FileNotFoundError: @@ -480,7 +511,7 @@ def version_info(): try: xelatex_version = head( - subprocess.check_output(["xelatex", "--version"], universal_newlines=True), + subprocess.check_output(("xelatex", "--version"), text=True), lines=2, ) except FileNotFoundError: @@ -499,118 +530,15 @@ def version_info(): ) -def parse_args(): - """Parse command-line arguments.""" - - parser = ArgumentParser( - description="Runs a build of the Python docs for various branches." - ) - parser.add_argument( - "--select-output", - choices=("no-html", "only-html", "only-html-en"), - help="Choose what outputs to build.", - ) - parser.add_argument( - "-q", - "--quick", - action="store_true", - help="Run a quick build (only HTML files).", - ) - parser.add_argument( - "-b", - "--branch", - metavar="3.12", - help="Version to build (defaults to all maintained branches).", - ) - parser.add_argument( - "-r", - "--build-root", - type=Path, - help="Path to a directory containing a checkout per branch.", - default=Path("/srv/docsbuild"), - ) - parser.add_argument( - "-w", - "--www-root", - type=Path, - help="Path where generated files will be copied.", - default=Path("/srv/docs.python.org"), - ) - parser.add_argument( - "--skip-cache-invalidation", - help="Skip Fastly cache invalidation.", - action="store_true", - ) - parser.add_argument( - "--group", - help="Group files on targets and www-root file should get.", - default="docs", - ) - parser.add_argument( - "--log-directory", - type=Path, - help="Directory used to store logs.", - default=Path("/var/log/docsbuild/"), - ) - parser.add_argument( - "--languages", - nargs="*", - help="Language translation, as a PEP 545 language tag like" - " 'fr' or 'pt-br'. " - "Builds all available languages by default.", - metavar="fr", - ) - parser.add_argument( - "--version", - action="store_true", - help="Get build_docs and dependencies version info", - ) - parser.add_argument( - "--theme", - default="python-docs-theme", - help="Python package to use for python-docs-theme: Useful to test branches:" - " --theme git+https://github.com/obulat/python-docs-theme@master", - ) - args = parser.parse_args() - if args.version: - version_info() - sys.exit(0) - del args.version - if args.log_directory: - args.log_directory = args.log_directory.resolve() - if args.build_root: - args.build_root = args.build_root.resolve() - if args.www_root: - args.www_root = args.www_root.resolve() - return args - - -def setup_logging(log_directory: Path, select_output: str | None): - """Setup logging to stderr if run by a human, or to a file if run from a cron.""" - log_format = "%(asctime)s %(levelname)s: %(message)s" - if sys.stderr.isatty(): - logging.basicConfig(format=log_format, stream=sys.stderr) - else: - log_directory.mkdir(parents=True, exist_ok=True) - if select_output is None: - filename = log_directory / "docsbuild.log" - else: - filename = log_directory / f"docsbuild-{select_output}.log" - handler = logging.handlers.WatchedFileHandler(filename) - handler.setFormatter(logging.Formatter(log_format)) - logging.getLogger().addHandler(handler) - logging.getLogger().setLevel(logging.DEBUG) - - -@dataclass +@dataclasses.dataclass class DocBuilder: """Builder for a CPython version and a language.""" version: Version - versions: Sequence[Version] language: Language - languages: Sequence[Language] cpython_repo: Repository + docs_by_version_content: bytes + switchers_content: bytes build_root: Path www_root: Path select_output: Literal["no-html", "only-html", "only-html-en"] | None @@ -618,10 +546,10 @@ class DocBuilder: group: str log_directory: Path skip_cache_invalidation: bool - theme: Path + theme: str @property - def html_only(self): + def html_only(self) -> bool: return ( self.select_output in {"only-html", "only-html-en"} or self.quick @@ -629,23 +557,23 @@ def html_only(self): ) @property - def includes_html(self): + def includes_html(self) -> bool: """Does the build we are running include HTML output?""" return self.select_output != "no-html" - def run(self, http: urllib3.PoolManager) -> bool: + def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() - start_timestamp = dt.now(tz=timezone.utc).replace(microsecond=0) + start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) logging.info("Running.") try: if self.language.html_only and not self.includes_html: logging.info("Skipping non-HTML build (language is HTML-only).") - return True + return None # skipped self.cpython_repo.switch(self.version.branch_or_tag) - if self.language.tag != "en": + if self.language.is_translation: self.clone_translation() - if trigger_reason := self.should_rebuild(): + if trigger_reason := self.should_rebuild(force_build): self.build_venv() self.build() self.copy_build_to_webroot(http) @@ -654,6 +582,8 @@ def run(self, http: urllib3.PoolManager) -> bool: build_duration=perf_counter() - start_time, trigger=trigger_reason, ) + else: + return None # skipped except Exception as err: logging.exception("Badly handled exception, human, please help.") if sentry_sdk: @@ -661,31 +591,28 @@ def run(self, http: urllib3.PoolManager) -> bool: return False return True + @property + def locale_dir(self) -> Path: + return self.build_root / self.version.name / "locale" + @property def checkout(self) -> Path: """Path to CPython git clone.""" return self.build_root / _checkout_name(self.select_output) - def clone_translation(self): + def clone_translation(self) -> None: self.translation_repo.update() self.translation_repo.switch(self.translation_branch) @property - def translation_repo(self): + def translation_repo(self) -> Repository: """See PEP 545 for translations repository naming convention.""" - locale_repo = f"https://github.com/python/python-docs-{self.language.tag}.git" - locale_clone_dir = ( - self.build_root - / self.version.name - / "locale" - / self.language.iso639_tag - / "LC_MESSAGES" - ) - return Repository(locale_repo, locale_clone_dir) + locale_clone_dir = self.locale_dir / self.language.iso639_tag / "LC_MESSAGES" + return Repository(self.language.locale_repo_url, locale_clone_dir) @property - def translation_branch(self): + def translation_branch(self) -> str: """Some CPython versions may be untranslated, being either too old or too new. @@ -698,36 +625,18 @@ def translation_branch(self): branches = re.findall(r"/([0-9]+\.[0-9]+)$", remote_branches, re.M) return locate_nearest_version(branches, self.version.name) - def build(self): + def build(self) -> None: """Build this version/language doc.""" logging.info("Build start.") start_time = perf_counter() sphinxopts = list(self.language.sphinxopts) - if self.language.tag != "en": - locale_dirs = self.build_root / self.version.name / "locale" - sphinxopts.extend( - ( - f"-D locale_dirs={locale_dirs}", - f"-D language={self.language.iso639_tag}", - "-D gettext_compact=0", - ) - ) - if self.language.tag == "ja": - # Since luatex doesn't support \ufffd, replace \ufffd with '?'. - # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b - # Luatex already fixed this issue, so we can remove this once Texlive - # is updated. - # (https://github.com/TeX-Live/luatex/commit/af5faf1) - subprocess.check_output( - "sed -i s/\N{REPLACEMENT CHARACTER}/?/g " - f"{locale_dirs}/ja/LC_MESSAGES/**/*.po", - shell=True, - ) - subprocess.check_output( - "sed -i s/\N{REPLACEMENT CHARACTER}/?/g " - f"{self.checkout}/Doc/**/*.rst", - shell=True, - ) + if self.language.is_translation: + sphinxopts.extend(( + f"-D locale_dirs={self.locale_dir}", + f"-D language={self.language.iso639_tag}", + "-D gettext_compact=0", + "-D translation_progress_classes=1", + )) if self.version.status == "EOL": sphinxopts.append("-D html_context.outdated=1") @@ -744,70 +653,95 @@ def build(self): blurb = self.venv / "bin" / "blurb" if self.includes_html: - # Disable CPython switchers, we handle them now: - run( - ["sed", "-i"] - + ([""] if sys.platform == "darwin" else []) - + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] - ) - self.version.setup_indexsidebar( - self.versions, - self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", + site_url = self.version.url + if self.language.is_translation: + site_url += f"{self.language.tag}/" + # Define a tag to enable opengraph socialcards previews + # (used in Doc/conf.py and requires matplotlib) + sphinxopts += ( + "-t create-social-cards", + f"-D ogp_site_url={site_url}", ) - run_with_logging( - [ - "make", - "-C", - self.checkout / "Doc", - "PYTHON=" + str(python), - "SPHINXBUILD=" + str(sphinxbuild), - "BLURB=" + str(blurb), - "VENVDIR=" + str(self.venv), - "SPHINXOPTS=" + " ".join(sphinxopts), - "SPHINXERRORHANDLING=", - maketarget, - ] - ) - run(["mkdir", "-p", self.log_directory]) - run(["chgrp", "-R", self.group, self.log_directory]) + + if self.version.as_tuple() < (3, 8): + # Disable CPython switchers, we handle them now: + text = (self.checkout / "Doc" / "Makefile").read_text(encoding="utf-8") + text = text.replace(" -A switchers=1", "") + (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") + + self.setup_indexsidebar() + run_with_logging(( + "make", + "-C", + self.checkout / "Doc", + f"PYTHON={python}", + f"SPHINXBUILD={sphinxbuild}", + f"BLURB={blurb}", + f"VENVDIR={self.venv}", + f"SPHINXOPTS={' '.join(sphinxopts)}", + "SPHINXERRORHANDLING=", + maketarget, + )) + self.log_directory.mkdir(parents=True, exist_ok=True) + chgrp(self.log_directory, group=self.group, recursive=True) if self.includes_html: setup_switchers( - self.versions, self.languages, self.checkout / "Doc" / "build" / "html" + self.switchers_content, self.checkout / "Doc" / "build" / "html" ) logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) - def build_venv(self): + def build_venv(self) -> None: """Build a venv for the specific Python version. So we can reuse them from builds to builds, while they contain different Sphinx versions. """ - venv_path = self.build_root / ("venv-" + self.version.name) - run([sys.executable, "-m", "venv", venv_path]) + requirements = list(self.version.requirements) + if self.includes_html: + # opengraph previews + requirements.append("matplotlib>=3") + + venv_path = self.build_root / f"venv-{self.version.name}" + venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( - [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] - + ["--upgrade-strategy=eager"] - + [self.theme] - + self.version.requirements, + ( + venv_path / "bin" / "python", + "-m", + "pip", + "install", + "--upgrade", + "--upgrade-strategy=eager", + self.theme, + *requirements, + ), cwd=self.checkout / "Doc", ) - run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) + run((venv_path / "bin" / "python", "-m", "pip", "freeze", "--all")) self.venv = venv_path + def setup_indexsidebar(self) -> None: + """Copy indexsidebar.html for Sphinx.""" + tmpl_src = HERE / "templates" + tmpl_dst = self.checkout / "Doc" / "tools" / "templates" + dbv_path = tmpl_dst / "_docs_by_version.html" + + shutil.copy(tmpl_src / "indexsidebar.html", tmpl_dst / "indexsidebar.html") + if self.version.status != "EOL": + dbv_path.write_bytes(self.docs_by_version_content) + else: + shutil.copy(tmpl_src / "_docs_by_version.html", dbv_path) + def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: """Copy a given build to the appropriate webroot with appropriate rights.""" logging.info("Publishing start.") start_time = perf_counter() self.www_root.mkdir(parents=True, exist_ok=True) - if self.language.tag == "en": + if not self.language.is_translation: target = self.www_root / self.version.name else: language_dir = self.www_root / self.language.tag language_dir.mkdir(parents=True, exist_ok=True) - try: - run(["chgrp", "-R", self.group, language_dir]) - except subprocess.CalledProcessError as err: - logging.warning("Can't change group of %s: %s", language_dir, str(err)) + chgrp(language_dir, group=self.group, recursive=True) language_dir.chmod(0o775) target = language_dir / self.version.name @@ -816,84 +750,47 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: target.chmod(0o775) except PermissionError as err: logging.warning("Can't change mod of %s: %s", target, str(err)) - try: - run(["chgrp", "-R", self.group, target]) - except subprocess.CalledProcessError as err: - logging.warning("Can't change group of %s: %s", target, str(err)) + chgrp(target, group=self.group, recursive=True) - changed = [] + changed = 0 if self.includes_html: # Copy built HTML files to webroot (default /srv/docs.python.org) - changed = changed_files(self.checkout / "Doc" / "build" / "html", target) + changed += changed_files(self.checkout / "Doc" / "build" / "html", target) logging.info("Copying HTML files to %s", target) - run( - [ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "build" / "html/", - ] + chgrp( + self.checkout / "Doc" / "build" / "html/", + group=self.group, + recursive=True, ) - run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) - run( - [ - "find", - self.checkout / "Doc" / "build" / "html", - "-type", - "d", - "-exec", - "chmod", - "o+x", - "{}", - ";", - ] - ) - run( - [ - "rsync", - "-a", - "--delete-delay", - "--filter", - "P archives/", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - ] - ) - - if not self.quick: + chmod_make_readable(self.checkout / "Doc" / "build" / "html") + run(( + "rsync", + "-a", + "--delete-delay", + "--filter", + "P archives/", + str(self.checkout / "Doc" / "build" / "html") + "/", + target, + )) + + dist_dir = self.checkout / "Doc" / "dist" + if dist_dir.is_dir(): # Copy archive files to /archives/ logging.debug("Copying dist files.") - run( - [ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "dist", - ] - ) - run( - [ - "chmod", - "-R", - "o+r", - self.checkout / "Doc" / "dist", - ] + chgrp(dist_dir, group=self.group, recursive=True) + chmod_make_readable(dist_dir) + archives_dir = target / "archives" + archives_dir.mkdir(parents=True, exist_ok=True) + archives_dir.chmod( + archives_dir.stat().st_mode | stat.S_IROTH | stat.S_IXOTH ) - run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) - run(["chown", ":" + self.group, target / "archives"]) - run( - [ - "cp", - "-a", - *(self.checkout / "Doc" / "dist").glob("*"), - target / "archives", - ] - ) - changed.append("archives/") - for file in (target / "archives").iterdir(): - changed.append("archives/" + file.name) + chgrp(archives_dir, group=self.group) + changed += 1 + for dist_file in dist_dir.iterdir(): + shutil.copy2(dist_file, archives_dir / dist_file.name) + changed += 1 - logging.info("%s files changed", len(changed)) + logging.info("%s files changed", changed) if changed and not self.skip_cache_invalidation: surrogate_key = f"{self.language.tag}/{self.version.name}" purge_surrogate_key(http, surrogate_key) @@ -901,13 +798,13 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: "Publishing done (%s).", format_seconds(perf_counter() - start_time) ) - def should_rebuild(self): + def should_rebuild(self, force: bool) -> str | Literal[False]: state = self.load_state() if not state: logging.info("Should rebuild: no previous state found.") return "no previous state" cpython_sha = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() - if self.language.tag != "en": + if self.language.is_translation: translation_sha = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() @@ -929,6 +826,9 @@ def should_rebuild(self): cpython_sha, ) return "Doc/ has changed" + if force: + logging.info("Should rebuild: forced.") + return "forced" logging.info("Nothing changed, no rebuild needed.") return False @@ -944,7 +844,9 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_start: dt, build_duration: float, trigger: str): + def save_state( + self, build_start: dt.datetime, build_duration: float, trigger: str + ) -> None: """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -965,7 +867,7 @@ def save_state(self, build_start: dt, build_duration: float, trigger: str): "triggered_by": trigger, "cpython_sha": self.cpython_repo.run("rev-parse", "HEAD").stdout.strip(), } - if self.language.tag != "en": + if self.language.is_translation: state["translation_sha"] = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() @@ -977,201 +879,239 @@ def save_state(self, build_start: dt, build_duration: float, trigger: str): logging.info("Saved new rebuild state for %s: %s", key, table.as_string()) -def symlink( - www_root: Path, - language: Language, - directory: str, - name: str, - group: str, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, +def chgrp( + path: Path, + /, + group: int | str | None, + *, + recursive: bool = False, + follow_symlinks: bool = True, ) -> None: - """Used by major_symlinks and dev_symlink to maintain symlinks.""" - if language.tag == "en": # English is rooted on /, no /en/ - path = www_root - else: - path = www_root / language.tag - link = path / name - directory_path = path / directory - if not directory_path.exists(): - return # No touching link, dest doc not built yet. + if sys.platform == "win32": + return - if not link.exists() or readlink(link) != directory: - # Link does not exist or points to the wrong target. - if link.exists(): - link.unlink() - link.symlink_to(directory) - run(["chown", "-h", f":{group}", str(link)]) - if not skip_cache_invalidation: - surrogate_key = f"{language.tag}/{name}" - purge_surrogate_key(http, surrogate_key) + from grp import getgrnam + try: + try: + group_id = int(group) + except ValueError: + group_id = getgrnam(group)[2] + except (LookupError, TypeError, ValueError): + return -def major_symlinks( - www_root: Path, - group: str, - versions: Iterable[Version], - languages: Iterable[Language], - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /2/ and /3/ symlinks for each language. + try: + os.chown(path, -1, group_id, follow_symlinks=follow_symlinks) + if recursive: + for p in path.rglob("*"): + os.chown(p, -1, group_id, follow_symlinks=follow_symlinks) + except OSError as err: + logging.warning("Can't change group of %s: %s", path, str(err)) + + +def chmod_make_readable(path: Path, /, mode: int = stat.S_IROTH) -> None: + if not path.is_dir(): + raise ValueError + + path.chmod(path.stat().st_mode | stat.S_IROTH | stat.S_IXOTH) # o+rx + for p in path.rglob("*"): + if p.is_dir(): + p.chmod(p.stat().st_mode | stat.S_IROTH | stat.S_IXOTH) # o+rx + else: + p.chmod(p.stat().st_mode | stat.S_IROTH) # o+r - Like: - - /3/ → /3.9/ - - /fr/3/ → /fr/3.9/ - - /es/3/ → /es/3.9/ - """ - logging.info("Creating major version symlinks...") - current_stable = Version.current_stable(versions).name - for language in languages: - symlink( - www_root, - language, - current_stable, - "3", - group, - skip_cache_invalidation, - http, - ) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) +def format_seconds(seconds: float) -> str: + hours, remainder = divmod(seconds, 3600) + minutes, seconds = divmod(remainder, 60) + hours, minutes, seconds = int(hours), int(minutes), round(seconds) -def dev_symlink( - www_root: Path, - group, - versions, - languages, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /dev/ symlinks for each language. + match (hours, minutes, seconds): + case 0, 0, s: + return f"{s}s" + case 0, m, s: + return f"{m}m {s}s" + case h, m, s: + return f"{h}h {m}m {s}s" - Like: - - /dev/ → /3.11/ - - /fr/dev/ → /fr/3.11/ - - /es/dev/ → /es/3.11/ - """ - logging.info("Creating development version symlinks...") - current_dev = Version.current_dev(versions).name - for language in languages: - symlink( - www_root, - language, - current_dev, - "dev", - group, - skip_cache_invalidation, - http, - ) + raise ValueError("unreachable") -def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: - """Remove one or many paths from docs.python.org's CDN. +def _checkout_name(select_output: str | None) -> str: + if select_output is not None: + return f"cpython-{select_output}" + return "cpython" - To be used when a file changes, so the CDN fetches the new one. - """ - base = "https://docs.python.org/" - for path in paths: - url = urljoin(base, str(path)) - logging.debug("Purging %s from CDN", url) - http.request("PURGE", url, timeout=30) +def main() -> int: + """Script entry point.""" + args = parse_args() + setup_logging(args.log_directory, args.select_output) + load_environment_variables() -def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: - """Remove paths from docs.python.org's CDN. + if args.select_output is None: + return build_docs_with_lock(args, "build_docs.lock") + if args.select_output == "no-html": + return build_docs_with_lock(args, "build_docs_archives.lock") + if args.select_output == "only-html": + return build_docs_with_lock(args, "build_docs_html.lock") + if args.select_output == "only-html-en": + return build_docs_with_lock(args, "build_docs_html_en.lock") + return EX_FAILURE - All paths matching the given 'Surrogate-Key' will be removed. - This is set by the Nginx server for every language-version pair. - To be used when a directory changes, so the CDN fetches the new one. - https://www.fastly.com/documentation/reference/api/purging/#purge-tag - """ - service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") - fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" - logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) - http.request( - "POST", - f"https://api.fastly.com/service/{service_id}/purge/{surrogate_key}", - headers={"Fastly-Key": fastly_key}, - timeout=30, + parser = argparse.ArgumentParser( + description="Runs a build of the Python docs for various branches.", + allow_abbrev=False, + ) + parser.add_argument( + "--select-output", + choices=("no-html", "only-html", "only-html-en"), + help="Choose what outputs to build.", + ) + parser.add_argument( + "-q", + "--quick", + action="store_true", + help="Run a quick build (only HTML files).", + ) + parser.add_argument( + "-b", + "--branches", + nargs="*", + metavar="3.12", + help="Versions to build (defaults to all maintained branches).", + ) + parser.add_argument( + "-r", + "--build-root", + type=Path, + help="Path to a directory containing a checkout per branch.", + default=Path("/srv/docsbuild"), + ) + parser.add_argument( + "-w", + "--www-root", + type=Path, + help="Path where generated files will be copied.", + default=Path("/srv/docs.python.org"), + ) + parser.add_argument( + "--force", + action="store_true", + help="Always build the chosen languages and versions, " + "regardless of existing state.", + ) + parser.add_argument( + "--skip-cache-invalidation", + help="Skip Fastly cache invalidation.", + action="store_true", + ) + parser.add_argument( + "--group", + help="Group files on targets and www-root file should get.", + default="docs", + ) + parser.add_argument( + "--log-directory", + type=Path, + help="Directory used to store logs.", + default=Path("/var/log/docsbuild/"), + ) + parser.add_argument( + "--languages", + nargs="*", + help="Language translation, as a PEP 545 language tag like" + " 'fr' or 'pt-br'. " + "Builds all available languages by default.", + metavar="fr", + ) + parser.add_argument( + "--version", + action="store_true", + help="Get build_docs and dependencies version info", ) + parser.add_argument( + "--theme", + default="python-docs-theme", + help="Python package to use for python-docs-theme: Useful to test branches:" + " --theme git+https://github.com/obulat/python-docs-theme@master", + ) + args = parser.parse_args() + if args.version: + version_info() + sys.exit(0) + del args.version + if args.log_directory: + args.log_directory = args.log_directory.resolve() + if args.build_root: + args.build_root = args.build_root.resolve() + if args.www_root: + args.www_root = args.www_root.resolve() + return args -def proofread_canonicals( - www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager -) -> None: - """In www_root we check that all canonical links point to existing contents. +def setup_logging(log_directory: Path, select_output: str | None) -> None: + """Setup logging to stderr if run by a human, or to a file if run from a cron.""" + log_format = "%(asctime)s %(levelname)s: %(message)s" + if sys.stderr.isatty() or "CI" in os.environ: + logging.basicConfig(format=log_format, stream=sys.stderr) + else: + log_directory.mkdir(parents=True, exist_ok=True) + if select_output is None: + filename = log_directory / "docsbuild.log" + else: + filename = log_directory / f"docsbuild-{select_output}.log" + handler = logging.handlers.WatchedFileHandler(filename) + handler.setFormatter(logging.Formatter(log_format)) + logging.getLogger().addHandler(handler) + logging.getLogger().setLevel(logging.DEBUG) - It can happen that a canonical is "broken": - - /3.11/whatsnew/3.11.html typically would link to - /3/whatsnew/3.11.html, which may not exist yet. - """ - logging.info("Checking canonical links...") - canonical_re = re.compile( - """""" - ) - for file in www_root.glob("**/*.html"): - html = file.read_text(encoding="UTF-8", errors="surrogateescape") - canonical = canonical_re.search(html) - if not canonical: - continue - target = canonical.group(1) - if not (www_root / target).exists(): - logging.info("Removing broken canonical from %s to %s", file, target) - html = html.replace(canonical.group(0), "") - file.write_text(html, encoding="UTF-8", errors="surrogateescape") - if not skip_cache_invalidation: - purge(http, str(file).replace("/srv/docs.python.org/", "")) - - -def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: - releases = http.request( - "GET", - "https://raw.githubusercontent.com/" - "python/devguide/main/include/release-cycle.json", - timeout=30, - ).json() - versions = [Version.from_json(name, release) for name, release in releases.items()] - versions.sort(key=Version.as_tuple) - return versions +def load_environment_variables() -> None: + dbs_user_config = platformdirs.user_config_path("docsbuild-scripts") + dbs_site_config = platformdirs.site_config_path("docsbuild-scripts") + if dbs_user_config.is_file(): + env_conf_file = dbs_user_config + elif dbs_site_config.is_file(): + env_conf_file = dbs_site_config + else: + logging.info( + "No environment variables configured. Configure in %s or %s.", + dbs_site_config, + dbs_user_config, + ) + return + logging.info("Reading environment variables from %s.", env_conf_file) + if env_conf_file == dbs_site_config: + logging.info("You can override settings in %s.", dbs_user_config) + elif dbs_site_config.is_file(): + logging.info("Overriding %s.", dbs_site_config) -def parse_languages_from_config() -> list[Language]: - """Read config.toml to discover languages to build.""" - config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) - languages = [] - defaults = config["defaults"] - for iso639_tag, section in config["languages"].items(): - languages.append( - Language( - iso639_tag, - section["name"], - section.get("in_prod", defaults["in_prod"]), - sphinxopts=section.get("sphinxopts", defaults["sphinxopts"]), - html_only=section.get("html_only", defaults["html_only"]), - ) - ) - return languages + env_config = env_conf_file.read_text(encoding="utf-8") + for key, value in tomlkit.parse(env_config).get("env", {}).items(): + logging.debug("Setting %s in environment.", key) + os.environ[key] = value -def format_seconds(seconds: float) -> str: - hours, remainder = divmod(seconds, 3600) - minutes, seconds = divmod(remainder, 60) - hours, minutes, seconds = int(hours), int(minutes), round(seconds) +def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: + try: + lock = zc.lockfile.LockFile(HERE / lockfile_name) + except zc.lockfile.LockError: + logging.info("Another builder is running... dying...") + return EX_FAILURE - match (hours, minutes, seconds): - case 0, 0, s: - return f"{s}s" - case 0, m, s: - return f"{m}m {s}s" - case h, m, s: - return f"{h}h {m}m {s}s" + try: + return build_docs(args) + finally: + lock.close() -def build_docs(args) -> bool: +def build_docs(args: argparse.Namespace) -> int: """Build all docs (each language and each version).""" logging.info("Full build start.") start_time = perf_counter() @@ -1183,12 +1123,19 @@ def build_docs(args) -> bool: # This runs languages in config.toml order and versions newest first. todo = [ (version, language) - for version in Version.filter(versions, args.branch) - for language in reversed(Language.filter(languages, args.languages)) + for version in versions.filter(args.branches) + for language in reversed(languages.filter(args.languages)) ] - del args.branch + del args.branches del args.languages - all_built_successfully = True + force_build = args.force + del args.force + + docs_by_version_content = render_docs_by_version(versions).encode() + switchers_content = render_switchers(versions, languages) + + build_succeeded = set() + any_build_failed = False cpython_repo = Repository( "https://github.com/python/cpython.git", args.build_root / _checkout_name(args.select_output), @@ -1204,11 +1151,21 @@ def build_docs(args) -> bool: scope = sentry_sdk.get_isolation_scope() scope.set_tag("version", version.name) scope.set_tag("language", language.tag) - cpython_repo.update() + cpython_repo.update() builder = DocBuilder( - version, versions, language, languages, cpython_repo, **vars(args) + version, + language, + cpython_repo, + docs_by_version_content, + switchers_content, + **vars(args), ) - all_built_successfully &= builder.run(http) + built_successfully = builder.run(http, force_build=force_build) + if built_successfully: + build_succeeded.add((version.name, language.tag)) + elif built_successfully is not None: + any_build_failed = True + logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") ) @@ -1221,62 +1178,263 @@ def build_docs(args) -> bool: args.skip_cache_invalidation, http, ) - major_symlinks( - args.www_root, - args.group, - versions, - languages, - args.skip_cache_invalidation, - http, - ) - dev_symlink( + make_symlinks( args.www_root, args.group, versions, languages, + build_succeeded, args.skip_cache_invalidation, http, ) - proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) + if build_succeeded: + # Only check canonicals if at least one version was built. + proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - return all_built_successfully + return EX_FAILURE if any_build_failed else EX_OK -def _checkout_name(select_output: str | None) -> str: - if select_output is not None: - return f"cpython-{select_output}" - return "cpython" +def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: + releases = http.request( + "GET", + "https://raw.githubusercontent.com/" + "python/devguide/main/include/release-cycle.json", + timeout=30, + ).json() + return Versions.from_json(releases) -def main(): - """Script entry point.""" - args = parse_args() - setup_logging(args.log_directory, args.select_output) +def parse_languages_from_config() -> Languages: + """Read config.toml to discover languages to build.""" + config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) + return Languages.from_json(config["defaults"], config["languages"]) - if args.select_output is None: - build_docs_with_lock(args, "build_docs.lock") - elif args.select_output == "no-html": - build_docs_with_lock(args, "build_docs_archives.lock") - elif args.select_output == "only-html": - build_docs_with_lock(args, "build_docs_html.lock") - elif args.select_output == "only-html-en": - build_docs_with_lock(args, "build_docs_html_en.lock") +def render_docs_by_version(versions: Versions) -> str: + """Generate content for _docs_by_version.html.""" + links = [f'
  • {v.title}
    • ' for v in reversed(versions)] + return "\n".join(links) -def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: - try: - lock = zc.lockfile.LockFile(HERE / lockfile_name) - except zc.lockfile.LockError: - logging.info("Another builder is running... dying...") - return EX_FAILURE - try: - return EX_OK if build_docs(args) else EX_FAILURE - finally: - lock.close() +def render_switchers(versions: Versions, languages: Languages) -> bytes: + language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 + version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] + + switchers_template_file = HERE / "templates" / "switchers.js" + template = Template(switchers_template_file.read_text(encoding="UTF-8")) + rendered_template = template.safe_substitute( + LANGUAGES=json.dumps(language_pairs), + VERSIONS=json.dumps(version_pairs), + ) + return rendered_template.encode("UTF-8") + + +def build_sitemap( + versions: Versions, languages: Languages, www_root: Path, group: str +) -> None: + """Build a sitemap with all live versions and translations.""" + if not www_root.exists(): + logging.info("Skipping sitemap generation (www root does not even exist).") + return + logging.info("Starting sitemap generation...") + template_path = HERE / "templates" / "sitemap.xml" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render(languages=languages, versions=versions) + sitemap_path = www_root / "sitemap.xml" + sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") + sitemap_path.chmod(0o664) + chgrp(sitemap_path, group=group) + + +def build_404(www_root: Path, group: str) -> None: + """Build a nice 404 error page to display in case PDFs are not built yet.""" + if not www_root.exists(): + logging.info("Skipping 404 page generation (www root does not even exist).") + return + logging.info("Copying 404 page...") + not_found_file = www_root / "404.html" + shutil.copyfile(HERE / "templates" / "404.html", not_found_file) + not_found_file.chmod(0o664) + chgrp(not_found_file, group=group) + + +def copy_robots_txt( + www_root: Path, + group: str, + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Copy robots.txt to www_root.""" + if not www_root.exists(): + logging.info("Skipping copying robots.txt (www root does not even exist).") + return + logging.info("Copying robots.txt...") + template_path = HERE / "templates" / "robots.txt" + robots_path = www_root / "robots.txt" + shutil.copyfile(template_path, robots_path) + robots_path.chmod(0o775) + chgrp(robots_path, group=group) + if not skip_cache_invalidation: + purge(http, "robots.txt") + + +def make_symlinks( + www_root: Path, + group: str, + versions: Versions, + languages: Languages, + successful_builds: Set[tuple[str, str]], + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Maintains the /2/, /3/, and /dev/ symlinks for each language. + + Like: + - /2/ → /2.7/ + - /3/ → /3.12/ + - /dev/ → /3.14/ + - /fr/3/ → /fr/3.12/ + - /es/dev/ → /es/3.14/ + """ + logging.info("Creating major and development version symlinks...") + for symlink_name, symlink_target in ( + ("3", versions.current_stable.name), + ("2", "2.7"), + ("dev", versions.current_dev.name), + ): + for language in languages: + if (symlink_target, language.tag) in successful_builds: + symlink( + www_root, + language.tag, + symlink_target, + symlink_name, + group, + skip_cache_invalidation, + http, + ) + + +def symlink( + www_root: Path, + language_tag: str, + directory: str, + name: str, + group: str, + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Used by major_symlinks and dev_symlink to maintain symlinks.""" + msg = "Creating symlink from /%s/ to /%s/" + if language_tag == "en": # English is rooted on /, no /en/ + path = www_root + logging.debug(msg, name, directory) + else: + path = www_root / language_tag + logging.debug(msg, f"{language_tag}/{name}", f"{language_tag}/{directory}") + link = path / name + directory_path = path / directory + if not directory_path.exists(): + return # No touching link, dest doc not built yet. + + if not link.exists() or os.readlink(link) != directory: + # Link does not exist or points to the wrong target. + link.unlink(missing_ok=True) + link.symlink_to(directory) + chgrp(link, group=group, follow_symlinks=False) + if not skip_cache_invalidation: + surrogate_key = f"{language_tag}/{name}" + purge_surrogate_key(http, surrogate_key) + + +def proofread_canonicals( + www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager +) -> None: + """In www_root we check that all canonical links point to existing contents. + + It can happen that a canonical is "broken": + + - /3.11/whatsnew/3.11.html typically would link to + /3/whatsnew/3.11.html, which may not exist yet. + """ + logging.info("Checking canonical links...") + worker_count = (os.cpu_count() or 1) + 2 + with concurrent.futures.ThreadPoolExecutor(worker_count) as executor: + futures = { + executor.submit(_check_canonical_rel, file, www_root) + for file in www_root.glob("**/*.html") + } + paths_to_purge = { + res.relative_to(www_root) # strip the leading /srv/docs.python.org + for fut in concurrent.futures.as_completed(futures) + if (res := fut.result()) is not None + } + if not skip_cache_invalidation: + purge(http, *paths_to_purge) + + +# Python 3.12 onwards doesn't use self-closing tags for +_canonical_re = re.compile( + b"""""" +) + + +def _check_canonical_rel(file: Path, www_root: Path) -> Path | None: + # Check for a canonical relation link in the HTML. + # If one exists, ensure that the target exists + # or otherwise remove the canonical link element. + html = file.read_bytes() + canonical = _canonical_re.search(html) + if canonical is None: + return None + target = canonical[1].decode(encoding="UTF-8", errors="surrogateescape") + if (www_root / target).exists(): + return None + logging.info("Removing broken canonical from %s to %s", file, target) + start, end = canonical.span() + file.write_bytes(html[:start] + html[end:]) + return file + + +def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: + """Remove one or many paths from docs.python.org's CDN. + + To be used when a file changes, so the CDN fetches the new one. + """ + base = "https://docs.python.org/" + for path in paths: + url = urljoin(base, str(path)) + logging.debug("Purging %s from CDN", url) + http.request("PURGE", url, timeout=30) + + +def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: + """Remove paths from docs.python.org's CDN. + + All paths matching the given 'Surrogate-Key' will be removed. + This is set by the Nginx server for every language-version pair. + To be used when a directory changes, so the CDN fetches the new one. + + https://www.fastly.com/documentation/reference/api/purging/#purge-tag + """ + unset = "__UNSET__" + service_id = os.environ.get("FASTLY_SERVICE_ID", unset) + fastly_key = os.environ.get("FASTLY_TOKEN", unset) + + if service_id == unset or fastly_key == unset: + logging.info("CDN secrets not set, skipping Surrogate-Key purge") + return + + logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) + http.request( + "POST", + f"https://api.fastly.com/service/{service_id}/purge/{surrogate_key}", + headers={"Fastly-Key": fastly_key}, + timeout=30, + ) if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) diff --git a/check_times.py b/check_times.py index 9310cd4..2b3d2f9 100644 --- a/check_times.py +++ b/check_times.py @@ -50,7 +50,7 @@ def calc_time(lines: list[str]) -> None: fmt_duration = format_seconds(state_data["last_build_duration"]) reason = state_data["triggered_by"] print( - f"{start:%Y-%m-%d %H:%M UTC} | {version: <7} | {language: <8} | {fmt_duration :<14} | {reason}" + f"{start:%Y-%m-%d %H:%M UTC} | {version: <7} | {language: <8} | {fmt_duration:<14} | {reason}" ) if line.endswith("Build start."): @@ -64,7 +64,7 @@ def calc_time(lines: list[str]) -> None: timestamp = f"{line[:16]} UTC" _, fmt_duration = line.removesuffix(").").split("(") print( - f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration :<14} | -----------" + f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration:<14} | -----------" ) if in_progress: diff --git a/check_versions.py b/check_versions.py index 70cade9..1a1016f 100644 --- a/check_versions.py +++ b/check_versions.py @@ -1,15 +1,15 @@ #!/usr/bin/env python -from pathlib import Path import argparse import asyncio import logging import re +from pathlib import Path +import git import httpx import urllib3 from tabulate import tabulate -import git import build_docs @@ -111,12 +111,10 @@ async def which_sphinx_is_used_in_production(): table = [ [ version.name, - *await asyncio.gather( - *[ - get_version_in_prod(language.tag, version.name) - for language in LANGUAGES - ] - ), + *await asyncio.gather(*[ + get_version_in_prod(language.tag, version.name) + for language in LANGUAGES + ]), ] for version in VERSIONS ] diff --git a/config.toml b/config.toml index 3716d7f..d628ca0 100644 --- a/config.toml +++ b/config.toml @@ -1,5 +1,12 @@ +# name: the English name for the language. +# translated_name: the 'local' name for the language. +# in_prod: If true, include in the language switcher. +# html_only: If true, only create HTML files. +# sphinxopts: Extra options to pass to SPHINXOPTS in the Makefile. + [defaults] # name has no default, it is mandatory. +translated_name = "" in_prod = true html_only = false sphinxopts = [ @@ -13,6 +20,7 @@ name = "English" [languages.es] name = "Spanish" +translated_name = "español" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -21,22 +29,30 @@ sphinxopts = [ [languages.fr] name = "French" +translated_name = "français" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', '-D latex_elements.fontenc=\\usepackage{fontspec}', ] +[languages.bn_IN] +name = "Bengali" +translated_name = "বাংলা" +in_prod = false + [languages.id] name = "Indonesian" +translated_name = "Indonesia" in_prod = false [languages.it] name = "Italian" -in_prod = true +translated_name = "italiano" [languages.ja] name = "Japanese" +translated_name = "日本語" sphinxopts = [ '-D latex_engine=lualatex', '-D latex_elements.inputenc=', @@ -60,6 +76,7 @@ sphinxopts = [ [languages.ko] name = "Korean" +translated_name = "한국어" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -69,21 +86,30 @@ sphinxopts = [ [languages.pl] name = "Polish" -in_prod = false +translated_name = "polski" [languages.pt_BR] name = "Brazilian Portuguese" +translated_name = "Português brasileiro" + +[languages.ro] +name = "Romanian" +translated_name = "Românește" +in_prod = false [languages.tr] name = "Turkish" +translated_name = "Türkçe" [languages.uk] name = "Ukrainian" +translated_name = "українська" in_prod = false html_only = true [languages.zh_CN] name = "Simplified Chinese" +translated_name = "简体中文" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -92,6 +118,7 @@ sphinxopts = [ [languages.zh_TW] name = "Traditional Chinese" +translated_name = "繁體中文" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', diff --git a/requirements.txt b/requirements.txt index 0cac810..535b36a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ jinja2 +platformdirs sentry-sdk>=2 tomlkit>=0.13 urllib3>=2 diff --git a/templates/_docs_by_version.html b/templates/_docs_by_version.html new file mode 100644 index 0000000..1a84cfb --- /dev/null +++ b/templates/_docs_by_version.html @@ -0,0 +1,11 @@ +{# +This file is only used in indexsidebar.html, where it is included in the docs +by version list. For non-end-of-life branches, build_docs.py overwrites this +list with the full list of versions. + +Keep the following two files synchronised: +* cpython/Doc/tools/templates/_docs_by_version.html +* docsbuild-scripts/templates/_docs_by_version.html +#} +
  • {% trans %}Stable{% endtrans %}
    • +
  • {% trans %}In development{% endtrans %}
    • diff --git a/templates/indexsidebar.html b/templates/indexsidebar.html index 3a56219..eea29e2 100644 --- a/templates/indexsidebar.html +++ b/templates/indexsidebar.html @@ -1,30 +1,17 @@ -{# -Beware, this file is rendered twice via Jinja2: -- First by build_docs.py, given 'current_version' and 'versions'. -- A 2nd time by Sphinx. -#} - -{% raw %}

    {% trans %}Download{% endtrans %}

    {% trans %}Download these documents{% endtrans %}

    -{% endraw %} -{% if current_version.status != "EOL" %} -{% raw %}

    {% trans %}Docs by version{% endtrans %}

    {% endraw %} +

    {% trans %}Docs by version{% endtrans %}

    -{% endif %} -{% raw %}

    {% trans %}Other resources{% endtrans %}

    -{% endraw %} diff --git a/templates/switchers.js b/templates/switchers.js index 29204ae..e54a278 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,197 +1,228 @@ -(function() { - 'use strict'; - - if (!String.prototype.startsWith) { - Object.defineProperty(String.prototype, 'startsWith', { - value: function(search, rawPos) { - const pos = rawPos > 0 ? rawPos|0 : 0; - return this.substring(pos, pos + search.length) === search; - } - }); - } - - // Parses versions in URL segments like: - // "3", "dev", "release/2.7" or "3.6rc2" - const version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; - - const all_versions = $VERSIONS; - const all_languages = $LANGUAGES; - - function quote_attr(str) { - return '"' + str.replace('"', '\\"') + '"'; - } - - function build_version_select(release) { - let buf = [''); - return buf.join(''); +"use strict"; + +// File URIs must begin with either one or three forward slashes +const _is_file_uri = (uri) => uri.startsWith("file:/"); + +const _IS_LOCAL = _is_file_uri(window.location.href); +const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ""; +const _CURRENT_VERSION = _CURRENT_RELEASE.split(".", 2).join("."); +const _CURRENT_LANGUAGE = (() => { + const _LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || "en"; + // Python 2.7 and 3.5--3.10 use ``LANGUAGE: 'None'`` for English + // in ``documentation_options.js``. + if (_LANGUAGE === "none") return "en"; + return _LANGUAGE; +})(); +const _CURRENT_PREFIX = (() => { + if (_IS_LOCAL) return null; + // Sphinx 7.2+ defines the content root data attribute in the HTML element. + const _CONTENT_ROOT = document.documentElement.dataset.content_root; + if (_CONTENT_ROOT !== undefined) { + return new URL(_CONTENT_ROOT, window.location).pathname; } + // Fallback for older versions of Sphinx (used in Python 3.10 and older). + const _NUM_PREFIX_PARTS = _CURRENT_LANGUAGE === "en" ? 2 : 3; + return window.location.pathname.split("/", _NUM_PREFIX_PARTS).join("/") + "/"; +})(); - function build_language_select(current_language) { - let buf = [''); - return buf.join(''); - } - - function navigate_to_first_existing(urls) { - // Navigate to the first existing URL in urls. - const url = urls.shift(); - if (urls.length == 0 || url.startsWith("file:///")) { - window.location.href = url; + item.parentElement.insertBefore(container, item); return; } - fetch(url) - .then(function(response) { - if (response.ok) { - window.location.href = url; - } else { - navigate_to_first_existing(urls); - } - }) - .catch(function(error) { - navigate_to_first_existing(urls); - }); } - - function on_version_switch() { - const selected_version = this.options[this.selectedIndex].value + '/'; - const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); - const new_url = url.replace('/' + current_language + current_version, - '/' + current_language + selected_version); - if (new_url != url) { - navigate_to_first_existing([ - new_url, - url.replace('/' + current_language + current_version, - '/' + selected_version), - '/' + current_language + selected_version, - '/' + selected_version, - '/' - ]); - } +}; + +/** + * @param {Map} versions + * @returns {HTMLSelectElement} + * @private + */ +const _create_version_select = (versions) => { + const select = document.createElement("select"); + select.className = "version-select"; + if (_IS_LOCAL) { + select.disabled = true; + select.title = "Version switching is disabled in local builds"; } - function on_language_switch() { - let selected_language = this.options[this.selectedIndex].value + '/'; - const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); - if (selected_language == 'en/') // Special 'default' case for English. - selected_language = ''; - let new_url = url.replace('/' + current_language + current_version, - '/' + selected_language + current_version); - if (new_url != url) { - navigate_to_first_existing([ - new_url, - '/' - ]); + for (const [version, title] of versions) { + const option = document.createElement("option"); + option.value = version; + if (version === _CURRENT_VERSION) { + option.text = _CURRENT_RELEASE; + option.selected = true; + } else { + option.text = title; } + select.add(option); } - // Returns the path segment of the language as a string, like 'fr/' - // or '' if not found. - function language_segment_from_url() { - const path = window.location.pathname; - const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' - const match = path.match(language_regexp); - if (match !== null) - return match[1]; - return ''; + return select; +}; + +/** + * @param {Map} languages + * @returns {HTMLSelectElement} + * @private + */ +const _create_language_select = (languages) => { + if (!languages.has(_CURRENT_LANGUAGE)) { + // In case we are browsing a language that is not yet in languages. + languages.set(_CURRENT_LANGUAGE, _CURRENT_LANGUAGE); } - // Returns the path segment of the version as a string, like '3.6/' - // or '' if not found. - function version_segment_from_url() { - const path = window.location.pathname; - const language_segment = language_segment_from_url(); - const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - const version_regexp = language_segment + '(' + version_segment + ')'; - const match = path.match(version_regexp); - if (match !== null) - return match[1]; - return '' + const select = document.createElement("select"); + select.className = "language-select"; + if (_IS_LOCAL) { + select.disabled = true; + select.title = "Language switching is disabled in local builds"; } - function create_placeholders_if_missing() { - const version_segment = version_segment_from_url(); - const language_segment = language_segment_from_url(); - const index = "/" + language_segment + version_segment; - - if (document.querySelectorAll('.version_switcher_placeholder').length > 0) { - return; - } + for (const [language, title] of languages) { + const option = document.createElement("option"); + option.value = language; + option.text = title; + if (language === _CURRENT_LANGUAGE) option.selected = true; + select.add(option); + } - const html = ' \ - \ -Documentation »'; - - const probable_places = [ - "body>div.related>ul>li:not(.right):contains('Documentation'):first", - "body>div.related>ul>li:not(.right):contains('documentation'):first", - ]; - - for (let i = 0; i < probable_places.length; i++) { - let probable_place = $(probable_places[i]); - if (probable_place.length == 1) { - probable_place.html(html); - document.getElementById('indexlink').href = index; - return; + return select; +}; + +/** + * Change the current page to the first existing URL in the list. + * @param {Array} urls + * @private + */ +const _navigate_to_first_existing = async (urls) => { + // Navigate to the first existing URL in urls. + for (const url of urls) { + try { + const response = await fetch(url, { method: "HEAD" }); + if (response.ok) { + window.location.href = url; + return url; } + } catch (err) { + console.error(`Error when fetching '${url}': ${err}`); } } - document.addEventListener('DOMContentLoaded', function() { - const language_segment = language_segment_from_url(); - const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - - create_placeholders_if_missing(); - - let placeholders = document.querySelectorAll('.version_switcher_placeholder'); - placeholders.forEach(function(placeholder) { - placeholder.innerHTML = version_select; - - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_version_switch); + // if all else fails, redirect to the d.p.o root + window.location.href = "/"; + return "/"; +}; + +/** + * Navigate to the selected version. + * @param {Event} event + * @returns {void} + * @private + */ +const _on_version_switch = async (event) => { + if (_IS_LOCAL) return; + + const selected_version = event.target.value; + // English has no language prefix. + const new_prefix_en = `/${selected_version}/`; + const new_prefix = + _CURRENT_LANGUAGE === "en" + ? new_prefix_en + : `/${_CURRENT_LANGUAGE}/${selected_version}/`; + if (_CURRENT_PREFIX !== new_prefix) { + // Try the following pages in order: + // 1. The current page in the current language with the new version + // 2. The current page in English with the new version + // 3. The documentation home in the current language with the new version + // 4. The documentation home in English with the new version + await _navigate_to_first_existing([ + window.location.href.replace(_CURRENT_PREFIX, new_prefix), + window.location.href.replace(_CURRENT_PREFIX, new_prefix_en), + new_prefix, + new_prefix_en, + ]); + } +}; + +/** + * Navigate to the selected language. + * @param {Event} event + * @returns {void} + * @private + */ +const _on_language_switch = async (event) => { + if (_IS_LOCAL) return; + + const selected_language = event.target.value; + // English has no language prefix. + const new_prefix = + selected_language === "en" + ? `/${_CURRENT_VERSION}/` + : `/${selected_language}/${_CURRENT_VERSION}/`; + if (_CURRENT_PREFIX !== new_prefix) { + // Try the following pages in order: + // 1. The current page in the new language with the current version + // 2. The documentation home in the new language with the current version + await _navigate_to_first_existing([ + window.location.href.replace(_CURRENT_PREFIX, new_prefix), + new_prefix, + ]); + } +}; + +/** + * Set up the version and language switchers. + * @returns {void} + * @private + */ +const _initialise_switchers = () => { + const versions = _ALL_VERSIONS; + const languages = _ALL_LANGUAGES; + + _create_placeholders_if_missing(); + + document + .querySelectorAll(".version_switcher_placeholder") + .forEach((placeholder) => { + const s = _create_version_select(versions); + s.addEventListener("change", _on_version_switch); + placeholder.append(s); }); - const language_select = build_language_select(current_language); - - placeholders = document.querySelectorAll('.language_switcher_placeholder'); - placeholders.forEach(function(placeholder) { - placeholder.innerHTML = language_select; - - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_language_switch); + document + .querySelectorAll(".language_switcher_placeholder") + .forEach((placeholder) => { + const s = _create_language_select(languages); + s.addEventListener("change", _on_language_switch); + placeholder.append(s); }); - }); -})(); +}; + +if (document.readyState !== "loading") { + _initialise_switchers(); +} else { + document.addEventListener("DOMContentLoaded", _initialise_switchers); +} diff --git a/tests/test_build_docs.py b/tests/test_build_docs.py index 4457e95..028da90 100644 --- a/tests/test_build_docs.py +++ b/tests/test_build_docs.py @@ -4,7 +4,7 @@ @pytest.mark.parametrize( - "seconds, expected", + ("seconds", "expected"), [ (0.4, "0s"), (0.5, "0s"), diff --git a/tests/test_build_docs_version.py b/tests/test_build_docs_version.py new file mode 100644 index 0000000..bd5e644 --- /dev/null +++ b/tests/test_build_docs_version.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import pytest + +from build_docs import Version + + +def test_equality() -> None: + # Arrange + version1 = Version(name="3.13", status="stable", branch_or_tag="3.13") + version2 = Version(name="3.13", status="stable", branch_or_tag="3.13") + + # Act / Assert + assert version1 == version2 + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("3.13", "-rrequirements.txt"), + ("3.10", "standard-imghdr"), + ("3.7", "sphinx==2.3.1"), + ("3.5", "sphinx==1.8.4"), + ], +) +def test_requirements(name: str, expected: str) -> None: + # Arrange + version = Version(name=name, status="stable", branch_or_tag="") + + # Act / Assert + assert expected in version.requirements + + +def test_requirements_error() -> None: + # Arrange + version = Version(name="2.8", status="ex-release", branch_or_tag="") + + # Act / Assert + with pytest.raises(ValueError, match="unreachable"): + _ = version.requirements + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + ("EOL", "never"), + ("security-fixes", "yearly"), + ("stable", "daily"), + ], +) +def test_changefreq(status: str, expected: str) -> None: + # Arrange + version = Version(name="3.13", status=status, branch_or_tag="") + + # Act / Assert + assert version.changefreq == expected + + +def test_url() -> None: + # Arrange + version = Version(name="3.13", status="stable", branch_or_tag="") + + # Act / Assert + assert version.url == "https://docs.python.org/3.13/" + + +def test_title() -> None: + # Arrange + version = Version(name="3.14", status="in development", branch_or_tag="") + + # Act / Assert + assert version.title == "Python 3.14 (in development)" + + +@pytest.mark.parametrize( + ("name", "status", "expected"), + [ + ("3.15", "in development", "dev (3.15)"), + ("3.14", "pre-release", "pre (3.14)"), + ("3.13", "stable", "3.13"), + ], +) +def test_picker_label(name: str, status: str, expected: str) -> None: + # Arrange + version = Version(name=name, status=status, branch_or_tag="") + + # Act / Assert + assert version.picker_label == expected diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py new file mode 100644 index 0000000..1d8f6dc --- /dev/null +++ b/tests/test_build_docs_versions.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import pytest + +from build_docs import Version, Versions + + +@pytest.fixture +def versions() -> Versions: + return Versions([ + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.11", status="security-fixes", branch_or_tag=""), + Version(name="3.10", status="security-fixes", branch_or_tag=""), + Version(name="3.9", status="security-fixes", branch_or_tag=""), + ]) + + +def test_reversed(versions: Versions) -> None: + # Act + output = list(reversed(versions)) + + # Assert + assert output[0].name == "3.9" + assert output[-1].name == "3.14" + + +def test_from_json() -> None: + # Arrange + json_data = { + "3.14": { + "branch": "main", + "pep": 745, + "status": "feature", + "first_release": "2025-10-01", + "end_of_life": "2030-10", + "release_manager": "Hugo van Kemenade", + }, + "3.13": { + "branch": "3.13", + "pep": 719, + "status": "bugfix", + "first_release": "2024-10-07", + "end_of_life": "2029-10", + "release_manager": "Thomas Wouters", + }, + } + + # Act + versions = list(Versions.from_json(json_data)) + + # Assert + assert versions == [ + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.14", status="in development", branch_or_tag=""), + ] + + +def test_from_json_error() -> None: + # Arrange + json_data = {"2.8": {"branch": "2.8", "pep": 404, "status": "ex-release"}} + + # Act / Assert + with pytest.raises( + ValueError, + match="Saw invalid version status 'ex-release', expected to be one of", + ): + Versions.from_json(json_data) + + +def test_current_stable(versions) -> None: + # Act + current_stable = versions.current_stable + + # Assert + assert current_stable.name == "3.13" + assert current_stable.status == "stable" + + +def test_current_dev(versions) -> None: + # Act + current_dev = versions.current_dev + + # Assert + assert current_dev.name == "3.14" + assert current_dev.status == "in development" + + +def test_filter_default(versions) -> None: + # Act + filtered = versions.filter() + + # Assert + assert filtered == [ + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + ] + + +def test_filter_one(versions) -> None: + # Act + filtered = versions.filter(["3.13"]) + + # Assert + assert filtered == [Version(name="3.13", status="security-fixes", branch_or_tag="")] + + +def test_filter_multiple(versions) -> None: + # Act + filtered = versions.filter(["3.13", "3.14"]) + + # Assert + assert filtered == [ + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="security-fixes", branch_or_tag=""), + ] diff --git a/tox.ini b/tox.ini index 40a034d..12efcdf 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ requires = tox>=4.2 env_list = lint - py{313, 312, 311, 310} + py{314, 313} [testenv] package = wheel @@ -12,8 +12,19 @@ skip_install = true deps = -r requirements.txt pytest + pytest-cov +pass_env = + FORCE_COLOR +set_env = + COVERAGE_CORE = sysmon commands = - {envpython} -m pytest {posargs} + {envpython} -m pytest \ + --cov . \ + --cov tests \ + --cov-report html \ + --cov-report term \ + --cov-report xml \ + {posargs} [testenv:lint] skip_install = true