[pull] master from python:master

README.md

@@ -10,7 +10,7 @@ translations in ``./www``, beware it can take a few hours:
     $ python3 ./build_docs.py --quick --build-root ./build_root --www-root ./www --log-directory ./logs --group $(id -g) --skip-cache-invalidation
 
 If you don't need to build all translations of all branches, add
-``--language en --branch master``.
+``--language en --branch main``.
 
 
 # Check current version
@@ -32,7 +32,8 @@ of Sphinx we're using where::
     3.7       sphinx==1.8.2  sphinx==1.8.2  ø                   needs_sphinx="1.6.6"  Sphinx==2.3.1  Sphinx==2.3.1
     3.8       sphinx==1.8.2  sphinx==1.8.2  ø                   needs_sphinx='1.8'    Sphinx==2.3.1  Sphinx==2.3.1
     3.9       sphinx==2.2.0  sphinx==2.2.0  sphinx==2.2.0       needs_sphinx='1.8'    Sphinx==2.3.1  Sphinx==2.3.1
-    master    sphinx==2.2.0  sphinx==2.2.0  sphinx==2.2.0       needs_sphinx='1.8'    Sphinx==2.3.1  Sphinx==2.3.1
+    3.10      sphinx==2.2.0  sphinx==2.2.0  sphinx==2.2.0       needs_sphinx='1.8'    Sphinx==2.3.1  Sphinx==2.3.1
+    main      sphinx==2.2.0  sphinx==2.2.0  sphinx==2.2.0       needs_sphinx='1.8'    Sphinx==2.3.1  Sphinx==2.3.1
     ========  =============  =============  ==================  ====================  =============  ===============
 
     Sphinx build as seen on docs.python.org:
@@ -47,4 +48,5 @@ of Sphinx we're using where::
     3.8       2.3.1    2.3.1  2.3.1    2.3.1  2.3.1  2.3.1  2.3.1    2.3.1  2.3.1
     3.9       2.3.1    2.3.1  2.3.1    2.3.1  2.3.1  2.3.1  2.3.1    2.3.1  2.3.1
     3.10      2.3.1    2.3.1  2.3.1    2.3.1  2.3.1  2.3.1  2.3.1    2.3.1  2.3.1
+    3.11      2.3.1    2.3.1  2.3.1    2.3.1  2.3.1  2.3.1  2.3.1    2.3.1  2.3.1
     ========  =======  =====  =======  =====  =====  =====  =======  =====  =====

build_docs.py

@@ -64,7 +64,14 @@
 class Version:
     STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"}
 
-    def __init__(self, name, branch, status, sphinx_version=DEFAULT_SPHINX_VERSION):
+    def __init__(
+        self,
+        name,
+        branch,
+        status,
+        sphinx_version=DEFAULT_SPHINX_VERSION,
+        sphinxopts=[],
+    ):
         if status not in self.STATUSES:
             raise ValueError(
                 "Version status expected to be in {}".format(", ".join(self.STATUSES))
@@ -73,6 +80,7 @@ def __init__(self, name, branch, status, sphinx_version=DEFAULT_SPHINX_VERSION):
         self.branch = branch
         self.status = status
         self.sphinx_version = sphinx_version
+        self.sphinxopts = list(sphinxopts)
 
     @property
     def changefreq(self):
@@ -100,9 +108,14 @@ def title(self):
     Version("3.5", "3.5", "EOL", sphinx_version="1.8.4"),
     Version("3.6", "3.6", "security-fixes", sphinx_version="2.3.1"),
     Version("3.7", "3.7", "security-fixes", sphinx_version="2.3.1"),
-    Version("3.8", "3.8", "stable", sphinx_version="2.4.4"),
+    Version("3.8", "3.8", "security-fixes", sphinx_version="2.4.4"),
     Version("3.9", "3.9", "stable", sphinx_version="2.4.4"),
-    Version("3.10", "master", "in development", sphinx_version="3.2.1"),
+    Version(
+        "3.10", "3.10", "pre-release", sphinx_version="3.2.1", sphinxopts=["-j4"]
+    ),
+    Version(
+        "3.11", "main", "in development", sphinx_version="3.2.1", sphinxopts=["-j4"]
+    ),
 ]
 
 XELATEX_DEFAULT = (
@@ -150,34 +163,29 @@ def title(self):
 }
 
 
-def shell_out(cmd, shell=False):
-    cmdstring = cmd if shell else shlex.join(cmd)
-    logging.debug("Running command: %s", cmdstring)
-    try:
-        output = subprocess.check_output(
-            cmd,
-            shell=shell,
-            stdin=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            encoding="utf-8",
-            errors="backslashreplace",
+def run(cmd) -> subprocess.CompletedProcess:
+    """Like subprocess.run, with logging before and after the command execution."""
+    cmdstring = shlex.join(cmd)
+    logging.debug("Run: %r", cmdstring)
+    result = subprocess.run(
+        cmd,
+        stdin=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        stdout=subprocess.PIPE,
+        encoding="utf-8",
+        errors="backslashreplace",
+    )
+    if result.returncode:
+        # Log last 20 lines, those are likely the interesting ones.
+        logging.error(
+            "Run KO: %r:\n%s",
+            cmdstring,
+            indent("\n".join(result.stdout.split("\n")[-20:]), "    "),
         )
-        if output:
-            logging.debug(
-                "Command executed successfully: %s\n%s",
-                cmdstring,
-                indent(output, "    "),
-            )
-        return output
-    except subprocess.CalledProcessError as e:
-        if sentry_sdk:
-            with sentry_sdk.push_scope() as scope:
-                scope.fingerprint = ["{{ default }}", str(cmd)]
-                sentry_sdk.capture_exception(e)
-        if e.output:
-            logging.error("Command %s failed:\n%s", cmdstring, indent(e.output, "    "))
-        else:
-            logging.error("Command %s failed.", cmdstring)
+    else:
+        logging.debug("Run OK: %r", cmdstring)
+    result.check_returncode()
+    return result
 
 
 def changed_files(left, right):
@@ -207,20 +215,18 @@ def git_clone(repository, directory, branch=None):
     try:
         if not os.path.isdir(os.path.join(directory, ".git")):
             raise AssertionError("Not a git repository.")
-        shell_out(["git", "-C", directory, "fetch"])
+        run(["git", "-C", directory, "fetch"])
         if branch:
-            shell_out(["git", "-C", directory, "checkout", branch])
-            shell_out(["git", "-C", directory, "reset", "--hard", "origin/" + branch])
+            run(["git", "-C", directory, "checkout", branch])
+            run(["git", "-C", directory, "reset", "--hard", "origin/" + branch])
     except (subprocess.CalledProcessError, AssertionError):
         if os.path.exists(directory):
             shutil.rmtree(directory)
         logging.info("Cloning %s into %s", repository, directory)
         os.makedirs(directory, mode=0o775)
-        shell_out(
-            ["git", "clone", "--depth=1", "--no-single-branch", repository, directory]
-        )
+        run(["git", "clone", "--depth=1", "--no-single-branch", repository, directory])
         if branch:
-            shell_out(["git", "-C", directory, "checkout", branch])
+            run(["git", "-C", directory, "checkout", branch])
 
 
 def version_to_tuple(version):
@@ -271,7 +277,7 @@ def translation_branch(locale_repo, locale_clone_dir, needed_version):
     returns the name of the nearest existing branch.
     """
     git_clone(locale_repo, locale_clone_dir)
-    remote_branches = shell_out(["git", "-C", locale_clone_dir, "branch", "-r"])
+    remote_branches = run(["git", "-C", locale_clone_dir, "branch", "-r"]).stdout
     branches = []
     for branch in remote_branches.split("\n"):
         if re.match(r".*/[0-9]+\.[0-9]+$", branch):
@@ -390,7 +396,7 @@ def build_one(
     logging.info(
         "Build start for version: %s, language: %s", version.name, language.tag
     )
-    sphinxopts = list(language.sphinxopts)
+    sphinxopts = list(language.sphinxopts) + list(version.sphinxopts)
     sphinxopts.extend(["-q"])
     if language.tag != "en":
         locale_dirs = os.path.join(build_root, version.name, "locale")
@@ -423,7 +429,7 @@ def build_one(
     sphinxbuild = os.path.join(venv, "bin/sphinx-build")
     blurb = os.path.join(venv, "bin/blurb")
     # Disable cpython switchers, we handle them now:
-    shell_out(
+    run(
         [
             "sed",
             "-i",
@@ -434,7 +440,7 @@ def build_one(
     setup_indexsidebar(
         os.path.join(checkout, "Doc", "tools", "templates", "indexsidebar.html")
     )
-    shell_out(
+    run(
         [
             "make",
             "-C",
@@ -448,7 +454,8 @@ def build_one(
             maketarget,
         ]
     )
-    shell_out(["chgrp", "-R", group, log_directory])
+    run(["mkdir", "-p", log_directory])
+    run(["chgrp", "-R", group, log_directory])
     setup_switchers(os.path.join(checkout, "Doc", "build", "html"))
     logging.info("Build done for version: %s, language: %s", version.name, language.tag)
 
@@ -464,15 +471,18 @@ def build_venv(build_root, version, theme):
         "sphinx=={}".format(version.sphinx_version),
     ]
     venv_path = os.path.join(build_root, "venv-with-sphinx-" + version.sphinx_version)
-    shell_out(["python3", "-m", "venv", venv_path])
-    shell_out(
+    run(["python3", "-m", "venv", venv_path])
+    run(
         [os.path.join(venv_path, "bin", "python"), "-m", "pip", "install"]
         + requirements
     )
     return venv_path
 
 
 def build_robots_txt(www_root, group, skip_cache_invalidation):
+    if not Path(www_root).exists():
+        logging.info("Skipping robots.txt generation (www root does not even exists).")
+        return
     robots_file = os.path.join(www_root, "robots.txt")
     with open(HERE / "templates" / "robots.txt") as robots_txt_template_file:
         with open(robots_file, "w") as robots_txt_file:
@@ -481,12 +491,22 @@ def build_robots_txt(www_root, group, skip_cache_invalidation):
                 template.render(languages=LANGUAGES, versions=VERSIONS) + "\n"
             )
     os.chmod(robots_file, 0o775)
-    shell_out(["chgrp", group, robots_file])
+    run(["chgrp", group, robots_file])
     if not skip_cache_invalidation:
-        shell_out(["curl", "-XPURGE", "https://docs.python.org/robots.txt"])
+        run(
+            [
+                "curl",
+                "--silent",
+                "-XPURGE",
+                "https://docs.python.org/robots.txt",
+            ]
+        )
 
 
 def build_sitemap(www_root):
+    if not Path(www_root).exists():
+        logging.info("Skipping sitemap generation (www root does not even exists).")
+        return
     with open(HERE / "templates" / "sitemap.xml") as sitemap_template_file:
         with open(os.path.join(www_root, "sitemap.xml"), "w") as sitemap_file:
             template = jinja2.Template(sitemap_template_file.read())
@@ -518,7 +538,7 @@ def copy_build_to_webroot(
         language_dir = os.path.join(www_root, language.tag)
         os.makedirs(language_dir, exist_ok=True)
         try:
-            shell_out(["chgrp", "-R", group, language_dir])
+            run(["chgrp", "-R", group, language_dir])
         except subprocess.CalledProcessError as err:
             logging.warning("Can't change group of %s: %s", language_dir, str(err))
         os.chmod(language_dir, 0o775)
@@ -530,15 +550,15 @@ def copy_build_to_webroot(
     except PermissionError as err:
         logging.warning("Can't change mod of %s: %s", target, str(err))
     try:
-        shell_out(["chgrp", "-R", group, target])
+        run(["chgrp", "-R", group, target])
     except subprocess.CalledProcessError as err:
         logging.warning("Can't change group of %s: %s", target, str(err))
 
     changed = changed_files(os.path.join(checkout, "Doc/build/html"), target)
     logging.info("Copying HTML files to %s", target)
-    shell_out(["chown", "-R", ":" + group, os.path.join(checkout, "Doc/build/html/")])
-    shell_out(["chmod", "-R", "o+r", os.path.join(checkout, "Doc/build/html/")])
-    shell_out(
+    run(["chown", "-R", ":" + group, os.path.join(checkout, "Doc/build/html/")])
+    run(["chmod", "-R", "o+r", os.path.join(checkout, "Doc/build/html/")])
+    run(
         [
             "find",
             os.path.join(checkout, "Doc/build/html/"),
@@ -552,9 +572,9 @@ def copy_build_to_webroot(
         ]
     )
     if quick:
-        shell_out(["rsync", "-a", os.path.join(checkout, "Doc/build/html/"), target])
+        run(["rsync", "-a", os.path.join(checkout, "Doc/build/html/"), target])
     else:
-        shell_out(
+        run(
             [
                 "rsync",
                 "-a",
@@ -567,18 +587,17 @@ def copy_build_to_webroot(
         )
     if not quick:
         logging.debug("Copying dist files")
-        shell_out(["chown", "-R", ":" + group, os.path.join(checkout, "Doc/dist/")])
-        shell_out(
-            ["chmod", "-R", "o+r", os.path.join(checkout, os.path.join("Doc/dist/"))]
-        )
-        shell_out(["mkdir", "-m", "o+rx", "-p", os.path.join(target, "archives")])
-        shell_out(["chown", ":" + group, os.path.join(target, "archives")])
-        shell_out(
-            "cp -a {src} {dst}".format(
-                src=os.path.join(checkout, "Doc/dist/*"),
-                dst=os.path.join(target, "archives"),
-            ),
-            shell=True,
+        run(["chown", "-R", ":" + group, os.path.join(checkout, "Doc/dist/")])
+        run(["chmod", "-R", "o+r", os.path.join(checkout, os.path.join("Doc/dist/"))])
+        run(["mkdir", "-m", "o+rx", "-p", os.path.join(target, "archives")])
+        run(["chown", ":" + group, os.path.join(target, "archives")])
+        run(
+            [
+                "cp",
+                "-a",
+                *[str(dist) for dist in (Path(checkout) / "Doc" / "dist").glob("*")],
+                os.path.join(target, "archives"),
+            ]
         )
         changed.append("archives/")
         for fn in os.listdir(os.path.join(target, "archives")):
@@ -587,16 +606,14 @@ def copy_build_to_webroot(
     logging.info("%s files changed", len(changed))
     if changed and not skip_cache_invalidation:
         targets_dir = www_root
-        prefixes = shell_out(["find", "-L", targets_dir, "-samefile", target])
+        prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout
         prefixes = prefixes.replace(targets_dir + "/", "")
         prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix]
         to_purge = prefixes[:]
         for prefix in prefixes:
             to_purge.extend(prefix + p for p in changed)
         logging.info("Running CDN purge")
-        shell_out(
-            ["curl", "-XPURGE", "https://docs.python.org/{%s}" % ",".join(to_purge)]
-        )
+        run(["curl", "-XPURGE", "https://docs.python.org/{%s}" % ",".join(to_purge)])
     logging.info(
         "Publishing done for version: %s, language: %s", version.name, language.tag
     )

requirements.txt

@@ -5,5 +5,7 @@
 #    pip-compile requirements.in
 #
 certifi==2020.6.20        # via sentry-sdk
+jinja2==2.11.3            # via -r requirements.in
+markupsafe==1.1.1         # via jinja2
 sentry-sdk==0.15.1        # via -r requirements.in
 urllib3==1.25.9           # via sentry-sdk

templates/switchers.js

@@ -26,7 +26,7 @@
   }
 
   function build_version_select(release) {
-    var buf = ['<select>'];
+    var buf = ['<select id="version_select">'];
     var major_minor = release.split(".").slice(0, 2).join(".");
 
     $.each(all_versions, function(version, title) {
@@ -41,7 +41,7 @@
   }
 
   function build_language_select(current_language) {
-    var buf = ['<select>'];
+    var buf = ['<select id="language_select">'];
 
     $.each(all_languages, function(language, title) {
       if (language == current_language)