From 1e7c64617412a4fa1588d013592a8dec8f77b9f8 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 13 Apr 2021 10:08:31 +0200 Subject: [PATCH 1/9] pip-compile requirements.in --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index dbe8732..19d5e46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,7 @@ # pip-compile requirements.in # certifi==2020.6.20 # via sentry-sdk +jinja2==2.11.3 # via -r requirements.in +markupsafe==1.1.1 # via jinja2 sentry-sdk==0.15.1 # via -r requirements.in urllib3==1.25.9 # via sentry-sdk From 0fffa546746c0421737f5781b2464edb89831418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 4 May 2021 08:52:53 +0200 Subject: [PATCH 2/9] Mark 3.8 as "security-fixes" (#105) Per PEP 569 today's release of 3.8.10 is the last regular bugfix release. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 3edaa90..dc7d6f0 100755 --- a/build_docs.py +++ b/build_docs.py @@ -100,7 +100,7 @@ def title(self): Version("3.5", "3.5", "EOL", sphinx_version="1.8.4"), Version("3.6", "3.6", "security-fixes", sphinx_version="2.3.1"), Version("3.7", "3.7", "security-fixes", sphinx_version="2.3.1"), - Version("3.8", "3.8", "stable", sphinx_version="2.4.4"), + Version("3.8", "3.8", "security-fixes", sphinx_version="2.4.4"), Version("3.9", "3.9", "stable", sphinx_version="2.4.4"), Version("3.10", "master", "in development", sphinx_version="3.2.1"), ] From 551cd8b3c99763a790f561e5cc67f53dc84ac025 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 4 May 2021 08:53:55 +0200 Subject: [PATCH 3/9] Stop a build on the first failing command. (#104) This avoid the following commands to fail too. This way the root cause is easier to spot, as it's the only error being logged. --- build_docs.py | 126 ++++++++++++++++++++++++-------------------------- 1 file changed, 61 insertions(+), 65 deletions(-) diff --git a/build_docs.py b/build_docs.py index dc7d6f0..7b4e785 100755 --- a/build_docs.py +++ b/build_docs.py @@ -150,34 +150,29 @@ def title(self): } -def shell_out(cmd, shell=False): - cmdstring = cmd if shell else shlex.join(cmd) - logging.debug("Running command: %s", cmdstring) - try: - output = subprocess.check_output( - cmd, - shell=shell, - stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding="utf-8", - errors="backslashreplace", +def run(cmd) -> subprocess.CompletedProcess: + """Like subprocess.run, with logging before and after the command execution.""" + cmdstring = shlex.join(cmd) + logging.debug("Run: %r", cmdstring) + result = subprocess.run( + cmd, + stdin=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + encoding="utf-8", + errors="backslashreplace", + ) + if result.returncode: + # Log last 20 lines, those are likely the interesting ones. + logging.error( + "Run KO: %r:\n%s", + cmdstring, + indent("\n".join(result.stdout.split("\n")[-20:]), " "), ) - if output: - logging.debug( - "Command executed successfully: %s\n%s", - cmdstring, - indent(output, " "), - ) - return output - except subprocess.CalledProcessError as e: - if sentry_sdk: - with sentry_sdk.push_scope() as scope: - scope.fingerprint = ["{{ default }}", str(cmd)] - sentry_sdk.capture_exception(e) - if e.output: - logging.error("Command %s failed:\n%s", cmdstring, indent(e.output, " ")) - else: - logging.error("Command %s failed.", cmdstring) + else: + logging.debug("Run OK: %r", cmdstring) + result.check_returncode() + return result def changed_files(left, right): @@ -207,20 +202,18 @@ def git_clone(repository, directory, branch=None): try: if not os.path.isdir(os.path.join(directory, ".git")): raise AssertionError("Not a git repository.") - shell_out(["git", "-C", directory, "fetch"]) + run(["git", "-C", directory, "fetch"]) if branch: - shell_out(["git", "-C", directory, "checkout", branch]) - shell_out(["git", "-C", directory, "reset", "--hard", "origin/" + branch]) + run(["git", "-C", directory, "checkout", branch]) + run(["git", "-C", directory, "reset", "--hard", "origin/" + branch]) except (subprocess.CalledProcessError, AssertionError): if os.path.exists(directory): shutil.rmtree(directory) logging.info("Cloning %s into %s", repository, directory) os.makedirs(directory, mode=0o775) - shell_out( - ["git", "clone", "--depth=1", "--no-single-branch", repository, directory] - ) + run(["git", "clone", "--depth=1", "--no-single-branch", repository, directory]) if branch: - shell_out(["git", "-C", directory, "checkout", branch]) + run(["git", "-C", directory, "checkout", branch]) def version_to_tuple(version): @@ -271,7 +264,7 @@ def translation_branch(locale_repo, locale_clone_dir, needed_version): returns the name of the nearest existing branch. """ git_clone(locale_repo, locale_clone_dir) - remote_branches = shell_out(["git", "-C", locale_clone_dir, "branch", "-r"]) + remote_branches = run(["git", "-C", locale_clone_dir, "branch", "-r"]).stdout branches = [] for branch in remote_branches.split("\n"): if re.match(r".*/[0-9]+\.[0-9]+$", branch): @@ -423,7 +416,7 @@ def build_one( sphinxbuild = os.path.join(venv, "bin/sphinx-build") blurb = os.path.join(venv, "bin/blurb") # Disable cpython switchers, we handle them now: - shell_out( + run( [ "sed", "-i", @@ -434,7 +427,7 @@ def build_one( setup_indexsidebar( os.path.join(checkout, "Doc", "tools", "templates", "indexsidebar.html") ) - shell_out( + run( [ "make", "-C", @@ -448,7 +441,7 @@ def build_one( maketarget, ] ) - shell_out(["chgrp", "-R", group, log_directory]) + run(["chgrp", "-R", group, log_directory]) setup_switchers(os.path.join(checkout, "Doc", "build", "html")) logging.info("Build done for version: %s, language: %s", version.name, language.tag) @@ -464,8 +457,8 @@ def build_venv(build_root, version, theme): "sphinx=={}".format(version.sphinx_version), ] venv_path = os.path.join(build_root, "venv-with-sphinx-" + version.sphinx_version) - shell_out(["python3", "-m", "venv", venv_path]) - shell_out( + run(["python3", "-m", "venv", venv_path]) + run( [os.path.join(venv_path, "bin", "python"), "-m", "pip", "install"] + requirements ) @@ -473,6 +466,9 @@ def build_venv(build_root, version, theme): def build_robots_txt(www_root, group, skip_cache_invalidation): + if not Path(www_root).exists(): + logging.info("Skipping robots.txt generation (www root does not even exists).") + return robots_file = os.path.join(www_root, "robots.txt") with open(HERE / "templates" / "robots.txt") as robots_txt_template_file: with open(robots_file, "w") as robots_txt_file: @@ -481,12 +477,15 @@ def build_robots_txt(www_root, group, skip_cache_invalidation): template.render(languages=LANGUAGES, versions=VERSIONS) + "\n" ) os.chmod(robots_file, 0o775) - shell_out(["chgrp", group, robots_file]) + run(["chgrp", group, robots_file]) if not skip_cache_invalidation: - shell_out(["curl", "-XPURGE", "https://docs.python.org/robots.txt"]) + run(["curl", "-XPURGE", "https://docs.python.org/robots.txt"]) def build_sitemap(www_root): + if not Path(www_root).exists(): + logging.info("Skipping sitemap generation (www root does not even exists).") + return with open(HERE / "templates" / "sitemap.xml") as sitemap_template_file: with open(os.path.join(www_root, "sitemap.xml"), "w") as sitemap_file: template = jinja2.Template(sitemap_template_file.read()) @@ -518,7 +517,7 @@ def copy_build_to_webroot( language_dir = os.path.join(www_root, language.tag) os.makedirs(language_dir, exist_ok=True) try: - shell_out(["chgrp", "-R", group, language_dir]) + run(["chgrp", "-R", group, language_dir]) except subprocess.CalledProcessError as err: logging.warning("Can't change group of %s: %s", language_dir, str(err)) os.chmod(language_dir, 0o775) @@ -530,15 +529,15 @@ def copy_build_to_webroot( except PermissionError as err: logging.warning("Can't change mod of %s: %s", target, str(err)) try: - shell_out(["chgrp", "-R", group, target]) + run(["chgrp", "-R", group, target]) except subprocess.CalledProcessError as err: logging.warning("Can't change group of %s: %s", target, str(err)) changed = changed_files(os.path.join(checkout, "Doc/build/html"), target) logging.info("Copying HTML files to %s", target) - shell_out(["chown", "-R", ":" + group, os.path.join(checkout, "Doc/build/html/")]) - shell_out(["chmod", "-R", "o+r", os.path.join(checkout, "Doc/build/html/")]) - shell_out( + run(["chown", "-R", ":" + group, os.path.join(checkout, "Doc/build/html/")]) + run(["chmod", "-R", "o+r", os.path.join(checkout, "Doc/build/html/")]) + run( [ "find", os.path.join(checkout, "Doc/build/html/"), @@ -552,9 +551,9 @@ def copy_build_to_webroot( ] ) if quick: - shell_out(["rsync", "-a", os.path.join(checkout, "Doc/build/html/"), target]) + run(["rsync", "-a", os.path.join(checkout, "Doc/build/html/"), target]) else: - shell_out( + run( [ "rsync", "-a", @@ -567,18 +566,17 @@ def copy_build_to_webroot( ) if not quick: logging.debug("Copying dist files") - shell_out(["chown", "-R", ":" + group, os.path.join(checkout, "Doc/dist/")]) - shell_out( - ["chmod", "-R", "o+r", os.path.join(checkout, os.path.join("Doc/dist/"))] - ) - shell_out(["mkdir", "-m", "o+rx", "-p", os.path.join(target, "archives")]) - shell_out(["chown", ":" + group, os.path.join(target, "archives")]) - shell_out( - "cp -a {src} {dst}".format( - src=os.path.join(checkout, "Doc/dist/*"), - dst=os.path.join(target, "archives"), - ), - shell=True, + run(["chown", "-R", ":" + group, os.path.join(checkout, "Doc/dist/")]) + run(["chmod", "-R", "o+r", os.path.join(checkout, os.path.join("Doc/dist/"))]) + run(["mkdir", "-m", "o+rx", "-p", os.path.join(target, "archives")]) + run(["chown", ":" + group, os.path.join(target, "archives")]) + run( + [ + "cp", + "-a", + *[str(dist) for dist in (Path(checkout) / "Doc" / "dist").glob("*")], + os.path.join(target, "archives"), + ] ) changed.append("archives/") for fn in os.listdir(os.path.join(target, "archives")): @@ -587,16 +585,14 @@ def copy_build_to_webroot( logging.info("%s files changed", len(changed)) if changed and not skip_cache_invalidation: targets_dir = www_root - prefixes = shell_out(["find", "-L", targets_dir, "-samefile", target]) + prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout prefixes = prefixes.replace(targets_dir + "/", "") prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix] to_purge = prefixes[:] for prefix in prefixes: to_purge.extend(prefix + p for p in changed) logging.info("Running CDN purge") - shell_out( - ["curl", "-XPURGE", "https://docs.python.org/{%s}" % ",".join(to_purge)] - ) + run(["curl", "-XPURGE", "https://docs.python.org/{%s}" % ",".join(to_purge)]) logging.info( "Publishing done for version: %s, language: %s", version.name, language.tag ) From 2ccb988702576940c49eb618f48c2506930ac201 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 4 May 2021 08:54:49 +0200 Subject: [PATCH 4/9] Prepare docsbuild-scripts for the cpython migration to the main branch. (#103) --- README.md | 4 ++-- build_docs.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5816469..07937ca 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ translations in ``./www``, beware it can take a few hours: $ python3 ./build_docs.py --quick --build-root ./build_root --www-root ./www --log-directory ./logs --group $(id -g) --skip-cache-invalidation If you don't need to build all translations of all branches, add -``--language en --branch master``. +``--language en --branch main``. # Check current version @@ -32,7 +32,7 @@ of Sphinx we're using where:: 3.7 sphinx==1.8.2 sphinx==1.8.2 ø needs_sphinx="1.6.6" Sphinx==2.3.1 Sphinx==2.3.1 3.8 sphinx==1.8.2 sphinx==1.8.2 ø needs_sphinx='1.8' Sphinx==2.3.1 Sphinx==2.3.1 3.9 sphinx==2.2.0 sphinx==2.2.0 sphinx==2.2.0 needs_sphinx='1.8' Sphinx==2.3.1 Sphinx==2.3.1 - master sphinx==2.2.0 sphinx==2.2.0 sphinx==2.2.0 needs_sphinx='1.8' Sphinx==2.3.1 Sphinx==2.3.1 + main sphinx==2.2.0 sphinx==2.2.0 sphinx==2.2.0 needs_sphinx='1.8' Sphinx==2.3.1 Sphinx==2.3.1 ======== ============= ============= ================== ==================== ============= =============== Sphinx build as seen on docs.python.org: diff --git a/build_docs.py b/build_docs.py index 7b4e785..be4e5b9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -102,7 +102,7 @@ def title(self): Version("3.7", "3.7", "security-fixes", sphinx_version="2.3.1"), Version("3.8", "3.8", "security-fixes", sphinx_version="2.4.4"), Version("3.9", "3.9", "stable", sphinx_version="2.4.4"), - Version("3.10", "master", "in development", sphinx_version="3.2.1"), + Version("3.10", "main", "in development", sphinx_version="3.2.1"), ] XELATEX_DEFAULT = ( From 4f620e14e1286f162f41fc0515d9410964c1ba88 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 4 May 2021 09:34:58 +0200 Subject: [PATCH 5/9] Avoid flooding the logs with curl progress meter. --- build_docs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index be4e5b9..0e90405 100755 --- a/build_docs.py +++ b/build_docs.py @@ -479,7 +479,14 @@ def build_robots_txt(www_root, group, skip_cache_invalidation): os.chmod(robots_file, 0o775) run(["chgrp", group, robots_file]) if not skip_cache_invalidation: - run(["curl", "-XPURGE", "https://docs.python.org/robots.txt"]) + run( + [ + "curl", + "--silent", + "-XPURGE", + "https://docs.python.org/robots.txt", + ] + ) def build_sitemap(www_root): From 7a073488e03f852a3fefa345efa5b1c09da17135 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Tue, 4 May 2021 10:38:30 +0200 Subject: [PATCH 6/9] Since Python 3.10 (a103e73ce8) we can build in parallel. --- build_docs.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0e90405..4b77559 100755 --- a/build_docs.py +++ b/build_docs.py @@ -64,7 +64,14 @@ class Version: STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"} - def __init__(self, name, branch, status, sphinx_version=DEFAULT_SPHINX_VERSION): + def __init__( + self, + name, + branch, + status, + sphinx_version=DEFAULT_SPHINX_VERSION, + sphinxopts=[], + ): if status not in self.STATUSES: raise ValueError( "Version status expected to be in {}".format(", ".join(self.STATUSES)) @@ -73,6 +80,7 @@ def __init__(self, name, branch, status, sphinx_version=DEFAULT_SPHINX_VERSION): self.branch = branch self.status = status self.sphinx_version = sphinx_version + self.sphinxopts = list(sphinxopts) @property def changefreq(self): @@ -102,7 +110,9 @@ def title(self): Version("3.7", "3.7", "security-fixes", sphinx_version="2.3.1"), Version("3.8", "3.8", "security-fixes", sphinx_version="2.4.4"), Version("3.9", "3.9", "stable", sphinx_version="2.4.4"), - Version("3.10", "main", "in development", sphinx_version="3.2.1"), + Version( + "3.10", "main", "in development", sphinx_version="3.2.1", sphinxopts=["-j4"] + ), ] XELATEX_DEFAULT = ( @@ -383,7 +393,7 @@ def build_one( logging.info( "Build start for version: %s, language: %s", version.name, language.tag ) - sphinxopts = list(language.sphinxopts) + sphinxopts = list(language.sphinxopts) + list(version.sphinxopts) sphinxopts.extend(["-q"]) if language.tag != "en": locale_dirs = os.path.join(build_root, version.name, "locale") From 6a8c80a3e83f4de9bfbce913c0f4627cba88c48b Mon Sep 17 00:00:00 2001 From: Mindiell <61205582+Mindiell@users.noreply.github.com> Date: Wed, 5 May 2021 08:53:33 +0200 Subject: [PATCH 7/9] switchers select tags should have a correct id (#109) --- templates/switchers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index df7a621..7a46ea7 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -26,7 +26,7 @@ } function build_version_select(release) { - var buf = ['']; var major_minor = release.split(".").slice(0, 2).join("."); $.each(all_versions, function(version, title) { @@ -41,7 +41,7 @@ } function build_language_select(current_language) { - var buf = ['']; $.each(all_languages, function(language, title) { if (language == current_language) From 1be708816ead2fd09ce35915fc6661d3fc9cf3ff Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 May 2021 16:29:19 +0800 Subject: [PATCH 8/9] Add 3.11 (#107) * Hello 3.10 and welcome 3.11! * bump README version --- README.md | 2 ++ build_docs.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 07937ca..c1ec571 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ of Sphinx we're using where:: 3.7 sphinx==1.8.2 sphinx==1.8.2 ø needs_sphinx="1.6.6" Sphinx==2.3.1 Sphinx==2.3.1 3.8 sphinx==1.8.2 sphinx==1.8.2 ø needs_sphinx='1.8' Sphinx==2.3.1 Sphinx==2.3.1 3.9 sphinx==2.2.0 sphinx==2.2.0 sphinx==2.2.0 needs_sphinx='1.8' Sphinx==2.3.1 Sphinx==2.3.1 + 3.10 sphinx==2.2.0 sphinx==2.2.0 sphinx==2.2.0 needs_sphinx='1.8' Sphinx==2.3.1 Sphinx==2.3.1 main sphinx==2.2.0 sphinx==2.2.0 sphinx==2.2.0 needs_sphinx='1.8' Sphinx==2.3.1 Sphinx==2.3.1 ======== ============= ============= ================== ==================== ============= =============== @@ -47,4 +48,5 @@ of Sphinx we're using where:: 3.8 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 3.9 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 3.10 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 + 3.11 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 ======== ======= ===== ======= ===== ===== ===== ======= ===== ===== diff --git a/build_docs.py b/build_docs.py index 4b77559..582b26a 100755 --- a/build_docs.py +++ b/build_docs.py @@ -111,7 +111,10 @@ def title(self): Version("3.8", "3.8", "security-fixes", sphinx_version="2.4.4"), Version("3.9", "3.9", "stable", sphinx_version="2.4.4"), Version( - "3.10", "main", "in development", sphinx_version="3.2.1", sphinxopts=["-j4"] + "3.10", "3.10", "pre-release", sphinx_version="3.2.1", sphinxopts=["-j4"] + ), + Version( + "3.11", "main", "in development", sphinx_version="3.2.1", sphinxopts=["-j4"] ), ] From d48a2714047a654e906ea4b96c45de5cc5ab606a Mon Sep 17 00:00:00 2001 From: Mindiell <61205582+Mindiell@users.noreply.github.com> Date: Wed, 5 May 2021 10:29:51 +0200 Subject: [PATCH 9/9] logs_directory is now created if necessary (#110) --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index 582b26a..bef921a 100755 --- a/build_docs.py +++ b/build_docs.py @@ -454,6 +454,7 @@ def build_one( maketarget, ] ) + run(["mkdir", "-p", log_directory]) run(["chgrp", "-R", group, log_directory]) setup_switchers(os.path.join(checkout, "Doc", "build", "html")) logging.info("Build done for version: %s, language: %s", version.name, language.tag)