From 36f8c1ebf61f5d3c970566e2da0a273ff29ca3d2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:29:32 +0000 Subject: [PATCH 001/102] Further simplify symlink removal (#224) Co-authored-by: Ezio Melotti --- build_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 658ba63..6596a25 100755 --- a/build_docs.py +++ b/build_docs.py @@ -998,8 +998,7 @@ def symlink( if not link.exists() or readlink(link) != directory: # Link does not exist or points to the wrong target. - if link.exists(): - link.unlink() + link.unlink(missing_ok=True) link.symlink_to(directory) run(["chown", "-h", f":{group}", str(link)]) if not skip_cache_invalidation: From 1807c70aff5a050077ede04fe65c7689110bf5f1 Mon Sep 17 00:00:00 2001 From: Kerim Kabirov Date: Mon, 28 Oct 2024 16:31:05 +0100 Subject: [PATCH 002/102] Add context labels to the version switcher (#223) --- templates/switchers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 29204ae..999ca10 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -26,7 +26,7 @@ } function build_version_select(release) { - let buf = ['']; const major_minor = release.split(".").slice(0, 2).join("."); Object.entries(all_versions).forEach(function([version, title]) { @@ -42,7 +42,7 @@ } function build_language_select(current_language) { - let buf = ['']; Object.entries(all_languages).forEach(function([language, title]) { if (language === current_language) { From 7e274eb87a36acf9ca35e3f1671ee4eb2d87d9f3 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:47:52 +0000 Subject: [PATCH 003/102] Remove String.startsWith polyfill --- templates/switchers.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 999ca10..ad31a03 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,15 +1,6 @@ (function() { 'use strict'; - if (!String.prototype.startsWith) { - Object.defineProperty(String.prototype, 'startsWith', { - value: function(search, rawPos) { - const pos = rawPos > 0 ? rawPos|0 : 0; - return this.substring(pos, pos + search.length) === search; - } - }); - } - // Parses versions in URL segments like: // "3", "dev", "release/2.7" or "3.6rc2" const version_regexs = [ From 27e2dc7fbf61c48bd58c79152b5a0480c2d02ef1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:53:09 +0000 Subject: [PATCH 004/102] Remove create_placeholders_if_missing() The placeholders have been in the theme since v2021.5 --- templates/switchers.js | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index ad31a03..ffedb7d 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -132,41 +132,11 @@ return '' } - function create_placeholders_if_missing() { - const version_segment = version_segment_from_url(); - const language_segment = language_segment_from_url(); - const index = "/" + language_segment + version_segment; - - if (document.querySelectorAll('.version_switcher_placeholder').length > 0) { - return; - } - - const html = ' \ - \ -Documentation »'; - - const probable_places = [ - "body>div.related>ul>li:not(.right):contains('Documentation'):first", - "body>div.related>ul>li:not(.right):contains('documentation'):first", - ]; - - for (let i = 0; i < probable_places.length; i++) { - let probable_place = $(probable_places[i]); - if (probable_place.length == 1) { - probable_place.html(html); - document.getElementById('indexlink').href = index; - return; - } - } - } - document.addEventListener('DOMContentLoaded', function() { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - create_placeholders_if_missing(); - let placeholders = document.querySelectorAll('.version_switcher_placeholder'); placeholders.forEach(function(placeholder) { placeholder.innerHTML = version_select; From 75594be5806eb6ed91aecac46a9a4036f734dc02 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:54:10 +0000 Subject: [PATCH 005/102] Use exact comparison operators --- templates/switchers.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index ffedb7d..664db12 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -55,7 +55,7 @@ function navigate_to_first_existing(urls) { // Navigate to the first existing URL in urls. const url = urls.shift(); - if (urls.length == 0 || url.startsWith("file:///")) { + if (urls.length === 0 || url.startsWith("file:///")) { window.location.href = url; return; } @@ -79,7 +79,7 @@ const current_version = version_segment_from_url(); const new_url = url.replace('/' + current_language + current_version, '/' + current_language + selected_version); - if (new_url != url) { + if (new_url !== url) { navigate_to_first_existing([ new_url, url.replace('/' + current_language + current_version, @@ -96,11 +96,11 @@ const url = window.location.href; const current_language = language_segment_from_url(); const current_version = version_segment_from_url(); - if (selected_language == 'en/') // Special 'default' case for English. + if (selected_language === 'en/') // Special 'default' case for English. selected_language = ''; let new_url = url.replace('/' + current_language + current_version, '/' + selected_language + current_version); - if (new_url != url) { + if (new_url !== url) { navigate_to_first_existing([ new_url, '/' From 8a71e4273a287deb62e8ed84cc9d837a1a13a0f4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:56:33 +0000 Subject: [PATCH 006/102] Remove an unneeded variable --- templates/switchers.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 664db12..70997e1 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -135,10 +135,9 @@ document.addEventListener('DOMContentLoaded', function() { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - let placeholders = document.querySelectorAll('.version_switcher_placeholder'); - placeholders.forEach(function(placeholder) { + const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); + document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { placeholder.innerHTML = version_select; let selectElement = placeholder.querySelector('select'); @@ -146,9 +145,7 @@ }); const language_select = build_language_select(current_language); - - placeholders = document.querySelectorAll('.language_switcher_placeholder'); - placeholders.forEach(function(placeholder) { + document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { placeholder.innerHTML = language_select; let selectElement = placeholder.querySelector('select'); From ce8b4742aa957612ce6734b78bdf6e242a6bd43c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:57:55 +0000 Subject: [PATCH 007/102] Remove anonymous function (IIFE) --- templates/switchers.js | 284 ++++++++++++++++++++--------------------- 1 file changed, 141 insertions(+), 143 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 70997e1..d436d5e 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,155 +1,153 @@ -(function() { - 'use strict'; - - // Parses versions in URL segments like: - // "3", "dev", "release/2.7" or "3.6rc2" - const version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; - - const all_versions = $VERSIONS; - const all_languages = $LANGUAGES; - - function quote_attr(str) { - return '"' + str.replace('"', '\\"') + '"'; - } +'use strict'; + +// Parses versions in URL segments like: +// "3", "dev", "release/2.7" or "3.6rc2" +const version_regexs = [ + '(?:\\d)', + '(?:\\d\\.\\d[\\w\\d\\.]*)', + '(?:dev)', + '(?:release/\\d.\\d[\\x\\d\\.]*)']; + +const all_versions = $VERSIONS; +const all_languages = $LANGUAGES; + +function quote_attr(str) { + return '"' + str.replace('"', '\\"') + '"'; +} + +function build_version_select(release) { + let buf = ['']; - const major_minor = release.split(".").slice(0, 2).join("."); + buf.push(''); + return buf.join(''); +} - Object.entries(all_versions).forEach(function([version, title]) { - if (version === major_minor) { - buf.push(''); - } else { - buf.push(''); - } - }); +function build_language_select(current_language) { + let buf = [''); - return buf.join(''); + Object.entries(all_languages).forEach(function([language, title]) { + if (language === current_language) { + buf.push(''); + } else { + buf.push(''); + } + }); + if (!(current_language in all_languages)) { + // In case we're browsing a language that is not yet in all_languages. + buf.push(''); + all_languages[current_language] = current_language; } - - function build_language_select(current_language) { - let buf = [''); + return buf.join(''); +} + +function navigate_to_first_existing(urls) { + // Navigate to the first existing URL in urls. + const url = urls.shift(); + if (urls.length === 0 || url.startsWith("file:///")) { + window.location.href = url; + return; + } + fetch(url) + .then(function(response) { + if (response.ok) { + window.location.href = url; } else { - buf.push(''); + navigate_to_first_existing(urls); } + }) + .catch(function(error) { + navigate_to_first_existing(urls); }); - if (!(current_language in all_languages)) { - // In case we're browsing a language that is not yet in all_languages. - buf.push(''); - all_languages[current_language] = current_language; - } - buf.push(''); - return buf.join(''); - } - - function navigate_to_first_existing(urls) { - // Navigate to the first existing URL in urls. - const url = urls.shift(); - if (urls.length === 0 || url.startsWith("file:///")) { - window.location.href = url; - return; - } - fetch(url) - .then(function(response) { - if (response.ok) { - window.location.href = url; - } else { - navigate_to_first_existing(urls); - } - }) - .catch(function(error) { - navigate_to_first_existing(urls); - }); - } - - function on_version_switch() { - const selected_version = this.options[this.selectedIndex].value + '/'; - const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); - const new_url = url.replace('/' + current_language + current_version, - '/' + current_language + selected_version); - if (new_url !== url) { - navigate_to_first_existing([ - new_url, - url.replace('/' + current_language + current_version, - '/' + selected_version), - '/' + current_language + selected_version, - '/' + selected_version, - '/' - ]); - } - } - - function on_language_switch() { - let selected_language = this.options[this.selectedIndex].value + '/'; - const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); - if (selected_language === 'en/') // Special 'default' case for English. - selected_language = ''; - let new_url = url.replace('/' + current_language + current_version, - '/' + selected_language + current_version); - if (new_url !== url) { - navigate_to_first_existing([ - new_url, - '/' - ]); - } - } - - // Returns the path segment of the language as a string, like 'fr/' - // or '' if not found. - function language_segment_from_url() { - const path = window.location.pathname; - const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' - const match = path.match(language_regexp); - if (match !== null) - return match[1]; - return ''; +} + +function on_version_switch() { + const selected_version = this.options[this.selectedIndex].value + '/'; + const url = window.location.href; + const current_language = language_segment_from_url(); + const current_version = version_segment_from_url(); + const new_url = url.replace('/' + current_language + current_version, + '/' + current_language + selected_version); + if (new_url !== url) { + navigate_to_first_existing([ + new_url, + url.replace('/' + current_language + current_version, + '/' + selected_version), + '/' + current_language + selected_version, + '/' + selected_version, + '/' + ]); } - - // Returns the path segment of the version as a string, like '3.6/' - // or '' if not found. - function version_segment_from_url() { - const path = window.location.pathname; - const language_segment = language_segment_from_url(); - const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - const version_regexp = language_segment + '(' + version_segment + ')'; - const match = path.match(version_regexp); - if (match !== null) - return match[1]; - return '' +} + +function on_language_switch() { + let selected_language = this.options[this.selectedIndex].value + '/'; + const url = window.location.href; + const current_language = language_segment_from_url(); + const current_version = version_segment_from_url(); + if (selected_language === 'en/') // Special 'default' case for English. + selected_language = ''; + let new_url = url.replace('/' + current_language + current_version, + '/' + selected_language + current_version); + if (new_url !== url) { + navigate_to_first_existing([ + new_url, + '/' + ]); } +} + +// Returns the path segment of the language as a string, like 'fr/' +// or '' if not found. +function language_segment_from_url() { + const path = window.location.pathname; + const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' + const match = path.match(language_regexp); + if (match !== null) + return match[1]; + return ''; +} + +// Returns the path segment of the version as a string, like '3.6/' +// or '' if not found. +function version_segment_from_url() { + const path = window.location.pathname; + const language_segment = language_segment_from_url(); + const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; + const version_regexp = language_segment + '(' + version_segment + ')'; + const match = path.match(version_regexp); + if (match !== null) + return match[1]; + return '' +} + +document.addEventListener('DOMContentLoaded', function() { + const language_segment = language_segment_from_url(); + const current_language = language_segment.replace(/\/+$/g, '') || 'en'; + + const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); + document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { + placeholder.innerHTML = version_select; + + let selectElement = placeholder.querySelector('select'); + selectElement.addEventListener('change', on_version_switch); + }); - document.addEventListener('DOMContentLoaded', function() { - const language_segment = language_segment_from_url(); - const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - - const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = version_select; - - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_version_switch); - }); - - const language_select = build_language_select(current_language); - document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = language_select; + const language_select = build_language_select(current_language); + document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { + placeholder.innerHTML = language_select; - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_language_switch); - }); + let selectElement = placeholder.querySelector('select'); + selectElement.addEventListener('change', on_language_switch); }); -})(); +}); From cdcd52305ee0b7b6d88f2cf859ae5d2f37e2ddd3 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:59:28 +0000 Subject: [PATCH 008/102] Name the initialisation function --- templates/switchers.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index d436d5e..8bdf3d5 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -130,8 +130,7 @@ function version_segment_from_url() { return match[1]; return '' } - -document.addEventListener('DOMContentLoaded', function() { +const _initialise_switchers = () => { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; @@ -150,4 +149,10 @@ document.addEventListener('DOMContentLoaded', function() { let selectElement = placeholder.querySelector('select'); selectElement.addEventListener('change', on_language_switch); }); -}); +}; + +if (document.readyState !== 'loading') { + _initialise_switchers(); +} else { + document.addEventListener('DOMContentLoaded', _initialise_switchers); +} From 84a3f9c5d8794fa20e738ddf195d2f6c0abb36f7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:09:38 +0000 Subject: [PATCH 009/102] Construct DOM nodes instead of parsing HTML --- templates/switchers.js | 90 ++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 8bdf3d5..094c2ce 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -15,41 +15,45 @@ function quote_attr(str) { return '"' + str.replace('"', '\\"') + '"'; } -function build_version_select(release) { - let buf = [''); - return buf.join(''); -} + select.add(option); + } -function build_language_select(current_language) { - let buf = [''); - return buf.join(''); -} + + const select = document.createElement('select'); + select.className = 'language-select'; + + for (const [language, title] in all_languages) { + const option = document.createElement('option'); + option.value = language; + option.text = title; + if (language === current_language) option.selected = true; + select.add(option); + } + + return select; +}; function navigate_to_first_existing(urls) { // Navigate to the first existing URL in urls. @@ -134,21 +138,23 @@ const _initialise_switchers = () => { const language_segment = language_segment_from_url(); const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - const version_select = build_version_select(DOCUMENTATION_OPTIONS.VERSION); - document.querySelectorAll('.version_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = version_select; - - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_version_switch); - }); - - const language_select = build_language_select(current_language); - document.querySelectorAll('.language_switcher_placeholder').forEach((placeholder) => { - placeholder.innerHTML = language_select; + const version_select = _create_version_select(DOCUMENTATION_OPTIONS.VERSION); + document + .querySelectorAll('.version_switcher_placeholder') + .forEach((placeholder) => { + const s = version_select.cloneNode(true); + s.addEventListener('change', on_version_switch); + placeholder.append(s); + }); - let selectElement = placeholder.querySelector('select'); - selectElement.addEventListener('change', on_language_switch); - }); + const language_select = _create_language_select(current_language); + document + .querySelectorAll('.language_switcher_placeholder') + .forEach((placeholder) => { + const s = language_select.cloneNode(true); + s.addEventListener('change', on_language_switch); + placeholder.append(s); + }); }; if (document.readyState !== 'loading') { From 2a5adf99c1af5d8b2f1874c696651ca72c2a0518 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:18:22 +0000 Subject: [PATCH 010/102] Use arrow functions --- templates/switchers.js | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 094c2ce..b5b46ba 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -55,7 +55,7 @@ const _create_language_select = (current_language) => { return select; }; -function navigate_to_first_existing(urls) { +const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. const url = urls.shift(); if (urls.length === 0 || url.startsWith("file:///")) { @@ -63,19 +63,20 @@ function navigate_to_first_existing(urls) { return; } fetch(url) - .then(function(response) { + .then((response) => { if (response.ok) { window.location.href = url; } else { navigate_to_first_existing(urls); } }) - .catch(function(error) { + .catch((err) => { + void err; navigate_to_first_existing(urls); }); -} +}; -function on_version_switch() { +const _on_version_switch = () => { const selected_version = this.options[this.selectedIndex].value + '/'; const url = window.location.href; const current_language = language_segment_from_url(); @@ -83,7 +84,7 @@ function on_version_switch() { const new_url = url.replace('/' + current_language + current_version, '/' + current_language + selected_version); if (new_url !== url) { - navigate_to_first_existing([ + _navigate_to_first_existing([ new_url, url.replace('/' + current_language + current_version, '/' + selected_version), @@ -92,9 +93,9 @@ function on_version_switch() { '/' ]); } -} +}; -function on_language_switch() { +const _on_language_switch = () => { let selected_language = this.options[this.selectedIndex].value + '/'; const url = window.location.href; const current_language = language_segment_from_url(); @@ -104,12 +105,12 @@ function on_language_switch() { let new_url = url.replace('/' + current_language + current_version, '/' + selected_language + current_version); if (new_url !== url) { - navigate_to_first_existing([ + _navigate_to_first_existing([ new_url, '/' ]); } -} +}; // Returns the path segment of the language as a string, like 'fr/' // or '' if not found. @@ -143,7 +144,7 @@ const _initialise_switchers = () => { .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { const s = version_select.cloneNode(true); - s.addEventListener('change', on_version_switch); + s.addEventListener('change', _on_version_switch); placeholder.append(s); }); @@ -152,7 +153,7 @@ const _initialise_switchers = () => { .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { const s = language_select.cloneNode(true); - s.addEventListener('change', on_language_switch); + s.addEventListener('change', _on_language_switch); placeholder.append(s); }); }; From b4b36c0983a8693cdfdd743268630678c737b158 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:18:37 +0000 Subject: [PATCH 011/102] Remove unused quote_attr() function --- templates/switchers.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index b5b46ba..d70fc8f 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -11,10 +11,6 @@ const version_regexs = [ const all_versions = $VERSIONS; const all_languages = $LANGUAGES; -function quote_attr(str) { - return '"' + str.replace('"', '\\"') + '"'; -} - const _create_version_select = (release) => { const major_minor = release.split('.').slice(0, 2).join('.'); const select = document.createElement('select'); From 2cac9f627ffc927ade8c09b3b0eec45181ff7c46 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:25:08 +0000 Subject: [PATCH 012/102] Run prettier --- templates/switchers.js | 44 +++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index d70fc8f..1b7a71d 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -6,7 +6,8 @@ const version_regexs = [ '(?:\\d)', '(?:\\d\\.\\d[\\w\\d\\.]*)', '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; + '(?:release/\\d.\\d[\\x\\d\\.]*)', +]; const all_versions = $VERSIONS; const all_languages = $LANGUAGES; @@ -54,7 +55,7 @@ const _create_language_select = (current_language) => { const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. const url = urls.shift(); - if (urls.length === 0 || url.startsWith("file:///")) { + if (urls.length === 0 || url.startsWith('file:///')) { window.location.href = url; return; } @@ -77,16 +78,20 @@ const _on_version_switch = () => { const url = window.location.href; const current_language = language_segment_from_url(); const current_version = version_segment_from_url(); - const new_url = url.replace('/' + current_language + current_version, - '/' + current_language + selected_version); + const new_url = url.replace( + '/' + current_language + current_version, + '/' + current_language + selected_version, + ); if (new_url !== url) { _navigate_to_first_existing([ new_url, - url.replace('/' + current_language + current_version, - '/' + selected_version), + url.replace( + '/' + current_language + current_version, + '/' + selected_version, + ), '/' + current_language + selected_version, '/' + selected_version, - '/' + '/', ]); } }; @@ -96,15 +101,15 @@ const _on_language_switch = () => { const url = window.location.href; const current_language = language_segment_from_url(); const current_version = version_segment_from_url(); - if (selected_language === 'en/') // Special 'default' case for English. + if (selected_language === 'en/') + // Special 'default' case for English. selected_language = ''; - let new_url = url.replace('/' + current_language + current_version, - '/' + selected_language + current_version); + let new_url = url.replace( + '/' + current_language + current_version, + '/' + selected_language + current_version, + ); if (new_url !== url) { - _navigate_to_first_existing([ - new_url, - '/' - ]); + _navigate_to_first_existing([new_url, '/']); } }; @@ -112,10 +117,10 @@ const _on_language_switch = () => { // or '' if not found. function language_segment_from_url() { const path = window.location.pathname; - const language_regexp = '/((?:' + Object.keys(all_languages).join("|") + ')/)' + const language_regexp = + '/((?:' + Object.keys(all_languages).join('|') + ')/)'; const match = path.match(language_regexp); - if (match !== null) - return match[1]; + if (match !== null) return match[1]; return ''; } @@ -127,9 +132,8 @@ function version_segment_from_url() { const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; const version_regexp = language_segment + '(' + version_segment + ')'; const match = path.match(version_regexp); - if (match !== null) - return match[1]; - return '' + if (match !== null) return match[1]; + return ''; } const _initialise_switchers = () => { const language_segment = language_segment_from_url(); From 7b5e7730954e7b788195ecddb12f8f7056b92675 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:29:30 +0000 Subject: [PATCH 013/102] Make the regular expression for version parts a constant --- templates/switchers.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 1b7a71d..6b12e32 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,13 +1,14 @@ 'use strict'; // Parses versions in URL segments like: -// "3", "dev", "release/2.7" or "3.6rc2" -const version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)', -]; +const _VERSION_PATTERN = ( + '((?:' + + '(?:\\d)' // e.g. "3" + +'|(?:\\d\\.\\d[\\w\\d\\.]*)' // e.g. "3.6rc2" + +'|(?:dev)' // e.g. "dev" + +'|(?:release/\\d.\\d[\\x\\d\\.]*)'// e.g. "release/2.7" + + ')/)' +); const all_versions = $VERSIONS; const all_languages = $LANGUAGES; @@ -128,9 +129,7 @@ function language_segment_from_url() { // or '' if not found. function version_segment_from_url() { const path = window.location.pathname; - const language_segment = language_segment_from_url(); - const version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - const version_regexp = language_segment + '(' + version_segment + ')'; + const version_regexp = language_segment_from_url() + _VERSION_PATTERN; const match = path.match(version_regexp); if (match !== null) return match[1]; return ''; From ab3b07962977ace9fea0c8c4034d53cb129872d0 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:05:32 +0000 Subject: [PATCH 014/102] Use constants from DOCUMENTATION_OPTIONS where possible --- templates/switchers.js | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 6b12e32..f98de05 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,5 +1,18 @@ 'use strict'; +const _CURRENT_VERSION = DOCUMENTATION_OPTIONS.VERSION; +const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; +const _CURRENT_PREFIX = (() => { + // Sphinx 7.2+ defines the content root data attribute in the HTML element. + const _CONTENT_ROOT = document.documentElement.dataset.content_root; + if (_CONTENT_ROOT !== undefined) { + return new URL(_CONTENT_ROOT, window.location).pathname; + } + // Fallback for older versions of Sphinx (used in Python 3.10 and older). + const _NUM_PREFIX_PARTS = _CURRENT_LANGUAGE === 'en' ? 2 : 3; + return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; +})(); + // Parses versions in URL segments like: const _VERSION_PATTERN = ( '((?:' @@ -77,20 +90,15 @@ const _navigate_to_first_existing = (urls) => { const _on_version_switch = () => { const selected_version = this.options[this.selectedIndex].value + '/'; const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); const new_url = url.replace( - '/' + current_language + current_version, - '/' + current_language + selected_version, + _CURRENT_PREFIX, + '/' + _CURRENT_LANGUAGE + selected_version, ); if (new_url !== url) { _navigate_to_first_existing([ new_url, - url.replace( - '/' + current_language + current_version, - '/' + selected_version, - ), - '/' + current_language + selected_version, + url.replace(_CURRENT_PREFIX, '/' + selected_version), + '/' + _CURRENT_LANGUAGE + selected_version, '/' + selected_version, '/', ]); @@ -100,14 +108,12 @@ const _on_version_switch = () => { const _on_language_switch = () => { let selected_language = this.options[this.selectedIndex].value + '/'; const url = window.location.href; - const current_language = language_segment_from_url(); - const current_version = version_segment_from_url(); if (selected_language === 'en/') // Special 'default' case for English. selected_language = ''; let new_url = url.replace( - '/' + current_language + current_version, - '/' + selected_language + current_version, + _CURRENT_PREFIX, + '/' + selected_language + _CURRENT_VERSION, ); if (new_url !== url) { _navigate_to_first_existing([new_url, '/']); @@ -135,10 +141,7 @@ function version_segment_from_url() { return ''; } const _initialise_switchers = () => { - const language_segment = language_segment_from_url(); - const current_language = language_segment.replace(/\/+$/g, '') || 'en'; - - const version_select = _create_version_select(DOCUMENTATION_OPTIONS.VERSION); + const version_select = _create_version_select(_CURRENT_VERSION); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { @@ -147,7 +150,7 @@ const _initialise_switchers = () => { placeholder.append(s); }); - const language_select = _create_language_select(current_language); + const language_select = _create_language_select(_CURRENT_LANGUAGE); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { From a5936462d9811cd843ddb9337208272d996a098d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:06:10 +0000 Subject: [PATCH 015/102] Remove now-unused segment_from_url() functions --- templates/switchers.js | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index f98de05..600c955 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -13,16 +13,6 @@ const _CURRENT_PREFIX = (() => { return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; })(); -// Parses versions in URL segments like: -const _VERSION_PATTERN = ( - '((?:' - + '(?:\\d)' // e.g. "3" - +'|(?:\\d\\.\\d[\\w\\d\\.]*)' // e.g. "3.6rc2" - +'|(?:dev)' // e.g. "dev" - +'|(?:release/\\d.\\d[\\x\\d\\.]*)'// e.g. "release/2.7" - + ')/)' -); - const all_versions = $VERSIONS; const all_languages = $LANGUAGES; @@ -120,26 +110,6 @@ const _on_language_switch = () => { } }; -// Returns the path segment of the language as a string, like 'fr/' -// or '' if not found. -function language_segment_from_url() { - const path = window.location.pathname; - const language_regexp = - '/((?:' + Object.keys(all_languages).join('|') + ')/)'; - const match = path.match(language_regexp); - if (match !== null) return match[1]; - return ''; -} - -// Returns the path segment of the version as a string, like '3.6/' -// or '' if not found. -function version_segment_from_url() { - const path = window.location.pathname; - const version_regexp = language_segment_from_url() + _VERSION_PATTERN; - const match = path.match(version_regexp); - if (match !== null) return match[1]; - return ''; -} const _initialise_switchers = () => { const version_select = _create_version_select(_CURRENT_VERSION); document From b62ea7a325fb2dc53350f07be9c0129de1f11c6a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:08:12 +0000 Subject: [PATCH 016/102] Use the event argument of select element callback functions --- templates/switchers.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 600c955..dbd4809 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -77,8 +77,8 @@ const _navigate_to_first_existing = (urls) => { }); }; -const _on_version_switch = () => { - const selected_version = this.options[this.selectedIndex].value + '/'; +const _on_version_switch = (event) => { + const selected_version = event.target.value + '/'; const url = window.location.href; const new_url = url.replace( _CURRENT_PREFIX, @@ -95,8 +95,8 @@ const _on_version_switch = () => { } }; -const _on_language_switch = () => { - let selected_language = this.options[this.selectedIndex].value + '/'; +const _on_language_switch = (event) => { + let selected_language = event.target.value + '/'; const url = window.location.href; if (selected_language === 'en/') // Special 'default' case for English. From e5b53bbf6d545d0d95cff2b9f74c1f602efb2511 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:43:21 +0000 Subject: [PATCH 017/102] Improve logic for the callback functions --- templates/switchers.js | 90 ++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index dbd4809..ababf04 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -58,55 +58,67 @@ const _create_language_select = (current_language) => { const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. - const url = urls.shift(); - if (urls.length === 0 || url.startsWith('file:///')) { - window.location.href = url; - return; + for (const url of urls) { + if (url.startsWith('file:///')) { + window.location.href = url; + return; + } + fetch(url) + .then((response) => { + if (response.ok) { + window.location.href = url; + return url; + } + }) + .catch((err) => { + console.error(`Error when fetching '${url}'!`); + console.error(err); + }); } - fetch(url) - .then((response) => { - if (response.ok) { - window.location.href = url; - } else { - navigate_to_first_existing(urls); - } - }) - .catch((err) => { - void err; - navigate_to_first_existing(urls); - }); + + // if all else fails, redirect to the d.p.o root + window.location.href = '/'; + return '/'; }; const _on_version_switch = (event) => { - const selected_version = event.target.value + '/'; - const url = window.location.href; - const new_url = url.replace( - _CURRENT_PREFIX, - '/' + _CURRENT_LANGUAGE + selected_version, - ); - if (new_url !== url) { + const selected_version = event.target.value; + // English has no language prefix. + const new_prefix_en = `/${selected_version}/`; + const new_prefix = + _CURRENT_LANGUAGE === 'en' + ? new_prefix_en + : `/${_CURRENT_LANGUAGE}/${selected_version}/`; + if (_CURRENT_PREFIX !== new_prefix) { + // Try the following pages in order: + // 1. The current page in the current language with the new version + // 2. The current page in English with the new version + // 3. The documentation home in the current language with the new version + // 4. The documentation home in English with the new version _navigate_to_first_existing([ - new_url, - url.replace(_CURRENT_PREFIX, '/' + selected_version), - '/' + _CURRENT_LANGUAGE + selected_version, - '/' + selected_version, - '/', + window.location.href.replace(_CURRENT_PREFIX, new_prefix), + window.location.href.replace(_CURRENT_PREFIX, new_prefix_en), + new_prefix, + new_prefix_en, ]); } }; const _on_language_switch = (event) => { - let selected_language = event.target.value + '/'; - const url = window.location.href; - if (selected_language === 'en/') - // Special 'default' case for English. - selected_language = ''; - let new_url = url.replace( - _CURRENT_PREFIX, - '/' + selected_language + _CURRENT_VERSION, - ); - if (new_url !== url) { - _navigate_to_first_existing([new_url, '/']); + const selected_language = event.target.value; + // English has no language prefix. + const new_prefix = + selected_language === 'en' + ? `/${_CURRENT_VERSION}/` + : `/${selected_language}/${_CURRENT_VERSION}/`; + if (_CURRENT_PREFIX !== new_prefix) { + // Try the following pages in order: + // 1. The current page in the new language with the current version + // 2. The documentation home in the new language with the current version + _navigate_to_first_existing([ + window.location.href.replace(_CURRENT_PREFIX, new_prefix), + new_prefix, + ]); } }; From 1179ee9d3b62ee38f978ffa6b97e311b2775d230 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:49:33 +0000 Subject: [PATCH 018/102] Iterate over arrays with for...of --- templates/switchers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index ababf04..af73ced 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -21,7 +21,7 @@ const _create_version_select = (release) => { const select = document.createElement('select'); select.className = 'version-select'; - for (const [version, title] in all_versions) { + for (const [version, title] of Object.entries(all_versions)) { const option = document.createElement('option'); option.value = version; if (version === major_minor) { @@ -45,7 +45,7 @@ const _create_language_select = (current_language) => { const select = document.createElement('select'); select.className = 'language-select'; - for (const [language, title] in all_languages) { + for (const [language, title] of Object.entries(all_languages)) { const option = document.createElement('option'); option.value = language; option.text = title; From d1b7ad8450b1ef58b307c61882c0336c60ef1276 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:52:54 +0000 Subject: [PATCH 019/102] Fix _CURRENT_VERSION for pre-releases --- templates/switchers.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/switchers.js b/templates/switchers.js index af73ced..afa26f4 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,6 +1,7 @@ 'use strict'; -const _CURRENT_VERSION = DOCUMENTATION_OPTIONS.VERSION; +const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ''; +const _CURRENT_VERSION = _CURRENT_RELEASE.split('.', 2).join('.'); const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; const _CURRENT_PREFIX = (() => { // Sphinx 7.2+ defines the content root data attribute in the HTML element. From 7efb6bf363a2f22338d0f9732e39895fa28abbe7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:00:10 +0000 Subject: [PATCH 020/102] Use _CURRENT_RELEASE in _create_version_select() --- templates/switchers.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index afa26f4..774f04e 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -17,16 +17,15 @@ const _CURRENT_PREFIX = (() => { const all_versions = $VERSIONS; const all_languages = $LANGUAGES; -const _create_version_select = (release) => { - const major_minor = release.split('.').slice(0, 2).join('.'); +const _create_version_select = () => { const select = document.createElement('select'); select.className = 'version-select'; for (const [version, title] of Object.entries(all_versions)) { const option = document.createElement('option'); option.value = version; - if (version === major_minor) { - option.text = release; + if (version === _CURRENT_VERSION) { + option.text = _CURRENT_RELEASE; option.selected = true; } else { option.text = title; @@ -124,7 +123,7 @@ const _on_language_switch = (event) => { }; const _initialise_switchers = () => { - const version_select = _create_version_select(_CURRENT_VERSION); + const version_select = _create_version_select(); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { From 359c393d92dbec5f383b974652f19259edf9fdbb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:02:14 +0000 Subject: [PATCH 021/102] Use _CURRENT_LANGUAGE in _create_language_select() --- templates/switchers.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 774f04e..740cc12 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -36,10 +36,10 @@ const _create_version_select = () => { return select; }; -const _create_language_select = (current_language) => { - if (!(current_language in all_languages)) { +const _create_language_select = () => { + if (!(_CURRENT_LANGUAGE in all_languages)) { // In case we are browsing a language that is not yet in all_languages. - all_languages[current_language] = current_language; + all_languages[_CURRENT_LANGUAGE] = _CURRENT_LANGUAGE; } const select = document.createElement('select'); @@ -49,7 +49,7 @@ const _create_language_select = (current_language) => { const option = document.createElement('option'); option.value = language; option.text = title; - if (language === current_language) option.selected = true; + if (language === _CURRENT_LANGUAGE) option.selected = true; select.add(option); } @@ -132,7 +132,7 @@ const _initialise_switchers = () => { placeholder.append(s); }); - const language_select = _create_language_select(_CURRENT_LANGUAGE); + const language_select = _create_language_select(); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { From e55b26c44c6410d2720d8d4b0b73d5140a29721e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:05:57 +0000 Subject: [PATCH 022/102] Add better handling for file URIs --- templates/switchers.js | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 740cc12..025daed 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,9 +1,14 @@ 'use strict'; +// File URIs must begin with either one or three forward slashes +const _is_file_uri = (uri) => uri.startsWith('file:/'); + +const _IS_LOCAL = _is_file_uri(window.location.href); const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ''; const _CURRENT_VERSION = _CURRENT_RELEASE.split('.', 2).join('.'); const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; const _CURRENT_PREFIX = (() => { + if (_IS_LOCAL) return null; // Sphinx 7.2+ defines the content root data attribute in the HTML element. const _CONTENT_ROOT = document.documentElement.dataset.content_root; if (_CONTENT_ROOT !== undefined) { @@ -20,6 +25,10 @@ const all_languages = $LANGUAGES; const _create_version_select = () => { const select = document.createElement('select'); select.className = 'version-select'; + if (_IS_LOCAL) { + select.disabled = true; + select.title = 'Version switching is disabled in local builds'; + } for (const [version, title] of Object.entries(all_versions)) { const option = document.createElement('option'); @@ -44,6 +53,10 @@ const _create_language_select = () => { const select = document.createElement('select'); select.className = 'language-select'; + if (_IS_LOCAL) { + select.disabled = true; + select.title = 'Language switching is disabled in local builds'; + } for (const [language, title] of Object.entries(all_languages)) { const option = document.createElement('option'); @@ -59,10 +72,6 @@ const _create_language_select = () => { const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { - if (url.startsWith('file:///')) { - window.location.href = url; - return; - } fetch(url) .then((response) => { if (response.ok) { @@ -82,6 +91,8 @@ const _navigate_to_first_existing = (urls) => { }; const _on_version_switch = (event) => { + if (_IS_LOCAL) return; + const selected_version = event.target.value; // English has no language prefix. const new_prefix_en = `/${selected_version}/`; @@ -105,6 +116,8 @@ const _on_version_switch = (event) => { }; const _on_language_switch = (event) => { + if (_IS_LOCAL) return; + const selected_language = event.target.value; // English has no language prefix. const new_prefix = From 4c495ccf8bb0eb512a4b7acba7b6d682babdf91c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:35:03 +0000 Subject: [PATCH 023/102] Remove placeholder classes when initialisation is complete --- templates/switchers.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 025daed..c6dd30e 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -143,6 +143,7 @@ const _initialise_switchers = () => { const s = version_select.cloneNode(true); s.addEventListener('change', _on_version_switch); placeholder.append(s); + placeholder.classList.remove('version_switcher_placeholder'); }); const language_select = _create_language_select(); @@ -152,6 +153,7 @@ const _initialise_switchers = () => { const s = language_select.cloneNode(true); s.addEventListener('change', _on_language_switch); placeholder.append(s); + placeholder.classList.remove('language_switcher_placeholder'); }); }; From 3031085ac20893336bd9f317b66d97db0ce21af1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:46:44 +0000 Subject: [PATCH 024/102] Use a Map for versions and languages --- templates/switchers.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index c6dd30e..2e4fd76 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -19,10 +19,10 @@ const _CURRENT_PREFIX = (() => { return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; })(); -const all_versions = $VERSIONS; -const all_languages = $LANGUAGES; +const _ALL_VERSIONS = new Map(Object.entries($VERSIONS)); +const _ALL_LANGUAGES = new Map(Object.entries($LANGUAGES)); -const _create_version_select = () => { +const _create_version_select = (versions) => { const select = document.createElement('select'); select.className = 'version-select'; if (_IS_LOCAL) { @@ -30,7 +30,7 @@ const _create_version_select = () => { select.title = 'Version switching is disabled in local builds'; } - for (const [version, title] of Object.entries(all_versions)) { + for (const [version, title] of versions) { const option = document.createElement('option'); option.value = version; if (version === _CURRENT_VERSION) { @@ -45,10 +45,10 @@ const _create_version_select = () => { return select; }; -const _create_language_select = () => { - if (!(_CURRENT_LANGUAGE in all_languages)) { - // In case we are browsing a language that is not yet in all_languages. - all_languages[_CURRENT_LANGUAGE] = _CURRENT_LANGUAGE; +const _create_language_select = (languages) => { + if (!languages.has(_CURRENT_LANGUAGE)) { + // In case we are browsing a language that is not yet in languages. + languages.set(_CURRENT_LANGUAGE, _CURRENT_LANGUAGE); } const select = document.createElement('select'); @@ -58,7 +58,7 @@ const _create_language_select = () => { select.title = 'Language switching is disabled in local builds'; } - for (const [language, title] of Object.entries(all_languages)) { + for (const [language, title] of languages) { const option = document.createElement('option'); option.value = language; option.text = title; @@ -136,7 +136,10 @@ const _on_language_switch = (event) => { }; const _initialise_switchers = () => { - const version_select = _create_version_select(); + const versions = _ALL_VERSIONS; + const languages = _ALL_LANGUAGES; + + const version_select = _create_version_select(versions); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { @@ -146,7 +149,7 @@ const _initialise_switchers = () => { placeholder.classList.remove('version_switcher_placeholder'); }); - const language_select = _create_language_select(); + const language_select = _create_language_select(languages); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { From 8cb67060a4fd9e4508b006df3052d91c2d545a3a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:51:02 +0000 Subject: [PATCH 025/102] Add JSDoc comments --- templates/switchers.js | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 2e4fd76..cd4cbc2 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -22,6 +22,11 @@ const _CURRENT_PREFIX = (() => { const _ALL_VERSIONS = new Map(Object.entries($VERSIONS)); const _ALL_LANGUAGES = new Map(Object.entries($LANGUAGES)); +/** + * @param {Map} versions + * @returns {HTMLSelectElement} + * @private + */ const _create_version_select = (versions) => { const select = document.createElement('select'); select.className = 'version-select'; @@ -45,6 +50,11 @@ const _create_version_select = (versions) => { return select; }; +/** + * @param {Map} languages + * @returns {HTMLSelectElement} + * @private + */ const _create_language_select = (languages) => { if (!languages.has(_CURRENT_LANGUAGE)) { // In case we are browsing a language that is not yet in languages. @@ -69,6 +79,11 @@ const _create_language_select = (languages) => { return select; }; +/** + * Change the current page to the first existing URL in the list. + * @param {Array} urls + * @private + */ const _navigate_to_first_existing = (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { @@ -90,6 +105,12 @@ const _navigate_to_first_existing = (urls) => { return '/'; }; +/** + * Callback for the version switcher. + * @param {Event} event + * @returns {void} + * @private + */ const _on_version_switch = (event) => { if (_IS_LOCAL) return; @@ -115,6 +136,12 @@ const _on_version_switch = (event) => { } }; +/** + * Callback for the language switcher. + * @param {Event} event + * @returns {void} + * @private + */ const _on_language_switch = (event) => { if (_IS_LOCAL) return; @@ -135,6 +162,11 @@ const _on_language_switch = (event) => { } }; +/** + * Initialisation function for the version and language switchers. + * @returns {void} + * @private + */ const _initialise_switchers = () => { const versions = _ALL_VERSIONS; const languages = _ALL_LANGUAGES; From 58c9634bcad42f8114e8665955d73dcbfbfab02e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 29 Oct 2024 22:22:46 +0000 Subject: [PATCH 026/102] Convert promises to async-await (#227) --- templates/switchers.js | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index cd4cbc2..b479b3c 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -84,20 +84,18 @@ const _create_language_select = (languages) => { * @param {Array} urls * @private */ -const _navigate_to_first_existing = (urls) => { +const _navigate_to_first_existing = async (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { - fetch(url) - .then((response) => { - if (response.ok) { - window.location.href = url; - return url; - } - }) - .catch((err) => { - console.error(`Error when fetching '${url}'!`); - console.error(err); - }); + try { + const response = await fetch(url, { method: 'HEAD' }); + if (response.ok) { + window.location.href = url; + return url; + } + } catch (err) { + console.error(`Error when fetching '${url}': ${err}`); + } } // if all else fails, redirect to the d.p.o root @@ -111,7 +109,7 @@ const _navigate_to_first_existing = (urls) => { * @returns {void} * @private */ -const _on_version_switch = (event) => { +const _on_version_switch = async (event) => { if (_IS_LOCAL) return; const selected_version = event.target.value; @@ -127,7 +125,7 @@ const _on_version_switch = (event) => { // 2. The current page in English with the new version // 3. The documentation home in the current language with the new version // 4. The documentation home in English with the new version - _navigate_to_first_existing([ + await _navigate_to_first_existing([ window.location.href.replace(_CURRENT_PREFIX, new_prefix), window.location.href.replace(_CURRENT_PREFIX, new_prefix_en), new_prefix, @@ -142,7 +140,7 @@ const _on_version_switch = (event) => { * @returns {void} * @private */ -const _on_language_switch = (event) => { +const _on_language_switch = async (event) => { if (_IS_LOCAL) return; const selected_language = event.target.value; @@ -155,7 +153,7 @@ const _on_language_switch = (event) => { // Try the following pages in order: // 1. The current page in the new language with the current version // 2. The documentation home in the new language with the current version - _navigate_to_first_existing([ + await _navigate_to_first_existing([ window.location.href.replace(_CURRENT_PREFIX, new_prefix), new_prefix, ]); From c1fc85cfe4aeb2e99cc09096e28b4d8ca76e3db2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 29 Oct 2024 22:32:11 +0000 Subject: [PATCH 027/102] Always create new select nodes (#228) --- templates/switchers.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index b479b3c..dd28044 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -169,21 +169,19 @@ const _initialise_switchers = () => { const versions = _ALL_VERSIONS; const languages = _ALL_LANGUAGES; - const version_select = _create_version_select(versions); document .querySelectorAll('.version_switcher_placeholder') .forEach((placeholder) => { - const s = version_select.cloneNode(true); + const s = _create_version_select(versions); s.addEventListener('change', _on_version_switch); placeholder.append(s); placeholder.classList.remove('version_switcher_placeholder'); }); - const language_select = _create_language_select(languages); document .querySelectorAll('.language_switcher_placeholder') .forEach((placeholder) => { - const s = language_select.cloneNode(true); + const s = _create_language_select(languages); s.addEventListener('change', _on_language_switch); placeholder.append(s); placeholder.classList.remove('language_switcher_placeholder'); From eff0dd956eb3e58cb294100aea1bf0bf12a53ba9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:36:29 +0000 Subject: [PATCH 028/102] Run prettier in GitHub Actions (#232) --- .github/workflows/lint.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d553e49..26f2cc1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ permissions: contents: read jobs: - lint: + pre-commit: runs-on: ubuntu-latest steps: @@ -20,3 +20,14 @@ jobs: python-version: "3.x" cache: pip - uses: pre-commit/action@v3.0.1 + + prettier: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Lint with Prettier + run: npx prettier templates/switchers.js --check --single-quote From 49641c17b23dac6ecf3fefbe4483dca530dfd19f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:42:25 +0000 Subject: [PATCH 029/102] Create switchers maps from arrays of pairs (#231) --- build_docs.py | 8 ++++---- templates/switchers.js | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build_docs.py b/build_docs.py index 6596a25..f14cc2b 100755 --- a/build_docs.py +++ b/build_docs.py @@ -388,16 +388,16 @@ def setup_switchers( - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - languages_map = dict(sorted((l.tag, l.name) for l in languages if l.in_prod)) - versions_map = {v.name: v.picker_label for v in reversed(versions)} + language_pairs = sorted((l.tag, l.name) for l in languages if l.in_prod) + version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] switchers_template_file = HERE / "templates" / "switchers.js" switchers_path = html_root / "_static" / "switchers.js" template = Template(switchers_template_file.read_text(encoding="UTF-8")) rendered_template = template.safe_substitute( - LANGUAGES=json.dumps(languages_map), - VERSIONS=json.dumps(versions_map), + LANGUAGES=json.dumps(language_pairs), + VERSIONS=json.dumps(version_pairs), ) switchers_path.write_text(rendered_template, encoding="UTF-8") diff --git a/templates/switchers.js b/templates/switchers.js index dd28044..d1a4768 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -19,8 +19,8 @@ const _CURRENT_PREFIX = (() => { return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; })(); -const _ALL_VERSIONS = new Map(Object.entries($VERSIONS)); -const _ALL_LANGUAGES = new Map(Object.entries($LANGUAGES)); +const _ALL_VERSIONS = new Map($VERSIONS); +const _ALL_LANGUAGES = new Map($LANGUAGES); /** * @param {Map} versions From 91d0994a794d255c767e42873ffee6502c88b91a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:45:36 +0000 Subject: [PATCH 030/102] Keep switcher placeholder classes --- templates/switchers.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index d1a4768..3eefa71 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -175,7 +175,6 @@ const _initialise_switchers = () => { const s = _create_version_select(versions); s.addEventListener('change', _on_version_switch); placeholder.append(s); - placeholder.classList.remove('version_switcher_placeholder'); }); document @@ -184,7 +183,6 @@ const _initialise_switchers = () => { const s = _create_language_select(languages); s.addEventListener('change', _on_language_switch); placeholder.append(s); - placeholder.classList.remove('language_switcher_placeholder'); }); }; From cc4f5531c333c0beaaa4e89e528787e877db6c43 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 31 Oct 2024 18:13:40 +0200 Subject: [PATCH 031/102] Run prettier via pre-commit (#234) --- .github/workflows/lint.yml | 13 +--------- .pre-commit-config.yaml | 20 ++++++++++----- templates/switchers.js | 52 +++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 45 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 26f2cc1..d553e49 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ permissions: contents: read jobs: - pre-commit: + lint: runs-on: ubuntu-latest steps: @@ -20,14 +20,3 @@ jobs: python-version: "3.x" cache: pip - uses: pre-commit/action@v3.0.1 - - prettier: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "22" - - name: Lint with Prettier - run: npx prettier templates/switchers.js --check --single-quote diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c3d2f5f..60a928c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -14,35 +14,41 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.7 + rev: v0.7.1 hooks: - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.2 + rev: 0.29.4 hooks: - id: check-github-workflows - repo: https://github.com/rhysd/actionlint - rev: v1.7.1 + rev: v1.7.3 hooks: - id: actionlint - repo: https://github.com/tox-dev/pyproject-fmt - rev: 2.2.3 + rev: v2.5.0 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.19 + rev: v0.22 hooks: - id: validate-pyproject - repo: https://github.com/tox-dev/tox-ini-fmt - rev: 1.4.0 + rev: 1.4.1 hooks: - id: tox-ini-fmt + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.3.3 + hooks: + - id: prettier + files: templates/switchers.js + - repo: meta hooks: - id: check-hooks-apply diff --git a/templates/switchers.js b/templates/switchers.js index 3eefa71..44c71bb 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -1,12 +1,12 @@ -'use strict'; +"use strict"; // File URIs must begin with either one or three forward slashes -const _is_file_uri = (uri) => uri.startsWith('file:/'); +const _is_file_uri = (uri) => uri.startsWith("file:/"); const _IS_LOCAL = _is_file_uri(window.location.href); -const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ''; -const _CURRENT_VERSION = _CURRENT_RELEASE.split('.', 2).join('.'); -const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || 'en'; +const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ""; +const _CURRENT_VERSION = _CURRENT_RELEASE.split(".", 2).join("."); +const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || "en"; const _CURRENT_PREFIX = (() => { if (_IS_LOCAL) return null; // Sphinx 7.2+ defines the content root data attribute in the HTML element. @@ -15,8 +15,8 @@ const _CURRENT_PREFIX = (() => { return new URL(_CONTENT_ROOT, window.location).pathname; } // Fallback for older versions of Sphinx (used in Python 3.10 and older). - const _NUM_PREFIX_PARTS = _CURRENT_LANGUAGE === 'en' ? 2 : 3; - return window.location.pathname.split('/', _NUM_PREFIX_PARTS).join('/') + '/'; + const _NUM_PREFIX_PARTS = _CURRENT_LANGUAGE === "en" ? 2 : 3; + return window.location.pathname.split("/", _NUM_PREFIX_PARTS).join("/") + "/"; })(); const _ALL_VERSIONS = new Map($VERSIONS); @@ -28,15 +28,15 @@ const _ALL_LANGUAGES = new Map($LANGUAGES); * @private */ const _create_version_select = (versions) => { - const select = document.createElement('select'); - select.className = 'version-select'; + const select = document.createElement("select"); + select.className = "version-select"; if (_IS_LOCAL) { select.disabled = true; - select.title = 'Version switching is disabled in local builds'; + select.title = "Version switching is disabled in local builds"; } for (const [version, title] of versions) { - const option = document.createElement('option'); + const option = document.createElement("option"); option.value = version; if (version === _CURRENT_VERSION) { option.text = _CURRENT_RELEASE; @@ -61,15 +61,15 @@ const _create_language_select = (languages) => { languages.set(_CURRENT_LANGUAGE, _CURRENT_LANGUAGE); } - const select = document.createElement('select'); - select.className = 'language-select'; + const select = document.createElement("select"); + select.className = "language-select"; if (_IS_LOCAL) { select.disabled = true; - select.title = 'Language switching is disabled in local builds'; + select.title = "Language switching is disabled in local builds"; } for (const [language, title] of languages) { - const option = document.createElement('option'); + const option = document.createElement("option"); option.value = language; option.text = title; if (language === _CURRENT_LANGUAGE) option.selected = true; @@ -88,7 +88,7 @@ const _navigate_to_first_existing = async (urls) => { // Navigate to the first existing URL in urls. for (const url of urls) { try { - const response = await fetch(url, { method: 'HEAD' }); + const response = await fetch(url, { method: "HEAD" }); if (response.ok) { window.location.href = url; return url; @@ -99,8 +99,8 @@ const _navigate_to_first_existing = async (urls) => { } // if all else fails, redirect to the d.p.o root - window.location.href = '/'; - return '/'; + window.location.href = "/"; + return "/"; }; /** @@ -116,7 +116,7 @@ const _on_version_switch = async (event) => { // English has no language prefix. const new_prefix_en = `/${selected_version}/`; const new_prefix = - _CURRENT_LANGUAGE === 'en' + _CURRENT_LANGUAGE === "en" ? new_prefix_en : `/${_CURRENT_LANGUAGE}/${selected_version}/`; if (_CURRENT_PREFIX !== new_prefix) { @@ -146,7 +146,7 @@ const _on_language_switch = async (event) => { const selected_language = event.target.value; // English has no language prefix. const new_prefix = - selected_language === 'en' + selected_language === "en" ? `/${_CURRENT_VERSION}/` : `/${selected_language}/${_CURRENT_VERSION}/`; if (_CURRENT_PREFIX !== new_prefix) { @@ -170,24 +170,24 @@ const _initialise_switchers = () => { const languages = _ALL_LANGUAGES; document - .querySelectorAll('.version_switcher_placeholder') + .querySelectorAll(".version_switcher_placeholder") .forEach((placeholder) => { const s = _create_version_select(versions); - s.addEventListener('change', _on_version_switch); + s.addEventListener("change", _on_version_switch); placeholder.append(s); }); document - .querySelectorAll('.language_switcher_placeholder') + .querySelectorAll(".language_switcher_placeholder") .forEach((placeholder) => { const s = _create_language_select(languages); - s.addEventListener('change', _on_language_switch); + s.addEventListener("change", _on_language_switch); placeholder.append(s); }); }; -if (document.readyState !== 'loading') { +if (document.readyState !== "loading") { _initialise_switchers(); } else { - document.addEventListener('DOMContentLoaded', _initialise_switchers); + document.addEventListener("DOMContentLoaded", _initialise_switchers); } From b2b548353b50a3316ba098b89382fe6b8f8474a9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 31 Oct 2024 23:04:00 +0000 Subject: [PATCH 032/102] Update JSDoc comments in switchers.js --- templates/switchers.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/switchers.js b/templates/switchers.js index 44c71bb..7a4fb63 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -104,7 +104,7 @@ const _navigate_to_first_existing = async (urls) => { }; /** - * Callback for the version switcher. + * Navigate to the selected version. * @param {Event} event * @returns {void} * @private @@ -135,7 +135,7 @@ const _on_version_switch = async (event) => { }; /** - * Callback for the language switcher. + * Navigate to the selected language. * @param {Event} event * @returns {void} * @private @@ -161,7 +161,7 @@ const _on_language_switch = async (event) => { }; /** - * Initialisation function for the version and language switchers. + * Set up the version and language switchers. * @returns {void} * @private */ From b5f5a52faf28ab13990ebbc577c1a3c206343850 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:15:46 +0200 Subject: [PATCH 033/102] Add instructions on manually rebuilding a branch --- README.md | 4 ++-- RELEASING.md | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 RELEASING.md diff --git a/README.md b/README.md index 767014c..b9881e7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This repository contains scripts for automatically building the Python documentation on [docs.python.org](https://docs.python.org). -# How to test it? +## How to test it? The following command should build all maintained versions and translations in `./www`, beware it can take a few hours: @@ -15,7 +15,7 @@ If you don't need to build all translations of all branches, add `--language en --branch main`. -# Check current version +## Check current version Install `tools_requirements.txt` then run `python check_versions.py ../cpython/` (pointing to a real CPython clone) to see which version diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..abe72cd --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,16 @@ +# Manually rebuild a branch + +Docs for [feature and bugfix branches](https://devguide.python.org/versions/) are +automatically built from a cron. + +Manual rebuilds are needed for new security releases, +and to add the end-of-life banner for newly end-of-life branches. + +To manually rebuild a branch, for example 3.11: + +```shell +ssh docs.nyc1.psf.io +sudo su --shell=/bin/bash docsbuild +screen -DUR # Rejoin screen session if it exists, otherwise create a new one +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 +``` From 10064ee59d8c6f2fbad2391055d746f1d5deb991 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 10 Dec 2024 23:10:01 +0200 Subject: [PATCH 034/102] Move to README --- README.md | 17 +++++++++++++++++ RELEASING.md | 16 ---------------- 2 files changed, 17 insertions(+), 16 deletions(-) delete mode 100644 RELEASING.md diff --git a/README.md b/README.md index b9881e7..1b76bcd 100644 --- a/README.md +++ b/README.md @@ -51,3 +51,20 @@ of Sphinx we're using where: 3.13 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 3.14 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= + +## Manually rebuild a branch + +Docs for [feature and bugfix branches](https://devguide.python.org/versions/) are +automatically built from a cron. + +Manual rebuilds are needed for new security releases, +and to add the end-of-life banner for newly end-of-life branches. + +To manually rebuild a branch, for example 3.11: + +```shell +ssh docs.nyc1.psf.io +sudo su --shell=/bin/bash docsbuild +screen -DUR # Rejoin screen session if it exists, otherwise create a new one +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 +``` diff --git a/RELEASING.md b/RELEASING.md deleted file mode 100644 index abe72cd..0000000 --- a/RELEASING.md +++ /dev/null @@ -1,16 +0,0 @@ -# Manually rebuild a branch - -Docs for [feature and bugfix branches](https://devguide.python.org/versions/) are -automatically built from a cron. - -Manual rebuilds are needed for new security releases, -and to add the end-of-life banner for newly end-of-life branches. - -To manually rebuild a branch, for example 3.11: - -```shell -ssh docs.nyc1.psf.io -sudo su --shell=/bin/bash docsbuild -screen -DUR # Rejoin screen session if it exists, otherwise create a new one -/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 -``` From e4a8aff9772738a63d0945042777d18c3d926930 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 28 Dec 2024 12:30:57 +0000 Subject: [PATCH 035/102] Enable the Polish translation in the language switcher (#237) --- config.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/config.toml b/config.toml index 3716d7f..3679c73 100644 --- a/config.toml +++ b/config.toml @@ -69,7 +69,6 @@ sphinxopts = [ [languages.pl] name = "Polish" -in_prod = false [languages.pt_BR] name = "Brazilian Portuguese" From b3f238f367d70c4f8cab4d3163d1a5eecc771c2f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 17 Jan 2025 12:51:16 +0000 Subject: [PATCH 036/102] Enable translation_progress_classes (#239) --- build_docs.py | 26 ++++++++++++++------------ config.toml | 1 - 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/build_docs.py b/build_docs.py index f14cc2b..b5cf717 100755 --- a/build_docs.py +++ b/build_docs.py @@ -192,7 +192,7 @@ def __gt__(self, other): return self.as_tuple() > other.as_tuple() -@dataclass(frozen=True, order=True) +@dataclass(order=True, frozen=True, kw_only=True) class Language: iso639_tag: str name: str @@ -710,6 +710,7 @@ def build(self): f"-D locale_dirs={locale_dirs}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", + "-D translation_progress_classes=1", ) ) if self.language.tag == "ja": @@ -1141,19 +1142,20 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: def parse_languages_from_config() -> list[Language]: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) - languages = [] defaults = config["defaults"] - for iso639_tag, section in config["languages"].items(): - languages.append( - Language( - iso639_tag, - section["name"], - section.get("in_prod", defaults["in_prod"]), - sphinxopts=section.get("sphinxopts", defaults["sphinxopts"]), - html_only=section.get("html_only", defaults["html_only"]), - ) + default_in_prod = defaults.get("in_prod", True) + default_sphinxopts = defaults.get("sphinxopts", []) + default_html_only = defaults.get("html_only", False) + return [ + Language( + iso639_tag=iso639_tag, + name=section["name"], + in_prod=section.get("in_prod", default_in_prod), + sphinxopts=section.get("sphinxopts", default_sphinxopts), + html_only=section.get("html_only", default_html_only), ) - return languages + for iso639_tag, section in config["languages"].items() + ] def format_seconds(seconds: float) -> str: diff --git a/config.toml b/config.toml index 3679c73..b0994ad 100644 --- a/config.toml +++ b/config.toml @@ -33,7 +33,6 @@ in_prod = false [languages.it] name = "Italian" -in_prod = true [languages.ja] name = "Japanese" From be699c482265555fda16dc8dccf1bcef25d23fb8 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 18 Jan 2025 15:38:54 +0000 Subject: [PATCH 037/102] Disable translation_progress_classes Sphinx does not properly convert command-line overrides to Boolean values, so ``-D translation_progress_classes=1`` fails with: Sphinx parallel build error: sphinx.errors.ConfigError: translation_progress_classes must be True, False, "translated" or "untranslated" --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index b5cf717..40384b8 100755 --- a/build_docs.py +++ b/build_docs.py @@ -710,7 +710,7 @@ def build(self): f"-D locale_dirs={locale_dirs}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", - "-D translation_progress_classes=1", + # "-D translation_progress_classes=1", ) ) if self.language.tag == "ja": From 4e7299dfd3719b08d42a162c0bbb9181218b5596 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 18 Jan 2025 18:47:45 +0000 Subject: [PATCH 038/102] Identify the language correctly for older versions (#240) --- templates/switchers.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/switchers.js b/templates/switchers.js index 7a4fb63..774366f 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -6,7 +6,13 @@ const _is_file_uri = (uri) => uri.startsWith("file:/"); const _IS_LOCAL = _is_file_uri(window.location.href); const _CURRENT_RELEASE = DOCUMENTATION_OPTIONS.VERSION || ""; const _CURRENT_VERSION = _CURRENT_RELEASE.split(".", 2).join("."); -const _CURRENT_LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || "en"; +const _CURRENT_LANGUAGE = (() => { + const _LANGUAGE = DOCUMENTATION_OPTIONS.LANGUAGE?.toLowerCase() || "en"; + // Python 2.7 and 3.5--3.10 use ``LANGUAGE: 'None'`` for English + // in ``documentation_options.js``. + if (_LANGUAGE === "none") return "en"; + return _LANGUAGE; +})(); const _CURRENT_PREFIX = (() => { if (_IS_LOCAL) return null; // Sphinx 7.2+ defines the content root data attribute in the HTML element. From 2efa5bce54a2d09573e62a580047b5c7419bbedc Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 20 Jan 2025 23:35:16 +0000 Subject: [PATCH 039/102] Install matplotlib for opengraph preview images (#242) --- build_docs.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 40384b8..a6b08bb 100755 --- a/build_docs.py +++ b/build_docs.py @@ -745,6 +745,10 @@ def build(self): blurb = self.venv / "bin" / "blurb" if self.includes_html: + # Define a tag to enable opengraph socialcards previews + # (used in Doc/conf.py and requires matplotlib) + sphinxopts.append("-t create-social-cards") + # Disable CPython switchers, we handle them now: run( ["sed", "-i"] @@ -783,13 +787,17 @@ def build_venv(self): So we can reuse them from builds to builds, while they contain different Sphinx versions. """ + requirements = [self.theme] + self.version.requirements + if self.includes_html: + # opengraph previews + requirements.append("matplotlib>=3") + venv_path = self.build_root / ("venv-" + self.version.name) run([sys.executable, "-m", "venv", venv_path]) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] + ["--upgrade-strategy=eager"] - + [self.theme] - + self.version.requirements, + + requirements, cwd=self.checkout / "Doc", ) run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) From 6aa487d97044d111c27deae12baa85755351e4fb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 19 Feb 2025 03:23:46 +0000 Subject: [PATCH 040/102] Include languages' translated names in the switcher (#245) Co-authored-by: Ezio Melotti Co-authored-by: Rafael Fontenelle Co-authored-by: W. H. Wang --- build_docs.py | 11 ++++++++++- config.toml | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index a6b08bb..8b7f3f9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -196,6 +196,7 @@ def __gt__(self, other): class Language: iso639_tag: str name: str + translated_name: str in_prod: bool sphinxopts: tuple html_only: bool = False @@ -204,6 +205,12 @@ class Language: def tag(self): return self.iso639_tag.replace("_", "-").lower() + @property + def switcher_label(self): + if self.translated_name: + return f"{self.name} | {self.translated_name}" + return self.name + @staticmethod def filter(languages, language_tags=None): """Filter a sequence of languages according to --languages.""" @@ -388,7 +395,7 @@ def setup_switchers( - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - language_pairs = sorted((l.tag, l.name) for l in languages if l.in_prod) + language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] switchers_template_file = HERE / "templates" / "switchers.js" @@ -1151,6 +1158,7 @@ def parse_languages_from_config() -> list[Language]: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) defaults = config["defaults"] + default_translated_name = defaults.get("translated_name", "") default_in_prod = defaults.get("in_prod", True) default_sphinxopts = defaults.get("sphinxopts", []) default_html_only = defaults.get("html_only", False) @@ -1158,6 +1166,7 @@ def parse_languages_from_config() -> list[Language]: Language( iso639_tag=iso639_tag, name=section["name"], + translated_name=section.get("translated_name", default_translated_name), in_prod=section.get("in_prod", default_in_prod), sphinxopts=section.get("sphinxopts", default_sphinxopts), html_only=section.get("html_only", default_html_only), diff --git a/config.toml b/config.toml index b0994ad..489c774 100644 --- a/config.toml +++ b/config.toml @@ -1,5 +1,12 @@ +# name: the English name for the language. +# translated_name: the 'local' name for the language. +# in_prod: If true, include in the language switcher. +# html_only: If true, only create HTML files. +# sphinxopts: Extra options to pass to SPHINXOPTS in the Makefile. + [defaults] # name has no default, it is mandatory. +translated_name = "" in_prod = true html_only = false sphinxopts = [ @@ -13,6 +20,7 @@ name = "English" [languages.es] name = "Spanish" +translated_name = "español" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -21,6 +29,7 @@ sphinxopts = [ [languages.fr] name = "French" +translated_name = "français" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -29,13 +38,16 @@ sphinxopts = [ [languages.id] name = "Indonesian" +translated_name = "Indonesia" in_prod = false [languages.it] name = "Italian" +translated_name = "italiano" [languages.ja] name = "Japanese" +translated_name = "日本語" sphinxopts = [ '-D latex_engine=lualatex', '-D latex_elements.inputenc=', @@ -59,6 +71,7 @@ sphinxopts = [ [languages.ko] name = "Korean" +translated_name = "한국어" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -68,20 +81,25 @@ sphinxopts = [ [languages.pl] name = "Polish" +translated_name = "polski" [languages.pt_BR] name = "Brazilian Portuguese" +translated_name = "Português brasileiro" [languages.tr] name = "Turkish" +translated_name = "Türkçe" [languages.uk] name = "Ukrainian" +translated_name = "українська" in_prod = false html_only = true [languages.zh_CN] name = "Simplified Chinese" +translated_name = "简体中文" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', @@ -90,6 +108,7 @@ sphinxopts = [ [languages.zh_TW] name = "Traditional Chinese" +translated_name = "繁體中文" sphinxopts = [ '-D latex_engine=xelatex', '-D latex_elements.inputenc=', From b66ca348eeea7d97eed0617cee091162320a8704 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 19 Feb 2025 12:53:56 +0000 Subject: [PATCH 041/102] Re-enable translation_progress_classes We now use Sphinx 8.2, which should properly convert command-line overrides to Boolean values. --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 8b7f3f9..ca57ca5 100755 --- a/build_docs.py +++ b/build_docs.py @@ -717,7 +717,7 @@ def build(self): f"-D locale_dirs={locale_dirs}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", - # "-D translation_progress_classes=1", + "-D translation_progress_classes=1", ) ) if self.language.tag == "ja": From 273cd3c5ab4242c37117a15938e439f289b4c995 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:21:34 +0200 Subject: [PATCH 042/102] Test Python 3.14 & remove Python 3.10-3.12 (#251) --- .github/workflows/test.yml | 2 +- build_docs.py | 8 ++++---- tox.ini | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d74e607..ebff180 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.13", "3.14"] os: [ubuntu-latest] steps: diff --git a/build_docs.py b/build_docs.py index ca57ca5..fb51cf9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -23,7 +23,7 @@ from __future__ import annotations from argparse import ArgumentParser, Namespace -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from contextlib import suppress, contextmanager from dataclasses import dataclass import filecmp @@ -42,7 +42,7 @@ from pathlib import Path from string import Template from time import perf_counter, sleep -from typing import Iterable, Literal +from typing import Literal from urllib.parse import urljoin import jinja2 @@ -479,7 +479,7 @@ def version_info(): """Handler for --version.""" try: platex_version = head( - subprocess.check_output(["platex", "--version"], universal_newlines=True), + subprocess.check_output(["platex", "--version"], text=True), lines=3, ) except FileNotFoundError: @@ -487,7 +487,7 @@ def version_info(): try: xelatex_version = head( - subprocess.check_output(["xelatex", "--version"], universal_newlines=True), + subprocess.check_output(["xelatex", "--version"], text=True), lines=2, ) except FileNotFoundError: diff --git a/tox.ini b/tox.ini index 40a034d..56c6420 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ requires = tox>=4.2 env_list = lint - py{313, 312, 311, 310} + py{314, 313} [testenv] package = wheel From 6902d1c2302411dc560655f7948bd2dee6dcf264 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 20 Feb 2025 15:15:09 +0200 Subject: [PATCH 043/102] Update pre-commit (#249) --- .github/workflows/lint.yml | 7 ++++--- .github/workflows/test.yml | 5 +++-- .pre-commit-config.yaml | 17 +++++++++++------ build_docs.py | 5 ++--- check_times.py | 4 ++-- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d553e49..cd9d07e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,19 +2,20 @@ name: Lint on: [push, pull_request, workflow_dispatch] +permissions: {} + env: FORCE_COLOR: 1 PIP_DISABLE_PIP_VERSION_CHECK: 1 -permissions: - contents: read - jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ebff180..2976bae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,7 @@ name: Test on: [push, pull_request, workflow_dispatch] -permissions: - contents: read +permissions: {} env: FORCE_COLOR: 1 @@ -19,6 +18,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60a928c..44948ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,37 +14,42 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.9.6 hooks: - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.4 + rev: 0.31.1 hooks: - id: check-github-workflows - repo: https://github.com/rhysd/actionlint - rev: v1.7.3 + rev: v1.7.7 hooks: - id: actionlint + - repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v1.3.1 + hooks: + - id: zizmor + - repo: https://github.com/tox-dev/pyproject-fmt rev: v2.5.0 hooks: - id: pyproject-fmt - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.22 + rev: v0.23 hooks: - id: validate-pyproject - repo: https://github.com/tox-dev/tox-ini-fmt - rev: 1.4.1 + rev: 1.5.0 hooks: - id: tox-ini-fmt - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.3.3 + rev: v3.5.1 hooks: - id: prettier files: templates/switchers.js diff --git a/build_docs.py b/build_docs.py index fb51cf9..7da3520 100755 --- a/build_docs.py +++ b/build_docs.py @@ -86,7 +86,7 @@ def __init__(self, name, *, status, branch_or_tag=None): if status not in self.STATUSES: raise ValueError( "Version status expected to be one of: " - f"{', '.join(self.STATUSES|set(self.SYNONYMS.keys()))}, got {status!r}." + f"{', '.join(self.STATUSES | set(self.SYNONYMS.keys()))}, got {status!r}." ) self.name = name self.branch_or_tag = branch_or_tag @@ -732,8 +732,7 @@ def build(self): shell=True, ) subprocess.check_output( - "sed -i s/\N{REPLACEMENT CHARACTER}/?/g " - f"{self.checkout}/Doc/**/*.rst", + f"sed -i s/\N{REPLACEMENT CHARACTER}/?/g {self.checkout}/Doc/**/*.rst", shell=True, ) diff --git a/check_times.py b/check_times.py index 9310cd4..2b3d2f9 100644 --- a/check_times.py +++ b/check_times.py @@ -50,7 +50,7 @@ def calc_time(lines: list[str]) -> None: fmt_duration = format_seconds(state_data["last_build_duration"]) reason = state_data["triggered_by"] print( - f"{start:%Y-%m-%d %H:%M UTC} | {version: <7} | {language: <8} | {fmt_duration :<14} | {reason}" + f"{start:%Y-%m-%d %H:%M UTC} | {version: <7} | {language: <8} | {fmt_duration:<14} | {reason}" ) if line.endswith("Build start."): @@ -64,7 +64,7 @@ def calc_time(lines: list[str]) -> None: timestamp = f"{line[:16]} UTC" _, fmt_duration = line.removesuffix(").").split("(") print( - f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration :<14} | -----------" + f"{timestamp: <20} | --FULL- | -BUILD-- | {fmt_duration:<14} | -----------" ) if in_progress: From 7583b1744007e74cf9aafce98fa59677a75ce28d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 5 Apr 2025 16:29:14 +0100 Subject: [PATCH 044/102] Define ``ogp_site_url`` for social media cards (#252) --- build_docs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 7da3520..23fd6c0 100755 --- a/build_docs.py +++ b/build_docs.py @@ -751,9 +751,15 @@ def build(self): blurb = self.venv / "bin" / "blurb" if self.includes_html: + site_url = self.version.url + if self.language.tag != "en": + site_url += f"{self.language.tag}/" # Define a tag to enable opengraph socialcards previews # (used in Doc/conf.py and requires matplotlib) - sphinxopts.append("-t create-social-cards") + sphinxopts += ( + "-t create-social-cards", + f"-D ogp_site_url={site_url}", + ) # Disable CPython switchers, we handle them now: run( From 667cd01ec2b694fdecae6a3416f16774d51d390a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 01:09:44 +0100 Subject: [PATCH 045/102] Sort imports --- build_docs.py | 12 ++++++------ check_versions.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build_docs.py b/build_docs.py index 23fd6c0..1ab2fdd 100755 --- a/build_docs.py +++ b/build_docs.py @@ -22,23 +22,23 @@ from __future__ import annotations -from argparse import ArgumentParser, Namespace -from collections.abc import Iterable, Sequence -from contextlib import suppress, contextmanager -from dataclasses import dataclass import filecmp import json import logging import logging.handlers -from functools import total_ordering -from os import getenv, readlink import re import shlex import shutil import subprocess import sys +from argparse import ArgumentParser, Namespace from bisect import bisect_left as bisect +from collections.abc import Iterable, Sequence +from contextlib import contextmanager, suppress +from dataclasses import dataclass from datetime import datetime as dt, timezone +from functools import total_ordering +from os import getenv, readlink from pathlib import Path from string import Template from time import perf_counter, sleep diff --git a/check_versions.py b/check_versions.py index 70cade9..343c85a 100644 --- a/check_versions.py +++ b/check_versions.py @@ -1,15 +1,15 @@ #!/usr/bin/env python -from pathlib import Path import argparse import asyncio import logging import re +from pathlib import Path +import git import httpx import urllib3 from tabulate import tabulate -import git import build_docs From 62903155bd36c7ce09e8136b5d804552f4ba477c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 01:50:33 +0100 Subject: [PATCH 046/102] Move ``parse_args()`` and ``setup_logging()`` after ``main()`` --- build_docs.py | 206 +++++++++++++++++++++++++------------------------- 1 file changed, 103 insertions(+), 103 deletions(-) diff --git a/build_docs.py b/build_docs.py index 1ab2fdd..3da84df 100755 --- a/build_docs.py +++ b/build_docs.py @@ -506,109 +506,6 @@ def version_info(): ) -def parse_args(): - """Parse command-line arguments.""" - - parser = ArgumentParser( - description="Runs a build of the Python docs for various branches." - ) - parser.add_argument( - "--select-output", - choices=("no-html", "only-html", "only-html-en"), - help="Choose what outputs to build.", - ) - parser.add_argument( - "-q", - "--quick", - action="store_true", - help="Run a quick build (only HTML files).", - ) - parser.add_argument( - "-b", - "--branch", - metavar="3.12", - help="Version to build (defaults to all maintained branches).", - ) - parser.add_argument( - "-r", - "--build-root", - type=Path, - help="Path to a directory containing a checkout per branch.", - default=Path("/srv/docsbuild"), - ) - parser.add_argument( - "-w", - "--www-root", - type=Path, - help="Path where generated files will be copied.", - default=Path("/srv/docs.python.org"), - ) - parser.add_argument( - "--skip-cache-invalidation", - help="Skip Fastly cache invalidation.", - action="store_true", - ) - parser.add_argument( - "--group", - help="Group files on targets and www-root file should get.", - default="docs", - ) - parser.add_argument( - "--log-directory", - type=Path, - help="Directory used to store logs.", - default=Path("/var/log/docsbuild/"), - ) - parser.add_argument( - "--languages", - nargs="*", - help="Language translation, as a PEP 545 language tag like" - " 'fr' or 'pt-br'. " - "Builds all available languages by default.", - metavar="fr", - ) - parser.add_argument( - "--version", - action="store_true", - help="Get build_docs and dependencies version info", - ) - parser.add_argument( - "--theme", - default="python-docs-theme", - help="Python package to use for python-docs-theme: Useful to test branches:" - " --theme git+https://github.com/obulat/python-docs-theme@master", - ) - args = parser.parse_args() - if args.version: - version_info() - sys.exit(0) - del args.version - if args.log_directory: - args.log_directory = args.log_directory.resolve() - if args.build_root: - args.build_root = args.build_root.resolve() - if args.www_root: - args.www_root = args.www_root.resolve() - return args - - -def setup_logging(log_directory: Path, select_output: str | None): - """Setup logging to stderr if run by a human, or to a file if run from a cron.""" - log_format = "%(asctime)s %(levelname)s: %(message)s" - if sys.stderr.isatty(): - logging.basicConfig(format=log_format, stream=sys.stderr) - else: - log_directory.mkdir(parents=True, exist_ok=True) - if select_output is None: - filename = log_directory / "docsbuild.log" - else: - filename = log_directory / f"docsbuild-{select_output}.log" - handler = logging.handlers.WatchedFileHandler(filename) - handler.setFormatter(logging.Formatter(log_format)) - logging.getLogger().addHandler(handler) - logging.getLogger().setLevel(logging.DEBUG) - - @dataclass class DocBuilder: """Builder for a CPython version and a language.""" @@ -1288,6 +1185,109 @@ def main(): build_docs_with_lock(args, "build_docs_html_en.lock") +def parse_args(): + """Parse command-line arguments.""" + + parser = ArgumentParser( + description="Runs a build of the Python docs for various branches." + ) + parser.add_argument( + "--select-output", + choices=("no-html", "only-html", "only-html-en"), + help="Choose what outputs to build.", + ) + parser.add_argument( + "-q", + "--quick", + action="store_true", + help="Run a quick build (only HTML files).", + ) + parser.add_argument( + "-b", + "--branch", + metavar="3.12", + help="Version to build (defaults to all maintained branches).", + ) + parser.add_argument( + "-r", + "--build-root", + type=Path, + help="Path to a directory containing a checkout per branch.", + default=Path("/srv/docsbuild"), + ) + parser.add_argument( + "-w", + "--www-root", + type=Path, + help="Path where generated files will be copied.", + default=Path("/srv/docs.python.org"), + ) + parser.add_argument( + "--skip-cache-invalidation", + help="Skip Fastly cache invalidation.", + action="store_true", + ) + parser.add_argument( + "--group", + help="Group files on targets and www-root file should get.", + default="docs", + ) + parser.add_argument( + "--log-directory", + type=Path, + help="Directory used to store logs.", + default=Path("/var/log/docsbuild/"), + ) + parser.add_argument( + "--languages", + nargs="*", + help="Language translation, as a PEP 545 language tag like" + " 'fr' or 'pt-br'. " + "Builds all available languages by default.", + metavar="fr", + ) + parser.add_argument( + "--version", + action="store_true", + help="Get build_docs and dependencies version info", + ) + parser.add_argument( + "--theme", + default="python-docs-theme", + help="Python package to use for python-docs-theme: Useful to test branches:" + " --theme git+https://github.com/obulat/python-docs-theme@master", + ) + args = parser.parse_args() + if args.version: + version_info() + sys.exit(0) + del args.version + if args.log_directory: + args.log_directory = args.log_directory.resolve() + if args.build_root: + args.build_root = args.build_root.resolve() + if args.www_root: + args.www_root = args.www_root.resolve() + return args + + +def setup_logging(log_directory: Path, select_output: str | None): + """Setup logging to stderr if run by a human, or to a file if run from a cron.""" + log_format = "%(asctime)s %(levelname)s: %(message)s" + if sys.stderr.isatty(): + logging.basicConfig(format=log_format, stream=sys.stderr) + else: + log_directory.mkdir(parents=True, exist_ok=True) + if select_output is None: + filename = log_directory / "docsbuild.log" + else: + filename = log_directory / f"docsbuild-{select_output}.log" + handler = logging.handlers.WatchedFileHandler(filename) + handler.setFormatter(logging.Formatter(log_format)) + logging.getLogger().addHandler(handler) + logging.getLogger().setLevel(logging.DEBUG) + + def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: try: lock = zc.lockfile.LockFile(HERE / lockfile_name) From 8d0b9c3237cc34e9ebc0480cb85bd3345f8da3a9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 01:52:49 +0100 Subject: [PATCH 047/102] Move ``build_docs()`` after ``build_docs_with_lock()`` --- build_docs.py | 608 +++++++++++++++++++++++++------------------------- 1 file changed, 304 insertions(+), 304 deletions(-) diff --git a/build_docs.py b/build_docs.py index 3da84df..fa0ce44 100755 --- a/build_docs.py +++ b/build_docs.py @@ -421,55 +421,6 @@ def setup_switchers( ofile.write(line) -def copy_robots_txt( - www_root: Path, - group, - skip_cache_invalidation, - http: urllib3.PoolManager, -) -> None: - """Copy robots.txt to www_root.""" - if not www_root.exists(): - logging.info("Skipping copying robots.txt (www root does not even exist).") - return - logging.info("Copying robots.txt...") - template_path = HERE / "templates" / "robots.txt" - robots_path = www_root / "robots.txt" - shutil.copyfile(template_path, robots_path) - robots_path.chmod(0o775) - run(["chgrp", group, robots_path]) - if not skip_cache_invalidation: - purge(http, "robots.txt") - - -def build_sitemap( - versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group -): - """Build a sitemap with all live versions and translations.""" - if not www_root.exists(): - logging.info("Skipping sitemap generation (www root does not even exist).") - return - logging.info("Starting sitemap generation...") - template_path = HERE / "templates" / "sitemap.xml" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render(languages=languages, versions=versions) - sitemap_path = www_root / "sitemap.xml" - sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") - sitemap_path.chmod(0o664) - run(["chgrp", group, sitemap_path]) - - -def build_404(www_root: Path, group): - """Build a nice 404 error page to display in case PDFs are not built yet.""" - if not www_root.exists(): - logging.info("Skipping 404 page generation (www root does not even exist).") - return - logging.info("Copying 404 page...") - not_found_file = www_root / "404.html" - shutil.copyfile(HERE / "templates" / "404.html", not_found_file) - not_found_file.chmod(0o664) - run(["chgrp", group, not_found_file]) - - def head(text, lines=10): """Return the first *lines* lines from the given text.""" return "\n".join(text.split("\n")[:lines]) @@ -895,188 +846,6 @@ def save_state(self, build_start: dt, build_duration: float, trigger: str): logging.info("Saved new rebuild state for %s: %s", key, table.as_string()) -def symlink( - www_root: Path, - language: Language, - directory: str, - name: str, - group: str, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Used by major_symlinks and dev_symlink to maintain symlinks.""" - if language.tag == "en": # English is rooted on /, no /en/ - path = www_root - else: - path = www_root / language.tag - link = path / name - directory_path = path / directory - if not directory_path.exists(): - return # No touching link, dest doc not built yet. - - if not link.exists() or readlink(link) != directory: - # Link does not exist or points to the wrong target. - link.unlink(missing_ok=True) - link.symlink_to(directory) - run(["chown", "-h", f":{group}", str(link)]) - if not skip_cache_invalidation: - surrogate_key = f"{language.tag}/{name}" - purge_surrogate_key(http, surrogate_key) - - -def major_symlinks( - www_root: Path, - group: str, - versions: Iterable[Version], - languages: Iterable[Language], - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /2/ and /3/ symlinks for each language. - - Like: - - /3/ → /3.9/ - - /fr/3/ → /fr/3.9/ - - /es/3/ → /es/3.9/ - """ - logging.info("Creating major version symlinks...") - current_stable = Version.current_stable(versions).name - for language in languages: - symlink( - www_root, - language, - current_stable, - "3", - group, - skip_cache_invalidation, - http, - ) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) - - -def dev_symlink( - www_root: Path, - group, - versions, - languages, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /dev/ symlinks for each language. - - Like: - - /dev/ → /3.11/ - - /fr/dev/ → /fr/3.11/ - - /es/dev/ → /es/3.11/ - """ - logging.info("Creating development version symlinks...") - current_dev = Version.current_dev(versions).name - for language in languages: - symlink( - www_root, - language, - current_dev, - "dev", - group, - skip_cache_invalidation, - http, - ) - - -def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: - """Remove one or many paths from docs.python.org's CDN. - - To be used when a file changes, so the CDN fetches the new one. - """ - base = "https://docs.python.org/" - for path in paths: - url = urljoin(base, str(path)) - logging.debug("Purging %s from CDN", url) - http.request("PURGE", url, timeout=30) - - -def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: - """Remove paths from docs.python.org's CDN. - - All paths matching the given 'Surrogate-Key' will be removed. - This is set by the Nginx server for every language-version pair. - To be used when a directory changes, so the CDN fetches the new one. - - https://www.fastly.com/documentation/reference/api/purging/#purge-tag - """ - service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") - fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") - - logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) - http.request( - "POST", - f"https://api.fastly.com/service/{service_id}/purge/{surrogate_key}", - headers={"Fastly-Key": fastly_key}, - timeout=30, - ) - - -def proofread_canonicals( - www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager -) -> None: - """In www_root we check that all canonical links point to existing contents. - - It can happen that a canonical is "broken": - - - /3.11/whatsnew/3.11.html typically would link to - /3/whatsnew/3.11.html, which may not exist yet. - """ - logging.info("Checking canonical links...") - canonical_re = re.compile( - """""" - ) - for file in www_root.glob("**/*.html"): - html = file.read_text(encoding="UTF-8", errors="surrogateescape") - canonical = canonical_re.search(html) - if not canonical: - continue - target = canonical.group(1) - if not (www_root / target).exists(): - logging.info("Removing broken canonical from %s to %s", file, target) - html = html.replace(canonical.group(0), "") - file.write_text(html, encoding="UTF-8", errors="surrogateescape") - if not skip_cache_invalidation: - purge(http, str(file).replace("/srv/docs.python.org/", "")) - - -def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: - releases = http.request( - "GET", - "https://raw.githubusercontent.com/" - "python/devguide/main/include/release-cycle.json", - timeout=30, - ).json() - versions = [Version.from_json(name, release) for name, release in releases.items()] - versions.sort(key=Version.as_tuple) - return versions - - -def parse_languages_from_config() -> list[Language]: - """Read config.toml to discover languages to build.""" - config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) - defaults = config["defaults"] - default_translated_name = defaults.get("translated_name", "") - default_in_prod = defaults.get("in_prod", True) - default_sphinxopts = defaults.get("sphinxopts", []) - default_html_only = defaults.get("html_only", False) - return [ - Language( - iso639_tag=iso639_tag, - name=section["name"], - translated_name=section.get("translated_name", default_translated_name), - in_prod=section.get("in_prod", default_in_prod), - sphinxopts=section.get("sphinxopts", default_sphinxopts), - html_only=section.get("html_only", default_html_only), - ) - for iso639_tag, section in config["languages"].items() - ] - - def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) @@ -1091,79 +860,6 @@ def format_seconds(seconds: float) -> str: return f"{h}h {m}m {s}s" -def build_docs(args) -> bool: - """Build all docs (each language and each version).""" - logging.info("Full build start.") - start_time = perf_counter() - http = urllib3.PoolManager() - versions = parse_versions_from_devguide(http) - languages = parse_languages_from_config() - # Reverse languages but not versions, because we take version-language - # pairs from the end of the list, effectively reversing it. - # This runs languages in config.toml order and versions newest first. - todo = [ - (version, language) - for version in Version.filter(versions, args.branch) - for language in reversed(Language.filter(languages, args.languages)) - ] - del args.branch - del args.languages - all_built_successfully = True - cpython_repo = Repository( - "https://github.com/python/cpython.git", - args.build_root / _checkout_name(args.select_output), - ) - while todo: - version, language = todo.pop() - logging.root.handlers[0].setFormatter( - logging.Formatter( - f"%(asctime)s %(levelname)s {language.tag}/{version.name}: %(message)s" - ) - ) - if sentry_sdk: - scope = sentry_sdk.get_isolation_scope() - scope.set_tag("version", version.name) - scope.set_tag("language", language.tag) - cpython_repo.update() - builder = DocBuilder( - version, versions, language, languages, cpython_repo, **vars(args) - ) - all_built_successfully &= builder.run(http) - logging.root.handlers[0].setFormatter( - logging.Formatter("%(asctime)s %(levelname)s: %(message)s") - ) - - build_sitemap(versions, languages, args.www_root, args.group) - build_404(args.www_root, args.group) - copy_robots_txt( - args.www_root, - args.group, - args.skip_cache_invalidation, - http, - ) - major_symlinks( - args.www_root, - args.group, - versions, - languages, - args.skip_cache_invalidation, - http, - ) - dev_symlink( - args.www_root, - args.group, - versions, - languages, - args.skip_cache_invalidation, - http, - ) - proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) - - logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - - return all_built_successfully - - def _checkout_name(select_output: str | None) -> str: if select_output is not None: return f"cpython-{select_output}" @@ -1301,5 +997,309 @@ def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: lock.close() +def build_docs(args) -> bool: + """Build all docs (each language and each version).""" + logging.info("Full build start.") + start_time = perf_counter() + http = urllib3.PoolManager() + versions = parse_versions_from_devguide(http) + languages = parse_languages_from_config() + # Reverse languages but not versions, because we take version-language + # pairs from the end of the list, effectively reversing it. + # This runs languages in config.toml order and versions newest first. + todo = [ + (version, language) + for version in Version.filter(versions, args.branch) + for language in reversed(Language.filter(languages, args.languages)) + ] + del args.branch + del args.languages + all_built_successfully = True + cpython_repo = Repository( + "https://github.com/python/cpython.git", + args.build_root / _checkout_name(args.select_output), + ) + while todo: + version, language = todo.pop() + logging.root.handlers[0].setFormatter( + logging.Formatter( + f"%(asctime)s %(levelname)s {language.tag}/{version.name}: %(message)s" + ) + ) + if sentry_sdk: + scope = sentry_sdk.get_isolation_scope() + scope.set_tag("version", version.name) + scope.set_tag("language", language.tag) + cpython_repo.update() + builder = DocBuilder( + version, versions, language, languages, cpython_repo, **vars(args) + ) + all_built_successfully &= builder.run(http) + logging.root.handlers[0].setFormatter( + logging.Formatter("%(asctime)s %(levelname)s: %(message)s") + ) + + build_sitemap(versions, languages, args.www_root, args.group) + build_404(args.www_root, args.group) + copy_robots_txt( + args.www_root, + args.group, + args.skip_cache_invalidation, + http, + ) + major_symlinks( + args.www_root, + args.group, + versions, + languages, + args.skip_cache_invalidation, + http, + ) + dev_symlink( + args.www_root, + args.group, + versions, + languages, + args.skip_cache_invalidation, + http, + ) + proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) + + logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) + + return all_built_successfully + + +def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: + releases = http.request( + "GET", + "https://raw.githubusercontent.com/" + "python/devguide/main/include/release-cycle.json", + timeout=30, + ).json() + versions = [Version.from_json(name, release) for name, release in releases.items()] + versions.sort(key=Version.as_tuple) + return versions + + +def parse_languages_from_config() -> list[Language]: + """Read config.toml to discover languages to build.""" + config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) + defaults = config["defaults"] + default_translated_name = defaults.get("translated_name", "") + default_in_prod = defaults.get("in_prod", True) + default_sphinxopts = defaults.get("sphinxopts", []) + default_html_only = defaults.get("html_only", False) + return [ + Language( + iso639_tag=iso639_tag, + name=section["name"], + translated_name=section.get("translated_name", default_translated_name), + in_prod=section.get("in_prod", default_in_prod), + sphinxopts=section.get("sphinxopts", default_sphinxopts), + html_only=section.get("html_only", default_html_only), + ) + for iso639_tag, section in config["languages"].items() + ] + + +def build_sitemap( + versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group +): + """Build a sitemap with all live versions and translations.""" + if not www_root.exists(): + logging.info("Skipping sitemap generation (www root does not even exist).") + return + logging.info("Starting sitemap generation...") + template_path = HERE / "templates" / "sitemap.xml" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render(languages=languages, versions=versions) + sitemap_path = www_root / "sitemap.xml" + sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") + sitemap_path.chmod(0o664) + run(["chgrp", group, sitemap_path]) + + +def build_404(www_root: Path, group): + """Build a nice 404 error page to display in case PDFs are not built yet.""" + if not www_root.exists(): + logging.info("Skipping 404 page generation (www root does not even exist).") + return + logging.info("Copying 404 page...") + not_found_file = www_root / "404.html" + shutil.copyfile(HERE / "templates" / "404.html", not_found_file) + not_found_file.chmod(0o664) + run(["chgrp", group, not_found_file]) + + +def copy_robots_txt( + www_root: Path, + group, + skip_cache_invalidation, + http: urllib3.PoolManager, +) -> None: + """Copy robots.txt to www_root.""" + if not www_root.exists(): + logging.info("Skipping copying robots.txt (www root does not even exist).") + return + logging.info("Copying robots.txt...") + template_path = HERE / "templates" / "robots.txt" + robots_path = www_root / "robots.txt" + shutil.copyfile(template_path, robots_path) + robots_path.chmod(0o775) + run(["chgrp", group, robots_path]) + if not skip_cache_invalidation: + purge(http, "robots.txt") + + +def major_symlinks( + www_root: Path, + group: str, + versions: Iterable[Version], + languages: Iterable[Language], + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Maintains the /2/ and /3/ symlinks for each language. + + Like: + - /3/ → /3.9/ + - /fr/3/ → /fr/3.9/ + - /es/3/ → /es/3.9/ + """ + logging.info("Creating major version symlinks...") + current_stable = Version.current_stable(versions).name + for language in languages: + symlink( + www_root, + language, + current_stable, + "3", + group, + skip_cache_invalidation, + http, + ) + symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) + + +def dev_symlink( + www_root: Path, + group, + versions, + languages, + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Maintains the /dev/ symlinks for each language. + + Like: + - /dev/ → /3.11/ + - /fr/dev/ → /fr/3.11/ + - /es/dev/ → /es/3.11/ + """ + logging.info("Creating development version symlinks...") + current_dev = Version.current_dev(versions).name + for language in languages: + symlink( + www_root, + language, + current_dev, + "dev", + group, + skip_cache_invalidation, + http, + ) + + +def symlink( + www_root: Path, + language: Language, + directory: str, + name: str, + group: str, + skip_cache_invalidation: bool, + http: urllib3.PoolManager, +) -> None: + """Used by major_symlinks and dev_symlink to maintain symlinks.""" + if language.tag == "en": # English is rooted on /, no /en/ + path = www_root + else: + path = www_root / language.tag + link = path / name + directory_path = path / directory + if not directory_path.exists(): + return # No touching link, dest doc not built yet. + + if not link.exists() or readlink(link) != directory: + # Link does not exist or points to the wrong target. + link.unlink(missing_ok=True) + link.symlink_to(directory) + run(["chown", "-h", f":{group}", str(link)]) + if not skip_cache_invalidation: + surrogate_key = f"{language.tag}/{name}" + purge_surrogate_key(http, surrogate_key) + + +def proofread_canonicals( + www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager +) -> None: + """In www_root we check that all canonical links point to existing contents. + + It can happen that a canonical is "broken": + + - /3.11/whatsnew/3.11.html typically would link to + /3/whatsnew/3.11.html, which may not exist yet. + """ + logging.info("Checking canonical links...") + canonical_re = re.compile( + """""" + ) + for file in www_root.glob("**/*.html"): + html = file.read_text(encoding="UTF-8", errors="surrogateescape") + canonical = canonical_re.search(html) + if not canonical: + continue + target = canonical.group(1) + if not (www_root / target).exists(): + logging.info("Removing broken canonical from %s to %s", file, target) + html = html.replace(canonical.group(0), "") + file.write_text(html, encoding="UTF-8", errors="surrogateescape") + if not skip_cache_invalidation: + purge(http, str(file).replace("/srv/docs.python.org/", "")) + + +def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: + """Remove one or many paths from docs.python.org's CDN. + + To be used when a file changes, so the CDN fetches the new one. + """ + base = "https://docs.python.org/" + for path in paths: + url = urljoin(base, str(path)) + logging.debug("Purging %s from CDN", url) + http.request("PURGE", url, timeout=30) + + +def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: + """Remove paths from docs.python.org's CDN. + + All paths matching the given 'Surrogate-Key' will be removed. + This is set by the Nginx server for every language-version pair. + To be used when a directory changes, so the CDN fetches the new one. + + https://www.fastly.com/documentation/reference/api/purging/#purge-tag + """ + service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") + fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") + + logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) + http.request( + "POST", + f"https://api.fastly.com/service/{service_id}/purge/{surrogate_key}", + headers={"Fastly-Key": fastly_key}, + timeout=30, + ) + + if __name__ == "__main__": sys.exit(main()) From e096701ea6bf82281dc1c73936bf8ca2d8b39bee Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:17:57 +0100 Subject: [PATCH 048/102] Improve type annotations for versions and languages (#253) --- build_docs.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index fa0ce44..21b3e6a 100755 --- a/build_docs.py +++ b/build_docs.py @@ -33,7 +33,6 @@ import sys from argparse import ArgumentParser, Namespace from bisect import bisect_left as bisect -from collections.abc import Iterable, Sequence from contextlib import contextmanager, suppress from dataclasses import dataclass from datetime import datetime as dt, timezone @@ -42,7 +41,6 @@ from pathlib import Path from string import Template from time import perf_counter, sleep -from typing import Literal from urllib.parse import urljoin import jinja2 @@ -50,6 +48,14 @@ import urllib3 import zc.lockfile +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Sequence + from typing import Literal, TypeAlias + + Versions: TypeAlias = Sequence["Version"] + Languages: TypeAlias = Sequence["Language"] + try: from os import EX_OK, EX_SOFTWARE as EX_FAILURE except ImportError: @@ -170,7 +176,7 @@ def picker_label(self): return f"pre ({self.name})" return self.name - def setup_indexsidebar(self, versions: Sequence[Version], dest_path: Path): + def setup_indexsidebar(self, versions: Versions, dest_path: Path): """Build indexsidebar.html for Sphinx.""" template_path = HERE / "templates" / "indexsidebar.html" template = jinja2.Template(template_path.read_text(encoding="UTF-8")) @@ -388,9 +394,7 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers( - versions: Sequence[Version], languages: Sequence[Language], html_root: Path -): +def setup_switchers(versions: Versions, languages: Languages, html_root: Path): """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher @@ -462,9 +466,9 @@ class DocBuilder: """Builder for a CPython version and a language.""" version: Version - versions: Sequence[Version] + versions: Versions language: Language - languages: Sequence[Language] + languages: Languages cpython_repo: Repository build_root: Path www_root: Path @@ -1070,7 +1074,7 @@ def build_docs(args) -> bool: return all_built_successfully -def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: +def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: releases = http.request( "GET", "https://raw.githubusercontent.com/" @@ -1082,7 +1086,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]: return versions -def parse_languages_from_config() -> list[Language]: +def parse_languages_from_config() -> Languages: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) defaults = config["defaults"] @@ -1103,9 +1107,7 @@ def parse_languages_from_config() -> list[Language]: ] -def build_sitemap( - versions: Iterable[Version], languages: Iterable[Language], www_root: Path, group -): +def build_sitemap(versions: Versions, languages: Languages, www_root: Path, group): """Build a sitemap with all live versions and translations.""" if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exist).") @@ -1155,8 +1157,8 @@ def copy_robots_txt( def major_symlinks( www_root: Path, group: str, - versions: Iterable[Version], - languages: Iterable[Language], + versions: Versions, + languages: Languages, skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: From 044aba7d5a27748d6d28f043446b3334fe76dd68 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:36:45 +0100 Subject: [PATCH 049/102] Create a dataclass for Versions (#254) --- build_docs.py | 123 ++++++++++++++++++++++++++++---------------------- 1 file changed, 69 insertions(+), 54 deletions(-) diff --git a/build_docs.py b/build_docs.py index 21b3e6a..3c8d435 100755 --- a/build_docs.py +++ b/build_docs.py @@ -50,10 +50,9 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Iterator, Sequence from typing import Literal, TypeAlias - Versions: TypeAlias = Sequence["Version"] Languages: TypeAlias = Sequence["Language"] try: @@ -71,6 +70,57 @@ HERE = Path(__file__).resolve().parent +@dataclass(frozen=True, slots=True) +class Versions: + _seq: Sequence[Version] + + def __iter__(self) -> Iterator[Version]: + return iter(self._seq) + + def __reversed__(self) -> Iterator[Version]: + return reversed(self._seq) + + @classmethod + def from_json(cls, data) -> Versions: + versions = sorted( + [Version.from_json(name, release) for name, release in data.items()], + key=Version.as_tuple, + ) + return cls(versions) + + def filter(self, branch: str = "") -> Sequence[Version]: + """Filter the given versions. + + If *branch* is given, only *versions* matching *branch* are returned. + + Else all live versions are returned (this means no EOL and no + security-fixes branches). + """ + if branch: + return [v for v in self if branch in (v.name, v.branch_or_tag)] + return [v for v in self if v.status not in {"EOL", "security-fixes"}] + + @property + def current_stable(self) -> Version: + """Find the current stable CPython version.""" + return max((v for v in self if v.status == "stable"), key=Version.as_tuple) + + @property + def current_dev(self) -> Version: + """Find the current CPython version in development.""" + return max(self, key=Version.as_tuple) + + def setup_indexsidebar(self, current: Version, dest_path: Path) -> None: + """Build indexsidebar.html for Sphinx.""" + template_path = HERE / "templates" / "indexsidebar.html" + template = jinja2.Template(template_path.read_text(encoding="UTF-8")) + rendered_template = template.render( + current_version=current, + versions=list(reversed(self)), + ) + dest_path.write_text(rendered_template, encoding="UTF-8") + + @total_ordering class Version: """Represents a CPython version and its documentation build dependencies.""" @@ -101,6 +151,17 @@ def __init__(self, name, *, status, branch_or_tag=None): def __repr__(self): return f"Version({self.name})" + def __eq__(self, other): + return self.name == other.name + + def __gt__(self, other): + return self.as_tuple() > other.as_tuple() + + @classmethod + def from_json(cls, name, values): + """Loads a version from devguide's json representation.""" + return cls(name, status=values["status"], branch_or_tag=values["branch"]) + @property def requirements(self): """Generate the right requirements for this version. @@ -144,29 +205,6 @@ def title(self): """The title of this version's doc, for the sidebar.""" return f"Python {self.name} ({self.status})" - @staticmethod - def filter(versions, branch=None): - """Filter the given versions. - - If *branch* is given, only *versions* matching *branch* are returned. - - Else all live versions are returned (this means no EOL and no - security-fixes branches). - """ - if branch: - return [v for v in versions if branch in (v.name, v.branch_or_tag)] - return [v for v in versions if v.status not in ("EOL", "security-fixes")] - - @staticmethod - def current_stable(versions): - """Find the current stable CPython version.""" - return max((v for v in versions if v.status == "stable"), key=Version.as_tuple) - - @staticmethod - def current_dev(versions): - """Find the current CPython version in development.""" - return max(versions, key=Version.as_tuple) - @property def picker_label(self): """Forge the label of a version picker.""" @@ -176,27 +214,6 @@ def picker_label(self): return f"pre ({self.name})" return self.name - def setup_indexsidebar(self, versions: Versions, dest_path: Path): - """Build indexsidebar.html for Sphinx.""" - template_path = HERE / "templates" / "indexsidebar.html" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render( - current_version=self, - versions=versions[::-1], - ) - dest_path.write_text(rendered_template, encoding="UTF-8") - - @classmethod - def from_json(cls, name, values): - """Loads a version from devguide's json representation.""" - return cls(name, status=values["status"], branch_or_tag=values["branch"]) - - def __eq__(self, other): - return self.name == other.name - - def __gt__(self, other): - return self.as_tuple() > other.as_tuple() - @dataclass(order=True, frozen=True, kw_only=True) class Language: @@ -619,8 +636,8 @@ def build(self): + ([""] if sys.platform == "darwin" else []) + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] ) - self.version.setup_indexsidebar( - self.versions, + self.versions.setup_indexsidebar( + self.version, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", ) run_with_logging( @@ -1013,7 +1030,7 @@ def build_docs(args) -> bool: # This runs languages in config.toml order and versions newest first. todo = [ (version, language) - for version in Version.filter(versions, args.branch) + for version in versions.filter(args.branch) for language in reversed(Language.filter(languages, args.languages)) ] del args.branch @@ -1081,9 +1098,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: "python/devguide/main/include/release-cycle.json", timeout=30, ).json() - versions = [Version.from_json(name, release) for name, release in releases.items()] - versions.sort(key=Version.as_tuple) - return versions + return Versions.from_json(releases) def parse_languages_from_config() -> Languages: @@ -1170,7 +1185,7 @@ def major_symlinks( - /es/3/ → /es/3.9/ """ logging.info("Creating major version symlinks...") - current_stable = Version.current_stable(versions).name + current_stable = versions.current_stable.name for language in languages: symlink( www_root, @@ -1200,7 +1215,7 @@ def dev_symlink( - /es/dev/ → /es/3.11/ """ logging.info("Creating development version symlinks...") - current_dev = Version.current_dev(versions).name + current_dev = versions.current_dev.name for language in languages: symlink( www_root, From a8070868f67096d5c05122c29d408ac4c1f1083d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:51:46 +0100 Subject: [PATCH 050/102] Create a dataclass for Languages (#255) --- build_docs.py | 70 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/build_docs.py b/build_docs.py index 3c8d435..ebb5a94 100755 --- a/build_docs.py +++ b/build_docs.py @@ -51,9 +51,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: from collections.abc import Iterator, Sequence - from typing import Literal, TypeAlias - - Languages: TypeAlias = Sequence["Language"] + from typing import Literal try: from os import EX_OK, EX_SOFTWARE as EX_FAILURE @@ -215,13 +213,50 @@ def picker_label(self): return self.name +@dataclass(frozen=True, slots=True) +class Languages: + _seq: Sequence[Language] + + def __iter__(self) -> Iterator[Language]: + return iter(self._seq) + + def __reversed__(self) -> Iterator[Language]: + return reversed(self._seq) + + @classmethod + def from_json(cls, defaults, languages) -> Languages: + default_translated_name = defaults.get("translated_name", "") + default_in_prod = defaults.get("in_prod", True) + default_sphinxopts = defaults.get("sphinxopts", []) + default_html_only = defaults.get("html_only", False) + langs = [ + Language( + iso639_tag=iso639_tag, + name=section["name"], + translated_name=section.get("translated_name", default_translated_name), + in_prod=section.get("in_prod", default_in_prod), + sphinxopts=section.get("sphinxopts", default_sphinxopts), + html_only=section.get("html_only", default_html_only), + ) + for iso639_tag, section in languages.items() + ] + return cls(langs) + + def filter(self, language_tags: Sequence[str] = ()) -> Sequence[Language]: + """Filter a sequence of languages according to --languages.""" + if language_tags: + language_tags = frozenset(language_tags) + return [l for l in self if l.tag in language_tags] + return list(self) + + @dataclass(order=True, frozen=True, kw_only=True) class Language: iso639_tag: str name: str translated_name: str in_prod: bool - sphinxopts: tuple + sphinxopts: Sequence[str] html_only: bool = False @property @@ -234,14 +269,6 @@ def switcher_label(self): return f"{self.name} | {self.translated_name}" return self.name - @staticmethod - def filter(languages, language_tags=None): - """Filter a sequence of languages according to --languages.""" - if language_tags: - languages_dict = {language.tag: language for language in languages} - return [languages_dict[tag] for tag in language_tags] - return languages - def run(cmd, cwd=None) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" @@ -1031,7 +1058,7 @@ def build_docs(args) -> bool: todo = [ (version, language) for version in versions.filter(args.branch) - for language in reversed(Language.filter(languages, args.languages)) + for language in reversed(languages.filter(args.languages)) ] del args.branch del args.languages @@ -1104,22 +1131,7 @@ def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: def parse_languages_from_config() -> Languages: """Read config.toml to discover languages to build.""" config = tomlkit.parse((HERE / "config.toml").read_text(encoding="UTF-8")) - defaults = config["defaults"] - default_translated_name = defaults.get("translated_name", "") - default_in_prod = defaults.get("in_prod", True) - default_sphinxopts = defaults.get("sphinxopts", []) - default_html_only = defaults.get("html_only", False) - return [ - Language( - iso639_tag=iso639_tag, - name=section["name"], - translated_name=section.get("translated_name", default_translated_name), - in_prod=section.get("in_prod", default_in_prod), - sphinxopts=section.get("sphinxopts", default_sphinxopts), - html_only=section.get("html_only", default_html_only), - ) - for iso639_tag, section in config["languages"].items() - ] + return Languages.from_json(config["defaults"], config["languages"]) def build_sitemap(versions: Versions, languages: Languages, www_root: Path, group): From 1a8a586d37840fa5ed43aed225fd9dcb6e8ca07f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 02:59:27 +0100 Subject: [PATCH 051/102] Convert some relative imports to absolute (#256) --- build_docs.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/build_docs.py b/build_docs.py index ebb5a94..a19f2f8 100755 --- a/build_docs.py +++ b/build_docs.py @@ -22,22 +22,22 @@ from __future__ import annotations +import argparse +import dataclasses +import datetime as dt import filecmp import json import logging import logging.handlers +import os import re import shlex import shutil import subprocess import sys -from argparse import ArgumentParser, Namespace from bisect import bisect_left as bisect from contextlib import contextmanager, suppress -from dataclasses import dataclass -from datetime import datetime as dt, timezone from functools import total_ordering -from os import getenv, readlink from pathlib import Path from string import Template from time import perf_counter, sleep @@ -68,7 +68,7 @@ HERE = Path(__file__).resolve().parent -@dataclass(frozen=True, slots=True) +@dataclasses.dataclass(frozen=True, slots=True) class Versions: _seq: Sequence[Version] @@ -213,7 +213,7 @@ def picker_label(self): return self.name -@dataclass(frozen=True, slots=True) +@dataclasses.dataclass(frozen=True, slots=True) class Languages: _seq: Sequence[Language] @@ -250,7 +250,7 @@ def filter(self, language_tags: Sequence[str] = ()) -> Sequence[Language]: return list(self) -@dataclass(order=True, frozen=True, kw_only=True) +@dataclasses.dataclass(order=True, frozen=True, kw_only=True) class Language: iso639_tag: str name: str @@ -337,7 +337,7 @@ def traverse(dircmp_result): return changed -@dataclass +@dataclasses.dataclass class Repository: """Git repository abstraction for our specific needs.""" @@ -505,7 +505,7 @@ def version_info(): ) -@dataclass +@dataclasses.dataclass class DocBuilder: """Builder for a CPython version and a language.""" @@ -539,7 +539,7 @@ def includes_html(self): def run(self, http: urllib3.PoolManager) -> bool: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() - start_timestamp = dt.now(tz=timezone.utc).replace(microsecond=0) + start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) logging.info("Running.") try: if self.language.html_only and not self.includes_html: @@ -861,7 +861,7 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_start: dt, build_duration: float, trigger: str): + def save_state(self, build_start: dt.datetime, build_duration: float, trigger: str): """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -932,8 +932,9 @@ def main(): def parse_args(): """Parse command-line arguments.""" - parser = ArgumentParser( - description="Runs a build of the Python docs for various branches." + parser = argparse.ArgumentParser( + description="Runs a build of the Python docs for various branches.", + allow_abbrev=False, ) parser.add_argument( "--select-output", @@ -1032,7 +1033,7 @@ def setup_logging(log_directory: Path, select_output: str | None): logging.getLogger().setLevel(logging.DEBUG) -def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: +def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: try: lock = zc.lockfile.LockFile(HERE / lockfile_name) except zc.lockfile.LockError: @@ -1045,7 +1046,7 @@ def build_docs_with_lock(args: Namespace, lockfile_name: str) -> int: lock.close() -def build_docs(args) -> bool: +def build_docs(args: argparse.Namespace) -> bool: """Build all docs (each language and each version).""" logging.info("Full build start.") start_time = perf_counter() @@ -1259,7 +1260,7 @@ def symlink( if not directory_path.exists(): return # No touching link, dest doc not built yet. - if not link.exists() or readlink(link) != directory: + if not link.exists() or os.readlink(link) != directory: # Link does not exist or points to the wrong target. link.unlink(missing_ok=True) link.symlink_to(directory) @@ -1318,8 +1319,8 @@ def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: https://www.fastly.com/documentation/reference/api/purging/#purge-tag """ - service_id = getenv("FASTLY_SERVICE_ID", "__UNSET__") - fastly_key = getenv("FASTLY_TOKEN", "__UNSET__") + service_id = os.environ.get("FASTLY_SERVICE_ID", "__UNSET__") + fastly_key = os.environ.get("FASTLY_TOKEN", "__UNSET__") logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) http.request( From a6c458d9ffaa135ec4a317012c21e9eae04814eb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 03:25:25 +0100 Subject: [PATCH 052/102] Only recreate a symlink when the relevant version has changed (#257) --- build_docs.py | 109 +++++++++++++++++++++----------------------------- 1 file changed, 45 insertions(+), 64 deletions(-) diff --git a/build_docs.py b/build_docs.py index a19f2f8..d03d687 100755 --- a/build_docs.py +++ b/build_docs.py @@ -50,7 +50,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Iterator, Sequence + from collections.abc import Iterator, Sequence, Set from typing import Literal try: @@ -1063,7 +1063,9 @@ def build_docs(args: argparse.Namespace) -> bool: ] del args.branch del args.languages - all_built_successfully = True + + build_succeeded = set() + build_failed = set() cpython_repo = Repository( "https://github.com/python/cpython.git", args.build_root / _checkout_name(args.select_output), @@ -1083,7 +1085,12 @@ def build_docs(args: argparse.Namespace) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - all_built_successfully &= builder.run(http) + built_successfully = builder.run(http) + if built_successfully: + build_succeeded.add((version.name, language.tag)) + else: + build_failed.add((version.name, language.tag)) + logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") ) @@ -1096,19 +1103,12 @@ def build_docs(args: argparse.Namespace) -> bool: args.skip_cache_invalidation, http, ) - major_symlinks( - args.www_root, - args.group, - versions, - languages, - args.skip_cache_invalidation, - http, - ) - dev_symlink( + make_symlinks( args.www_root, args.group, versions, languages, + build_succeeded, args.skip_cache_invalidation, http, ) @@ -1116,7 +1116,7 @@ def build_docs(args: argparse.Namespace) -> bool: logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - return all_built_successfully + return len(build_failed) == 0 def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: @@ -1182,68 +1182,46 @@ def copy_robots_txt( purge(http, "robots.txt") -def major_symlinks( +def make_symlinks( www_root: Path, group: str, versions: Versions, languages: Languages, + successful_builds: Set[tuple[str, str]], skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: - """Maintains the /2/ and /3/ symlinks for each language. + """Maintains the /2/, /3/, and /dev/ symlinks for each language. Like: - - /3/ → /3.9/ - - /fr/3/ → /fr/3.9/ - - /es/3/ → /es/3.9/ + - /2/ → /2.7/ + - /3/ → /3.12/ + - /dev/ → /3.14/ + - /fr/3/ → /fr/3.12/ + - /es/dev/ → /es/3.14/ """ - logging.info("Creating major version symlinks...") - current_stable = versions.current_stable.name - for language in languages: - symlink( - www_root, - language, - current_stable, - "3", - group, - skip_cache_invalidation, - http, - ) - symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http) - - -def dev_symlink( - www_root: Path, - group, - versions, - languages, - skip_cache_invalidation: bool, - http: urllib3.PoolManager, -) -> None: - """Maintains the /dev/ symlinks for each language. - - Like: - - /dev/ → /3.11/ - - /fr/dev/ → /fr/3.11/ - - /es/dev/ → /es/3.11/ - """ - logging.info("Creating development version symlinks...") - current_dev = versions.current_dev.name - for language in languages: - symlink( - www_root, - language, - current_dev, - "dev", - group, - skip_cache_invalidation, - http, - ) + logging.info("Creating major and development version symlinks...") + for symlink_name, symlink_target in ( + ("3", versions.current_stable.name), + ("2", "2.7"), + ("dev", versions.current_dev.name), + ): + for language in languages: + if (symlink_target, language.tag) in successful_builds: + symlink( + www_root, + language.tag, + symlink_target, + symlink_name, + group, + skip_cache_invalidation, + http, + ) def symlink( www_root: Path, - language: Language, + language_tag: str, directory: str, name: str, group: str, @@ -1251,10 +1229,13 @@ def symlink( http: urllib3.PoolManager, ) -> None: """Used by major_symlinks and dev_symlink to maintain symlinks.""" - if language.tag == "en": # English is rooted on /, no /en/ + msg = "Creating symlink from %s to %s" + if language_tag == "en": # English is rooted on /, no /en/ path = www_root + logging.debug(msg, name, directory) else: - path = www_root / language.tag + path = www_root / language_tag + logging.debug(msg, f"{language_tag}/{name}", f"{language_tag}/{directory}") link = path / name directory_path = path / directory if not directory_path.exists(): @@ -1266,7 +1247,7 @@ def symlink( link.symlink_to(directory) run(["chown", "-h", f":{group}", str(link)]) if not skip_cache_invalidation: - surrogate_key = f"{language.tag}/{name}" + surrogate_key = f"{language_tag}/{name}" purge_surrogate_key(http, surrogate_key) From 3d068f403607669b9408115dce9918a9b915dc62 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 05:41:28 +0100 Subject: [PATCH 053/102] Adjust symlink log message (#259) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index d03d687..b6a58a9 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1229,7 +1229,7 @@ def symlink( http: urllib3.PoolManager, ) -> None: """Used by major_symlinks and dev_symlink to maintain symlinks.""" - msg = "Creating symlink from %s to %s" + msg = "Creating symlink from /%s/ to /%s/" if language_tag == "en": # English is rooted on /, no /en/ path = www_root logging.debug(msg, name, directory) From 0ef872fd7e7b00e6e1193a73ebaf37d7eff7b444 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 05:43:29 +0100 Subject: [PATCH 054/102] Skip CDN requests when secrets are unset (#260) --- build_docs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index b6a58a9..4d7bf8d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1300,8 +1300,13 @@ def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: https://www.fastly.com/documentation/reference/api/purging/#purge-tag """ - service_id = os.environ.get("FASTLY_SERVICE_ID", "__UNSET__") - fastly_key = os.environ.get("FASTLY_TOKEN", "__UNSET__") + unset = "__UNSET__" + service_id = os.environ.get("FASTLY_SERVICE_ID", unset) + fastly_key = os.environ.get("FASTLY_TOKEN", unset) + + if service_id == unset or fastly_key == unset: + logging.info("CDN secrets not set, skipping Surrogate-Key purge") + return logging.info("Purging Surrogate-Key '%s' from CDN", surrogate_key) http.request( From 6d537969095c21f34406ff954535020b690f0d3d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 06:00:37 +0100 Subject: [PATCH 055/102] Add ``--language`` as a synonym of ``--languages`` --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 4d7bf8d..8f5d763 100755 --- a/build_docs.py +++ b/build_docs.py @@ -984,7 +984,7 @@ def parse_args(): default=Path("/var/log/docsbuild/"), ) parser.add_argument( - "--languages", + "--languages", "--language", nargs="*", help="Language translation, as a PEP 545 language tag like" " 'fr' or 'pt-br'. " From 77a363be7d4e4574bb950e02cb504434bdb2e830 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 06:29:32 +0100 Subject: [PATCH 056/102] Add Ruff configuration file (#262) --- .pre-commit-config.yaml | 2 +- .ruff.toml | 7 ++ build_docs.py | 143 ++++++++++++++++++---------------------- check_versions.py | 10 ++- 4 files changed, 76 insertions(+), 86 deletions(-) create mode 100644 .ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 44948ef..4ad4370 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.6 + rev: v0.11.5 hooks: - id: ruff-format diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..e837d03 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,7 @@ +target-version = "py313" # Pin Ruff to Python 3.13 +line-length = 88 +output-format = "full" + +[format] +preview = true +docstring-code-format = true diff --git a/build_docs.py b/build_docs.py index 8f5d763..0ca807e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -608,14 +608,12 @@ def build(self): sphinxopts = list(self.language.sphinxopts) if self.language.tag != "en": locale_dirs = self.build_root / self.version.name / "locale" - sphinxopts.extend( - ( - f"-D locale_dirs={locale_dirs}", - f"-D language={self.language.iso639_tag}", - "-D gettext_compact=0", - "-D translation_progress_classes=1", - ) - ) + sphinxopts.extend(( + f"-D locale_dirs={locale_dirs}", + f"-D language={self.language.iso639_tag}", + "-D gettext_compact=0", + "-D translation_progress_classes=1", + )) if self.language.tag == "ja": # Since luatex doesn't support \ufffd, replace \ufffd with '?'. # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b @@ -667,20 +665,18 @@ def build(self): self.version, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", ) - run_with_logging( - [ - "make", - "-C", - self.checkout / "Doc", - "PYTHON=" + str(python), - "SPHINXBUILD=" + str(sphinxbuild), - "BLURB=" + str(blurb), - "VENVDIR=" + str(self.venv), - "SPHINXOPTS=" + " ".join(sphinxopts), - "SPHINXERRORHANDLING=", - maketarget, - ] - ) + run_with_logging([ + "make", + "-C", + self.checkout / "Doc", + "PYTHON=" + str(python), + "SPHINXBUILD=" + str(sphinxbuild), + "BLURB=" + str(blurb), + "VENVDIR=" + str(self.venv), + "SPHINXOPTS=" + " ".join(sphinxopts), + "SPHINXERRORHANDLING=", + maketarget, + ]) run(["mkdir", "-p", self.log_directory]) run(["chgrp", "-R", self.group, self.log_directory]) if self.includes_html: @@ -743,69 +739,57 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: # Copy built HTML files to webroot (default /srv/docs.python.org) changed = changed_files(self.checkout / "Doc" / "build" / "html", target) logging.info("Copying HTML files to %s", target) - run( - [ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "build" / "html/", - ] - ) + run([ + "chown", + "-R", + ":" + self.group, + self.checkout / "Doc" / "build" / "html/", + ]) run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) - run( - [ - "find", - self.checkout / "Doc" / "build" / "html", - "-type", - "d", - "-exec", - "chmod", - "o+x", - "{}", - ";", - ] - ) - run( - [ - "rsync", - "-a", - "--delete-delay", - "--filter", - "P archives/", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - ] - ) + run([ + "find", + self.checkout / "Doc" / "build" / "html", + "-type", + "d", + "-exec", + "chmod", + "o+x", + "{}", + ";", + ]) + run([ + "rsync", + "-a", + "--delete-delay", + "--filter", + "P archives/", + str(self.checkout / "Doc" / "build" / "html") + "/", + target, + ]) if not self.quick: # Copy archive files to /archives/ logging.debug("Copying dist files.") - run( - [ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "dist", - ] - ) - run( - [ - "chmod", - "-R", - "o+r", - self.checkout / "Doc" / "dist", - ] - ) + run([ + "chown", + "-R", + ":" + self.group, + self.checkout / "Doc" / "dist", + ]) + run([ + "chmod", + "-R", + "o+r", + self.checkout / "Doc" / "dist", + ]) run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) run(["chown", ":" + self.group, target / "archives"]) - run( - [ - "cp", - "-a", - *(self.checkout / "Doc" / "dist").glob("*"), - target / "archives", - ] - ) + run([ + "cp", + "-a", + *(self.checkout / "Doc" / "dist").glob("*"), + target / "archives", + ]) changed.append("archives/") for file in (target / "archives").iterdir(): changed.append("archives/" + file.name) @@ -984,7 +968,8 @@ def parse_args(): default=Path("/var/log/docsbuild/"), ) parser.add_argument( - "--languages", "--language", + "--languages", + "--language", nargs="*", help="Language translation, as a PEP 545 language tag like" " 'fr' or 'pt-br'. " diff --git a/check_versions.py b/check_versions.py index 343c85a..1a1016f 100644 --- a/check_versions.py +++ b/check_versions.py @@ -111,12 +111,10 @@ async def which_sphinx_is_used_in_production(): table = [ [ version.name, - *await asyncio.gather( - *[ - get_version_in_prod(language.tag, version.name) - for language in LANGUAGES - ] - ), + *await asyncio.gather(*[ + get_version_in_prod(language.tag, version.name) + for language in LANGUAGES + ]), ] for version in VERSIONS ] From a6a666d9f5007351803eb83b4525864d314732f9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 07:17:01 +0100 Subject: [PATCH 057/102] Account for skipped builds in ``build_docs()`` (#261) --- build_docs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build_docs.py b/build_docs.py index 0ca807e..08ce611 100755 --- a/build_docs.py +++ b/build_docs.py @@ -536,7 +536,7 @@ def includes_html(self): """Does the build we are running include HTML output?""" return self.select_output != "no-html" - def run(self, http: urllib3.PoolManager) -> bool: + def run(self, http: urllib3.PoolManager) -> bool | None: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) @@ -544,7 +544,7 @@ def run(self, http: urllib3.PoolManager) -> bool: try: if self.language.html_only and not self.includes_html: logging.info("Skipping non-HTML build (language is HTML-only).") - return True + return None # skipped self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() @@ -557,6 +557,8 @@ def run(self, http: urllib3.PoolManager) -> bool: build_duration=perf_counter() - start_time, trigger=trigger_reason, ) + else: + return None # skipped except Exception as err: logging.exception("Badly handled exception, human, please help.") if sentry_sdk: @@ -1073,7 +1075,7 @@ def build_docs(args: argparse.Namespace) -> bool: built_successfully = builder.run(http) if built_successfully: build_succeeded.add((version.name, language.tag)) - else: + elif built_successfully is not None: build_failed.add((version.name, language.tag)) logging.root.handlers[0].setFormatter( From e80b7296bdff2d7034d397bdd73b3852e8e1c4c1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 14:10:00 +0100 Subject: [PATCH 058/102] Improve performance for ``proofread_canonicals()`` (#258) --- build_docs.py | 51 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index 08ce611..63b561c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -23,6 +23,7 @@ from __future__ import annotations import argparse +import concurrent.futures import dataclasses import datetime as dt import filecmp @@ -1249,21 +1250,41 @@ def proofread_canonicals( /3/whatsnew/3.11.html, which may not exist yet. """ logging.info("Checking canonical links...") - canonical_re = re.compile( - """""" - ) - for file in www_root.glob("**/*.html"): - html = file.read_text(encoding="UTF-8", errors="surrogateescape") - canonical = canonical_re.search(html) - if not canonical: - continue - target = canonical.group(1) - if not (www_root / target).exists(): - logging.info("Removing broken canonical from %s to %s", file, target) - html = html.replace(canonical.group(0), "") - file.write_text(html, encoding="UTF-8", errors="surrogateescape") - if not skip_cache_invalidation: - purge(http, str(file).replace("/srv/docs.python.org/", "")) + worker_count = (os.cpu_count() or 1) + 2 + with concurrent.futures.ThreadPoolExecutor(worker_count) as executor: + futures = { + executor.submit(_check_canonical_rel, file, www_root) + for file in www_root.glob("**/*.html") + } + paths_to_purge = { + res.relative_to(www_root) # strip the leading /srv/docs.python.org + for fut in concurrent.futures.as_completed(futures) + if (res := fut.result()) is not None + } + if not skip_cache_invalidation: + purge(http, *paths_to_purge) + + +def _check_canonical_rel(file: Path, www_root: Path): + # Check for a canonical relation link in the HTML. + # If one exists, ensure that the target exists + # or otherwise remove the canonical link element. + prefix = b'' + pfx_len = len(prefix) + sfx_len = len(suffix) + html = file.read_bytes() + try: + start = html.index(prefix) + end = html.index(suffix, start + pfx_len) + except ValueError: + return None + target = html[start + pfx_len : end].decode(errors="surrogateescape") + if (www_root / target).exists(): + return None + logging.info("Removing broken canonical from %s to %s", file, target) + file.write_bytes(html[:start] + html[end + sfx_len :]) + return file def purge(http: urllib3.PoolManager, *paths: Path | str) -> None: From 946b6bc68b239931579e56f43f45f6e15d3e60c4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:19:21 +0100 Subject: [PATCH 059/102] Account for recent removal of self-closing tags (#264) --- build_docs.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/build_docs.py b/build_docs.py index 63b561c..ff9eb59 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1265,25 +1265,26 @@ def proofread_canonicals( purge(http, *paths_to_purge) +# Python 3.12 onwards doesn't use self-closing tags for +_canonical_re = re.compile( + b"""""" +) + + def _check_canonical_rel(file: Path, www_root: Path): # Check for a canonical relation link in the HTML. # If one exists, ensure that the target exists # or otherwise remove the canonical link element. - prefix = b'' - pfx_len = len(prefix) - sfx_len = len(suffix) html = file.read_bytes() - try: - start = html.index(prefix) - end = html.index(suffix, start + pfx_len) - except ValueError: + canonical = _canonical_re.search(html) + if canonical is None: return None - target = html[start + pfx_len : end].decode(errors="surrogateescape") + target = canonical[1].decode(encoding="UTF-8", errors="surrogateescape") if (www_root / target).exists(): return None logging.info("Removing broken canonical from %s to %s", file, target) - file.write_bytes(html[:start] + html[end + sfx_len :]) + start, end = canonical.span() + file.write_bytes(html[:start] + html[end:]) return file From 2113fd7daaed924663c986384048defcab68b342 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 11 Apr 2025 17:46:23 +0300 Subject: [PATCH 060/102] Allow passing multiple branches to build via CLI (#235) --- .coveragerc | 7 ++++ .github/workflows/test.yml | 7 ++++ README.md | 7 +++- build_docs.py | 19 +++++----- tests/test_build_docs_versions.py | 62 +++++++++++++++++++++++++++++++ tox.ini | 13 ++++++- 6 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 .coveragerc create mode 100644 tests/test_build_docs_versions.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..0f12707 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +# .coveragerc to control coverage.py + +[report] +# Regexes for lines to exclude from consideration +exclude_also = + # Don't complain if non-runnable code isn't run: + if __name__ == .__main__.: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2976bae..e272e76 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,3 +33,10 @@ jobs: - name: Tox tests run: | uvx --with tox-uv tox -e py + + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + flags: ${{ matrix.os }} + name: ${{ matrix.os }} Python ${{ matrix.python-version }} + token: ${{ secrets.CODECOV_ORG_TOKEN }} diff --git a/README.md b/README.md index 1b76bcd..d78bdb7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ +# docsbuild-scripts + +[![GitHub Actions status](https://github.com/python/docsbuild-scripts/actions/workflows/test.yml/badge.svg)](https://github.com/python/docsbuild-scripts/actions/workflows/test.yml) +[![Codecov](https://codecov.io/gh/python/docsbuild-scripts/branch/main/graph/badge.svg)](https://codecov.io/gh/python/docsbuild-scripts) + This repository contains scripts for automatically building the Python documentation on [docs.python.org](https://docs.python.org). @@ -12,7 +17,7 @@ python3 ./build_docs.py --quick --build-root ./build_root --www-root ./www --log ``` If you don't need to build all translations of all branches, add -`--language en --branch main`. +`--languages en --branches main`. ## Check current version diff --git a/build_docs.py b/build_docs.py index ff9eb59..d4578eb 100755 --- a/build_docs.py +++ b/build_docs.py @@ -87,16 +87,17 @@ def from_json(cls, data) -> Versions: ) return cls(versions) - def filter(self, branch: str = "") -> Sequence[Version]: + def filter(self, branches: Sequence[str] = ()) -> Sequence[Version]: """Filter the given versions. - If *branch* is given, only *versions* matching *branch* are returned. + If *branches* is given, only *versions* matching *branches* are returned. Else all live versions are returned (this means no EOL and no security-fixes branches). """ - if branch: - return [v for v in self if branch in (v.name, v.branch_or_tag)] + if branches: + branches = frozenset(branches) + return [v for v in self if {v.name, v.branch_or_tag} & branches] return [v for v in self if v.status not in {"EOL", "security-fixes"}] @property @@ -936,9 +937,10 @@ def parse_args(): ) parser.add_argument( "-b", - "--branch", + "--branches", + nargs="*", metavar="3.12", - help="Version to build (defaults to all maintained branches).", + help="Versions to build (defaults to all maintained branches).", ) parser.add_argument( "-r", @@ -972,7 +974,6 @@ def parse_args(): ) parser.add_argument( "--languages", - "--language", nargs="*", help="Language translation, as a PEP 545 language tag like" " 'fr' or 'pt-br'. " @@ -1046,10 +1047,10 @@ def build_docs(args: argparse.Namespace) -> bool: # This runs languages in config.toml order and versions newest first. todo = [ (version, language) - for version in versions.filter(args.branch) + for version in versions.filter(args.branches) for language in reversed(languages.filter(args.languages)) ] - del args.branch + del args.branches del args.languages build_succeeded = set() diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py new file mode 100644 index 0000000..5ed9bcb --- /dev/null +++ b/tests/test_build_docs_versions.py @@ -0,0 +1,62 @@ +from build_docs import Versions, Version + + +def test_filter_default() -> None: + # Arrange + versions = Versions([ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + Version("3.11", status="security"), + Version("3.10", status="security"), + Version("3.9", status="security"), + ]) + + # Act + filtered = versions.filter() + + # Assert + assert filtered == [ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + ] + + +def test_filter_one() -> None: + # Arrange + versions = Versions([ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + Version("3.11", status="security"), + Version("3.10", status="security"), + Version("3.9", status="security"), + ]) + + # Act + filtered = versions.filter(["3.13"]) + + # Assert + assert filtered == [Version("3.13", status="security")] + + +def test_filter_multiple() -> None: + # Arrange + versions = Versions([ + Version("3.14", status="feature"), + Version("3.13", status="bugfix"), + Version("3.12", status="bugfix"), + Version("3.11", status="security"), + Version("3.10", status="security"), + Version("3.9", status="security"), + ]) + + # Act + filtered = versions.filter(["3.13", "3.14"]) + + # Assert + assert filtered == [ + Version("3.14", status="feature"), + Version("3.13", status="security"), + ] diff --git a/tox.ini b/tox.ini index 56c6420..12efcdf 100644 --- a/tox.ini +++ b/tox.ini @@ -12,8 +12,19 @@ skip_install = true deps = -r requirements.txt pytest + pytest-cov +pass_env = + FORCE_COLOR +set_env = + COVERAGE_CORE = sysmon commands = - {envpython} -m pytest {posargs} + {envpython} -m pytest \ + --cov . \ + --cov tests \ + --cov-report html \ + --cov-report term \ + --cov-report xml \ + {posargs} [testenv:lint] skip_install = true From bd0e222e4e0844f2790d61e5d66e0243b97073c8 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Fri, 11 Apr 2025 14:49:11 -0400 Subject: [PATCH 061/102] Move environment variables to a configuration file (#269) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- build_docs.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 48 insertions(+) diff --git a/build_docs.py b/build_docs.py index d4578eb..22fc5bd 100755 --- a/build_docs.py +++ b/build_docs.py @@ -5,6 +5,26 @@ Without any arguments builds docs for all active versions and languages. +Environment variables for: + +- `SENTRY_DSN` (Error reporting) +- `FASTLY_SERVICE_ID` / `FASTLY_TOKEN` (CDN purges) +- `PYTHON_DOCS_ENABLE_ANALYTICS` (Enable Plausible for online docs) + +are read from the site configuration path for your platform +(/etc/xdg/docsbuild-scripts on linux) if available, +and can be overriden by writing a file to the user config dir +for your platform ($HOME/.config/docsbuild-scripts on linux). +The contents of the file is parsed as toml: + +```toml +[env] +SENTRY_DSN = "https://0a0a0a0a0a0a0a0a0a0a0a@sentry.io/69420" +FASTLY_SERVICE_ID = "deadbeefdeadbeefdead" +FASTLY_TOKEN = "secureme!" +PYTHON_DOCS_ENABLE_ANALYTICS = "1" +``` + Languages are stored in `config.toml` while versions are discovered from the devguide. @@ -48,6 +68,7 @@ import tomlkit import urllib3 import zc.lockfile +from platformdirs import user_config_path, site_config_path TYPE_CHECKING = False if TYPE_CHECKING: @@ -906,6 +927,7 @@ def main(): """Script entry point.""" args = parse_args() setup_logging(args.log_directory, args.select_output) + load_environment_variables() if args.select_output is None: build_docs_with_lock(args, "build_docs.lock") @@ -1022,6 +1044,31 @@ def setup_logging(log_directory: Path, select_output: str | None): logging.getLogger().setLevel(logging.DEBUG) +def load_environment_variables() -> None: + _user_config_path = user_config_path("docsbuild-scripts") + _site_config_path = site_config_path("docsbuild-scripts") + if _user_config_path.is_file(): + ENV_CONF_FILE = _user_config_path + elif _site_config_path.is_file(): + ENV_CONF_FILE = _site_config_path + else: + logging.info( + "No environment variables configured. " + f"Configure in {_site_config_path} or {_user_config_path}." + ) + return + + logging.info(f"Reading environment variables from {ENV_CONF_FILE}.") + if ENV_CONF_FILE == _site_config_path: + logging.info(f"You can override settings in {_user_config_path}.") + elif _site_config_path.is_file(): + logging.info(f"Overriding {_site_config_path}.") + with open(ENV_CONF_FILE, "r") as f: + for key, value in tomlkit.parse(f.read()).get("env", {}).items(): + logging.debug(f"Setting {key} in environment.") + os.environ[key] = value + + def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: try: lock = zc.lockfile.LockFile(HERE / lockfile_name) diff --git a/requirements.txt b/requirements.txt index 0cac810..535b36a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ jinja2 +platformdirs sentry-sdk>=2 tomlkit>=0.13 urllib3>=2 From 10e20e45c8aaa473b667ec9e2bb4f3a277685488 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:25:54 +0100 Subject: [PATCH 062/102] Require imghdr for building Python 3.8-3.10 (#267) --- build_docs.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/build_docs.py b/build_docs.py index 22fc5bd..1b790c6 100755 --- a/build_docs.py +++ b/build_docs.py @@ -196,23 +196,30 @@ def requirements(self): See https://github.com/python/cpython/issues/91483 """ - if self.name == "3.5": - return ["jieba", "blurb", "sphinx==1.8.4", "jinja2<3.1", "docutils<=0.17.1"] - if self.name in {"3.7", "3.6", "2.7"}: - return ["jieba", "blurb", "sphinx==2.3.1", "jinja2<3.1", "docutils<=0.17.1"] - - return [ + dependencies = [ "jieba", # To improve zh search. "PyStemmer~=2.2.0", # To improve performance for word stemming. "-rrequirements.txt", ] + if self.as_tuple() >= (3, 11): + return dependencies + if self.as_tuple() >= (3, 8): + # Restore the imghdr module for Python 3.8-3.10. + return dependencies + ["standard-imghdr"] + + # Requirements/constraints for Python 3.7 and older, pre-requirements.txt + reqs = ["jieba", "blurb", "jinja2<3.1", "docutils<=0.17.1", "standard-imghdr"] + if self.name in {"3.7", "3.6", "2.7"}: + return reqs + ["sphinx==2.3.1"] + if self.name == "3.5": + return reqs + ["sphinx==1.8.4"] @property def changefreq(self): """Estimate this version change frequency, for the sitemap.""" return {"EOL": "never", "security-fixes": "yearly"}.get(self.status, "daily") - def as_tuple(self): + def as_tuple(self) -> tuple[int, ...]: """This version name as tuple, for easy comparisons.""" return version_to_tuple(self.name) @@ -407,7 +414,7 @@ def update(self): self.clone() or self.fetch() -def version_to_tuple(version): +def version_to_tuple(version) -> tuple[int, ...]: """Transform a version string to a tuple, for easy comparisons.""" return tuple(int(part) for part in version.split(".")) From 776f413c7e174017d59aa38ff9f75dc135f749e1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:30:16 +0100 Subject: [PATCH 063/102] Add ``--force`` to always rebuild (#268) --- README.md | 2 +- build_docs.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d78bdb7..2c84353 100644 --- a/README.md +++ b/README.md @@ -71,5 +71,5 @@ To manually rebuild a branch, for example 3.11: ssh docs.nyc1.psf.io sudo su --shell=/bin/bash docsbuild screen -DUR # Rejoin screen session if it exists, otherwise create a new one -/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --branch 3.11 +/srv/docsbuild/venv/bin/python /srv/docsbuild/scripts/build_docs.py --force --branch 3.11 ``` diff --git a/build_docs.py b/build_docs.py index 1b790c6..9727a85 100755 --- a/build_docs.py +++ b/build_docs.py @@ -566,7 +566,7 @@ def includes_html(self): """Does the build we are running include HTML output?""" return self.select_output != "no-html" - def run(self, http: urllib3.PoolManager) -> bool | None: + def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: """Build and publish a Python doc, for a language, and a version.""" start_time = perf_counter() start_timestamp = dt.datetime.now(tz=dt.UTC).replace(microsecond=0) @@ -578,7 +578,7 @@ def run(self, http: urllib3.PoolManager) -> bool | None: self.cpython_repo.switch(self.version.branch_or_tag) if self.language.tag != "en": self.clone_translation() - if trigger_reason := self.should_rebuild(): + if trigger_reason := self.should_rebuild(force_build): self.build_venv() self.build() self.copy_build_to_webroot(http) @@ -834,7 +834,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: "Publishing done (%s).", format_seconds(perf_counter() - start_time) ) - def should_rebuild(self): + def should_rebuild(self, force: bool): state = self.load_state() if not state: logging.info("Should rebuild: no previous state found.") @@ -862,6 +862,9 @@ def should_rebuild(self): cpython_sha, ) return "Doc/ has changed" + if force: + logging.info("Should rebuild: forced.") + return "forced" logging.info("Nothing changed, no rebuild needed.") return False @@ -985,6 +988,12 @@ def parse_args(): help="Path where generated files will be copied.", default=Path("/srv/docs.python.org"), ) + parser.add_argument( + "--force", + action="store_true", + help="Always build the chosen languages and versions, " + "regardless of existing state.", + ) parser.add_argument( "--skip-cache-invalidation", help="Skip Fastly cache invalidation.", @@ -1128,7 +1137,7 @@ def build_docs(args: argparse.Namespace) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - built_successfully = builder.run(http) + built_successfully = builder.run(http, force_build=args.force) if built_successfully: build_succeeded.add((version.name, language.tag)) elif built_successfully is not None: From 82e2a41997ce7bb02b113d429b894119ac2c8437 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:38:18 +0100 Subject: [PATCH 064/102] Delete ``force`` from ``args`` namespace (#270) --- build_docs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 9727a85..309d786 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1115,6 +1115,8 @@ def build_docs(args: argparse.Namespace) -> bool: ] del args.branches del args.languages + force_build = args.force + del args.force build_succeeded = set() build_failed = set() @@ -1137,7 +1139,7 @@ def build_docs(args: argparse.Namespace) -> bool: builder = DocBuilder( version, versions, language, languages, cpython_repo, **vars(args) ) - built_successfully = builder.run(http, force_build=args.force) + built_successfully = builder.run(http, force_build=force_build) if built_successfully: build_succeeded.add((version.name, language.tag)) elif built_successfully is not None: From 4a5d2ca799ae7a7b9240cde7a987b8c63da070b1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 12 Apr 2025 02:14:44 +0300 Subject: [PATCH 065/102] Test for GNU sed instead of macOS (#263) --- build_docs.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 309d786..97eee6d 100755 --- a/build_docs.py +++ b/build_docs.py @@ -687,10 +687,25 @@ def build(self): f"-D ogp_site_url={site_url}", ) + def is_gnu_sed() -> bool: + """Check if we are using GNU sed.""" + try: + subprocess.run( + ["sed", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + return True + except subprocess.CalledProcessError: + return False + except FileNotFoundError: + return False + # Disable CPython switchers, we handle them now: run( ["sed", "-i"] - + ([""] if sys.platform == "darwin" else []) + + ([] if is_gnu_sed() else [""]) + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] ) self.versions.setup_indexsidebar( From c341248dbc162b2ab9c527d95e8de0a9b6958285 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 00:19:50 +0100 Subject: [PATCH 066/102] Ensure that `Doc/dist` exists (#271) --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index 97eee6d..c0a954e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -817,6 +817,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: if not self.quick: # Copy archive files to /archives/ logging.debug("Copying dist files.") + (self.checkout / "Doc" / "dist").mkdir(exist_ok=True) run([ "chown", "-R", From 7a9bca9945f149486a366d4050f658f3f34dc81b Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 01:14:16 +0100 Subject: [PATCH 067/102] Remove use of the ``sed`` command (#272) --- build_docs.py | 42 ++++++------------------------------------ 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/build_docs.py b/build_docs.py index c0a954e..ac4f351 100755 --- a/build_docs.py +++ b/build_docs.py @@ -646,21 +646,6 @@ def build(self): "-D gettext_compact=0", "-D translation_progress_classes=1", )) - if self.language.tag == "ja": - # Since luatex doesn't support \ufffd, replace \ufffd with '?'. - # https://gist.github.com/zr-tex8r/e0931df922f38fbb67634f05dfdaf66b - # Luatex already fixed this issue, so we can remove this once Texlive - # is updated. - # (https://github.com/TeX-Live/luatex/commit/af5faf1) - subprocess.check_output( - "sed -i s/\N{REPLACEMENT CHARACTER}/?/g " - f"{locale_dirs}/ja/LC_MESSAGES/**/*.po", - shell=True, - ) - subprocess.check_output( - f"sed -i s/\N{REPLACEMENT CHARACTER}/?/g {self.checkout}/Doc/**/*.rst", - shell=True, - ) if self.version.status == "EOL": sphinxopts.append("-D html_context.outdated=1") @@ -687,27 +672,12 @@ def build(self): f"-D ogp_site_url={site_url}", ) - def is_gnu_sed() -> bool: - """Check if we are using GNU sed.""" - try: - subprocess.run( - ["sed", "--version"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=True, - ) - return True - except subprocess.CalledProcessError: - return False - except FileNotFoundError: - return False - - # Disable CPython switchers, we handle them now: - run( - ["sed", "-i"] - + ([] if is_gnu_sed() else [""]) - + ["s/ *-A switchers=1//", self.checkout / "Doc" / "Makefile"] - ) + if self.version.as_tuple() < (3, 8): + # Disable CPython switchers, we handle them now: + text = (self.checkout / "Doc" / "Makefile").read_text(encoding="utf-8") + text = text.replace(" -A switchers=1", "") + (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") + self.versions.setup_indexsidebar( self.version, self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", From feef5f169a1a2155097d3b44eb649f34405b0653 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 01:38:36 +0100 Subject: [PATCH 068/102] Only copy archive files if the dist directory exists (#273) --- build_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index ac4f351..654c945 100755 --- a/build_docs.py +++ b/build_docs.py @@ -784,10 +784,9 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: target, ]) - if not self.quick: + if not self.quick and (self.checkout / "Doc" / "dist").is_dir(): # Copy archive files to /archives/ logging.debug("Copying dist files.") - (self.checkout / "Doc" / "dist").mkdir(exist_ok=True) run([ "chown", "-R", From 76192128301d3b553b9b6f287b880d454c245606 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 03:05:54 +0100 Subject: [PATCH 069/102] Add more type hints (#274) --- build_docs.py | 109 ++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 48 deletions(-) diff --git a/build_docs.py b/build_docs.py index 654c945..0eb900e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -72,7 +72,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Iterator, Sequence, Set + from collections.abc import Collection, Iterator, Sequence, Set from typing import Literal try: @@ -101,7 +101,7 @@ def __reversed__(self) -> Iterator[Version]: return reversed(self._seq) @classmethod - def from_json(cls, data) -> Versions: + def from_json(cls, data: dict) -> Versions: versions = sorted( [Version.from_json(name, release) for name, release in data.items()], key=Version.as_tuple, @@ -158,7 +158,9 @@ class Version: "prerelease": "pre-release", } - def __init__(self, name, *, status, branch_or_tag=None): + def __init__( + self, name: str, *, status: str, branch_or_tag: str | None = None + ) -> None: status = self.SYNONYMS.get(status, status) if status not in self.STATUSES: raise ValueError( @@ -169,22 +171,22 @@ def __init__(self, name, *, status, branch_or_tag=None): self.branch_or_tag = branch_or_tag self.status = status - def __repr__(self): + def __repr__(self) -> str: return f"Version({self.name})" - def __eq__(self, other): + def __eq__(self, other: Version) -> bool: return self.name == other.name - def __gt__(self, other): + def __gt__(self, other: Version) -> bool: return self.as_tuple() > other.as_tuple() @classmethod - def from_json(cls, name, values): + def from_json(cls, name: str, values: dict) -> Version: """Loads a version from devguide's json representation.""" return cls(name, status=values["status"], branch_or_tag=values["branch"]) @property - def requirements(self): + def requirements(self) -> list[str]: """Generate the right requirements for this version. Since CPython 3.8 a Doc/requirements.txt file can be used. @@ -213,9 +215,10 @@ def requirements(self): return reqs + ["sphinx==2.3.1"] if self.name == "3.5": return reqs + ["sphinx==1.8.4"] + raise ValueError("unreachable") @property - def changefreq(self): + def changefreq(self) -> str: """Estimate this version change frequency, for the sitemap.""" return {"EOL": "never", "security-fixes": "yearly"}.get(self.status, "daily") @@ -224,17 +227,17 @@ def as_tuple(self) -> tuple[int, ...]: return version_to_tuple(self.name) @property - def url(self): + def url(self) -> str: """The doc URL of this version in production.""" return f"https://docs.python.org/{self.name}/" @property - def title(self): + def title(self) -> str: """The title of this version's doc, for the sidebar.""" return f"Python {self.name} ({self.status})" @property - def picker_label(self): + def picker_label(self) -> str: """Forge the label of a version picker.""" if self.status == "in development": return f"dev ({self.name})" @@ -254,7 +257,7 @@ def __reversed__(self) -> Iterator[Language]: return reversed(self._seq) @classmethod - def from_json(cls, defaults, languages) -> Languages: + def from_json(cls, defaults: dict, languages: dict) -> Languages: default_translated_name = defaults.get("translated_name", "") default_in_prod = defaults.get("in_prod", True) default_sphinxopts = defaults.get("sphinxopts", []) @@ -290,17 +293,19 @@ class Language: html_only: bool = False @property - def tag(self): + def tag(self) -> str: return self.iso639_tag.replace("_", "-").lower() @property - def switcher_label(self): + def switcher_label(self) -> str: if self.translated_name: return f"{self.name} | {self.translated_name}" return self.name -def run(cmd, cwd=None) -> subprocess.CompletedProcess: +def run( + cmd: Sequence[str | Path], cwd: Path | None = None +) -> subprocess.CompletedProcess: """Like subprocess.run, with logging before and after the command execution.""" cmd = list(map(str, cmd)) cmdstring = shlex.join(cmd) @@ -326,7 +331,7 @@ def run(cmd, cwd=None) -> subprocess.CompletedProcess: return result -def run_with_logging(cmd, cwd=None): +def run_with_logging(cmd: Sequence[str | Path], cwd: Path | None = None) -> None: """Like subprocess.check_call, with logging before the command execution.""" cmd = list(map(str, cmd)) logging.debug("Run: '%s'", shlex.join(cmd)) @@ -348,13 +353,13 @@ def run_with_logging(cmd, cwd=None): raise subprocess.CalledProcessError(return_code, cmd[0]) -def changed_files(left, right): +def changed_files(left: Path, right: Path) -> list[str]: """Compute a list of different files between left and right, recursively. Resulting paths are relative to left. """ changed = [] - def traverse(dircmp_result): + def traverse(dircmp_result: filecmp.dircmp) -> None: base = Path(dircmp_result.left).relative_to(left) for file in dircmp_result.diff_files: changed.append(str(base / file)) @@ -374,11 +379,11 @@ class Repository: remote: str directory: Path - def run(self, *args): + def run(self, *args: str) -> subprocess.CompletedProcess: """Run git command in the clone repository.""" return run(("git", "-C", self.directory) + args) - def get_ref(self, pattern): + def get_ref(self, pattern: str) -> str: """Return the reference of a given tag or branch.""" try: # Maybe it's a branch @@ -387,7 +392,7 @@ def get_ref(self, pattern): # Maybe it's a tag return self.run("show-ref", "-s", "tags/" + pattern).stdout.strip() - def fetch(self): + def fetch(self) -> subprocess.CompletedProcess: """Try (and retry) to run git fetch.""" try: return self.run("fetch") @@ -396,12 +401,12 @@ def fetch(self): sleep(5) return self.run("fetch") - def switch(self, branch_or_tag): + def switch(self, branch_or_tag: str) -> None: """Reset and cleans the repository to the given branch or tag.""" self.run("reset", "--hard", self.get_ref(branch_or_tag), "--") self.run("clean", "-dfqx") - def clone(self): + def clone(self) -> bool: """Maybe clone the repository, if not already cloned.""" if (self.directory / ".git").is_dir(): return False # Already cloned @@ -410,21 +415,23 @@ def clone(self): run(["git", "clone", self.remote, self.directory]) return True - def update(self): + def update(self) -> None: self.clone() or self.fetch() -def version_to_tuple(version) -> tuple[int, ...]: +def version_to_tuple(version: str) -> tuple[int, ...]: """Transform a version string to a tuple, for easy comparisons.""" return tuple(int(part) for part in version.split(".")) -def tuple_to_version(version_tuple): +def tuple_to_version(version_tuple: tuple[int, ...]) -> str: """Reverse version_to_tuple.""" return ".".join(str(part) for part in version_tuple) -def locate_nearest_version(available_versions, target_version): +def locate_nearest_version( + available_versions: Collection[str], target_version: str +) -> str: """Look for the nearest version of target_version in available_versions. Versions are to be given as tuples, like (3, 7) for 3.7. @@ -468,7 +475,7 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers(versions: Versions, languages: Languages, html_root: Path): +def setup_switchers(versions: Versions, languages: Languages, html_root: Path) -> None: """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher @@ -499,12 +506,12 @@ def setup_switchers(versions: Versions, languages: Languages, html_root: Path): ofile.write(line) -def head(text, lines=10): +def head(text: str, lines: int = 10) -> str: """Return the first *lines* lines from the given text.""" return "\n".join(text.split("\n")[:lines]) -def version_info(): +def version_info() -> None: """Handler for --version.""" try: platex_version = head( @@ -554,7 +561,7 @@ class DocBuilder: theme: Path @property - def html_only(self): + def html_only(self) -> bool: return ( self.select_output in {"only-html", "only-html-en"} or self.quick @@ -562,7 +569,7 @@ def html_only(self): ) @property - def includes_html(self): + def includes_html(self) -> bool: """Does the build we are running include HTML output?""" return self.select_output != "no-html" @@ -601,12 +608,12 @@ def checkout(self) -> Path: """Path to CPython git clone.""" return self.build_root / _checkout_name(self.select_output) - def clone_translation(self): + def clone_translation(self) -> None: self.translation_repo.update() self.translation_repo.switch(self.translation_branch) @property - def translation_repo(self): + def translation_repo(self) -> Repository: """See PEP 545 for translations repository naming convention.""" locale_repo = f"https://github.com/python/python-docs-{self.language.tag}.git" @@ -620,7 +627,7 @@ def translation_repo(self): return Repository(locale_repo, locale_clone_dir) @property - def translation_branch(self): + def translation_branch(self) -> str: """Some CPython versions may be untranslated, being either too old or too new. @@ -633,7 +640,7 @@ def translation_branch(self): branches = re.findall(r"/([0-9]+\.[0-9]+)$", remote_branches, re.M) return locate_nearest_version(branches, self.version.name) - def build(self): + def build(self) -> None: """Build this version/language doc.""" logging.info("Build start.") start_time = perf_counter() @@ -702,7 +709,7 @@ def build(self): ) logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) - def build_venv(self): + def build_venv(self) -> None: """Build a venv for the specific Python version. So we can reuse them from builds to builds, while they contain @@ -819,7 +826,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: "Publishing done (%s).", format_seconds(perf_counter() - start_time) ) - def should_rebuild(self, force: bool): + def should_rebuild(self, force: bool) -> str | Literal[False]: state = self.load_state() if not state: logging.info("Should rebuild: no previous state found.") @@ -865,7 +872,9 @@ def load_state(self) -> dict: except (KeyError, FileNotFoundError): return {} - def save_state(self, build_start: dt.datetime, build_duration: float, trigger: str): + def save_state( + self, build_start: dt.datetime, build_duration: float, trigger: str + ) -> None: """Save current CPython sha1 and current translation sha1. Using this we can deduce if a rebuild is needed or not. @@ -911,6 +920,8 @@ def format_seconds(seconds: float) -> str: case h, m, s: return f"{h}h {m}m {s}s" + raise ValueError("unreachable") + def _checkout_name(select_output: str | None) -> str: if select_output is not None: @@ -918,7 +929,7 @@ def _checkout_name(select_output: str | None) -> str: return "cpython" -def main(): +def main() -> None: """Script entry point.""" args = parse_args() setup_logging(args.log_directory, args.select_output) @@ -934,7 +945,7 @@ def main(): build_docs_with_lock(args, "build_docs_html_en.lock") -def parse_args(): +def parse_args() -> argparse.Namespace: """Parse command-line arguments.""" parser = argparse.ArgumentParser( @@ -1028,7 +1039,7 @@ def parse_args(): return args -def setup_logging(log_directory: Path, select_output: str | None): +def setup_logging(log_directory: Path, select_output: str | None) -> None: """Setup logging to stderr if run by a human, or to a file if run from a cron.""" log_format = "%(asctime)s %(levelname)s: %(message)s" if sys.stderr.isatty(): @@ -1174,7 +1185,9 @@ def parse_languages_from_config() -> Languages: return Languages.from_json(config["defaults"], config["languages"]) -def build_sitemap(versions: Versions, languages: Languages, www_root: Path, group): +def build_sitemap( + versions: Versions, languages: Languages, www_root: Path, group: str +) -> None: """Build a sitemap with all live versions and translations.""" if not www_root.exists(): logging.info("Skipping sitemap generation (www root does not even exist).") @@ -1189,7 +1202,7 @@ def build_sitemap(versions: Versions, languages: Languages, www_root: Path, grou run(["chgrp", group, sitemap_path]) -def build_404(www_root: Path, group): +def build_404(www_root: Path, group: str) -> None: """Build a nice 404 error page to display in case PDFs are not built yet.""" if not www_root.exists(): logging.info("Skipping 404 page generation (www root does not even exist).") @@ -1203,8 +1216,8 @@ def build_404(www_root: Path, group): def copy_robots_txt( www_root: Path, - group, - skip_cache_invalidation, + group: str, + skip_cache_invalidation: bool, http: urllib3.PoolManager, ) -> None: """Copy robots.txt to www_root.""" @@ -1322,7 +1335,7 @@ def proofread_canonicals( ) -def _check_canonical_rel(file: Path, www_root: Path): +def _check_canonical_rel(file: Path, www_root: Path) -> Path | None: # Check for a canonical relation link in the HTML. # If one exists, ensure that the target exists # or otherwise remove the canonical link element. From 2fee30f3edc36216b0e329a93703d038b960d446 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:05:11 +0100 Subject: [PATCH 070/102] Refresh versions table in README (#276) --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2c84353..8f914c6 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,9 @@ of Sphinx we're using where: 3.9 ø sphinx==2.4.4 needs_sphinx='1.8' 3.10 ø sphinx==3.4.3 needs_sphinx='3.2' 3.11 ø sphinx~=7.2.0 needs_sphinx='4.2' - 3.12 ø sphinx~=8.1.0 needs_sphinx='7.2.6' - 3.13 ø sphinx~=8.1.0 needs_sphinx='7.2.6' - 3.14 ø sphinx~=8.1.0 needs_sphinx='7.2.6' + 3.12 ø sphinx~=8.2.0 needs_sphinx='8.2.0' + 3.13 ø sphinx~=8.2.0 needs_sphinx='8.2.0' + 3.14 ø sphinx~=8.2.0 needs_sphinx='8.2.0' ========= ============= ================== ==================== Sphinx build as seen on docs.python.org: @@ -52,9 +52,9 @@ of Sphinx we're using where: 3.9 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 2.4.4 3.10 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.4.3 3.11 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 7.2.6 - 3.12 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 - 3.13 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 - 3.14 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 8.1.3 + 3.12 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 + 3.13 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 + 3.14 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 8.2.3 ========= ===== ===== ===== ===== ===== ===== ===== ===== ======= ===== ===== ======= ======= ## Manually rebuild a branch From 57136bc88571e8115b30f94ca9adaebb2cd7cd38 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:06:16 +0100 Subject: [PATCH 071/102] Use Ruff linting (#277) --- .pre-commit-config.yaml | 1 + .ruff.toml | 22 ++++++++++++++++ build_docs.py | 42 ++++++++++++++++--------------- tests/test_build_docs.py | 2 +- tests/test_build_docs_versions.py | 2 +- 5 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4ad4370..c0eb053 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.11.5 hooks: + - id: ruff - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema diff --git a/.ruff.toml b/.ruff.toml index e837d03..6862f11 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -5,3 +5,25 @@ output-format = "full" [format] preview = true docstring-code-format = true + +[lint] +preview = true +select = [ + "C4", # flake8-comprehensions + "B", # flake8-bugbear + "E", # pycodestyle + "F", # pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "I", # isort + "N", # pep8-naming + "PERF", # perflint + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "TC", # flake8-type-checking + "UP", # pyupgrade + "W", # pycodestyle +] +ignore = [ + "E501", # Ignore line length errors (we use auto-formatting) +] diff --git a/build_docs.py b/build_docs.py index 0eb900e..7329bbe 100755 --- a/build_docs.py +++ b/build_docs.py @@ -65,10 +65,10 @@ from urllib.parse import urljoin import jinja2 +import platformdirs import tomlkit import urllib3 import zc.lockfile -from platformdirs import user_config_path, site_config_path TYPE_CHECKING = False if TYPE_CHECKING: @@ -76,7 +76,8 @@ from typing import Literal try: - from os import EX_OK, EX_SOFTWARE as EX_FAILURE + from os import EX_OK + from os import EX_SOFTWARE as EX_FAILURE except ImportError: EX_OK, EX_FAILURE = 0, 1 @@ -279,7 +280,7 @@ def filter(self, language_tags: Sequence[str] = ()) -> Sequence[Language]: """Filter a sequence of languages according to --languages.""" if language_tags: language_tags = frozenset(language_tags) - return [l for l in self if l.tag in language_tags] + return [l for l in self if l.tag in language_tags] # NoQA: E741 return list(self) @@ -480,7 +481,7 @@ def setup_switchers(versions: Versions, languages: Languages, html_root: Path) - - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) + language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] switchers_template_file = HERE / "templates" / "switchers.js" @@ -1057,28 +1058,29 @@ def setup_logging(log_directory: Path, select_output: str | None) -> None: def load_environment_variables() -> None: - _user_config_path = user_config_path("docsbuild-scripts") - _site_config_path = site_config_path("docsbuild-scripts") - if _user_config_path.is_file(): - ENV_CONF_FILE = _user_config_path - elif _site_config_path.is_file(): - ENV_CONF_FILE = _site_config_path + dbs_user_config = platformdirs.user_config_path("docsbuild-scripts") + dbs_site_config = platformdirs.site_config_path("docsbuild-scripts") + if dbs_user_config.is_file(): + env_conf_file = dbs_user_config + elif dbs_site_config.is_file(): + env_conf_file = dbs_site_config else: logging.info( "No environment variables configured. " - f"Configure in {_site_config_path} or {_user_config_path}." + f"Configure in {dbs_site_config} or {dbs_user_config}." ) return - logging.info(f"Reading environment variables from {ENV_CONF_FILE}.") - if ENV_CONF_FILE == _site_config_path: - logging.info(f"You can override settings in {_user_config_path}.") - elif _site_config_path.is_file(): - logging.info(f"Overriding {_site_config_path}.") - with open(ENV_CONF_FILE, "r") as f: - for key, value in tomlkit.parse(f.read()).get("env", {}).items(): - logging.debug(f"Setting {key} in environment.") - os.environ[key] = value + logging.info(f"Reading environment variables from {env_conf_file}.") + if env_conf_file == dbs_site_config: + logging.info(f"You can override settings in {dbs_user_config}.") + elif dbs_site_config.is_file(): + logging.info(f"Overriding {dbs_site_config}.") + + env_config = env_conf_file.read_text(encoding="utf-8") + for key, value in tomlkit.parse(env_config).get("env", {}).items(): + logging.debug(f"Setting {key} in environment.") + os.environ[key] = value def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: diff --git a/tests/test_build_docs.py b/tests/test_build_docs.py index 4457e95..028da90 100644 --- a/tests/test_build_docs.py +++ b/tests/test_build_docs.py @@ -4,7 +4,7 @@ @pytest.mark.parametrize( - "seconds, expected", + ("seconds", "expected"), [ (0.4, "0s"), (0.5, "0s"), diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py index 5ed9bcb..662838e 100644 --- a/tests/test_build_docs_versions.py +++ b/tests/test_build_docs_versions.py @@ -1,4 +1,4 @@ -from build_docs import Versions, Version +from build_docs import Version, Versions def test_filter_default() -> None: From 3deb0c3248f9016419868ed25a87d617ad786d04 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:10:16 +0100 Subject: [PATCH 072/102] Use strict mode for ``flake8-type-checking`` --- .ruff.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ruff.toml b/.ruff.toml index 6862f11..52976fe 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -27,3 +27,7 @@ select = [ ignore = [ "E501", # Ignore line length errors (we use auto-formatting) ] + +[lint.flake8-type-checking] +exempt-modules = [] +strict = true From 8c8b000889e51c078045a61b822ced8e6f226609 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 04:25:36 +0100 Subject: [PATCH 073/102] Enable flake8-logging-format in Ruff (#278) --- .ruff.toml | 1 + build_docs.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 52976fe..47cbf74 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -15,6 +15,7 @@ select = [ "F", # pyflakes "FA", # flake8-future-annotations "FLY", # flynt + "G", # flake8-logging-format "I", # isort "N", # pep8-naming "PERF", # perflint diff --git a/build_docs.py b/build_docs.py index 7329bbe..a204560 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1066,20 +1066,21 @@ def load_environment_variables() -> None: env_conf_file = dbs_site_config else: logging.info( - "No environment variables configured. " - f"Configure in {dbs_site_config} or {dbs_user_config}." + "No environment variables configured. Configure in %s or %s.", + dbs_site_config, + dbs_user_config, ) return - logging.info(f"Reading environment variables from {env_conf_file}.") + logging.info("Reading environment variables from %s.", env_conf_file) if env_conf_file == dbs_site_config: - logging.info(f"You can override settings in {dbs_user_config}.") + logging.info("You can override settings in %s.", dbs_user_config) elif dbs_site_config.is_file(): - logging.info(f"Overriding {dbs_site_config}.") + logging.info("Overriding %s.", dbs_site_config) env_config = env_conf_file.read_text(encoding="utf-8") for key, value in tomlkit.parse(env_config).get("env", {}).items(): - logging.debug(f"Setting {key} in environment.") + logging.debug("Setting %s in environment.", key) os.environ[key] = value From 88404600ea006ee14ea1bdbe716c925acba67bbb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 15:46:32 +0100 Subject: [PATCH 074/102] Add a simple integration test (#275) --- .github/workflows/test.yml | 87 ++++++++++++++++++++++++++------------ build_docs.py | 2 +- 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e272e76..d7ebf7f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,9 @@ name: Test -on: [push, pull_request, workflow_dispatch] +on: + push: + pull_request: + workflow_dispatch: permissions: {} @@ -8,35 +11,63 @@ env: FORCE_COLOR: 1 jobs: - test: - runs-on: ${{ matrix.os }} + integration: + name: Integration test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Set up requirements + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Build documentation + run: > + python ./build_docs.py + --quick + --build-root ./build_root + --www-root ./www + --log-directory ./logs + --group "$(id -g)" + --skip-cache-invalidation + --languages en + --branches 3.14 + + unit: + name: Unit tests + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.13", "3.14"] - os: [ubuntu-latest] + python-version: + - "3.13" + - "3.14" steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - - - name: Install uv - uses: hynek/setup-cached-uv@v2 - - - name: Tox tests - run: | - uvx --with tox-uv tox -e py - - - name: Upload coverage - uses: codecov/codecov-action@v5 - with: - flags: ${{ matrix.os }} - name: ${{ matrix.os }} Python ${{ matrix.python-version }} - token: ${{ secrets.CODECOV_ORG_TOKEN }} + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Install uv + uses: hynek/setup-cached-uv@v2 + + - name: Tox tests + run: uvx --with tox-uv tox -e py + + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + name: Python ${{ matrix.python-version }} + token: ${{ secrets.CODECOV_ORG_TOKEN }} diff --git a/build_docs.py b/build_docs.py index a204560..2301918 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1043,7 +1043,7 @@ def parse_args() -> argparse.Namespace: def setup_logging(log_directory: Path, select_output: str | None) -> None: """Setup logging to stderr if run by a human, or to a file if run from a cron.""" log_format = "%(asctime)s %(levelname)s: %(message)s" - if sys.stderr.isatty(): + if sys.stderr.isatty() or "CI" in os.environ: logging.basicConfig(format=log_format, stream=sys.stderr) else: log_directory.mkdir(parents=True, exist_ok=True) From 5cfd94e84ead25ce6baa3d859a53c30ea37eac7e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 15:51:56 +0100 Subject: [PATCH 075/102] Convert ``Version`` to a dataclass (#279) --- build_docs.py | 53 +++++++++++++------------------ tests/test_build_docs_versions.py | 48 ++++++++++++++-------------- 2 files changed, 46 insertions(+), 55 deletions(-) diff --git a/build_docs.py b/build_docs.py index 2301918..1efdabf 100755 --- a/build_docs.py +++ b/build_docs.py @@ -58,7 +58,6 @@ import sys from bisect import bisect_left as bisect from contextlib import contextmanager, suppress -from functools import total_ordering from pathlib import Path from string import Template from time import perf_counter, sleep @@ -103,11 +102,23 @@ def __reversed__(self) -> Iterator[Version]: @classmethod def from_json(cls, data: dict) -> Versions: - versions = sorted( - [Version.from_json(name, release) for name, release in data.items()], - key=Version.as_tuple, - ) - return cls(versions) + """Load versions from the devguide's JSON representation.""" + permitted = ", ".join(sorted(Version.STATUSES | Version.SYNONYMS.keys())) + + versions = [] + for name, release in data.items(): + branch = release["branch"] + status = release["status"] + status = Version.SYNONYMS.get(status, status) + if status not in Version.STATUSES: + msg = ( + f"Saw invalid version status {status!r}, " + f"expected to be one of {permitted}." + ) + raise ValueError(msg) + versions.append(Version(name=name, status=status, branch_or_tag=branch)) + + return cls(sorted(versions, key=Version.as_tuple)) def filter(self, branches: Sequence[str] = ()) -> Sequence[Version]: """Filter the given versions. @@ -143,10 +154,14 @@ def setup_indexsidebar(self, current: Version, dest_path: Path) -> None: dest_path.write_text(rendered_template, encoding="UTF-8") -@total_ordering +@dataclasses.dataclass(frozen=True, kw_only=True, slots=True) class Version: """Represents a CPython version and its documentation build dependencies.""" + name: str + status: Literal["EOL", "security-fixes", "stable", "pre-release", "in development"] + branch_or_tag: str + STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"} # Those synonyms map branch status vocabulary found in the devguide @@ -159,33 +174,9 @@ class Version: "prerelease": "pre-release", } - def __init__( - self, name: str, *, status: str, branch_or_tag: str | None = None - ) -> None: - status = self.SYNONYMS.get(status, status) - if status not in self.STATUSES: - raise ValueError( - "Version status expected to be one of: " - f"{', '.join(self.STATUSES | set(self.SYNONYMS.keys()))}, got {status!r}." - ) - self.name = name - self.branch_or_tag = branch_or_tag - self.status = status - - def __repr__(self) -> str: - return f"Version({self.name})" - def __eq__(self, other: Version) -> bool: return self.name == other.name - def __gt__(self, other: Version) -> bool: - return self.as_tuple() > other.as_tuple() - - @classmethod - def from_json(cls, name: str, values: dict) -> Version: - """Loads a version from devguide's json representation.""" - return cls(name, status=values["status"], branch_or_tag=values["branch"]) - @property def requirements(self) -> list[str]: """Generate the right requirements for this version. diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py index 662838e..42b5392 100644 --- a/tests/test_build_docs_versions.py +++ b/tests/test_build_docs_versions.py @@ -4,12 +4,12 @@ def test_filter_default() -> None: # Arrange versions = Versions([ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), - Version("3.11", status="security"), - Version("3.10", status="security"), - Version("3.9", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.11", status="security-fixes", branch_or_tag=""), + Version(name="3.10", status="security-fixes", branch_or_tag=""), + Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) # Act @@ -17,39 +17,39 @@ def test_filter_default() -> None: # Assert assert filtered == [ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), ] def test_filter_one() -> None: # Arrange versions = Versions([ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), - Version("3.11", status="security"), - Version("3.10", status="security"), - Version("3.9", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.11", status="security-fixes", branch_or_tag=""), + Version(name="3.10", status="security-fixes", branch_or_tag=""), + Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) # Act filtered = versions.filter(["3.13"]) # Assert - assert filtered == [Version("3.13", status="security")] + assert filtered == [Version(name="3.13", status="security-fixes", branch_or_tag="")] def test_filter_multiple() -> None: # Arrange versions = Versions([ - Version("3.14", status="feature"), - Version("3.13", status="bugfix"), - Version("3.12", status="bugfix"), - Version("3.11", status="security"), - Version("3.10", status="security"), - Version("3.9", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="stable", branch_or_tag=""), + Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.11", status="security-fixes", branch_or_tag=""), + Version(name="3.10", status="security-fixes", branch_or_tag=""), + Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) # Act @@ -57,6 +57,6 @@ def test_filter_multiple() -> None: # Assert assert filtered == [ - Version("3.14", status="feature"), - Version("3.13", status="security"), + Version(name="3.14", status="in development", branch_or_tag=""), + Version(name="3.13", status="security-fixes", branch_or_tag=""), ] From 9f3437707aa698ecc560a5d911cfbad9803541e7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 19:00:31 +0100 Subject: [PATCH 076/102] Extract ``switchers.js`` rendering into a new function. (#280) Previously, identical content for ``switchers.js`` was rendered for each version-language pair. This commit pre-renders the content of the file, then writing it to disk for each version-language pair. --- build_docs.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/build_docs.py b/build_docs.py index 1efdabf..08f457e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -467,23 +467,13 @@ def edit(file: Path): temporary.rename(file) -def setup_switchers(versions: Versions, languages: Languages, html_root: Path) -> None: +def setup_switchers(script_content: bytes, html_root: Path) -> None: """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher - Cross-link various versions in a version switcher """ - language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 - version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] - - switchers_template_file = HERE / "templates" / "switchers.js" switchers_path = html_root / "_static" / "switchers.js" - - template = Template(switchers_template_file.read_text(encoding="UTF-8")) - rendered_template = template.safe_substitute( - LANGUAGES=json.dumps(language_pairs), - VERSIONS=json.dumps(version_pairs), - ) - switchers_path.write_text(rendered_template, encoding="UTF-8") + switchers_path.write_text(script_content, encoding="UTF-8") for file in html_root.glob("**/*.html"): depth = len(file.relative_to(html_root).parts) - 1 @@ -541,8 +531,8 @@ class DocBuilder: version: Version versions: Versions language: Language - languages: Languages cpython_repo: Repository + switchers_content: bytes build_root: Path www_root: Path select_output: Literal["no-html", "only-html", "only-html-en"] | None @@ -697,7 +687,7 @@ def build(self) -> None: run(["chgrp", "-R", self.group, self.log_directory]) if self.includes_html: setup_switchers( - self.versions, self.languages, self.checkout / "Doc" / "build" / "html" + self.switchers_content, self.checkout / "Doc" / "build" / "html" ) logging.info("Build done (%s).", format_seconds(perf_counter() - start_time)) @@ -1108,6 +1098,8 @@ def build_docs(args: argparse.Namespace) -> bool: force_build = args.force del args.force + switchers_content = render_switchers(versions, languages) + build_succeeded = set() build_failed = set() cpython_repo = Repository( @@ -1127,7 +1119,12 @@ def build_docs(args: argparse.Namespace) -> bool: scope.set_tag("language", language.tag) cpython_repo.update() builder = DocBuilder( - version, versions, language, languages, cpython_repo, **vars(args) + version, + versions, + language, + cpython_repo, + switchers_content, + **vars(args), ) built_successfully = builder.run(http, force_build=force_build) if built_successfully: @@ -1179,6 +1176,19 @@ def parse_languages_from_config() -> Languages: return Languages.from_json(config["defaults"], config["languages"]) +def render_switchers(versions: Versions, languages: Languages) -> bytes: + language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 + version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] + + switchers_template_file = HERE / "templates" / "switchers.js" + template = Template(switchers_template_file.read_text(encoding="UTF-8")) + rendered_template = template.safe_substitute( + LANGUAGES=json.dumps(language_pairs), + VERSIONS=json.dumps(version_pairs), + ) + return rendered_template.encode("UTF-8") + + def build_sitemap( versions: Versions, languages: Languages, www_root: Path, group: str ) -> None: From 638d5e6634869c769ac49ac655c92c47d9f4a662 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 19:02:28 +0100 Subject: [PATCH 077/102] Correct the type annotation for ``DocBuilder.theme`` (#281) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 08f457e..07c98ae 100755 --- a/build_docs.py +++ b/build_docs.py @@ -540,7 +540,7 @@ class DocBuilder: group: str log_directory: Path skip_cache_invalidation: bool - theme: Path + theme: str @property def html_only(self) -> bool: From f30fec8a7a811e7f325cadd705f7db4578e2d7a4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 12 Apr 2025 22:07:23 +0100 Subject: [PATCH 078/102] Skip checking canonical links if nothing was built (#282) --- build_docs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index 07c98ae..faa8648 100755 --- a/build_docs.py +++ b/build_docs.py @@ -1101,7 +1101,7 @@ def build_docs(args: argparse.Namespace) -> bool: switchers_content = render_switchers(versions, languages) build_succeeded = set() - build_failed = set() + any_build_failed = False cpython_repo = Repository( "https://github.com/python/cpython.git", args.build_root / _checkout_name(args.select_output), @@ -1130,7 +1130,7 @@ def build_docs(args: argparse.Namespace) -> bool: if built_successfully: build_succeeded.add((version.name, language.tag)) elif built_successfully is not None: - build_failed.add((version.name, language.tag)) + any_build_failed = True logging.root.handlers[0].setFormatter( logging.Formatter("%(asctime)s %(levelname)s: %(message)s") @@ -1153,11 +1153,13 @@ def build_docs(args: argparse.Namespace) -> bool: args.skip_cache_invalidation, http, ) - proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) + if build_succeeded: + # Only check canonicals if at least one version was built. + proofread_canonicals(args.www_root, args.skip_cache_invalidation, http) logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - return len(build_failed) == 0 + return any_build_failed def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: From 982c0f8b71f6fa10a8378eb3e3a60ed50fea4e5a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 13 Apr 2025 11:32:47 +0100 Subject: [PATCH 079/102] Fix TypeError in ``setup_switchers()`` (#284) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index faa8648..fef379e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -473,7 +473,7 @@ def setup_switchers(script_content: bytes, html_root: Path) -> None: - Cross-link various versions in a version switcher """ switchers_path = html_root / "_static" / "switchers.js" - switchers_path.write_text(script_content, encoding="UTF-8") + switchers_path.write_bytes(script_content) for file in html_root.glob("**/*.html"): depth = len(file.relative_to(html_root).parts) - 1 From cc5e153b4afc6bbc7dd5aac99ee8f396d9b9b7a4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sun, 13 Apr 2025 13:50:33 +0100 Subject: [PATCH 080/102] Ensure exit status is returned by ``build_docs.py`` (#286) --- build_docs.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/build_docs.py b/build_docs.py index fef379e..e40259c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -911,20 +911,21 @@ def _checkout_name(select_output: str | None) -> str: return "cpython" -def main() -> None: +def main() -> int: """Script entry point.""" args = parse_args() setup_logging(args.log_directory, args.select_output) load_environment_variables() if args.select_output is None: - build_docs_with_lock(args, "build_docs.lock") - elif args.select_output == "no-html": - build_docs_with_lock(args, "build_docs_archives.lock") - elif args.select_output == "only-html": - build_docs_with_lock(args, "build_docs_html.lock") - elif args.select_output == "only-html-en": - build_docs_with_lock(args, "build_docs_html_en.lock") + return build_docs_with_lock(args, "build_docs.lock") + if args.select_output == "no-html": + return build_docs_with_lock(args, "build_docs_archives.lock") + if args.select_output == "only-html": + return build_docs_with_lock(args, "build_docs_html.lock") + if args.select_output == "only-html-en": + return build_docs_with_lock(args, "build_docs_html_en.lock") + return EX_FAILURE def parse_args() -> argparse.Namespace: @@ -1073,12 +1074,12 @@ def build_docs_with_lock(args: argparse.Namespace, lockfile_name: str) -> int: return EX_FAILURE try: - return EX_OK if build_docs(args) else EX_FAILURE + return build_docs(args) finally: lock.close() -def build_docs(args: argparse.Namespace) -> bool: +def build_docs(args: argparse.Namespace) -> int: """Build all docs (each language and each version).""" logging.info("Full build start.") start_time = perf_counter() @@ -1159,7 +1160,7 @@ def build_docs(args: argparse.Namespace) -> bool: logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time)) - return any_build_failed + return EX_FAILURE if any_build_failed else EX_OK def parse_versions_from_devguide(http: urllib3.PoolManager) -> Versions: @@ -1397,4 +1398,4 @@ def purge_surrogate_key(http: urllib3.PoolManager, surrogate_key: str) -> None: if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) From 1e856ea3a38944e9743c11110d67599c3159cb07 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 13 Apr 2025 19:08:49 +0300 Subject: [PATCH 081/102] Run unit tests on three operating systems (#285) Co-authored-by: Ezio Melotti --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7ebf7f..8a9324f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,13 +41,12 @@ jobs: unit: name: Unit tests - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: - - "3.13" - - "3.14" + python-version: ["3.13", "3.14"] + os: [windows-latest, macos-latest, ubuntu-latest] steps: - uses: actions/checkout@v4 @@ -69,5 +68,6 @@ jobs: - name: Upload coverage uses: codecov/codecov-action@v5 with: + flags: ${{ matrix.os }} name: Python ${{ matrix.python-version }} token: ${{ secrets.CODECOV_ORG_TOKEN }} From 9f45c4e38487289db9dacaea600ecec2720b1507 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:44:32 +0300 Subject: [PATCH 082/102] Test the ``Version`` class (#287) --- tests/test_build_docs_version.py | 88 ++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/test_build_docs_version.py diff --git a/tests/test_build_docs_version.py b/tests/test_build_docs_version.py new file mode 100644 index 0000000..bd5e644 --- /dev/null +++ b/tests/test_build_docs_version.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import pytest + +from build_docs import Version + + +def test_equality() -> None: + # Arrange + version1 = Version(name="3.13", status="stable", branch_or_tag="3.13") + version2 = Version(name="3.13", status="stable", branch_or_tag="3.13") + + # Act / Assert + assert version1 == version2 + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("3.13", "-rrequirements.txt"), + ("3.10", "standard-imghdr"), + ("3.7", "sphinx==2.3.1"), + ("3.5", "sphinx==1.8.4"), + ], +) +def test_requirements(name: str, expected: str) -> None: + # Arrange + version = Version(name=name, status="stable", branch_or_tag="") + + # Act / Assert + assert expected in version.requirements + + +def test_requirements_error() -> None: + # Arrange + version = Version(name="2.8", status="ex-release", branch_or_tag="") + + # Act / Assert + with pytest.raises(ValueError, match="unreachable"): + _ = version.requirements + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + ("EOL", "never"), + ("security-fixes", "yearly"), + ("stable", "daily"), + ], +) +def test_changefreq(status: str, expected: str) -> None: + # Arrange + version = Version(name="3.13", status=status, branch_or_tag="") + + # Act / Assert + assert version.changefreq == expected + + +def test_url() -> None: + # Arrange + version = Version(name="3.13", status="stable", branch_or_tag="") + + # Act / Assert + assert version.url == "https://docs.python.org/3.13/" + + +def test_title() -> None: + # Arrange + version = Version(name="3.14", status="in development", branch_or_tag="") + + # Act / Assert + assert version.title == "Python 3.14 (in development)" + + +@pytest.mark.parametrize( + ("name", "status", "expected"), + [ + ("3.15", "in development", "dev (3.15)"), + ("3.14", "pre-release", "pre (3.14)"), + ("3.13", "stable", "3.13"), + ], +) +def test_picker_label(name: str, status: str, expected: str) -> None: + # Arrange + version = Version(name=name, status=status, branch_or_tag="") + + # Act / Assert + assert version.picker_label == expected From 9bcde435e586cca62c694678b0966b4b84d93047 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:45:44 +0100 Subject: [PATCH 083/102] Simplify generation of ``indexsidebar.html`` (#283) Remove the double-rendering of ``indexsidebar.html``. For end-of-life versions, change the sidebar to contain a link to the current stable version. For non-EOL versions, overwrite the new ``_docs_by_version.html`` file with the list of links. --- build_docs.py | 40 +++++++++++++++++++-------------- templates/_docs_by_version.html | 11 +++++++++ templates/indexsidebar.html | 23 +++++-------------- 3 files changed, 39 insertions(+), 35 deletions(-) create mode 100644 templates/_docs_by_version.html diff --git a/build_docs.py b/build_docs.py index e40259c..9b498fe 100755 --- a/build_docs.py +++ b/build_docs.py @@ -143,16 +143,6 @@ def current_dev(self) -> Version: """Find the current CPython version in development.""" return max(self, key=Version.as_tuple) - def setup_indexsidebar(self, current: Version, dest_path: Path) -> None: - """Build indexsidebar.html for Sphinx.""" - template_path = HERE / "templates" / "indexsidebar.html" - template = jinja2.Template(template_path.read_text(encoding="UTF-8")) - rendered_template = template.render( - current_version=current, - versions=list(reversed(self)), - ) - dest_path.write_text(rendered_template, encoding="UTF-8") - @dataclasses.dataclass(frozen=True, kw_only=True, slots=True) class Version: @@ -529,9 +519,9 @@ class DocBuilder: """Builder for a CPython version and a language.""" version: Version - versions: Versions language: Language cpython_repo: Repository + docs_by_version_content: bytes switchers_content: bytes build_root: Path www_root: Path @@ -667,10 +657,7 @@ def build(self) -> None: text = text.replace(" -A switchers=1", "") (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") - self.versions.setup_indexsidebar( - self.version, - self.checkout / "Doc" / "tools" / "templates" / "indexsidebar.html", - ) + self.setup_indexsidebar() run_with_logging([ "make", "-C", @@ -713,6 +700,18 @@ def build_venv(self) -> None: run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) self.venv = venv_path + def setup_indexsidebar(self) -> None: + """Copy indexsidebar.html for Sphinx.""" + tmpl_src = HERE / "templates" + tmpl_dst = self.checkout / "Doc" / "tools" / "templates" + dbv_path = tmpl_dst / "_docs_by_version.html" + + shutil.copy(tmpl_src / "indexsidebar.html", tmpl_dst / "indexsidebar.html") + if self.version.status != "EOL": + dbv_path.write_bytes(self.docs_by_version_content) + else: + shutil.copy(tmpl_src / "_docs_by_version.html", dbv_path) + def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: """Copy a given build to the appropriate webroot with appropriate rights.""" logging.info("Publishing start.") @@ -1099,6 +1098,7 @@ def build_docs(args: argparse.Namespace) -> int: force_build = args.force del args.force + docs_by_version_content = render_docs_by_version(versions).encode() switchers_content = render_switchers(versions, languages) build_succeeded = set() @@ -1118,12 +1118,12 @@ def build_docs(args: argparse.Namespace) -> int: scope = sentry_sdk.get_isolation_scope() scope.set_tag("version", version.name) scope.set_tag("language", language.tag) - cpython_repo.update() + cpython_repo.update() builder = DocBuilder( version, - versions, language, cpython_repo, + docs_by_version_content, switchers_content, **vars(args), ) @@ -1179,6 +1179,12 @@ def parse_languages_from_config() -> Languages: return Languages.from_json(config["defaults"], config["languages"]) +def render_docs_by_version(versions: Versions) -> str: + """Generate content for _docs_by_version.html.""" + links = [f'
  • {v.title}
    • ' for v in reversed(versions)] + return "\n".join(links) + + def render_switchers(versions: Versions, languages: Languages) -> bytes: language_pairs = sorted((l.tag, l.switcher_label) for l in languages if l.in_prod) # NoQA: E741 version_pairs = [(v.name, v.picker_label) for v in reversed(versions)] diff --git a/templates/_docs_by_version.html b/templates/_docs_by_version.html new file mode 100644 index 0000000..1a84cfb --- /dev/null +++ b/templates/_docs_by_version.html @@ -0,0 +1,11 @@ +{# +This file is only used in indexsidebar.html, where it is included in the docs +by version list. For non-end-of-life branches, build_docs.py overwrites this +list with the full list of versions. + +Keep the following two files synchronised: +* cpython/Doc/tools/templates/_docs_by_version.html +* docsbuild-scripts/templates/_docs_by_version.html +#} +
  • {% trans %}Stable{% endtrans %}
    • +
  • {% trans %}In development{% endtrans %}
    • diff --git a/templates/indexsidebar.html b/templates/indexsidebar.html index 3a56219..eea29e2 100644 --- a/templates/indexsidebar.html +++ b/templates/indexsidebar.html @@ -1,30 +1,17 @@ -{# -Beware, this file is rendered twice via Jinja2: -- First by build_docs.py, given 'current_version' and 'versions'. -- A 2nd time by Sphinx. -#} - -{% raw %}

    {% trans %}Download{% endtrans %}

    {% trans %}Download these documents{% endtrans %}

    -{% endraw %} -{% if current_version.status != "EOL" %} -{% raw %}

    {% trans %}Docs by version{% endtrans %}

    {% endraw %} +

    {% trans %}Docs by version{% endtrans %}

    -{% endif %} -{% raw %}

    {% trans %}Other resources{% endtrans %}

    -{% endraw %} From efd1e17c947f426af9301ec0387f2e56bf3f86bf Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:45:53 +0100 Subject: [PATCH 084/102] Use ``venv.create()`` to create virtual environments (#289) --- build_docs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 9b498fe..733473e 100755 --- a/build_docs.py +++ b/build_docs.py @@ -56,6 +56,7 @@ import shutil import subprocess import sys +import venv from bisect import bisect_left as bisect from contextlib import contextmanager, suppress from pathlib import Path @@ -690,7 +691,7 @@ def build_venv(self) -> None: requirements.append("matplotlib>=3") venv_path = self.build_root / ("venv-" + self.version.name) - run([sys.executable, "-m", "venv", venv_path]) + venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] + ["--upgrade-strategy=eager"] From 776e64996819c11cec1a4154342630f3e7ea4c1f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:56:15 +0100 Subject: [PATCH 085/102] Create a function to perform ``chgrp`` operations (#290) --- build_docs.py | 70 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/build_docs.py b/build_docs.py index 733473e..2f164f8 100755 --- a/build_docs.py +++ b/build_docs.py @@ -671,8 +671,8 @@ def build(self) -> None: "SPHINXERRORHANDLING=", maketarget, ]) - run(["mkdir", "-p", self.log_directory]) - run(["chgrp", "-R", self.group, self.log_directory]) + self.log_directory.mkdir(parents=True, exist_ok=True) + chgrp(self.log_directory, group=self.group, recursive=True) if self.includes_html: setup_switchers( self.switchers_content, self.checkout / "Doc" / "build" / "html" @@ -723,10 +723,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: else: language_dir = self.www_root / self.language.tag language_dir.mkdir(parents=True, exist_ok=True) - try: - run(["chgrp", "-R", self.group, language_dir]) - except subprocess.CalledProcessError as err: - logging.warning("Can't change group of %s: %s", language_dir, str(err)) + chgrp(language_dir, group=self.group, recursive=True) language_dir.chmod(0o775) target = language_dir / self.version.name @@ -735,22 +732,18 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: target.chmod(0o775) except PermissionError as err: logging.warning("Can't change mod of %s: %s", target, str(err)) - try: - run(["chgrp", "-R", self.group, target]) - except subprocess.CalledProcessError as err: - logging.warning("Can't change group of %s: %s", target, str(err)) + chgrp(target, group=self.group, recursive=True) changed = [] if self.includes_html: # Copy built HTML files to webroot (default /srv/docs.python.org) changed = changed_files(self.checkout / "Doc" / "build" / "html", target) logging.info("Copying HTML files to %s", target) - run([ - "chown", - "-R", - ":" + self.group, + chgrp( self.checkout / "Doc" / "build" / "html/", - ]) + group=self.group, + recursive=True, + ) run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) run([ "find", @@ -776,12 +769,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: if not self.quick and (self.checkout / "Doc" / "dist").is_dir(): # Copy archive files to /archives/ logging.debug("Copying dist files.") - run([ - "chown", - "-R", - ":" + self.group, - self.checkout / "Doc" / "dist", - ]) + chgrp(self.checkout / "Doc" / "dist", group=self.group, recursive=True) run([ "chmod", "-R", @@ -789,7 +777,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: self.checkout / "Doc" / "dist", ]) run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) - run(["chown", ":" + self.group, target / "archives"]) + chgrp(target / "archives", group=self.group) run([ "cp", "-a", @@ -889,6 +877,36 @@ def save_state( logging.info("Saved new rebuild state for %s: %s", key, table.as_string()) +def chgrp( + path: Path, + /, + group: int | str | None, + *, + recursive: bool = False, + follow_symlinks: bool = True, +) -> None: + if sys.platform == "win32": + return + + from grp import getgrnam + + try: + try: + group_id = int(group) + except ValueError: + group_id = getgrnam(group)[2] + except (LookupError, TypeError, ValueError): + return + + try: + os.chown(path, -1, group_id, follow_symlinks=follow_symlinks) + if recursive: + for p in path.rglob("*"): + os.chown(p, -1, group_id, follow_symlinks=follow_symlinks) + except OSError as err: + logging.warning("Can't change group of %s: %s", path, str(err)) + + def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) @@ -1213,7 +1231,7 @@ def build_sitemap( sitemap_path = www_root / "sitemap.xml" sitemap_path.write_text(rendered_template + "\n", encoding="UTF-8") sitemap_path.chmod(0o664) - run(["chgrp", group, sitemap_path]) + chgrp(sitemap_path, group=group) def build_404(www_root: Path, group: str) -> None: @@ -1225,7 +1243,7 @@ def build_404(www_root: Path, group: str) -> None: not_found_file = www_root / "404.html" shutil.copyfile(HERE / "templates" / "404.html", not_found_file) not_found_file.chmod(0o664) - run(["chgrp", group, not_found_file]) + chgrp(not_found_file, group=group) def copy_robots_txt( @@ -1243,7 +1261,7 @@ def copy_robots_txt( robots_path = www_root / "robots.txt" shutil.copyfile(template_path, robots_path) robots_path.chmod(0o775) - run(["chgrp", group, robots_path]) + chgrp(robots_path, group=group) if not skip_cache_invalidation: purge(http, "robots.txt") @@ -1311,7 +1329,7 @@ def symlink( # Link does not exist or points to the wrong target. link.unlink(missing_ok=True) link.symlink_to(directory) - run(["chown", "-h", f":{group}", str(link)]) + chgrp(link, group=group, follow_symlinks=False) if not skip_cache_invalidation: surrogate_key = f"{language_tag}/{name}" purge_surrogate_key(http, surrogate_key) From 6164fac89acf0e3890d70bdd60320cd9b5de77e2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 21:29:27 +0100 Subject: [PATCH 086/102] Create a function for ``chmod`` operations (#291) --- build_docs.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/build_docs.py b/build_docs.py index 2f164f8..d1afdf2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -54,6 +54,7 @@ import re import shlex import shutil +import stat import subprocess import sys import venv @@ -744,18 +745,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: group=self.group, recursive=True, ) - run(["chmod", "-R", "o+r", self.checkout / "Doc" / "build" / "html"]) - run([ - "find", - self.checkout / "Doc" / "build" / "html", - "-type", - "d", - "-exec", - "chmod", - "o+x", - "{}", - ";", - ]) + chmod_make_readable(self.checkout / "Doc" / "build" / "html") run([ "rsync", "-a", @@ -770,12 +760,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: # Copy archive files to /archives/ logging.debug("Copying dist files.") chgrp(self.checkout / "Doc" / "dist", group=self.group, recursive=True) - run([ - "chmod", - "-R", - "o+r", - self.checkout / "Doc" / "dist", - ]) + chmod_make_readable(self.checkout / "Doc" / "dist") run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) chgrp(target / "archives", group=self.group) run([ @@ -907,6 +892,18 @@ def chgrp( logging.warning("Can't change group of %s: %s", path, str(err)) +def chmod_make_readable(path: Path, /, mode: int = stat.S_IROTH) -> None: + if not path.is_dir(): + raise ValueError + + path.chmod(path.stat().st_mode | stat.S_IROTH | stat.S_IXOTH) # o+rx + for p in path.rglob("*"): + if p.is_dir(): + p.chmod(p.stat().st_mode | stat.S_IROTH | stat.S_IXOTH) # o+rx + else: + p.chmod(p.stat().st_mode | stat.S_IROTH) # o+r + + def format_seconds(seconds: float) -> str: hours, remainder = divmod(seconds, 3600) minutes, seconds = divmod(remainder, 60) From e1dcdfcaaac76fbb6a4608e169bef9d2e5f6f5a2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 22:53:25 +0100 Subject: [PATCH 087/102] Remove subprocess calls for copying archive files (#292) --- .github/workflows/test.yml | 6 ++++++ build_docs.py | 27 ++++++++++++++------------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a9324f..850fec7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,12 @@ jobs: --languages en --branches 3.14 + - name: Upload documentation + uses: actions/upload-artifact@v4 + with: + name: www-root + path: ./www + unit: name: Unit tests runs-on: ${{ matrix.os }} diff --git a/build_docs.py b/build_docs.py index d1afdf2..83bc26c 100755 --- a/build_docs.py +++ b/build_docs.py @@ -756,22 +756,23 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: target, ]) - if not self.quick and (self.checkout / "Doc" / "dist").is_dir(): + dist_dir = self.checkout / "Doc" / "dist" + if dist_dir.is_dir(): # Copy archive files to /archives/ logging.debug("Copying dist files.") - chgrp(self.checkout / "Doc" / "dist", group=self.group, recursive=True) - chmod_make_readable(self.checkout / "Doc" / "dist") - run(["mkdir", "-m", "o+rx", "-p", target / "archives"]) - chgrp(target / "archives", group=self.group) - run([ - "cp", - "-a", - *(self.checkout / "Doc" / "dist").glob("*"), - target / "archives", - ]) + chgrp(dist_dir, group=self.group, recursive=True) + chmod_make_readable(dist_dir) + archives_dir = target / "archives" + archives_dir.mkdir(parents=True, exist_ok=True) + archives_dir.chmod( + archives_dir.stat().st_mode | stat.S_IROTH | stat.S_IXOTH + ) + chgrp(archives_dir, group=self.group) + for dist_file in dist_dir.iterdir(): + shutil.copy2(dist_file, archives_dir / dist_file.name) changed.append("archives/") - for file in (target / "archives").iterdir(): - changed.append("archives/" + file.name) + for file in archives_dir.iterdir(): + changed.append(f"archives/{file.name}") logging.info("%s files changed", len(changed)) if changed and not self.skip_cache_invalidation: From d6f84292abef7150bdd0a9467d0fca08df7188f5 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 22:58:51 +0100 Subject: [PATCH 088/102] Set explicit artefact retention time --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 850fec7..b64e8a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,7 @@ jobs: with: name: www-root path: ./www + retention-days: 2 unit: name: Unit tests From 783e70445fa48a8d3d7268e75eb45b2926b04c05 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 23:22:12 +0100 Subject: [PATCH 089/102] Convert ``changed`` to an integer count --- build_docs.py | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/build_docs.py b/build_docs.py index 83bc26c..d20aea4 100755 --- a/build_docs.py +++ b/build_docs.py @@ -337,23 +337,15 @@ def run_with_logging(cmd: Sequence[str | Path], cwd: Path | None = None) -> None raise subprocess.CalledProcessError(return_code, cmd[0]) -def changed_files(left: Path, right: Path) -> list[str]: - """Compute a list of different files between left and right, recursively. - Resulting paths are relative to left. - """ - changed = [] +def changed_files(left: Path, right: Path) -> int: + """Compute the number of different files in the two directory trees.""" - def traverse(dircmp_result: filecmp.dircmp) -> None: - base = Path(dircmp_result.left).relative_to(left) - for file in dircmp_result.diff_files: - changed.append(str(base / file)) - if file == "index.html": - changed.append(str(base) + "/") - for dircomp in dircmp_result.subdirs.values(): - traverse(dircomp) + def traverse(dircmp_result: filecmp.dircmp) -> int: + changed = len(dircmp_result.diff_files) + changed += sum(map(traverse, dircmp_result.subdirs.values())) + return changed - traverse(filecmp.dircmp(left, right)) - return changed + return traverse(filecmp.dircmp(left, right)) @dataclasses.dataclass @@ -735,10 +727,10 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: logging.warning("Can't change mod of %s: %s", target, str(err)) chgrp(target, group=self.group, recursive=True) - changed = [] + changed = 0 if self.includes_html: # Copy built HTML files to webroot (default /srv/docs.python.org) - changed = changed_files(self.checkout / "Doc" / "build" / "html", target) + changed += changed_files(self.checkout / "Doc" / "build" / "html", target) logging.info("Copying HTML files to %s", target) chgrp( self.checkout / "Doc" / "build" / "html/", @@ -768,13 +760,12 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: archives_dir.stat().st_mode | stat.S_IROTH | stat.S_IXOTH ) chgrp(archives_dir, group=self.group) + changed += 1 for dist_file in dist_dir.iterdir(): shutil.copy2(dist_file, archives_dir / dist_file.name) - changed.append("archives/") - for file in archives_dir.iterdir(): - changed.append(f"archives/{file.name}") + changed += 1 - logging.info("%s files changed", len(changed)) + logging.info("%s files changed", changed) if changed and not self.skip_cache_invalidation: surrogate_key = f"{self.language.tag}/{self.version.name}" purge_surrogate_key(http, surrogate_key) From 9082a8eed43f3c2ec4f8101176d3932b4949ecde Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 23:32:48 +0100 Subject: [PATCH 090/102] Use f-strings instead of string concatenation --- build_docs.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build_docs.py b/build_docs.py index d20aea4..847bfe2 100755 --- a/build_docs.py +++ b/build_docs.py @@ -363,10 +363,10 @@ def get_ref(self, pattern: str) -> str: """Return the reference of a given tag or branch.""" try: # Maybe it's a branch - return self.run("show-ref", "-s", "origin/" + pattern).stdout.strip() + return self.run("show-ref", "-s", f"origin/{pattern}").stdout.strip() except subprocess.CalledProcessError: # Maybe it's a tag - return self.run("show-ref", "-s", "tags/" + pattern).stdout.strip() + return self.run("show-ref", "-s", f"tags/{pattern}").stdout.strip() def fetch(self) -> subprocess.CompletedProcess: """Try (and retry) to run git fetch.""" @@ -656,11 +656,11 @@ def build(self) -> None: "make", "-C", self.checkout / "Doc", - "PYTHON=" + str(python), - "SPHINXBUILD=" + str(sphinxbuild), - "BLURB=" + str(blurb), - "VENVDIR=" + str(self.venv), - "SPHINXOPTS=" + " ".join(sphinxopts), + f"PYTHON={python}", + f"SPHINXBUILD={sphinxbuild}", + f"BLURB={blurb}", + f"VENVDIR={self.venv}", + f"SPHINXOPTS={' '.join(sphinxopts)}", "SPHINXERRORHANDLING=", maketarget, ]) @@ -683,7 +683,7 @@ def build_venv(self) -> None: # opengraph previews requirements.append("matplotlib>=3") - venv_path = self.build_root / ("venv-" + self.version.name) + venv_path = self.build_root / f"venv-{self.version.name}" venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] From fe84a0bb4c442bcdb32154768c160468b1e90d6e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 15 Apr 2025 23:41:47 +0100 Subject: [PATCH 091/102] Use tuples for subprocess argument lists --- build_docs.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/build_docs.py b/build_docs.py index 847bfe2..d31ebdd 100755 --- a/build_docs.py +++ b/build_docs.py @@ -388,7 +388,7 @@ def clone(self) -> bool: return False # Already cloned logging.info("Cloning %s into %s", self.remote, self.directory) self.directory.mkdir(mode=0o775, parents=True, exist_ok=True) - run(["git", "clone", self.remote, self.directory]) + run(("git", "clone", self.remote, self.directory)) return True def update(self) -> None: @@ -481,7 +481,7 @@ def version_info() -> None: """Handler for --version.""" try: platex_version = head( - subprocess.check_output(["platex", "--version"], text=True), + subprocess.check_output(("platex", "--version"), text=True), lines=3, ) except FileNotFoundError: @@ -489,7 +489,7 @@ def version_info() -> None: try: xelatex_version = head( - subprocess.check_output(["xelatex", "--version"], text=True), + subprocess.check_output(("xelatex", "--version"), text=True), lines=2, ) except FileNotFoundError: @@ -652,7 +652,7 @@ def build(self) -> None: (self.checkout / "Doc" / "Makefile").write_text(text, encoding="utf-8") self.setup_indexsidebar() - run_with_logging([ + run_with_logging(( "make", "-C", self.checkout / "Doc", @@ -663,7 +663,7 @@ def build(self) -> None: f"SPHINXOPTS={' '.join(sphinxopts)}", "SPHINXERRORHANDLING=", maketarget, - ]) + )) self.log_directory.mkdir(parents=True, exist_ok=True) chgrp(self.log_directory, group=self.group, recursive=True) if self.includes_html: @@ -678,7 +678,7 @@ def build_venv(self) -> None: So we can reuse them from builds to builds, while they contain different Sphinx versions. """ - requirements = [self.theme] + self.version.requirements + requirements = list(self.version.requirements) if self.includes_html: # opengraph previews requirements.append("matplotlib>=3") @@ -686,12 +686,19 @@ def build_venv(self) -> None: venv_path = self.build_root / f"venv-{self.version.name}" venv.create(venv_path, symlinks=os.name != "nt", with_pip=True) run( - [venv_path / "bin" / "python", "-m", "pip", "install", "--upgrade"] - + ["--upgrade-strategy=eager"] - + requirements, + ( + venv_path / "bin" / "python", + "-m", + "pip", + "install", + "--upgrade", + "--upgrade-strategy=eager", + self.theme, + *requirements, + ), cwd=self.checkout / "Doc", ) - run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"]) + run((venv_path / "bin" / "python", "-m", "pip", "freeze", "--all")) self.venv = venv_path def setup_indexsidebar(self) -> None: @@ -738,7 +745,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: recursive=True, ) chmod_make_readable(self.checkout / "Doc" / "build" / "html") - run([ + run(( "rsync", "-a", "--delete-delay", @@ -746,7 +753,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: "P archives/", str(self.checkout / "Doc" / "build" / "html") + "/", target, - ]) + )) dist_dir = self.checkout / "Doc" / "dist" if dist_dir.is_dir(): From d1f0418ea5aadb14a670151eb16ce0e2ba4d6cda Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 00:02:39 +0100 Subject: [PATCH 092/102] Add more helper properties in the ``Language`` class (#293) --- build_docs.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/build_docs.py b/build_docs.py index d31ebdd..bd94ba1 100755 --- a/build_docs.py +++ b/build_docs.py @@ -280,6 +280,14 @@ class Language: def tag(self) -> str: return self.iso639_tag.replace("_", "-").lower() + @property + def is_translation(self) -> bool: + return self.tag != "en" + + @property + def locale_repo_url(self) -> str: + return f"https://github.com/python/python-docs-{self.tag}.git" + @property def switcher_label(self) -> str: if self.translated_name: @@ -549,7 +557,7 @@ def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: logging.info("Skipping non-HTML build (language is HTML-only).") return None # skipped self.cpython_repo.switch(self.version.branch_or_tag) - if self.language.tag != "en": + if self.language.is_translation: self.clone_translation() if trigger_reason := self.should_rebuild(force_build): self.build_venv() @@ -569,6 +577,10 @@ def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: return False return True + @property + def locale_dir(self) -> Path: + return self.build_root / self.version.name / "locale" + @property def checkout(self) -> Path: """Path to CPython git clone.""" @@ -582,15 +594,8 @@ def clone_translation(self) -> None: def translation_repo(self) -> Repository: """See PEP 545 for translations repository naming convention.""" - locale_repo = f"https://github.com/python/python-docs-{self.language.tag}.git" - locale_clone_dir = ( - self.build_root - / self.version.name - / "locale" - / self.language.iso639_tag - / "LC_MESSAGES" - ) - return Repository(locale_repo, locale_clone_dir) + locale_clone_dir = self.locale_dir / self.language.iso639_tag / "LC_MESSAGES" + return Repository(self.language.locale_repo_url, locale_clone_dir) @property def translation_branch(self) -> str: @@ -611,10 +616,9 @@ def build(self) -> None: logging.info("Build start.") start_time = perf_counter() sphinxopts = list(self.language.sphinxopts) - if self.language.tag != "en": - locale_dirs = self.build_root / self.version.name / "locale" + if self.language.is_translation: sphinxopts.extend(( - f"-D locale_dirs={locale_dirs}", + f"-D locale_dirs={self.locale_dir}", f"-D language={self.language.iso639_tag}", "-D gettext_compact=0", "-D translation_progress_classes=1", @@ -636,7 +640,7 @@ def build(self) -> None: if self.includes_html: site_url = self.version.url - if self.language.tag != "en": + if self.language.is_translation: site_url += f"{self.language.tag}/" # Define a tag to enable opengraph socialcards previews # (used in Doc/conf.py and requires matplotlib) @@ -718,7 +722,7 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: logging.info("Publishing start.") start_time = perf_counter() self.www_root.mkdir(parents=True, exist_ok=True) - if self.language.tag == "en": + if not self.language.is_translation: target = self.www_root / self.version.name else: language_dir = self.www_root / self.language.tag @@ -786,7 +790,7 @@ def should_rebuild(self, force: bool) -> str | Literal[False]: logging.info("Should rebuild: no previous state found.") return "no previous state" cpython_sha = self.cpython_repo.run("rev-parse", "HEAD").stdout.strip() - if self.language.tag != "en": + if self.language.is_translation: translation_sha = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() @@ -849,7 +853,7 @@ def save_state( "triggered_by": trigger, "cpython_sha": self.cpython_repo.run("rev-parse", "HEAD").stdout.strip(), } - if self.language.tag != "en": + if self.language.is_translation: state["translation_sha"] = self.translation_repo.run( "rev-parse", "HEAD" ).stdout.strip() From c64294658cf6ceffd6c169a95646ee14ef148b3f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 00:53:19 +0100 Subject: [PATCH 093/102] Constrain blurb to version 1.1 or older (#294) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index bd94ba1..d95b408 100755 --- a/build_docs.py +++ b/build_docs.py @@ -194,7 +194,7 @@ def requirements(self) -> list[str]: return dependencies + ["standard-imghdr"] # Requirements/constraints for Python 3.7 and older, pre-requirements.txt - reqs = ["jieba", "blurb", "jinja2<3.1", "docutils<=0.17.1", "standard-imghdr"] + reqs = ["jieba", "blurb<1.2", "jinja2<3.1", "docutils<0.18", "standard-imghdr"] if self.name in {"3.7", "3.6", "2.7"}: return reqs + ["sphinx==2.3.1"] if self.name == "3.5": From d53c8a6e4d64571d4902523d8d42ed4af5cb2907 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 01:10:42 +0100 Subject: [PATCH 094/102] Add constraints for sphinxcontrib dependencies (#295) --- build_docs.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index d95b408..e1ca9a3 100755 --- a/build_docs.py +++ b/build_docs.py @@ -183,9 +183,9 @@ def requirements(self) -> list[str]: """ dependencies = [ + "-rrequirements.txt", "jieba", # To improve zh search. "PyStemmer~=2.2.0", # To improve performance for word stemming. - "-rrequirements.txt", ] if self.as_tuple() >= (3, 11): return dependencies @@ -194,7 +194,19 @@ def requirements(self) -> list[str]: return dependencies + ["standard-imghdr"] # Requirements/constraints for Python 3.7 and older, pre-requirements.txt - reqs = ["jieba", "blurb<1.2", "jinja2<3.1", "docutils<0.18", "standard-imghdr"] + reqs = [ + "blurb<1.2", + "docutils<=0.17.1", + "jieba", + "jinja2<3.1", + "sphinxcontrib-applehelp<=1.0.2", + "sphinxcontrib-devhelp<=1.0.2", + "sphinxcontrib-htmlhelp<=2.0", + "sphinxcontrib-jsmath<=1.0.1", + "sphinxcontrib-qthelp<=1.0.3", + "sphinxcontrib-serializinghtml<=1.1.5", + "standard-imghdr", + ] if self.name in {"3.7", "3.6", "2.7"}: return reqs + ["sphinx==2.3.1"] if self.name == "3.5": From 3254c48f3cdab522896cc8d5d9d654dde11f3614 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 01:13:23 +0100 Subject: [PATCH 095/102] Constrain alabaster to version 0.7.12 or older --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index e1ca9a3..c257e2f 100755 --- a/build_docs.py +++ b/build_docs.py @@ -195,6 +195,7 @@ def requirements(self) -> list[str]: # Requirements/constraints for Python 3.7 and older, pre-requirements.txt reqs = [ + "alabaster<0.7.12", "blurb<1.2", "docutils<=0.17.1", "jieba", From 9a365d64d329aa3c286691dccd7ae475fc425829 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:24:10 +0100 Subject: [PATCH 096/102] Constrain python-docs-theme to version 2023.3.1 or older --- build_docs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build_docs.py b/build_docs.py index c257e2f..6895618 100755 --- a/build_docs.py +++ b/build_docs.py @@ -200,6 +200,7 @@ def requirements(self) -> list[str]: "docutils<=0.17.1", "jieba", "jinja2<3.1", + "python-docs-theme<=2023.3.1", "sphinxcontrib-applehelp<=1.0.2", "sphinxcontrib-devhelp<=1.0.2", "sphinxcontrib-htmlhelp<=2.0", From c215a786f60a9e24311b038155f5646e548287e2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:25:43 +0100 Subject: [PATCH 097/102] Restore _create_placeholders_if_missing() (#296) --- templates/switchers.js | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 774366f..324fd65 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -28,6 +28,30 @@ const _CURRENT_PREFIX = (() => { const _ALL_VERSIONS = new Map($VERSIONS); const _ALL_LANGUAGES = new Map($LANGUAGES); +/** + * Required for Python 3.7 and earlier. + * @returns {void} + * @private + */ +const _create_placeholders_if_missing = () => { + if (document.querySelectorAll(".version_switcher_placeholder").length) return; + + const items = document.querySelectorAll("body>div.related>ul>li:not(.right)"); + for (const item of items) { + if (item.innerText.toLowerCase().includes("documentation")) { + const container = document.createElement("li"); + container.className = "switchers"; + for (const placeholder_name of ["language", "version"]) { + const placeholder = document.createElement("div"); + placeholder.className = `${placeholder_name}_switcher_placeholder`; + container.appendChild(placeholder); + } + item.parentElement.insertBefore(container, item); + return; + } + } +}; + /** * @param {Map} versions * @returns {HTMLSelectElement} @@ -175,6 +199,8 @@ const _initialise_switchers = () => { const versions = _ALL_VERSIONS; const languages = _ALL_LANGUAGES; + _create_placeholders_if_missing(); + document .querySelectorAll(".version_switcher_placeholder") .forEach((placeholder) => { From 65735049bc9de117d9493a7677259cecc1f4de22 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:32:19 +0100 Subject: [PATCH 098/102] Require pipes for building Python 3.5 (#297) --- build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_docs.py b/build_docs.py index 6895618..c75f096 100755 --- a/build_docs.py +++ b/build_docs.py @@ -212,7 +212,7 @@ def requirements(self) -> list[str]: if self.name in {"3.7", "3.6", "2.7"}: return reqs + ["sphinx==2.3.1"] if self.name == "3.5": - return reqs + ["sphinx==1.8.4"] + return reqs + ["sphinx==1.8.4", "standard-pipes"] raise ValueError("unreachable") @property From c1be80ec82c318076b7c10b2e854674ca7d4ea62 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:49:00 +0100 Subject: [PATCH 099/102] Add inline styles in _create_placeholders_if_missing() --- templates/switchers.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/switchers.js b/templates/switchers.js index 324fd65..e54a278 100644 --- a/templates/switchers.js +++ b/templates/switchers.js @@ -41,9 +41,12 @@ const _create_placeholders_if_missing = () => { if (item.innerText.toLowerCase().includes("documentation")) { const container = document.createElement("li"); container.className = "switchers"; + container.style.display = "inline-flex"; for (const placeholder_name of ["language", "version"]) { const placeholder = document.createElement("div"); placeholder.className = `${placeholder_name}_switcher_placeholder`; + placeholder.style.marginRight = "5px"; + placeholder.style.paddingLeft = "5px"; container.appendChild(placeholder); } item.parentElement.insertBefore(container, item); From 84a749ebd9f7957baee6121f79aa86f930456218 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 18 Apr 2025 22:27:20 +0300 Subject: [PATCH 100/102] Add more tests for `Versions` class (#288) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- .coveragerc | 1 + .pre-commit-config.yaml | 1 + tests/test_build_docs_versions.py | 104 +++++++++++++++++++++++------- 3 files changed, 82 insertions(+), 24 deletions(-) diff --git a/.coveragerc b/.coveragerc index 0f12707..f970781 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,3 +5,4 @@ exclude_also = # Don't complain if non-runnable code isn't run: if __name__ == .__main__.: + if TYPE_CHECKING: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c0eb053..869a979 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,7 @@ repos: rev: v0.11.5 hooks: - id: ruff + args: [--fix] - id: ruff-format - repo: https://github.com/python-jsonschema/check-jsonschema diff --git a/tests/test_build_docs_versions.py b/tests/test_build_docs_versions.py index 42b5392..1d8f6dc 100644 --- a/tests/test_build_docs_versions.py +++ b/tests/test_build_docs_versions.py @@ -1,9 +1,13 @@ +from __future__ import annotations + +import pytest + from build_docs import Version, Versions -def test_filter_default() -> None: - # Arrange - versions = Versions([ +@pytest.fixture +def versions() -> Versions: + return Versions([ Version(name="3.14", status="in development", branch_or_tag=""), Version(name="3.13", status="stable", branch_or_tag=""), Version(name="3.12", status="stable", branch_or_tag=""), @@ -12,28 +16,90 @@ def test_filter_default() -> None: Version(name="3.9", status="security-fixes", branch_or_tag=""), ]) + +def test_reversed(versions: Versions) -> None: # Act - filtered = versions.filter() + output = list(reversed(versions)) # Assert - assert filtered == [ - Version(name="3.14", status="in development", branch_or_tag=""), + assert output[0].name == "3.9" + assert output[-1].name == "3.14" + + +def test_from_json() -> None: + # Arrange + json_data = { + "3.14": { + "branch": "main", + "pep": 745, + "status": "feature", + "first_release": "2025-10-01", + "end_of_life": "2030-10", + "release_manager": "Hugo van Kemenade", + }, + "3.13": { + "branch": "3.13", + "pep": 719, + "status": "bugfix", + "first_release": "2024-10-07", + "end_of_life": "2029-10", + "release_manager": "Thomas Wouters", + }, + } + + # Act + versions = list(Versions.from_json(json_data)) + + # Assert + assert versions == [ Version(name="3.13", status="stable", branch_or_tag=""), - Version(name="3.12", status="stable", branch_or_tag=""), + Version(name="3.14", status="in development", branch_or_tag=""), ] -def test_filter_one() -> None: +def test_from_json_error() -> None: # Arrange - versions = Versions([ + json_data = {"2.8": {"branch": "2.8", "pep": 404, "status": "ex-release"}} + + # Act / Assert + with pytest.raises( + ValueError, + match="Saw invalid version status 'ex-release', expected to be one of", + ): + Versions.from_json(json_data) + + +def test_current_stable(versions) -> None: + # Act + current_stable = versions.current_stable + + # Assert + assert current_stable.name == "3.13" + assert current_stable.status == "stable" + + +def test_current_dev(versions) -> None: + # Act + current_dev = versions.current_dev + + # Assert + assert current_dev.name == "3.14" + assert current_dev.status == "in development" + + +def test_filter_default(versions) -> None: + # Act + filtered = versions.filter() + + # Assert + assert filtered == [ Version(name="3.14", status="in development", branch_or_tag=""), Version(name="3.13", status="stable", branch_or_tag=""), Version(name="3.12", status="stable", branch_or_tag=""), - Version(name="3.11", status="security-fixes", branch_or_tag=""), - Version(name="3.10", status="security-fixes", branch_or_tag=""), - Version(name="3.9", status="security-fixes", branch_or_tag=""), - ]) + ] + +def test_filter_one(versions) -> None: # Act filtered = versions.filter(["3.13"]) @@ -41,17 +107,7 @@ def test_filter_one() -> None: assert filtered == [Version(name="3.13", status="security-fixes", branch_or_tag="")] -def test_filter_multiple() -> None: - # Arrange - versions = Versions([ - Version(name="3.14", status="in development", branch_or_tag=""), - Version(name="3.13", status="stable", branch_or_tag=""), - Version(name="3.12", status="stable", branch_or_tag=""), - Version(name="3.11", status="security-fixes", branch_or_tag=""), - Version(name="3.10", status="security-fixes", branch_or_tag=""), - Version(name="3.9", status="security-fixes", branch_or_tag=""), - ]) - +def test_filter_multiple(versions) -> None: # Act filtered = versions.filter(["3.13", "3.14"]) From 0e83b79b2d2488ec9226f28c93e215b5217e6730 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 22 May 2025 14:35:36 +0200 Subject: [PATCH 101/102] =?UTF-8?q?Hello=20=E0=A6=AC=E0=A6=BE=E0=A6=82?= =?UTF-8?q?=E0=A6=B2=E0=A6=BE=20(#301)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config.toml b/config.toml index 489c774..f25052a 100644 --- a/config.toml +++ b/config.toml @@ -36,6 +36,11 @@ sphinxopts = [ '-D latex_elements.fontenc=\\usepackage{fontspec}', ] +[languages.bn_IN] +name = "Bengali" +translated_name = "বাংলা" +in_prod = false + [languages.id] name = "Indonesian" translated_name = "Indonesia" From 15f94c0b7caec8b12f4bab3227967fd859ff6bad Mon Sep 17 00:00:00 2001 From: Octavian Mustafa Date: Mon, 16 Jun 2025 21:44:24 +0300 Subject: [PATCH 102/102] Start building Romanian translations (#302) --- config.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config.toml b/config.toml index f25052a..d628ca0 100644 --- a/config.toml +++ b/config.toml @@ -92,6 +92,11 @@ translated_name = "polski" name = "Brazilian Portuguese" translated_name = "Português brasileiro" +[languages.ro] +name = "Romanian" +translated_name = "Românește" +in_prod = false + [languages.tr] name = "Turkish" translated_name = "Türkçe"