diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000..6b26875f1 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1 @@ +1d7b1abd152e4cb5e6a46e52e6b7e3bf8d366486 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2d3e69fe8..0b7c7b289 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,9 @@ source/guides/github-actions-ci-cd-sample/* @webknjaz source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst @webknjaz + +# Sphinx extension +pug_sphinx_extensions/ @FFY00 + +# build-details.json +source/specifications/build-details/ @FFY00 +source/specifications/specs/build-details-*.json @FFY00 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..fbc581cd6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: "Community Guidelines" + url: "https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md" + about: "Please make sure to follow the PSF Code of Conduct when participating in this repository." diff --git a/.github/ISSUE_TEMPLATE/general.yml b/.github/ISSUE_TEMPLATE/general.yml new file mode 100644 index 000000000..d41731613 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general.yml @@ -0,0 +1,37 @@ +name: General Issue +description: Please fill out the form below to submit an issue. +labels: [] +assignees: [] + +body: + - type: markdown + attributes: + value: | + **Thanks for taking a minute to file an issue!** + + Read the [PSF Code of Conduct][CoC] first. + + ⚠ + Verify first that your issue is not [already reported on + GitHub][issue search]. + + _Please fill out the form below with as many precise + details as possible._ + + [CoC]: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md + [issue search]: ../search?q=is%3Aissue&type=issues + + - type: textarea + attributes: + label: Issue Description + description: Please provide a detailed description of your issue. + placeholder: Describe your issue here... + validations: + required: true + + - type: checkboxes + attributes: + label: Code of Conduct + options: + - label: I am aware that participants in this repository must follow the PSF Code of Conduct. + required: true diff --git a/.github/sphinx_lint_matcher.json b/.github/sphinx_lint_matcher.json new file mode 100644 index 000000000..41896d8f1 --- /dev/null +++ b/.github/sphinx_lint_matcher.json @@ -0,0 +1,15 @@ +{ + "problemMatcher": [ + { + "owner": "sphinx-lint-problem-matcher", + "pattern": [ + { + "regexp": "^(.*):(\\d+):\\s+(.*)$", + "file": 1, + "line": 2, + "message": 3 + } + ] + } + ] +} diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml new file mode 100644 index 000000000..8870bb70b --- /dev/null +++ b/.github/workflows/cron.yml @@ -0,0 +1,14 @@ +--- + +name: Cron + +on: + schedule: + - cron: "0 6 * * *" # daily at 6am + +jobs: + test: + if: github.repository_owner == 'pypa' # suppress noise in forks + uses: ./.github/workflows/test.yml + +... diff --git a/.github/workflows/pr-preview-links.yml b/.github/workflows/pr-preview-links.yml new file mode 100644 index 000000000..291ec3ad2 --- /dev/null +++ b/.github/workflows/pr-preview-links.yml @@ -0,0 +1,22 @@ +name: Read the Docs PR preview + +on: + pull_request_target: + types: + - opened + +permissions: + contents: read + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/actions/preview@b8bba1484329bda1a3abe986df7ebc80a8950333 # v1.5 + with: + project-slug: "python-packaging-user-guide" diff --git a/.github/workflows/test-translations.yml b/.github/workflows/test-translations.yml new file mode 100644 index 000000000..537a8df72 --- /dev/null +++ b/.github/workflows/test-translations.yml @@ -0,0 +1,87 @@ +name: Test translations + +on: + workflow_dispatch: + pull_request: + paths: + - '**.po' + branches: + - translation/source + push: + paths: + - '**.po' + branches: + - translation/source + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +env: + I18N_BRANCH: translation/source + +jobs: + matrix: + runs-on: ubuntu-latest + outputs: + languages: ${{ steps.languages.outputs.languages }} + + steps: + - name: Grab the repo src + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ env.I18N_BRANCH }} + persist-credentials: false + + - name: List languages + id: languages + working-directory: locales + run: | + list=$(find * -maxdepth 0 -type d | jq -nRc '[inputs]') + echo "languages=$list" >> $GITHUB_OUTPUT + + + test-translation: + runs-on: ubuntu-latest + needs: matrix + strategy: + fail-fast: false + matrix: + language: ${{fromJson(needs.matrix.outputs.languages)}} + + steps: + - name: Grab the repo src + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ env.I18N_BRANCH }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: >- + 3.10 + + - name: Install Python tooling + run: python -m pip install --upgrade nox virtualenv sphinx-lint + + - name: Set Sphinx problem matcher + uses: sphinx-doc/github-problem-matcher@1f74d6599f4a5e89a20d3c99aab4e6a70f7bda0f # v1.1 + + - name: Build translated docs in ${{ matrix.language }} + run: nox -s build -- -q -D language=${LANGUAGE} + env: + LANGUAGE: ${{ matrix.language }} + + - name: Set Sphinx Lint problem matcher + if: always() + run: echo '::add-matcher::.github/sphinx_lint_matcher.json' + + - name: Lint translation file + if: always() + run: sphinx-lint locales/${LANGUAGE}/LC_MESSAGES/messages.po + env: + LANGUAGE: ${{ matrix.language }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 09b428e16..1f67bad8e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,14 +1,28 @@ name: Test -on: [push, pull_request] +on: + merge_group: + push: + branches-ignore: + - gh-readonly-queue/** # Temporary merge queue-related GH-made branches + pull_request: + types: + - opened # default + - synchronize # default + - reopened # default + - ready_for_review # used in PRs created from GitHub Actions workflows + workflow_call: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true +permissions: {} + jobs: build: name: ${{ matrix.noxenv }} + if: ${{ github.repository_owner == 'pypa' || github.event_name != 'schedule' }} runs-on: ubuntu-latest strategy: matrix: @@ -17,21 +31,16 @@ jobs: - linkcheck steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: - python-version: 3.9 + persist-credentials: false - - name: pip cache - uses: actions/cache@v2 + - name: Set up Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - path: ~/.cache/pip - key: - ${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('noxfile.py') }} - restore-keys: | - ${{ matrix.os }}-${{ matrix.python-version }}- + python-version: "3.11" + cache: 'pip' + cache-dependency-path: 'requirements.txt' - name: Install dependencies run: | @@ -40,3 +49,21 @@ jobs: - name: Nox ${{ matrix.noxenv }} run: | python -m nox -s ${{ matrix.noxenv }} + + + check: + # This job does nothing and is only used for the branch protection + # or multi-stage CI jobs, like making sure that all tests pass before + # a publishing job is started. + if: always() + + needs: + - build + + runs-on: ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/translation.yml b/.github/workflows/translation.yml index 7b1bf40d4..67fcb5edf 100644 --- a/.github/workflows/translation.yml +++ b/.github/workflows/translation.yml @@ -15,17 +15,22 @@ env: jobs: build: runs-on: ubuntu-latest + if: github.repository_owner == 'pypa' + + permissions: + contents: write # to push to I18N_BRANCH steps: - name: Grab the repo src - uses: actions/checkout@v2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 # To reach the common commit + persist-credentials: true # For `git push` - name: Set up git user as [bot] # Refs: # * https://github.community/t/github-actions-bot-email-address/17204/6 # * https://github.com/actions/checkout/issues/13#issuecomment-724415212 - uses: fregante/setup-git-user@v1.0.1 + uses: fregante/setup-git-user@024bc0b8e177d7e77203b48dab6fb45666854b35 # v2.0.2 - name: Switch to the translation source branch run: | @@ -47,10 +52,12 @@ jobs: run: | sh -x - git merge '${{ github.event.repository.default_branch }}' + git merge "${DEFAULT_BRANCH}" + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: >- 3.10 @@ -65,6 +72,7 @@ jobs: run: | git_hash=$(git rev-parse --short "${GITHUB_SHA}") git add --force locales/messages.pot + git diff-index --quiet HEAD || \ git commit \ -m "Update messages.pot as of version ${git_hash}" \ locales/messages.pot diff --git a/.github/workflows/update-uv-build-version.yml b/.github/workflows/update-uv-build-version.yml new file mode 100644 index 000000000..d204bd391 --- /dev/null +++ b/.github/workflows/update-uv-build-version.yml @@ -0,0 +1,43 @@ +--- + +name: Update uv build version + +on: + schedule: + - cron: "0 6 * * 1" # mondays at 6am + workflow_dispatch: + +jobs: + update-uv-build-version: + name: Update uv_build version + if: github.repository_owner == 'pypa' # suppress noise in forks + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + - name: Set up uv + uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0 + - name: Update uv_build version + id: update_script + run: uv run scripts/update_uv_build_version.py + - # If there are no changes, no pull request will be created and the action exits silently. + name: Create Pull Request + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: Update uv_build version to ${{ steps.update_script.outputs.version }} + title: Update uv_build version to ${{ steps.update_script.outputs.version }} + draft: true # Trigger CI by un-drafting the PR, otherwise `GITHUB_TOKEN` PRs don't trigger CI. + body: | + Automated update of uv_build version bounds for uv ${{ steps.update_script.outputs.version }}. + + This PR was created automatically by the cron workflow, ping `@konstin` for problems. + branch: bot/update-uv-build-version + delete-branch: true + +... diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 000000000..6c8c62f7d --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,38 @@ +# From https://woodruffw.github.io/zizmor/usage/#use-in-github-actions + +name: GitHub Actions Security Analysis with zizmor 🌈 + +on: + push: + branches: ["main"] + pull_request: + branches: ["**"] + +jobs: + zizmor: + name: zizmor latest via PyPI + runs-on: ubuntu-latest + permissions: + security-events: write + # required for workflows in private repositories + contents: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - name: Install the latest version of uv + uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0 + + - name: Run zizmor 🌈 + run: uvx zizmor --format sarif source/guides/github-actions-ci-cd-sample/* > results.sarif + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload SARIF file + uses: github/codeql-action/upload-sarif@f443b600d91635bebf5b0d9ebc620189c0d6fba5 # v4.30.8 + with: + sarif_file: results.sarif + category: zizmor diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..47b864808 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +ci: + autoupdate_schedule: quarterly + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + - id: end-of-file-fixer + - id: mixed-line-ending + - id: trailing-whitespace + +- repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + args: ["-L", "ned,ist,oder", "--skip", "*.po"] + +- repo: local + hooks: + - id: disallow-caps + name: Disallow improper capitalization + language: pygrep + entry: PyBind|Numpy|Cmake|CCache|Github|PyTest + exclude: .pre-commit-config.yaml + exclude_types: ["pofile"] + +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.14.10 + hooks: + - id: ruff + - id: ruff-format diff --git a/.readthedocs.yaml b/.readthedocs.yaml index fe012dd98..90e9a9bbc 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,11 @@ sphinx: # https://github.com/pyca/cryptography/issues/5863#issuecomment-792343136 builder: dirhtml +build: + os: ubuntu-22.04 + tools: + python: "3.11" + python: - version: 3.8 install: - requirements: requirements.txt diff --git a/extra/specifications/schemas/build-details-v1.0.schema.json b/extra/specifications/schemas/build-details-v1.0.schema.json new file mode 100644 index 000000000..9954ddab7 --- /dev/null +++ b/extra/specifications/schemas/build-details-v1.0.schema.json @@ -0,0 +1,261 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://packaging.python.org/en/latest/specifications/schemas/build-details-v1.0.schema.json", + "type": "object", + "title": "build-details.json β€” a static description file with build details of Python installations", + "required": [ + "schema_version", + "base_prefix", + "platform", + "language", + "implementation" + ], + "additionalProperties": false, + "properties": { + "schema_version": { + "type": "string", + "description": "Schema version.\n\nThis is a string following the format ``.``, where ```` and ```` are unpaded numbers and represent the **major** and **minor** components of the version. Versions may be arithmetically compared by intrepreting the version string as a decimal number.\n\nFor this specification version, this value is constant and **MUST** be ``1.0``.\n\nFuture versions of this schema **MUST** use a higher version number. Future versions of this schema **MUST NOT** use the same **major** version component as other schema version unless its specification is deemed backwards-compatible with them β€” it can't change, or extend, any parts of the current specification in such a way as the semantics of the interpreted data differ, or that data valid under the new specification is invalid under the older specification, with the exception of additional properties (errors caused by ``additionalProperties``).", + "const": "1.0" + }, + "base_prefix": { + "type": "string", + "description": "Base prefix of the Python installation.\n\nEither an absolute path, or a path relative to directory where this file is contained.", + "examples": [ + "/usr", + "../.." + ] + }, + "base_interpreter": { + "type": "string", + "description": "The path to the Python interprer of the base installation.\n\nEither an absolute path, or a path relative to ``base_prefix``.\n\nThis field **MUST** be present if the installation provides an interpreter executable.", + "examples": [ + "/usr/bin/python", + "bin/python" + ] + }, + "platform": { + "type": "string", + "description": "System platform string.\n\nThis field **SHOULD** be equivalent to ``sysconfig.get_platform()``.", + "examples": [ + "linux-x86_64" + ] + }, + "language": { + "type": "object", + "description": "Object containing details related to the Python language specification.", + "required": [ + "version" + ], + "additionalProperties": false, + "properties": { + "version": { + "type": "string", + "description": "String representation the Python language version β€” a version string consisting only of the *major* and *minor* components.\n\nThis field **SHOULD** be equivalent to ``sysconfig.get_python_version()``.", + "examples": ["3.14"] + }, + "version_info": { + "type": "object", + "description": "Object in the format of :py:data:`sys.version_info`.\n\nThis section **SHOULD** be equivalent to :py:data:`sys.version_info`.", + "required": ["major", "minor", "micro", "releaselevel", "serial"], + "additionalProperties": false, + "examples": [ + { + "major": 3, + "minor": 14, + "micro": 1, + "releaselevel": "final", + "serial": 0 + } + ], + "properties": { + "major": { + "type": "number" + }, + "minor": { + "type": "number" + }, + "micro": { + "type": "number" + }, + "releaselevel": { + "type": "string", + "enum": ["alpha", "beta", "candidate", "final"] + }, + "serial": { + "type": "number" + } + } + } + } + }, + "implementation": { + "type": "object", + "description": "Object containing details related to Python implementation.\n\nThis section **SHOULD** be equivalent to :py:data:`sys.implementation`. It follows specification defined in PEP 421, meaning that on top of the required keys, implementation-specific keys can also exist, but must be prefixed with an underscore.", + "required": [ + "name", + "version", + "hexversion", + "cache_tag" + ], + "additionalProperties": true, + "properties": { + "name": { + "type": "string", + "description": "Lower-case name of the Python implementation.", + "examples": ["cpython", "pypy"] + }, + "version": { + "type": "object", + "description": "Object in the format of :py:data:`sys.version_info`, containing the implementation version.", + "required": ["major", "minor", "micro", "releaselevel", "serial"], + "additionalProperties": false, + "examples": [ + { + "major": 3, + "minor": 14, + "micro": 1, + "releaselevel": "final", + "serial": 0 + }, + { + "major": 7, + "minor": 3, + "micro": 16, + "releaselevel": "final", + "serial": 0 + } + ], + "properties": { + "major": { + "type": "number" + }, + "minor": { + "type": "number" + }, + "micro": { + "type": "number" + }, + "releaselevel": { + "type": "string", + "enum": ["alpha", "beta", "candidate", "final"] + }, + "serial": { + "type": "number" + } + } + } + } + }, + "abi": { + "type": "object", + "description": "Object containing details related to ABI.", + "required": [ + "flags" + ], + "additionalProperties": false, + "properties": { + "flags": { + "type": "array", + "description": "Build configuration flags, used to calculate the extension suffix.\n\nThe flags **MUST** be defined in the order they appear on the extension suffix.", + "additionalProperties": true, + "examples": [ + ["t", "d"] + ] + }, + "extension_suffix": { + "type": "string", + "description": "Suffix used for extensions built against the current implementation version.\n\nThis field **MUST** be present if the Python implementation supports extensions, otherwise this entry will be missing.", + "examples": [ + ".cpython-314-x86_64-linux-gnu.so" + ] + }, + "stable_abi_suffix": { + "type": "string", + "description": "Suffix used for extensions built against the stable ABI.\n\nThis field **MUST** be present if the Python implementation has a stable ABI extension suffix, otherwise this entry will be missing.", + "examples": [ + ".abi3.so" + ] + } + } + }, + "suffixes": { + "type": "object", + "description": "Valid module suffixes grouped by type.\n\nThis section **MUST** be present if the Python installation supports importing external files, and it **SHOULD** be equivalent to the ``importlib.machinery.*_SUFFIXES`` attributes.\n\nAdditionally, if a Python implementation provides extension kinds other than the ones listed on ``importlib.machinery`` module, they **MAY** add a sub-section for them.", + "examples": [ + { + "source": [".py"], + "bytecode": [".pyc"], + "optimized_bytecode": [".pyc"], + "debug_bytecode": [".pyc"], + "extensions": [".cpython-313-x86_64-linux-gnu.so", ".abi3.so", ".so"] + } + ] + }, + "libpython": { + "type": "object", + "description": "Object containing details related to the ``libpython`` library.\n\nThis section **MUST** by present if Python installation provides a ``libpython`` library, otherwise this section will be missing.", + "additionalProperties": false, + "properties": { + "dynamic": { + "type": "string", + "description": "The path to the dynamic ``libpython`` library.\n\nEither an absolute path, or a path relative to ``base_prefix``.\n\nThis field **MUST** be present if the Python installation provides a dynamic ``libpython`` library, otherwise this entry will be missing.", + "examples": [ + "/usr/lib/libpython3.14.so.1.0", + "lib/libpython3.14.so.1.0" + ] + }, + "dynamic_stableabi": { + "type": "string", + "description": "The path to the dynamic ``libpython`` library for the stable ABI.\n\nEither an absolute path, or a path relative to ``base_prefix``.\n\nThis field **MUST** be present if the Python installation provides a dynamic ``libpython`` library targeting the Stable ABI, otherwise this entry will be missing.\n\nIf this key is present ``dynamic`` **MUST** also be set.", + "examples": [ + "/usr/lib/libpython3.so", + "lib/libpython3.so" + ] + }, + "static": { + "type": "string", + "description": "The path to the static ``libpython`` library.\n\nEither an absolute path, or a path relative to ``base_prefix``.\n\nThis field **MUST** be present if the Python installation provides a static ``libpython`` library, otherwise this entry will be missing.", + "examples": [ + "/usr/lib/python3.14/config-3.14-x86_64-linux-gnu/libpython3.14.a", + "lib/python3.14/config-3.14-x86_64-linux-gnu/libpython3.14.a" + ] + }, + "link_extensions": { + "type": "boolean", + "description": "Should extensions built against a dynamic ``libpython`` link to it?\n\nThis field **MUST** be present if the Python installation provides a dynamic ``libpython`` library, otherwise this entry will be missing." + } + } + }, + "c_api": { + "type": "object", + "description": "Object containing details related to the Python C API.\n\nThis section **MUST** be present if the Python implementation provides a C API, otherwise this section will be missing.", + "required": [ + "headers" + ], + "additionalProperties": false, + "properties": { + "headers": { + "type": "string", + "description": "The path to the C API headers.\n\nEither an absolute path, or a path relative to ``base_prefix``.", + "examples": [ + "/usr/include/python3.14", + "include/python3.14" + ] + }, + "pkgconfig_path": { + "type": "string", + "description": "The path to the pkg-config definition files.\n\nEither an absolute path, or a path relative to ``base_prefix``.\n\nThis field **MUST** be present if the Python implementation provides pkg-config definition files, otherwise this section will be missing.", + "examples": [ + "/usr/lib/pkgconfig", + "lib/pkgconfig" + ] + } + } + }, + "arbitrary_data": { + "type": "object", + "description": "Object containing extra arbitrary data.\n\nThis is meant to be used as an escape-hatch, to include any relevant data that is not covered by this specification. Implementations may choose what data to provide in this section.", + "additionalProperties": true + } + } +} diff --git a/extra/specifications/schemas/direct-url.schema.json b/extra/specifications/schemas/direct-url.schema.json new file mode 100644 index 000000000..d1f4c860a --- /dev/null +++ b/extra/specifications/schemas/direct-url.schema.json @@ -0,0 +1,99 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://packaging.python.org/en/latest/specifications/schemas/direct-url.schema.json", + "title": "Direct URL Data", + "description": "Data structure that can represent URLs to python projects and distribution artifacts such as VCS source trees, local source trees, source distributions and wheels.", + "definitions": { + "url": { + "type": "string", + "format": "uri" + }, + "DirInfo": { + "type": "object", + "properties": { + "editable": { + "type": ["boolean", "null"] + } + } + }, + "VCSInfo": { + "type": "object", + "properties": { + "vcs": { + "type": "string", + "enum": ["git", "hg", "bzr", "svn"] + }, + "requested_revision": { + "type": "string" + }, + "commit_id": { + "type": "string" + }, + "resolved_revision": { + "type": "string" + } + }, + "required": ["vcs", "commit_id"] + }, + "ArchiveInfo": { + "type": "object", + "properties": { + "hash": { + "type": "string", + "pattern": "^\\w+=[a-f0-9]+$", + "deprecated": true + }, + "hashes": { + "type": "object", + "patternProperties": { + "^[a-f0-9]+$": { + "type": "string" + } + } + } + } + } + }, + "allOf": [ + { + "type": "object", + "properties": { + "url": { + "$ref": "#/definitions/url" + } + }, + "required": ["url"] + }, + { + "anyOf": [ + { + "type": "object", + "properties": { + "dir_info": { + "$ref": "#/definitions/DirInfo" + } + }, + "required": ["dir_info"] + }, + { + "type": "object", + "properties": { + "vcs_info": { + "$ref": "#/definitions/VCSInfo" + } + }, + "required": ["vcs_info"] + }, + { + "type": "object", + "properties": { + "archive_info": { + "$ref": "#/definitions/ArchiveInfo" + } + }, + "required": ["archive_info"] + } + ] + } + ] +} diff --git a/extra/specifications/schemas/pylock.schema.json b/extra/specifications/schemas/pylock.schema.json new file mode 100644 index 000000000..90404e33d --- /dev/null +++ b/extra/specifications/schemas/pylock.schema.json @@ -0,0 +1,345 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://packaging.python.org/en/latest/specifications/schemas/pylock.schema.json", + "additionalProperties": false, + "definitions": { + "tool": { + "type": "object", + "markdownDescription": "Similar usage as that of the `[tool]` table from the [pyproject.toml specification](https://packaging.python.org/en/latest/specifications/pyproject-toml/#pyproject-toml-spec), but at the package version level instead of at the lock file level (which is also available via `[tool]`).", + "additionalProperties": { + "type": "object", + "additionalProperties": true + } + }, + "url": { + "type": "string", + "markdownDescription": "The URL to the source tree." + }, + "path": { + "type": "string", + "markdownDescription": "The path to the local directory of the source tree." + }, + "upload-time": { + "markdownDescription": "The time the file was uploaded (UTC). Must be specified as a datetime literal." + }, + "size": { + "type": "integer", + "markdownDescription": "The size of the archive file." + }, + "hashes": { + "type": "object", + "description": "Known hash values of the file where the key is the hash algorithm and the value is the hash value.", + "additionalProperties": { + "type": "string" + } + }, + "subdirectory": { + "type": "string", + "markdownDescription": "The subdirectory within the [source tree](https://packaging.python.org/en/latest/specifications/source-distribution-format/#source-distribution-format-source-tree) where the project root of the project is (e.g. the location of the `pyproject.toml` file)." + }, + "vcs": { + "type": "object", + "markdownDescription": "Record the version control system details for the [source tree](https://packaging.python.org/en/latest/specifications/source-distribution-format/#source-distribution-format-source-tree) it contains.", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "markdownDescription": "The type of version control system used." + }, + "url": { + "$ref": "#/definitions/url" + }, + "path": { + "$ref": "#/definitions/path" + }, + "requested-revision": { + "type": "string", + "markdownDescription": "The branch/tag/ref/commit/revision/etc. that the user requested." + }, + "commit-id": { + "type": "string", + "markdownDescription": "The exact commit/revision number that is to be installed." + }, + "subdirectory": { + "$ref": "#/definitions/subdirectory" + } + } + }, + "directory": { + "type": "object", + "markdownDescription": "Record the local directory details for the [source tree](https://packaging.python.org/en/latest/specifications/source-distribution-format/#source-distribution-format-source-tree) it contains.", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "markdownDescription": "The local directory where the source tree is." + }, + "editable": { + "type": "boolean", + "default": false, + "markdownDescription": "A flag representing whether the source tree was an editable install at lock time." + }, + "subdirectory": { + "$ref": "#/definitions/subdirectory" + } + } + }, + "archive": { + "type": "object", + "additionalProperties": false, + "markdownDescription": "A direct reference to an archive file to install from (this can include wheels and sdists, as well as other archive formats containing a source tree).", + "properties": { + "url": { + "$ref": "#/definitions/url" + }, + "path": { + "$ref": "#/definitions/path" + }, + "size": { + "$ref": "#/definitions/size" + }, + "upload-time": { + "$ref": "#/definitions/upload-time" + }, + "hashes": { + "$ref": "#/definitions/hashes" + }, + "subdirectory": { + "$ref": "#/definitions/subdirectory" + } + } + }, + "sdist": { + "type": "object", + "additionalProperties": false, + "markdownDescription": "Details of a [source distribution file name](https://packaging.python.org/en/latest/specifications/source-distribution-format/#source-distribution-format-sdist) for the package.", + "properties": { + "name": { + "type": "string", + "markdownDescription": "The file name of the [source distribution file name](https://packaging.python.org/en/latest/specifications/source-distribution-format/#source-distribution-format-sdist) file." + }, + "upload-time": { + "$ref": "#/definitions/upload-time" + }, + "url": { + "$ref": "#/definitions/url" + }, + "path": { + "$ref": "#/definitions/path" + }, + "size": { + "$ref": "#/definitions/size" + }, + "hashes": { + "$ref": "#/definitions/hashes" + } + } + }, + "wheels": { + "type": "array", + "markdownDescription": "For recording the wheel files as specified by [Binary distribution format](https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format) for the package.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "markdownDescription": "The file name of the [Binary distribution format](https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format) file." + }, + "upload-time": { + "$ref": "#/definitions/upload-time" + }, + "url": { + "$ref": "#/definitions/url" + }, + "path": { + "$ref": "#/definitions/path" + }, + "size": { + "$ref": "#/definitions/size" + }, + "hashes": { + "$ref": "#/definitions/hashes" + } + } + } + }, + "1.0": { + "required": ["lock-version", "created-by", "packages"], + "properties": { + "lock-version": { + "type": "string", + "enum": ["1.0"], + "description": "Record the file format version that the file adheres to." + }, + "environments": { + "type": "array", + "markdownDescription": "A list of [environment markers](https://packaging.python.org/en/latest/specifications/dependency-specifiers/#dependency-specifiers-environment-markers) for which the lock file is considered compatible with.", + "items": { + "type": "string", + "description": "Environment marker" + } + }, + "requires-python": { + "type": "string", + "markdownDescription": "Specifies the [Requires-Python](https://packaging.python.org/en/latest/specifications/core-metadata/#core-metadata-requires-python) for the minimum Python version compatible for any environment supported by the lock file (i.e. the minimum viable Python version for the lock file)." + }, + "extras": { + "type": "array", + "markdownDescription": "The list of [extras](https://packaging.python.org/en/latest/specifications/core-metadata/#core-metadata-provides-extra) supported by this lock file.", + "default": [], + "items": { + "type": "string", + "description": "Extra name" + } + }, + "dependency-groups": { + "type": "array", + "markdownDescription": "The list of [dependency groups](https://packaging.python.org/en/latest/specifications/dependency-groups/#dependency-groups) publicly supported by this lock file (i.e. dependency groups users are expected to be able to specify via a tool’s UI).", + "default": [], + "items": { + "type": "string", + "description": "Dependency group name" + } + }, + "default-groups": { + "type": "array", + "markdownDescription": "The name of synthetic dependency groups to represent what should be installed by default (e.g. what `project.dependencies` implicitly represents).", + "default": [], + "items": { + "type": "string", + "description": "Dependency group name" + } + }, + "created-by": { + "type": "string", + "markdownDescription": "Records the name of the tool used to create the lock file." + }, + "packages": { + "type": "array", + "markdownDescription": "An array containing all packages that may be installed.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "allOf": [ + { + "if": { + "required": ["vcs"] + }, + "then": { + "not": { + "required": ["directory", "archive", "sdist", "wheels"] + } + } + }, + { + "if": { + "required": ["directory"] + }, + "then": { + "not": { + "required": ["vcs", "archive", "sdist", "wheels"] + } + } + }, + { + "if": { + "required": ["sdist"] + }, + "then": { + "not": { + "required": ["vcs", "directory", "archive"] + } + } + }, + { + "if": { + "required": ["wheels"] + }, + "then": { + "not": { + "required": ["vcs", "directory", "archive"] + } + } + } + ], + "properties": { + "name": { + "type": "string", + "markdownDescription": "The name of the package, [normalized](https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization)." + }, + "version": { + "type": "string", + "description": "The version of the package." + }, + "marker": { + "type": "string", + "markdownDescription": "The [environment marker](https://packaging.python.org/en/latest/specifications/dependency-specifiers/#dependency-specifiers-environment-markers) which specify when the package should be installed." + }, + "requires-python": { + "type": "string", + "markdownDescription": "Holds the [version specifiers](https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers) for Python version compatibility for the package." + }, + "dependencies": { + "type": "array", + "markdownDescription": "Records the other entries in `[[packages]]` which are direct dependencies of this package.", + "items": { + "type": "object", + "markdownDescription": "A table which contains the minimum information required to tell which other package entry it corresponds to where doing a key-by-key comparison would find the appropriate package with no ambiguity (e.g. if there are two entries for the `spam` package, then you can include the version number like `{name = \"spam\", version = \"1.0.0\"}`, or by source like `{name = \"spam\", vcs = { url = \"...\"}`).", + "additionalProperties": true + } + }, + "vcs": { + "$ref": "#/definitions/vcs" + }, + "directory": { + "$ref": "#/definitions/directory" + }, + "archive": { + "$ref": "#/definitions/archive" + }, + "index": { + "type": "string", + "markdownDescription": "The base URL for the package index from [simple repository API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#simple-repository-api) where the sdist and/or wheels were found (e.g. `https://pypi.org/simple/`)." + }, + "sdist": { + "$ref": "#/definitions/sdist" + }, + "wheels": { + "$ref": "#/definitions/wheels" + }, + "attestation-identities": { + "type": "array", + "markdownDescription": "A recording of the attestations for any file recorded for this package.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { + "type": "string", + "markdownDescription": "The unique identity of the Trusted Publisher." + } + } + } + }, + "tool": { + "$ref": "#/definitions/tool" + } + } + } + }, + "tool": { + "$ref": "#/definitions/tool" + } + } + } + }, + "oneOf": [ + { + "$ref": "#/definitions/1.0" + } + ], + "type": "object" +} diff --git a/noxfile.py b/noxfile.py index fe7872995..484a8d39a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,9 +3,8 @@ # Attribution-ShareAlike license: # http://creativecommons.org/licenses/by-sa/3.0. -import shutil -import nox +import nox nox.options.sessions = [] @@ -18,13 +17,16 @@ def translation(session): session.install("-r", "requirements.txt") target_dir = "locales" session.run( - "sphinx-build", - "-b", "gettext", # build gettext-style message catalogs (.pot file) - "-d", ".nox/.doctrees/", # path to put the cache + "sphinx-build", + "-b", + "gettext", # build gettext-style message catalogs (.pot file) + "-d", + session.cache_dir / ".doctrees", # path to put the cache "source/", # where the rst files are located - target_dir, # where to put the .pot file + target_dir, # where to put the .pot file ) + @nox.session() def build(session, autobuild=False): """ @@ -32,13 +34,9 @@ def build(session, autobuild=False): """ session.install("-r", "requirements.txt") - target_build_dir = "build" - - shutil.rmtree(target_build_dir, ignore_errors=True) - if autobuild: command = "sphinx-autobuild" - extra_args = "-H", "0.0.0.0" + extra_args = "--host", "0.0.0.0" else: # NOTE: This branch adds options that are unsupported by autobuild command = "sphinx-build" @@ -48,14 +46,19 @@ def build(session, autobuild=False): ) session.run( - command, *extra_args, - "-j", "auto", # parallelize the build - "-b", "html", # use HTML builder + command, + *extra_args, + "-j", + "auto", # parallelize the build + "-b", + "html", # use HTML builder + "-d", + session.cache_dir / ".doctrees", # path to put the cache "-n", # nitpicky warn about all missing references "-W", # Treat warnings as errors. *session.posargs, "source", # where the rst files are located - target_build_dir, # where to put the html output + "build", # where to put the html output ) @@ -75,10 +78,30 @@ def linkcheck(session): """ session.install("-r", "requirements.txt") session.run( - "sphinx-build", - "-b", "linkcheck", # use linkcheck builder + "sphinx-build", + "-b", + "linkcheck", # use linkcheck builder + "-d", + session.cache_dir / ".doctrees", # path to put the cache "--color", - "-n", "-W", "--keep-going", # be strict - "source", # where the rst files are located - "build", # where to put the check output + "-n", + "-W", + "--keep-going", # be strict + "source", # where the rst files are located + "build", # where to put the check output + *session.posargs, + ) + + +@nox.session() +def checkqa(session): + """ + Format the guide using pre-commit. + """ + session.install("pre-commit") + session.run( + "pre-commit", + "run", + "--all-files", + "--show-diff-on-failure", ) diff --git a/pug_sphinx_extensions/__init__.py b/pug_sphinx_extensions/__init__.py new file mode 100644 index 000000000..00d91da3c --- /dev/null +++ b/pug_sphinx_extensions/__init__.py @@ -0,0 +1,86 @@ +import os +import pathlib +import urllib + +import sphinx.application +import sphinx.util.logging + + +DOMAIN = "packaging.python.org" + + +logger = sphinx.util.logging.getLogger(__name__) + + +def resolve_local_html_link(app: sphinx.application.Sphinx, url_path: str) -> str: + """Takes path of a link pointing an HTML render of the current project, + and returns local path of the referenced document. + + Support links to renders from both the `html` and `dirhtml` builders. + + Example: + + .. code-block:: python + + >>> resolve_local_html_link('https://packaging.python.org/en/latest/flow/') + '{srcdir}/flow.rst' + >>> resolve_local_html_link('https://packaging.python.org/en/latest/flow.html') + '{srcdir}/flow.rst' + >>> resolve_local_html_link('https://packaging.python.org/en/latest/specifications/schemas/') + '{srcdir}/specifications/schemas/index.rst' + >>> resolve_local_html_link('https://packaging.python.org/en/latest/specifications/schemas/build-details-v1.0.schema.json') + '{html_extra_path0}/specifications/schemas/build-details-v1.0.schema.json' + + """ + # Search for document in html_extra_path + for entry in app.config.html_extra_path: + candidate = (app.confdir / entry / url_path).resolve() + if candidate.is_dir(): + candidate = candidate / "index.html" + if candidate.exists(): + return os.fspath(candidate) + # Convert html path to source path + url_path = url_path.removesuffix("/") # Normalize + if url_path.endswith(".html"): + document = url_path.removesuffix(".html") + elif (candidate := f"{url_path}/index") in app.project.docnames: + document = candidate + else: + document = url_path + return app.env.doc2path(document) + + +def rewrite_local_uri(app: sphinx.application.Sphinx, uri: str) -> str: + """Replace remote URIs targeting https://packaging.python.org/en/latest/... + with local ones, so that local changes are taken into account by linkcheck. + + Additionally, resolve local relative links to html_extra_path. + """ + local_uri = uri + parsed = urllib.parse.urlparse(uri) + # Links to https://packaging.python.org/en/latest/... + if parsed.hostname == DOMAIN and parsed.path.startswith("/en/latest/"): + document = parsed.path.removeprefix("/en/latest/") + local_uri = resolve_local_html_link(app, document) + logger.verbose( + f"{uri!s} is a remote URL that points to local sources, " + "replacing it with a local URL in linkcheck to take new changes " + "into account (pass -vv for more info)" + ) + logger.debug(f"Replacing linkcheck URL {uri!r} with {local_uri!r}") + # Local relative links + if not parsed.scheme and not parsed.netloc and parsed.path: + full_path = pathlib.Path(app.env.docname).parent / parsed.path + local_uri = resolve_local_html_link(app, os.fspath(full_path)) + if local_uri != uri: + logger.verbose(f"Local linkcheck URL {uri!r} resolved as {local_uri!r}") + return local_uri + + +def setup(app: sphinx.application.Sphinx) -> dict[str, bool]: + app.connect("linkcheck-process-uri", rewrite_local_uri) + + return { + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/requirements.txt b/requirements.txt index 57fd523b0..5c710c86b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ -sphinx==4.5.0 -sphinx-autobuild==0.7.1 -sphinx-inline-tabs==2021.4.11b9 -python-docs-theme==2022.1 -sphinx-copybutton==0.5.0 -git+https://github.com/pypa/pypa-docs-theme.git#egg=pypa-docs-theme +furo==2024.8.6 +sphinx==7.2.6 +sphinx-autobuild==2021.3.14 +sphinx-inline-tabs==2023.4.21 +sphinx-copybutton==0.5.2 +sphinx-toolbox==3.5.0 +sphinx-jsonschema==1.19.1 diff --git a/scripts/update_uv_build_version.py b/scripts/update_uv_build_version.py new file mode 100644 index 000000000..69fefba27 --- /dev/null +++ b/scripts/update_uv_build_version.py @@ -0,0 +1,64 @@ +# /// script +# requires-python = ">= 3.12" +# dependencies = [ +# "httpx>=0.28.1,<0.29", +# "packaging>=25.0", +# ] +# /// +import os +import re +from pathlib import Path + +import httpx +from packaging.utils import parse_wheel_filename +from packaging.version import Version + + +def main(): + response = httpx.get( + "https://pypi.org/simple/uv-build/", + headers={"Accept": "application/vnd.pypi.simple.v1+json"}, + ) + response.raise_for_status() + data = response.json() + current_release = None + for file in data["files"]: + if not file["filename"].endswith(".whl"): + continue + _name, version, _build, _tags = parse_wheel_filename(file["filename"]) + if version.is_prerelease: + continue + if current_release is None or version > current_release: + current_release = version + + [major, minor, _patch] = current_release.release + if major != 0: + raise NotImplementedError("The script needs to be updated for uv 1.x") + upper_bound = Version(f"{major}.{minor + 1}.0") + + repository_root = Path(__file__).parent.parent + existing = repository_root.joinpath( + "source/shared/build-backend-tabs.rst" + ).read_text() + replacement = f'requires = ["uv_build >= {current_release}, <{upper_bound}"]' + searcher = re.compile(re.escape('requires = ["uv_build') + ".*" + re.escape('"]')) + if not searcher.search(existing): + raise RuntimeError("Could not `uv-build` entry") + updated = searcher.sub(replacement, existing) + + if existing != updated: + print("Updating source/shared/build-backend-tabs.rst") + Path("source/shared/build-backend-tabs.rst").write_text(updated) + if github_output := os.environ.get("GITHUB_OUTPUT"): + with open(github_output, "a") as f: + f.write(f"version={current_release}\n") + f.write("updated=true\n") + else: + print("Already up-to-date source/shared/build-backend-tabs.rst") + if github_output := os.environ.get("GITHUB_OUTPUT"): + with open(github_output, "a") as f: + f.write("updated=false\n") + + +if __name__ == "__main__": + main() diff --git a/source/assets/py.png b/source/assets/py.png new file mode 100644 index 000000000..93e4a02c3 Binary files /dev/null and b/source/assets/py.png differ diff --git a/source/conf.py b/source/conf.py index 2dc8b6419..ccb828b6e 100644 --- a/source/conf.py +++ b/source/conf.py @@ -1,447 +1,233 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# The Python Packaging Guide documentation build configuration file, created by -# sphinx-quickstart on Sun Dec 13 14:07:23 2009. -# -# This file is execfile()d with the current directory set to its containing -# dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.extlinks', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx_inline_tabs', - 'sphinx_copybutton', -] +# -- Project information --------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -# config for copy button -copybutton_prompt_text = r">>> |\.\.\. |\$ |> " -copybutton_prompt_is_regexp = True +import os +import pathlib +import sys -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The encoding of source files. -# -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# -- Project information ----------------------------------------------------- - -github_url = 'https://github.com' -github_repo_org = 'pypa' -github_repo_name = 'packaging.python.org' -github_repo_slug = f'{github_repo_org}/{github_repo_name}' -github_repo_url = f'{github_url}/{github_repo_slug}' -github_repo_issues_url = f'{github_url}/{github_repo_slug}/issues' -github_sponsors_url = f'{github_url}/sponsors' - -# General information about the project. -project = u'Python Packaging User Guide' -copyright = u'2013–2020, PyPA' -author = 'Python Packaging Authority' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '' -# The full version, including alpha/beta/rc tags. -release = '' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -locale_dirs = ['../locales'] - -gettext_auto_build = True # Build MO files from PO on each build - -# making all documents use single text domain -gettext_compact = "messages" +_ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.append(os.fspath(_ROOT)) -gettext_location = True # Include location info in the translation files +# Some options are only enabled for the main packaging.python.org deployment builds +RTD_BUILD = bool(os.getenv("READTHEDOCS")) +RTD_PR_BUILD = RTD_BUILD and os.getenv("READTHEDOCS_VERSION_TYPE") == "external" +RTD_URL = os.getenv("READTHEDOCS_CANONICAL_URL") +RTD_CANONICAL_BUILD = ( + RTD_BUILD and not RTD_PR_BUILD and "packaging.python.org" in RTD_URL +) -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# -# today = '' -# -# Else, today_fmt is used as the format for a strftime call. -# -# today_fmt = '%B %d, %Y' +project = "Python Packaging User Guide" -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +copyright = "2013–2020, PyPA" +author = "Python Packaging Authority" -# The reST default role (used for this markup: `text`) to use for all -# documents. -# Ref: python-attrs/attrs#571 -default_role = 'any' # makes single backticks autofind targets +# -- General configuration ------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -# If true, '()' will be appended to :func: etc. cross-reference text. -# -# add_function_parentheses = True +root_doc = "index" -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# -# add_module_names = False +extensions = [ + "pug_sphinx_extensions", + "sphinx.ext.extlinks", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx_inline_tabs", + "sphinx_copybutton", + "sphinx_toolbox.collapse", + "sphinx-jsonschema", +] -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# -# show_authors = False +nitpicky = True +nitpick_ignore = [ + ("envvar", "PATH"), + ("py:func", "find_packages"), + ("py:func", "setup"), + ("py:func", "importlib.metadata.entry_points"), + ("py:class", "importlib.metadata.EntryPoint"), + ("py:func", "setuptools.find_namespace_packages"), + ("py:func", "setuptools.find_packages"), + ("py:func", "setuptools.setup"), +] -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +default_role = "any" +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] +# -- Options for internationalization -------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-internationalization -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False +language = "en" -# If true, `todo` and `todoList` produce output, else they produce nothing. -# todo_include_todos = False +locale_dirs = ["../locales"] +gettext_auto_build = True +gettext_compact = "messages" +gettext_location = True -# -- Options for HTML output ---------------------------------------------- +# -- Options for HTML output ----------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'pypa_theme' +html_title = "Python Packaging User Guide" +html_theme = "furo" -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. html_theme_options = { - 'collapsiblesidebar': True, - 'externalrefs': True, - 'navigation_depth': 2, - 'issues_url': github_repo_issues_url, + "source_edit_link": "https://github.com/pypa/packaging.python.org/edit/main/source/{filename}", + "source_view_link": "https://github.com/pypa/packaging.python.org/blob/main/source/{filename}?plain=true", } -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = ['themes'] - -# The name for this set of Sphinx documents. -# " v documentation" by default. -# -html_title = 'Python Packaging User Guide' - -# A shorter title for the navigation bar. Default is the same as html_title. -# -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# -# html_logo = None - -# The name of an image file (relative to this directory) to use as a favicon of -# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# -# html_extra_path = [] - -# If not None, a 'Last updated on:' timestamp is inserted at every page -# bottom, using the given strftime format. -# The empty string is equivalent to '%b %d, %Y'. -# -html_last_updated_fmt = '' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# - -# Custom sidebar templates, filenames relative to this file. -html_sidebars = { - '**': ['globaltoc.html', 'relations.html'], - 'index': ['globaltoc.html'] -} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# -# html_additional_pages = {} - -# If false, no module index is generated. -# -# html_domain_indices = True - -# If false, no index is generated. -# -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' -# -# html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# 'ja' uses this config value. -# 'zh' user can custom change `jieba` dictionary path. -# -# html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -# -# html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'pythonpackagingguide-authdoc' - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} +html_favicon = "assets/py.png" +html_last_updated_fmt = "" -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'pythonpackagingguide.tex', 'Python Packaging User Guide', - 'Python Packaging Authority', 'manual'), +_metrics_js_files = [ + ( + "https://analytics.python.org/js/script.outbound-links.js", + {"data-domain": "packaging.python.org", "defer": "defer"}, + ), +] +html_js_files = [] +if RTD_CANONICAL_BUILD: + # Enable collection of the visitor metrics reported at + # https://plausible.io/packaging.python.org + html_js_files.extend(_metrics_js_files) + +html_extra_path = [ + "../extra", ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# -# latex_use_parts = False - -# If true, show page references after internal links. -# -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# -# latex_appendices = [] +# -- Options for HTML help output ------------------------------------------------------ +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-help-output -# It false, will not define \strong, \code, itleref, \crossref ... but only -# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added -# packages. -# -# latex_keep_old_macro_names = True +htmlhelp_basename = "pythonpackagingguide-authdoc" -# If false, no module index is generated. -# -# latex_domain_indices = True +# -- Options for LaTeX output ---------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-latex-output +latex_elements = {} +latex_documents = [ + ( + root_doc, + "pythonpackagingguide.tex", + "Python Packaging User Guide", + "Python Packaging Authority", + "manual", + ), +] -# -- Options for manual page output --------------------------------------- +# -- Options for manual page output ---------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-manual-page-output -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'pythonpackagingguide', 'Python Packaging User Guide', - [author], 1) + (root_doc, "pythonpackagingguide", "Python Packaging User Guide", [author], 1) ] -# If true, show URL addresses after external links. -# -# man_show_urls = False - +# -- Options for Texinfo output -------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-texinfo-output -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pythonpackagingguide', 'Python Packaging User Guide', - author, 'pythonpackagingguide', 'One line description of project.', - 'Miscellaneous'), + ( + root_doc, + "pythonpackagingguide", + "Python Packaging User Guide", + author, + "pythonpackagingguide", + "One line description of project.", + "Miscellaneous", + ), ] -# Documents to append as an appendix to all manuals. -# -# texinfo_appendices = [] +# -- Options for the linkcheck builder ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-the-linkcheck-builder -# If false, no module index is generated. -# -# texinfo_domain_indices = True +linkcheck_ignore = [ + r"http://localhost:\d+", + r"https://test\.pypi\.org/project/example-package-YOUR-USERNAME-HERE", + r"https://pypi\.org/manage/.*", + r"https://test\.pypi\.org/manage/.*", + # Temporarily ignored. Ref: + # https://github.com/pypa/packaging.python.org/pull/1308#issuecomment-1775347690 + r"https://www\.breezy-vcs\.org/.*", + # Ignore while StackOverflow is blocking GitHub CI. Ref: + # https://github.com/pypa/packaging.python.org/pull/1474 + r"https://stackoverflow\.com/.*", + r"https://pyscaffold\.org/.*", + r"https://anaconda\.org", + r"https://www\.cisa\.gov/sbom", + r"https://developers\.redhat\.com/products/softwarecollections/overview", + r"https://math-atlas\.sourceforge\.net/?", + r"https://click\.palletsprojects\.com/.*", + r"https://typer\.tiangolo\.com/.*", + r"https://www.npmjs.com/.*", +] +linkcheck_retries = 5 +# Ignore anchors for common targets when we know they likely won't be found +linkcheck_anchors_ignore_for_url = [ + # GitHub synthesises anchors in JavaScript, so Sphinx can't find them in the HTML + r"https://github\.com/", + r"https://docs\.github\.com/", + # While PyPI has its botscraping defenses active, Sphinx can't resolve the anchors + # https://github.com/pypa/packaging.python.org/issues/1744 + r"https://pypi\.org/", +] -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# -# texinfo_show_urls = 'footnote' +# -- Options for extlinks ---------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html#configuration -# If true, do not generate a @detailmenu in the "Top" node's menu. -# -# texinfo_no_detailmenu = False +github_url = "https://github.com" +github_repo_org = "pypa" +github_repo_name = "packaging.python.org" +github_repo_slug = f"{github_repo_org}/{github_repo_name}" +github_repo_url = f"{github_url}/{github_repo_slug}" +github_repo_issues_url = f"{github_url}/{github_repo_slug}/issues" +github_sponsors_url = f"{github_url}/sponsors" -# -- Options for extlinks extension --------------------------------------- extlinks = { - 'issue': (f'{github_repo_issues_url}/%s', '#'), # noqa: WPS323 - 'pr': (f'{github_repo_url}/pull/%s', 'PR #'), # noqa: WPS323 - 'commit': (f'{github_repo_url}/commit/%s', ''), # noqa: WPS323 - 'gh': (f'{github_url}/%s', 'GitHub: '), # noqa: WPS323 - 'user': (f'{github_sponsors_url}/%s', '@'), # noqa: WPS323 + "issue": (f"{github_repo_issues_url}/%s", "#%s"), + "pr": (f"{github_repo_url}/pull/%s", "PR #%s"), + "commit": (f"{github_repo_url}/commit/%s", "%s"), + "gh": (f"{github_url}/%s", "GitHub: %s"), + "user": (f"{github_sponsors_url}/%s", "@%s"), } -linkcheck_ignore = [ - "http://localhost:\d+", - # This is an example that showing to the reader - "https://test.pypi.org/project/example-package-YOUR-USERNAME-HERE", - # Ignoring it as it will redirect to login page if reader hasn't logged in. - "https://pypi.org/manage/*", - "https://test.pypi.org/manage/*", -] +# -- Options for intersphinx ---------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration -# Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - 'boltons': ('https://boltons.readthedocs.io/en/latest/', None), - 'build': ('https://pypa-build.readthedocs.io/en/stable/', None), - 'cffi': ('https://cffi.readthedocs.io/en/latest/', None), - 'conda': ('https://conda.io/en/latest/', None), - 'devpi': ('https://devpi.net/docs/devpi/devpi/latest/+doc', None), - 'dh-virtualenv': ('https://dh-virtualenv.readthedocs.io/en/latest/', None), - 'distlib': ('https://distlib.readthedocs.io/en/latest/', None), - 'flexx': ('https://flexx.readthedocs.io/en/latest/', None), - 'nox': ('https://nox.thea.codes/en/latest/', None), - 'openstack': ('https://docs.openstack.org/glance/latest/', None), - 'packaging': ('https://packaging.pypa.io/en/latest/', None), - 'packaging.python.org': ('https://packaging.python.org/', None), - 'pip': ('https://pip.pypa.io/en/latest/', None), - 'pipenv': ('https://pipenv.pypa.io/en/latest/', None), - 'piwheels': ('https://piwheels.readthedocs.io/en/latest/', None), - 'pybind11': ('https://pybind11.readthedocs.io/en/stable/', None), - 'pynsist': ('https://pynsist.readthedocs.io/en/latest/', None), - 'pypa': ('https://www.pypa.io/en/latest/', None), - 'python': ('https://docs.python.org/3', None), - 'python-guide': ('https://docs.python-guide.org', None), - 'python2': ('https://docs.python.org/2', None), - 'setuptools': ('https://setuptools.readthedocs.io/en/latest/', None), - 'spack': ('https://spack.readthedocs.io/en/latest/', None), - 'sphinx': ('https://www.sphinx-doc.org/en/master', None), - 'tox': ('https://tox.readthedocs.io/en/latest/', None), - 'twine': ('https://twine.readthedocs.io/en/stable/', None), - 'virtualenv': ('https://virtualenv.pypa.io/en/stable/', None), + "boltons": ("https://boltons.readthedocs.io/en/latest/", None), + "bottle": ("https://bottlepy.org/docs/dev/", None), + "build": ("https://pypa-build.readthedocs.io/en/stable/", None), + "cffi": ("https://cffi.readthedocs.io/en/latest/", None), + "conda": ("https://conda.io/en/latest/", None), + "devpi": ("https://devpi.net/docs/devpi/devpi/latest/+doc", None), + "dh-virtualenv": ("https://dh-virtualenv.readthedocs.io/en/latest/", None), + "distlib": ("https://distlib.readthedocs.io/en/latest/", None), + "flexx": ("https://flexx.readthedocs.io/en/latest/", None), + "flit": ("https://flit.pypa.io/en/stable/", None), + "nox": ("https://nox.thea.codes/en/latest/", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "openstack": ("https://docs.openstack.org/glance/latest/", None), + "packaging": ("https://packaging.pypa.io/en/latest/", None), + "pip": ("https://pip.pypa.io/en/latest/", None), + "pipenv": ("https://pipenv.pypa.io/en/latest/", None), + "piwheels": ("https://piwheels.readthedocs.io/en/latest/", None), + "pybind11": ("https://pybind11.readthedocs.io/en/stable/", None), + "pynsist": ("https://pynsist.readthedocs.io/en/latest/", None), + "pypa": ("https://www.pypa.io/en/latest/", None), + "python": ("https://docs.python.org/3", None), + "python-guide": ("https://docs.python-guide.org", None), + "setuptools": ("https://setuptools.pypa.io/en/latest/", None), + "spack": ("https://spack.readthedocs.io/en/latest/", None), + "sphinx": ("https://www.sphinx-doc.org/en/master", None), + "tox": ("https://tox.wiki/en/latest/", None), + "twine": ("https://twine.readthedocs.io/en/stable/", None), + "virtualenv": ("https://virtualenv.pypa.io/en/stable/", None), } - -# If this is True, todo and todolist produce output, else they produce nothing. -# The default is False. +# -- Options for todo extension -------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/todo.html#configuration todo_include_todos = True -nitpicky = True +# -- Options for sphinx-copybutton ----------------------------------------------------- +# https://sphinx-copybutton.readthedocs.io/en/latest/use.html -# NOTE: consider having a separate ignore file -# Ref: https://stackoverflow.com/a/30624034/595220 -nitpick_ignore = [ - ('envvar', 'PATH'), - ('py:func', 'find_packages'), - ('py:func', 'setup'), - ('py:func', 'importlib.metadata.entry_points'), # remove when 3.10 is released - ('py:class', 'importlib.metadata.EntryPoint'), # remove when 3.10 is released - ('py:func', 'setuptools.find_namespace_packages'), - ('py:func', 'setuptools.find_packages'), - ('py:func', 'setuptools.setup'), -] +copybutton_prompt_text = r">>> |\.\.\. |\$ |> " +copybutton_prompt_is_regexp = True diff --git a/source/contribute.rst b/source/contribute.rst index c5edb0792..f512dd30d 100644 --- a/source/contribute.rst +++ b/source/contribute.rst @@ -11,7 +11,7 @@ including: * Reviewing new contributions * Revising existing content * Writing new content -* Translate the guide +* Translating the guide Most of the work on the |PyPUG| takes place on the `project's GitHub repository`__. To get started, check out the list of @@ -32,9 +32,12 @@ Documentation types =================== This project consists of four distinct documentation types with specific -purposes. When proposing new additions to the project please pick the +purposes. The project aspires to follow the `DiΓ‘taxis process`_ +for creating quality documentation. When proposing new additions to the project please pick the appropriate documentation type. +.. _DiΓ‘taxis process: https://diataxis.fr/ + Tutorials --------- @@ -42,7 +45,7 @@ Tutorials are focused on teaching the reader new concepts by accomplishing a goal. They are opinionated step-by-step guides. They do not include extraneous warnings or information. `example tutorial-style document`_. -.. _example tutorial-style document: https://docs.djangoproject.com/en/1.11/intro/ +.. _example tutorial-style document: https://docs.djangoproject.com/en/dev/intro/ Guides ------ @@ -63,7 +66,7 @@ document `. Specifications -------------- -Specifications are reference documention focused on comprehensively documenting +Specifications are reference documentation focused on comprehensively documenting an agreed-upon interface for interoperability between packaging tools. :doc:`example specification-style document `. @@ -75,7 +78,7 @@ We use `Weblate`_ to manage translations of this project. Please visit the `packaging.python.org`_ project on Weblate to contribute. If you are experiencing issues while you are working on translations, -please open an issue on `Github`_. +please open an issue on `GitHub`_. .. tip:: @@ -83,7 +86,7 @@ please open an issue on `Github`_. .. _Weblate: https://weblate.org/ .. _packaging.python.org: https://hosted.weblate.org/projects/pypa/packaging-python-org/ -.. _Github: https://github.com/pypa/packaging.python.org/issues +.. _GitHub: https://github.com/pypa/packaging.python.org/issues .. _reStructuredText syntax: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html Adding a language @@ -100,7 +103,6 @@ If you are not familiar with reStructuredText (RST) syntax, please read `this gu before translating on Weblate. **Do not translate the text in reference directly** - When translating the text in reference, please do not translate them directly. | Wrong: Translate the following text directly: @@ -131,11 +133,11 @@ need: python -m pip install --user nox -2. Python 3.8. Our build scripts are usually tested with Python 3.8 only. +2. Python 3.11. Our build scripts are usually tested with Python 3.11 only. See the :doc:`Hitchhiker's Guide to Python installation instructions ` - to install Python 3.8 on your operating system. + to install Python 3.11 on your operating system. -To build the guide, run the following bash command in the source folder: +To build the guide, run the following shell command in the project's root folder: .. code-block:: bash diff --git a/source/discussions/deploying-python-applications.rst b/source/discussions/deploying-python-applications.rst index de781d9fd..e10f36f9c 100644 --- a/source/discussions/deploying-python-applications.rst +++ b/source/discussions/deploying-python-applications.rst @@ -6,9 +6,6 @@ Deploying Python applications :Page Status: Incomplete :Last Reviewed: 2021-8-24 -.. contents:: Contents - :local: - Overview ======== @@ -26,7 +23,7 @@ Supporting multiple hardware platforms For Python-only distributions, it *should* be straightforward to deploy on all platforms where Python can run. - For distributions with binary extensions, deployment is major headache. Not only + For distributions with binary extensions, deployment is a major headache. Not only must the extensions be built on all the combinations of operating system and hardware platform, but they must also be tested, preferably on continuous integration platforms. The issues are similar to the "multiple Python @@ -73,7 +70,7 @@ directory, independent of any other Python installation on the computer. A big advantage of Pynsist is that the Windows packages can be built on Linux. There are several examples for different kinds of programs (console, GUI) in -the `documentation `. The tool is released +the :any:`documentation `. The tool is released under the MIT-licence. Application bundles @@ -92,12 +89,12 @@ py2exe ^^^^^^ `py2exe `__ is a distutils extension which -allows to build standalone Windows executable programs (32-bit and 64-bit) -from Python scripts. Python versions included in the official development +allows to build standalone Windows executable programs (32-bit and 64-bit) +from Python scripts. Python versions included in the official development cycle are supported (refers to `Status of Python branches`__). py2exe can build console executables and windows (GUI) executables. Building windows services, and DLL/EXE COM servers might work but it is not actively supported. -The distutils extension is released under the MIT-licence and Mozilla +The distutils extension is released under the MIT-licence and Mozilla Public License 2.0. .. __: https://devguide.python.org/#status-of-python-branches @@ -108,8 +105,8 @@ macOS py2app ^^^^^^ -`py2app `__ is a Python setuptools -command which will allow you to make standalone macOS application +`py2app `__ is a Python setuptools +command which will allow you to make standalone macOS application bundles and plugins from Python scripts. Note that py2app MUST be used on macOS to build applications, it cannot create Mac applications on other platforms. py2app is released under the MIT-license. @@ -120,11 +117,11 @@ Unix (including Linux and macOS) pex ^^^ -`pex `__ is a library for generating .pex -(Python EXecutable) files which are executable Python environments in the -spirit of virtualenvs. pex is an expansion upon the ideas outlined in :pep:`441` -and makes the deployment of Python applications as simple as cp. pex files may -even include multiple platform-specific Python distributions, meaning that a +`pex `__ is a library for generating .pex +(Python EXecutable) files which are executable Python environments in the +spirit of virtualenvs. pex is an expansion upon the ideas outlined in :pep:`441` +and makes the deployment of Python applications as simple as cp. pex files may +even include multiple platform-specific Python distributions, meaning that a single pex file can be portable across Linux and macOS. pex is released under the Apache License 2.0. diff --git a/source/discussions/distribution-package-vs-import-package.rst b/source/discussions/distribution-package-vs-import-package.rst new file mode 100644 index 000000000..65e7019c6 --- /dev/null +++ b/source/discussions/distribution-package-vs-import-package.rst @@ -0,0 +1,110 @@ +.. _distribution-package-vs-import-package: + +======================================= +Distribution package vs. import package +======================================= + +A number of different concepts are commonly referred to by the word +"package". This page clarifies the differences between two distinct but +related meanings in Python packaging, "distribution package" and "import +package". + +What's a distribution package? +============================== + +A distribution package is a piece of software that you can install. +Most of the time, this is synonymous with "project". When you type ``pip +install pkg``, or when you write ``dependencies = ["pkg"]`` in your +``pyproject.toml``, ``pkg`` is the name of a distribution package. When +you search or browse the PyPI_, the most widely known centralized source for +installing Python libraries and tools, what you see is a list of distribution +packages. Alternatively, the term "distribution package" can be used to +refer to a specific file that contains a certain version of a project. + +Note that in the Linux world, a "distribution package", +most commonly abbreviated as "distro package" or just "package", +is something provided by the system package manager of the `Linux distribution `_, +which is a different meaning. + + +What's an import package? +========================= + +An import package is a Python module. Thus, when you write ``import +pkg`` or ``from pkg import func`` in your Python code, ``pkg`` is the +name of an import package. More precisely, import packages are special +Python modules that can contain submodules. For example, the ``numpy`` +package contains modules like ``numpy.linalg`` and +``numpy.fft``. Usually, an import package is a directory on the file +system, containing modules as ``.py`` files and subpackages as +subdirectories. + +You can use an import package as soon as you have installed a distribution +package that provides it. + + +What are the links between distribution packages and import packages? +===================================================================== + +Most of the time, a distribution package provides one single import +package (or non-package module), with a matching name. For example, +``pip install numpy`` lets you ``import numpy``. + +However, this is only a convention. PyPI and other package indices *do not +enforce any relationship* between the name of a distribution package and the +import packages it provides. (A consequence of this is that you cannot blindly +install the PyPI package ``foo`` if you see ``import foo``; this may install an +unintended, and potentially even malicious package.) + +A distribution package could provide an import package with a different +name. An example of this is the popular Pillow_ library for image +processing. Its distribution package name is ``Pillow``, but it provides +the import package ``PIL``. This is for historical reasons: Pillow +started as a fork of the PIL library, thus it kept the import name +``PIL`` so that existing PIL users could switch to Pillow with little +effort. More generally, a fork of an existing library is a common reason +for differing names between the distribution package and the import +package. + +On a given package index (like PyPI), distribution package names must be +unique. On the other hand, import packages have no such requirement. +Import packages with the same name can be provided by several +distribution packages. Again, forks are a common reason for this. + +Conversely, a distribution package can provide several import packages, +although this is less common. An example is the attrs_ distribution +package, which provides both an ``attrs`` import package with a newer +API, and an ``attr`` import package with an older but supported API. + + +How do distribution package names and import package names compare? +=================================================================== + +Import packages should have valid Python identifiers as their name (the +:ref:`exact rules ` are found in the Python +documentation) [#non-identifier-mod-name]_. In particular, they use underscores ``_`` as word +separator and they are case-sensitive. + +On the other hand, distribution packages can use hyphens ``-`` or +underscores ``_``. They can also contain dots ``.``, which is sometimes +used for packaging a subpackage of a :ref:`namespace package +`. For most purposes, they are insensitive +to case and to ``-`` vs. ``_`` differences, e.g., ``pip install +Awesome_Package`` is the same as ``pip install awesome-package`` (the +precise rules are given in the :ref:`name normalization specification +`). + + + +--------------------------- + +.. [#non-identifier-mod-name] Although it is technically possible + to import packages/modules that do not have a valid Python identifier as + their name, using :doc:`importlib `, + this is vanishingly rare and strongly discouraged. + + +.. _distro: https://en.wikipedia.org/wiki/Linux_distribution +.. _PyPI: https://pypi.org +.. _Pillow: https://pypi.org/project/Pillow +.. _attrs: https://pypi.org/project/attrs diff --git a/source/discussions/downstream-packaging.rst b/source/discussions/downstream-packaging.rst new file mode 100644 index 000000000..3f4795fa8 --- /dev/null +++ b/source/discussions/downstream-packaging.rst @@ -0,0 +1,481 @@ +.. _downstream-packaging: + +=============================== +Supporting downstream packaging +=============================== + +:Page Status: Draft +:Last Reviewed: 2025-? + +While PyPI and the Python packaging tools such as :ref:`pip` are the primary +means of distributing Python packages, they are also often made available as part +of other packaging ecosystems. These repackaging efforts are collectively called +*downstream* packaging (your own efforts are called *upstream* packaging), +and include such projects as Linux distributions, Conda, Homebrew and MacPorts. +They generally aim to provide improved support for use cases that cannot be handled +via Python packaging tools alone, such as native integration with a specific operating +system, or assured compatibility with specific versions of non-Python software. + +This discussion attempts to explain how downstream packaging is usually done, +and what additional challenges downstream packagers typically face. It aims +to provide some optional guidelines that project maintainers may choose to +follow which help make downstream packaging *significantly* easier +(without imposing any major maintenance hassles on the upstream project). +Note that this is not an all-or-nothing proposal β€” anything that upstream +maintainers can do is useful, even if it's only a small part. Downstream +maintainers are also willing to prepare patches to resolve these issues. +Having these patches merged can be very helpful, since it removes the need +for different downstreams to carry and keep rebasing the same patches, +and the risk of applying inconsistent solutions to the same problem. + +Establishing a good relationship between software maintainers and downstream +packagers can bring mutual benefits. Downstreams are often willing to share +their experience, time and hardware to improve your package. They are +sometimes in a better position to see how your package is used in practice, +and to provide information about its relationships with other packages that +would otherwise require significant effort to obtain. +Packagers can often find bugs before your users hit them in production, +provide bug reports of good quality, and supply patches whenever they can. +For example, they are regularly active in ensuring the packages they redistribute +are updated for any compatibility issues that arise when a new Python version +is released. + +Please note that downstream builds include not only binary redistribution, +but also source builds done on user systems (in source-first distributions +such as Gentoo Linux, for example). + + +.. _provide-complete-source-distributions: + +Provide complete source distributions +------------------------------------- + +Why? +~~~~ + +The vast majority of downstream packagers prefer to build packages from source, +rather than use the upstream-provided binary packages. In some cases, using +sources is actually required for the package to be included in the distribution. +This is also true of pure Python packages that provide universal wheels. +The reasons for using source distributions may include: + +- Being able to audit the source code of all packages. + +- Being able to run the test suite and build documentation. + +- Being able to easily apply patches, including backporting commits + from the project's repository and sending patches back to the project. + +- Being able to build on a specific platform that is not covered + by upstream builds. + +- Being able to build against specific versions of system libraries. + +- Having a consistent build process across all Python packages. + +While it is usually possible to build packages from a Git repository, there are +a few important reasons to provide a static archive file instead: + +- Fetching a single file is often more efficient, more reliable and better + supported than e.g. using a Git clone. This can help users with poor + Internet connectivity. + +- Downstreams often use hashes to verify the authenticity of source files + on subsequent builds, which require that they remain bitwise identical over + time. For example, automatically generated Git archives do not guarantee + this, as the compressed data may change if gzip is upgraded on the server. + +- Archive files can be mirrored, reducing both upstream and downstream + bandwidth use. The actual builds can afterwards be performed in firewalled + or offline environments, that can only access source files provided + by the local mirror or redistributed earlier. + +- Explicitly publishing archive files can ensure that any dependencies on version control + system metadata are resolved when creating the source archive. For example, automatically + generated Git archives omit all of the commit tag information, potentially resulting in + incorrect version details in the resulting builds. + +How? +~~~~ + +Ideally, **a source distribution archive published on PyPI should include all the files +from the package's Git repository** that are necessary to build the package +itself, run its test suite, build and install its documentation, and any other +files that may be useful to end users, such as shell completions, editor +support files, and so on. + +This point applies only to the files belonging to the package itself. +The downstream packaging process, much like Python package managers, will +provision the necessary Python dependencies, system tools and external +libraries that are needed by your package and its build scripts. However, +the files listing these dependencies (for example, ``requirements*.txt`` files) +should also be included, to help downstreams determine the needed dependencies, +and check for changes in them. + +Some projects have concerns related to Python package managers using source +distributions from PyPI. They do not wish to increase their size with files +that are not used by these tools, or they do not wish to publish source +distributions at all, as they enable a problematic or outright nonfunctional +fallback to building the particular project from source. In these cases, a good +compromise may be to publish a separate source archive for downstream use +elsewhere, for example by attaching it to a GitHub release. Alternatively, +large files, such as test data, can be split into separate archives. + +On the other hand, some projects (NumPy_, for instance) decide to include tests +in their installed packages. This has the added advantage of permitting users to +run tests after installing them, for example to check for regressions +after upgrading a dependency. Yet another approach is to split tests or test +data into a separate Python package. Such an approach was taken by +the cryptography_ project, with the large test vectors being split +to cryptography-vectors_ package. + +A good idea is to use your source distribution in the release workflow. +For example, the :ref:`build` tool does exactly that β€” it first builds a source +distribution, and then uses it to build a wheel. This ensures that the source +distribution actually works, and that it won't accidentally install fewer files +than the official wheels. + +Ideally, also use the source distribution to run tests, build documentation, +and so on, or add specific tests to make sure that all necessary files were +actually included. Understandably, this requires more effort, so it's fine +not do that β€” downstream packagers will report any missing files promptly. + + +.. _no-internet-access-in-builds: + +Do not use the Internet during the build process +------------------------------------------------ + +Why? +~~~~ + +Downstream builds are frequently done in sandboxed environments that cannot +access the Internet. The package sources are unpacked into this environment, +and all the necessary dependencies are installed. + +Even if this is not the case, and assuming that you took sufficient care to +properly authenticate downloads, using the Internet is discouraged for a number +of reasons: + +- The Internet connection may be unstable (e.g. due to poor reception) + or suffer from temporary problems that could cause the process to fail + or hang. + +- The remote resources may become temporarily or even permanently + unavailable, making the build no longer possible. This is especially + problematic when someone needs to build an old package version. + +- The remote resources may change, making the build not reproducible. + +- Accessing remote servers poses a privacy issue and a potential + security issue, as it exposes information about the system building + the package. + +- The user may be using a service with a limited data plan, in which + uncontrolled Internet access may result in additional charges or other + inconveniences. + +How? +~~~~ + +If the package is implementing any custom build *backend* actions that use +the Internet, for example by automatically downloading vendored dependencies +or fetching Git submodules, its source distribution should either include all +of these files or allow provisioning them externally, and the Internet must not +be used if the files are already present. + +Note that this point does not apply to Python dependencies that are specified +in the package metadata, and are fetched during the build and installation +process by *frontends* (such as :ref:`build` or :ref:`pip`). Downstreams use +frontends that use local provisioning for Python dependencies. + +Ideally, custom build scripts should not even attempt to access the Internet +at all, unless explicitly requested to. If any resources are missing and need +to be fetched, they should ask the user for permission first. If that is not +feasible, the next best thing is to provide an opt-out switch to disable +all Internet access. This could be done e.g. by checking whether +a ``NO_NETWORK`` environment variable is set to a non-empty value. + +Since downstreams frequently also run tests and build documentation, the above +should ideally extend to these processes as well. + +Please also remember that if you are fetching remote resources, you absolutely +must *verify their authenticity* (usually against a hash), to protect against +the file being substituted by a malicious party. + + +.. _support-system-dependencies-in-builds: + +Support building against system dependencies +-------------------------------------------- + +Why? +~~~~ + +Some Python projects have non-Python dependencies, such as libraries written +in C or C++. Trying to use the system versions of these dependencies +in upstream packaging may cause a number of problems for end users: + +- The published wheels require a binary-compatible version of the used + library to be present on the user's system. If the library is missing + or an incompatible version is installed, the Python package may fail with errors + that are not clear to inexperienced users, or even misbehave at runtime. + +- Building from a source distribution requires a source-compatible version + of the dependency to be present, along with its development headers + and other auxiliary files that some systems package separately + from the library itself. + +- Even for an experienced user, installing a compatible dependency version + may be very hard. For example, the used Linux distribution may not provide + the required version, or some other package may require an incompatible + version. + +- The linkage between the Python package and its system dependency is not + recorded by the packaging system. The next system update may upgrade + the library to a newer version that breaks binary compatibility with + the Python package, and requires user intervention to fix. + +For these reasons, you may reasonably decide to either statically link +your dependencies, or to provide local copies in the installed package. +You may also vendor the dependency in your source distribution. Sometimes +these dependencies are also repackaged on PyPI, and can be declared as +project dependencies like any other Python package. + +However, none of these issues apply to downstream packaging, and downstreams +have good reasons to prefer dynamically linking to system dependencies. +In particular: + +- In many cases, reliably sharing dynamic dependencies between components is a large part + of the *purpose* of a downstream packaging ecosystem. Helping to support that makes it + easier for users of those systems to access upstream projects in their preferred format. + +- Static linking and vendoring obscures the use of external dependencies, + making source auditing harder. + +- Dynamic linking makes it possible to quickly and systematically replace the used + libraries across an entire downstream packaging ecosystem, which can be particularly + important when they turn out to contain a security vulnerability or critical bug. + +- Using system dependencies makes the package benefit from downstream + customization that can improve the user experience on a particular platform, + without the downstream maintainers having to consistently patch + the dependencies vendored in different packages. This can include + compatibility improvements and security hardening. + +- Static linking and vendoring can result in multiple different versions of the + same library being loaded in the same process (for example, attempting to + import two Python packages that link to different versions of the same library). + This sometimes works without incident, but it can also lead to anything from library + loading errors, to subtle runtime bugs, to catastrophic failures (like suddenly + crashing and losing data). + +- Last but not least, static linking and vendoring results in duplication, + and may increase the use of both disk space and memory. + +How? +~~~~ + +A good compromise between the needs of both parties is to provide a switch +between using vendored and system dependencies. Ideally, if the package has +multiple vendored dependencies, it should provide both individual switches +for each dependency, and a general switch to control the default for them, +e.g. via a ``USE_SYSTEM_DEPS`` environment variable. + +If the user requests using system dependencies, and a particular dependency +is either missing or incompatible, the build should fail with an explanatory +message rather than fall back to a vendored version. This gives the packager +the opportunity to notice their mistake and a chance to consciously decide +how to solve it. + +It is reasonable for upstream projects to leave *testing* of building with +system dependencies to their downstream repackagers. The goal of these guidelines +is to facilitate more effective collaboration between upstream projects and downstream +repackagers, not to suggest upstream projects take on tasks that downstream repackagers +are better equipped to handle. + +.. _support-downstream-testing: + +Support downstream testing +-------------------------- + +Why? +~~~~ + +A variety of downstream projects run some degree of testing on the packaged +Python projects. Depending on the particular case, this can range from minimal +smoke testing to comprehensive runs of the complete test suite. There can +be various reasons for doing this, for example: + +- Verifying that the downstream packaging did not introduce any bugs. + +- Testing on additional platforms that are not covered by upstream testing. + +- Finding subtle bugs that can only be reproduced with particular hardware, + system package versions, and so on. + +- Testing the released package against newer (or older) dependency versions than + the ones present during upstream release testing. + +- Testing the package in an environment closely resembling the production + setup. This can detect issues caused by non-trivial interactions between + different installed packages, including packages that are not dependencies + of your package, but nevertheless can cause issues. + +- Testing the released package against newer Python versions (including + newer point releases), or less tested Python implementations such as PyPy. + +Admittedly, sometimes downstream testing may yield false positives or bug +reports about scenarios the upstream project is not interested in supporting. +However, perhaps even more often it does provide early notice of problems, +or find non-trivial bugs that would otherwise cause issues for the upstream +project's users. While mistakes do happen, the majority of downstream packagers +are doing their best to double-check their results, and help upstream +maintainers triage and fix the bugs that they reported. + +How? +~~~~ + +There are a number of things that upstream projects can do to help downstream +repackagers test their packages efficiently and effectively, including some of the suggestions +already mentioned above. These are typically improvements that make the test suite more +reliable and easier to use for everyone, not just downstream packagers. +Some specific suggestions are: + +- Include the test files and fixtures in the source distribution, or make it + possible to easily download them separately. + +- Do not write to the package directories during testing. Downstream test + setups sometimes run tests on top of the installed package, and modifications + performed during testing and temporary test files may end up being part + of the installed package! + +- Make the test suite work offline. Mock network interactions, using + packages such as responses_ or vcrpy_. If that is not possible, make it + possible to easily disable the tests using Internet access, e.g. via a pytest_ + marker. Use pytest-socket_ to verify that your tests work offline. This + often makes your own test workflows faster and more reliable as well. + +- Make your tests work without a specialized setup, or perform the necessary + setup as part of test fixtures. Do not ever assume that you can connect + to system services such as databases β€” in an extreme case, you could crash + a production service! + +- If your package has optional dependencies, make their tests optional as + well. Either skip them if the needed packages are not installed, or add + markers to make deselecting easy. + +- More generally, add markers to tests with special requirements. These can + include e.g. significant space usage, significant memory usage, long runtime, + incompatibility with parallel testing. + +- Do not assume that the test suite will be run with ``-Werror``. Downstreams + often need to disable that, as it causes false positives, e.g. due to newer + dependency versions. Assert for warnings using ``pytest.warns()`` rather + than ``pytest.raises()``! + +- Aim to make your test suite reliable and reproducible. Avoid flaky tests. + Avoid depending on specific platform details, don't rely on exact results + of floating-point computation, or timing of operations, and so on. Fuzzing + has its advantages, but you want to have static test cases for completeness + as well. + +- Split tests by their purpose, and make it easy to skip categories that are + irrelevant or problematic. Since the primary purpose of downstream testing + is to ensure that the package itself works, downstreams are not generally interested + in tasks such as checking code coverage, code formatting, typechecking or running + benchmarks. These tests can fail as dependencies are upgraded or the system + is under load, without actually affecting the package itself. + +- If your test suite takes significant time to run, support testing + in parallel. Downstreams often maintain a large number of packages, + and testing them all takes a lot of time. Using pytest-xdist_ can help them + avoid bottlenecks. + +- Ideally, support running your test suite via ``pytest``. pytest_ has many + command-line arguments that are truly helpful to downstreams, such as + the ability to conveniently deselect tests, rerun flaky tests + (via pytest-rerunfailures_), add a timeout to prevent tests from hanging + (via pytest-timeout_) or run tests in parallel (via pytest-xdist_). + Note that test suites don't need to be *written* with ``pytest`` to be + *executed* with ``pytest``: ``pytest`` is able to find and execute almost + all test cases that are compatible with the standard library's ``unittest`` + test discovery. + + +.. _aim-for-stable-releases: + +Aim for stable releases +----------------------- + +Why? +~~~~ + +Many downstreams provide stable release channels in addition to the main +package streams. The goal of these channels is to provide more conservative +upgrades to users with higher stability needs. These users often prefer +to trade having the newest features available for lower risk of issues. + +While the exact policies differ, an important criterion for including a new +package version in a stable release channel is for it to be available in testing +for some time already, and have no known major regressions. For example, +in Gentoo Linux a package is usually marked stable after being available +in testing for a month, and being tested against the versions of its +dependencies that are marked stable at the time. + +However, there are circumstances which demand more prompt action. For example, +if a security vulnerability or a major bug is found in the version that is +currently available in the stable channel, the downstream is facing a need +to resolve it. In this case, they need to consider various options, such as: + +- putting a new version in the stable channel early, + +- adding patches to the version currently published, + +- or even downgrading the stable channel to an earlier release. + +Each of these options involves certain risks and a certain amount of work, +and packagers needs to weigh them to determine the course of action. + +How? +~~~~ + +There are some things that upstreams can do to tailor their workflow to stable +release channels. These actions often are beneficial to the package's users +as well. Some specific suggestions are: + +- Adjust the release frequency to the rate of code changes. Packages that + are released rarely often bring significant changes with every release, + and a higher risk of accidental regressions. + +- Avoid mixing bug fixes and new features, if possible. In particular, if there + are known bug fixes merged already, consider making a new release before + merging feature branches. + +- Consider making prereleases after major changes, to provide more testing + opportunities for users and downstreams willing to opt-in. + +- If your project is subject to very intense development, consider splitting + one or more branches that include a more conservative subset of commits, + and are released separately. For example, Django_ currently maintains three + release branches in addition to main. + +- Even if you don't wish to maintain additional branches permanently, consider + making additional patch releases with minimal changes to the previous + version, especially when a security vulnerability is discovered. + +- Split your changes into focused commits that address one problem at a time, + to make it easier to cherry-pick changes to earlier releases when necessary. + + +.. _responses: https://pypi.org/project/responses/ +.. _vcrpy: https://pypi.org/project/vcrpy/ +.. _pytest-socket: https://pypi.org/project/pytest-socket/ +.. _pytest-xdist: https://pypi.org/project/pytest-xdist/ +.. _pytest: https://pytest.org/ +.. _pytest-rerunfailures: https://pypi.org/project/pytest-rerunfailures/ +.. _pytest-timeout: https://pypi.org/project/pytest-timeout/ +.. _Django: https://www.djangoproject.com/ +.. _NumPy: https://numpy.org/ +.. _cryptography: https://pypi.org/project/cryptography/ +.. _cryptography-vectors: https://pypi.org/project/cryptography-vectors/ diff --git a/source/discussions/index.rst b/source/discussions/index.rst index bc1154507..b1b84f97a 100644 --- a/source/discussions/index.rst +++ b/source/discussions/index.rst @@ -8,8 +8,13 @@ specific topic. If you're just trying to get stuff done, see .. toctree:: :maxdepth: 1 + versioning deploying-python-applications pip-vs-easy-install install-requires-vs-requirements - wheel-vs-egg + distribution-package-vs-import-package + package-formats src-layout-vs-flat-layout + setup-py-deprecated + single-source-version + downstream-packaging diff --git a/source/discussions/install-requires-vs-requirements.rst b/source/discussions/install-requires-vs-requirements.rst index 95578bca8..99e1552b8 100644 --- a/source/discussions/install-requires-vs-requirements.rst +++ b/source/discussions/install-requires-vs-requirements.rst @@ -4,9 +4,6 @@ install_requires vs requirements files ====================================== -.. contents:: Contents - :local: - install_requires ---------------- @@ -38,8 +35,9 @@ v2 of 'B', so it would be like so: 'B>=2' ] -It may also be known that project A follows semantic versioning, and that v2 of -'A' will indicate a break in compatibility, so it makes sense to not allow v2: +It may also be known that project 'A' introduced a change in its v2 +that breaks the compatibility of your project with v2 of 'A' and later, +so it makes sense to not allow v2: :: diff --git a/source/discussions/package-formats.rst b/source/discussions/package-formats.rst new file mode 100644 index 000000000..6d4dee35c --- /dev/null +++ b/source/discussions/package-formats.rst @@ -0,0 +1,193 @@ +.. _package-formats: + +=============== +Package Formats +=============== + +This page discusses the file formats that are used to distribute Python packages +and the differences between them. + +You will find files in two formats on package indices such as PyPI_: **source +distributions**, or **sdists** for short, and **binary distributions**, commonly +called **wheels**. For example, the `PyPI page for pip 23.3.1 `_ +lets you download two files, ``pip-23.3.1.tar.gz`` and +``pip-23.3.1-py3-none-any.whl``. The former is an sdist, the latter is a +wheel. As explained below, these serve different purposes. When publishing a +package on PyPI (or elsewhere), you should always upload both an sdist and one +or more wheel. + + +What is a source distribution? +============================== + +Conceptually, a source distribution is an archive of the source code in raw +form. Concretely, an sdist is a ``.tar.gz`` archive containing the source code +plus an additional special file called ``PKG-INFO``, which holds the project +metadata. The presence of this file helps packaging tools to be more efficient +by not needing to compute the metadata themselves. The ``PKG-INFO`` file follows +the format specified in :ref:`core-metadata` and is not intended to be written +by hand [#core-metadata-format]_. + +You can thus inspect the contents of an sdist by unpacking it using standard +tools to work with tar archives, such as ``tar -xvf`` on UNIX platforms (like +Linux and macOS), or :ref:`the command line interface of Python's tarfile module +` on any platform. + +Sdists serve several purposes in the packaging ecosystem. When :ref:`pip`, the +standard Python package installer, cannot find a wheel to install, it will fall +back on downloading a source distribution, compiling a wheel from it, and +installing the wheel. Furthermore, sdists are often used as the package source +by downstream packagers (such as Linux distributions, Conda, Homebrew and +MacPorts on macOS, ...), who, for various reasons, may prefer them over, e.g., +pulling from a Git repository. + +A source distribution is recognized by its file name, which has the form +:samp:`{package_name}-{version}.tar.gz`, e.g., ``pip-23.3.1.tar.gz``. + +.. TODO: provide clear guidance on whether sdists should contain docs and tests. + Discussion: https://discuss.python.org/t/should-sdists-include-docs-and-tests/14578 + +If you want technical details on the sdist format, read the :ref:`sdist +specification `. + + +What is a wheel? +================ + +Conceptually, a wheel contains exactly the files that need to be copied when +installing the package. + +There is a big difference between sdists and wheels for packages with +:term:`extension modules `, written in compiled languages like +C, C++ and Rust, which need to be compiled into platform-dependent machine code. +With these packages, wheels do not contain source code (like C source files) but +compiled, executable code (like ``.so`` files on Linux or DLLs on Windows). + +Furthermore, while there is only one sdist per version of a project, there may +be many wheels. Again, this is most relevant in the context of extension +modules. The compiled code of an extension module is tied to an operating system +and processor architecture, and often also to the version of the Python +interpreter (unless the :ref:`Python stable ABI ` is used). + +For pure-Python packages, the difference between sdists and wheels is less +marked. There is normally one single wheel, for all platforms and Python +versions. Python is an interpreted language, which does not need ahead-of-time +compilation, so wheels contain ``.py`` files just like sdists. + +If you are wondering about ``.pyc`` bytecode files: they are not included in +wheels, since they are cheap to generate, and including them would unnecessarily +force a huge number of packages to distribute one wheel per Python version +instead of one single wheel. Instead, installers like :ref:`pip` generate them +while installing the package. + +With that being said, there are still important differences between sdists and +wheels, even for pure Python projects. Wheels are meant to contain exactly what +is to be installed, and nothing more. In particular, wheels should never include +tests and documentation, while sdists commonly do. Also, the wheel format is +more complex than sdist. For example, it includes a special file -- called +``RECORD`` -- that lists all files in the wheel along with a hash of their +content, as a safety check of the download's integrity. + +At a glance, you might wonder if wheels are really needed for "plain and basic" +pure Python projects. Keep in mind that due to the flexibility of sdists, +installers like pip cannot install from sdists directly -- they need to first +build a wheel, by invoking the :term:`build backend` that the sdist specifies +(the build backend may do all sorts of transformations while building the wheel, +such as compiling C extensions). For this reason, even for a pure Python +project, you should always upload *both* an sdist and a wheel to PyPI or other +package indices. This makes installation much faster for your users, since a +wheel is directly installable. By only including files that must be installed, +wheels also make for smaller downloads. + +On the technical level, a wheel is a ZIP archive (unlike sdists which are TAR +archives). You can inspect its contents by unpacking it as a normal ZIP archive, +e.g., using ``unzip`` on UNIX platforms like Linux and macOS, ``Expand-Archive`` +in Powershell on Windows, or :ref:`the command line interface of Python's +zipfile module `. This can be very useful to check +that the wheel includes all the files you need it to. + +Inside a wheel, you will find the package's files, plus an additional directory +called :samp:`{package_name}-{version}.dist-info`. This directory contains +various files, including a ``METADATA`` file which is the equivalent of +``PKG-INFO`` in sdists, as well as ``RECORD``. This can be useful to ensure no +files are missing from your wheels. + +The file name of a wheel (ignoring some rarely used features) looks like this: +:samp:`{package_name}-{version}-{python_tag}-{abi_tag}-{platform_tag}.whl`. +This naming convention identifies which platforms and Python versions the wheel +is compatible with. For example, the name ``pip-23.3.1-py3-none-any.whl`` means +that: + +- (``py3``) This wheel can be installed on any implementation of Python 3, + whether CPython, the most widely used Python implementation, or an alternative + implementation like PyPy_; +- (``none``) It does not depend on the Python version; +- (``any``) It does not depend on the platform. + +The pattern ``py3-none-any`` is common for pure Python projects. Packages with +extension modules typically ship multiple wheels with more complex tags. + +All technical details on the wheel format can be found in the :ref:`wheel +specification `. + + +.. _egg-format: +.. _`Wheel vs Egg`: + +What about eggs? +================ + +"Egg" is an old package format that has been replaced with the wheel format. It +should not be used anymore. Since August 2023, PyPI `rejects egg uploads +`_. + +Here's a breakdown of the important differences between wheel and egg. + +* The egg format was introduced by :ref:`setuptools` in 2004, whereas the wheel + format was introduced by :pep:`427` in 2012. + +* Wheel has an :doc:`official standard specification + `. Egg did not. + +* Wheel is a :term:`distribution ` format, i.e a packaging + format. [#wheel-importable]_ Egg was both a distribution format and a runtime + installation format (if left zipped), and was designed to be importable. + +* Wheel archives do not include ``.pyc`` files. Therefore, when the distribution + only contains Python files (i.e. no compiled extensions), and is compatible + with Python 2 and 3, it's possible for a wheel to be "universal", similar to + an :term:`sdist `. + +* Wheel uses standard :ref:`.dist-info directories + `. Egg used ``.egg-info``. + +* Wheel has a :ref:`richer file naming convention `. A + single wheel archive can indicate its compatibility with a number of Python + language versions and implementations, ABIs, and system architectures. + +* Wheel is versioned. Every wheel file contains the version of the wheel + specification and the implementation that packaged it. + +* Wheel is internally organized by `sysconfig path type + `_, + therefore making it easier to convert to other formats. + +-------------------------------------------------------------------------------- + +.. [#core-metadata-format] This format is email-based. Although this would + be unlikely to be chosen today, backwards compatibility considerations lead to + it being kept as the canonical format. From the user point of view, this + is mostly invisible, since the metadata is specified by the user in a way + understood by the build backend, typically ``[project]`` in ``pyproject.toml``, + and translated by the build backend into ``PKG-INFO``. + +.. [#wheel-importable] Circumstantially, in some cases, wheels can be used + as an importable runtime format, although :ref:`this is not officially supported + at this time `. + + + +.. _pip-pypi: https://pypi.org/project/pip/23.3.1/#files +.. _pypi: https://pypi.org +.. _pypi-eggs-deprecation: https://blog.pypi.org/posts/2023-06-26-deprecate-egg-uploads/ +.. _pypy: https://pypy.org diff --git a/source/discussions/pip-vs-easy-install.rst b/source/discussions/pip-vs-easy-install.rst index 4fa590cf3..2bb75d3be 100644 --- a/source/discussions/pip-vs-easy-install.rst +++ b/source/discussions/pip-vs-easy-install.rst @@ -11,7 +11,7 @@ It was notable at the time for installing :term:`packages :term:`PyPI ` using requirement specifiers, and automatically installing dependencies. -:ref:`pip` came later in 2008, as alternative to :ref:`easy_install `, although still +:ref:`pip` came later in 2008, as an alternative to :ref:`easy_install `, although still largely built on top of :ref:`setuptools` components. It was notable at the time for *not* installing packages as :term:`Eggs ` or from :term:`Eggs ` (but rather simply as 'flat' packages from :term:`sdists ` +and :term:`wheels ` +is to use the :ref:`build` tool with a command like +``python -m build`` +which triggers the generation of both distribution formats. +If necessary the ``--sdist`` and ``--wheel`` options can be used +to generate only one or the other. +Note that the build tool needs to be installed separately. + +The command ``python setup.py install`` was deprecated +in setuptools version *58.3.0*. + + +What about other commands? +========================== + +What are some replacements for the other ``python setup.py`` commands? + + +``python setup.py test`` +------------------------ + +The recommendation is to use a test runner such as pytest_. + +.. _pytest: https://docs.pytest.org/ + + +``python setup.py check``, ``python setup.py register``, and ``python setup.py upload`` +--------------------------------------------------------------------------------------- + +A trusted replacement is :ref:`twine`: + +* ``python -m twine check --strict dist/*`` +* ``python -m twine register dist/*.whl`` [#not-pypi]_ +* ``python -m twine upload dist/*`` + +.. [#not-pypi] Not necessary, nor supported on :term:`PyPI `. + But might be necessary on other :term:`package indexes ` (for example :ref:`devpi`). + + +``python setup.py --version`` +----------------------------- + +A possible replacement solution (among others) is to rely on setuptools-scm_: + +* ``python -m setuptools_scm`` + +.. _setuptools-scm: https://setuptools-scm.readthedocs.io/en/latest/usage#as-cli-tool + + +Remaining commands +------------------ + +This guide does not make suggestions of replacement solutions for those commands: + +.. hlist:: + :columns: 4 + + * ``alias`` + * ``bdist`` + * ``bdist_dumb`` + * ``bdist_egg`` + * ``bdist_rpm`` + * ``build`` + * ``build_clib`` + * ``build_ext`` + * ``build_py`` + * ``build_scripts`` + * ``clean`` + * ``dist_info`` + * ``easy_install`` + * ``editable_wheel`` + * ``egg_info`` + * ``install_data`` + * ``install_egg_info`` + * ``install_headers`` + * ``install_lib`` + * ``install_scripts`` + * ``rotate`` + * ``saveopts`` + * ``setopt`` + * ``upload_docs`` + + +What about custom commands? +=========================== + +Likewise, custom :file:`setup.py` commands are deprecated. +The recommendation is to migrate those custom commands +to a task runner tool or any other similar tool. +Some examples of such tools are: +chuy, make, nox or tox, pydoit, pyinvoke, taskipy, and thx. + + +What about custom build steps? +============================== + +Custom build steps that for example +either overwrite existing steps such as ``build_py``, ``build_ext``, and ``bdist_wheel`` +or add new build steps are not deprecated. +Those will be automatically called as expected. + + +Should ``setup.py`` be deleted? +=============================== + +Although the usage of :file:`setup.py` as an executable script is deprecated, +its usage as a configuration file for setuptools is absolutely fine. +There is likely no modification needed in :file:`setup.py`. + + +Is ``pyproject.toml`` mandatory? +================================ + +While it is not technically necessary yet, +it is **STRONGLY RECOMMENDED** for a project to have a :file:`pyproject.toml` file +at the root of its source tree with a content like this: + +.. code:: toml + + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + + +The guide :ref:`modernize-setup-py-project` has more details about this. + +The standard fallback behavior for a :term:`build frontend ` +in the absence of a :file:`pyproject.toml` file and its ``[build-system]`` table +is to assume that the :term:`build backend ` is setuptools. + + +Why? What does it all mean? +=========================== + +One way to look at it is that the scope of setuptools +has now been reduced to the role of a build backend. + + +Where to read more about this? +============================== + +* `Why you shouldn't invoke setup.py directly `__ by Paul Ganssle + +* :doc:`setuptools:deprecated/commands` diff --git a/source/discussions/single-source-version.rst b/source/discussions/single-source-version.rst new file mode 100644 index 000000000..c7dc8d1e1 --- /dev/null +++ b/source/discussions/single-source-version.rst @@ -0,0 +1,62 @@ +.. _single-source-version: + +=================================== +Single-sourcing the Project Version +=================================== + +:Page Status: Complete +:Last Reviewed: 2024-10-07 + +Many Python :term:`distribution packages ` publish a single +Python :term:`import package ` where it is desired that the runtime +``__version__`` attribute on the import package report the same version specifier +as :func:`importlib.metadata.version` reports for the distribution package +(as described in :ref:`runtime-version-access`). + +It is also frequently desired that this version information be derived from a version +control system *tag* (such as ``v1.2.3``) rather than being manually updated in the +source code. + +Some projects may choose to simply live with the data entry duplication, and rely +on automated testing to ensure the different values do not diverge. + +Alternatively, a project's chosen build system may offer a way to define a single +source of truth for the version number. + +In general, the options are: + +1) If the code is in a version control system (VCS), such as Git, then the version can be extracted from the VCS. + +2) The version can be hard-coded into the :file:`pyproject.toml` file -- and the build system can copy it + into other locations it may be required. + +3) The version string can be hard-coded into the source code -- either in a special purpose file, + such as :file:`_version.txt` (which must then be shipped as part of the project's source distribution + package), or as an attribute in a particular module, such as :file:`__init__.py`. The build + system can then extract it from the runtime location at build time. + +Consult your build system's documentation for their recommended method. + +When the intention is that a distribution package and its associated import package +share the same version, it is recommended that the project include an automated test +case that ensures ``import_name.__version__`` and ``importlib.metadata.version("dist-name")`` +report the same value (note: for many projects, ``import_name`` and ``dist-name`` will +be the same name). + + +.. _Build system version handling: + +Build System Version Handling +----------------------------- + +The following are links to some build system's documentation for handling version strings. + +* `Flit `_ + +* `Hatchling `_ + +* `PDM `_ + +* `Setuptools `_ + + - `setuptools_scm `_ diff --git a/source/discussions/src-layout-vs-flat-layout.rst b/source/discussions/src-layout-vs-flat-layout.rst index 2bf9fcdb9..c38968345 100644 --- a/source/discussions/src-layout-vs-flat-layout.rst +++ b/source/discussions/src-layout-vs-flat-layout.rst @@ -1,3 +1,5 @@ +.. _src-layout-vs-flat-layout: + ========================= src layout vs flat layout ========================= @@ -77,3 +79,27 @@ layout and the flat layout: ``tox.ini``) and packaging/tooling configuration files (eg: ``setup.py``, ``noxfile.py``) on the import path. This would make certain imports work in editable installations but not regular installations. + +.. _running-cli-from-source-src-layout: + +Running a command-line interface from source with src-layout +============================================================ + +Due to the firstly mentioned specialty of the src layout, a command-line +interface can not be run directly from the :term:`source tree `, +but requires installation of the package in +:doc:`Development Mode ` +for testing purposes. Since this can be unpractical in some situations, +a workaround could be to prepend the package folder to Python's +:py:data:`sys.path` when called via its :file:`__main__.py` file: + +.. code-block:: python + + import os + import sys + + if not __package__: + # Make CLI runnable from source tree with + # python src/package + package_source_path = os.path.dirname(os.path.dirname(__file__)) + sys.path.insert(0, package_source_path) diff --git a/source/discussions/versioning.rst b/source/discussions/versioning.rst new file mode 100644 index 000000000..eeea3578c --- /dev/null +++ b/source/discussions/versioning.rst @@ -0,0 +1,253 @@ +.. _versioning: +.. _`Choosing a versioning scheme`: + +========== +Versioning +========== + +This discussion covers all aspects of versioning Python packages. + + +Valid version numbers +===================== + +Different Python projects may use different versioning schemes based on the +needs of that particular project, but in order to be compatible with tools like +:ref:`pip`, all of them are required to comply with a flexible format for +version identifiers, for which the authoritative reference is the +:ref:`specification of version specifiers `. Here are some +examples of version numbers [#version-examples]_: + +- A simple version (final release): ``1.2.0`` +- A development release: ``1.2.0.dev1`` +- An alpha release: ``1.2.0a1`` +- A beta release: ``1.2.0b1`` +- A release candidate: ``1.2.0rc1`` +- A post-release: ``1.2.0.post1`` +- A post-release of an alpha release (possible, but discouraged): ``1.2.0a1.post1`` +- A simple version with only two components: ``23.12`` +- A simple version with just one component: ``42`` +- A version with an epoch: ``1!1.0`` + +Projects can use a cycle of pre-releases to support testing by their users +before a final release. In order, the steps are: alpha releases, beta releases, +release candidates, final release. Pip and other modern Python package +installers ignore pre-releases by default when deciding which versions of +dependencies to install, unless explicitly requested (e.g., with +``pip install pkg==1.1a3`` or ``pip install --pre pkg``). + +The purpose of development releases is to support releases made early during a +development cycle, for example, a nightly build, or a build from the latest +source in a Linux distribution. + +Post-releases are used to address minor errors in a final release that do not +affect the distributed software, such as correcting an error in the release +notes. They should not be used for bug fixes; these should be done with a new +final release (e.g., incrementing the third component when using semantic +versioning). + +Finally, epochs, a rarely used feature, serve to fix the sorting order when +changing the versioning scheme. For example, if a project is using calendar +versioning, with versions like 23.12, and switches to semantic versioning, with +versions like 1.0, the comparison between 1.0 and 23.12 will go the wrong way. +To correct this, the new version numbers should have an explicit epoch, as in +"1!1.0", in order to be treated as more recent than the old version numbers. + + + +Semantic versioning vs. calendar versioning +=========================================== + +A versioning scheme is a formalized way to interpret the segments of a version +number, and to decide which should be the next version number for a new release +of a package. Two versioning schemes are commonly used for Python packages, +semantic versioning and calendar versioning. + +.. caution:: + + The decision which version number to choose is up to a + project's maintainer. This effectively means that version + bumps reflect the maintainer's view. That view may differ + from the end-users' perception of what said formalized + versioning scheme promises them. + + There are known exceptions for selecting the next version + number. The maintainers may consciously choose to break the + assumption that the last version segment only contains + backwards-compatible changes. + One such case is when a security vulnerability needs to be + addressed. Security releases often come in patch versions + but contain breaking changes inevitably. + + +Semantic versioning +------------------- + +The idea of *semantic versioning* (or SemVer) is to use 3-part version numbers, +*major.minor.patch*, where the project author increments: + +- *major* when they make incompatible API changes, +- *minor* when they add functionality in a backwards-compatible manner, and +- *patch*, when they make backwards-compatible bug fixes. + +A majority of Python projects use a scheme that resembles semantic +versioning. However, most projects, especially larger ones, do not strictly +adhere to semantic versioning, since many changes are technically breaking +changes but affect only a small fraction of users. Such projects tend to +increment the major number when the incompatibility is high, or to signal a +shift in the project, rather than for any tiny incompatibility +[#semver-strictness]_. Conversely, a bump of the major version number +is sometimes used to signal significant but backwards-compatible new +features. + +For those projects that do use strict semantic versioning, this approach allows +users to make use of :ref:`compatible release version specifiers +`, with the ``~=`` operator. For +example, ``name ~= X.Y`` is roughly equivalent to ``name >= X.Y, == X.*``, i.e., +it requires at least release X.Y, and allows any later release with greater Y as +long as X is the same. Likewise, ``name ~= X.Y.Z`` is roughly equivalent to +``name >= X.Y.Z, == X.Y.*``, i.e., it requires at least X.Y.Z and allows a later +release with same X and Y but higher Z. + +Python projects adopting semantic versioning should abide by clauses 1-8 of the +`Semantic Versioning 2.0.0 specification `_. + +The popular :doc:`Sphinx ` documentation generator is an example +project that uses strict semantic versioning (:doc:`Sphinx versioning policy +`). The famous :doc:`NumPy ` +scientific computing package explicitly uses "loose" semantic versioning, where +releases incrementing the minor version can contain backwards-incompatible API +changes (:doc:`NumPy versioning policy `). + + +Calendar versioning +------------------- + +Semantic versioning is not a suitable choice for all projects, such as those +with a regular time-based release cadence and a deprecation process that +provides warnings for a number of releases prior to removal of a feature. + +A key advantage of date-based versioning, or `calendar versioning `_ +(CalVer), is that it is straightforward to tell how old the base feature set of +a particular release is given just the version number. + +Calendar version numbers typically take the form *year.month* (for example, +23.12 for December 2023). + +:doc:`Pip `, the standard Python package installer, uses calendar +versioning. + + +Other schemes +------------- + +Serial versioning refers to the simplest possible versioning scheme, which +consists of a single number incremented every release. While serial versioning +is very easy to manage as a developer, it is the hardest to track as an end +user, as serial version numbers convey little or no information regarding API +backwards compatibility. + +Combinations of the above schemes are possible. For example, a project may +combine date-based versioning with serial versioning to create a *year.serial* +numbering scheme that readily conveys the approximate age of a release, but +doesn't otherwise commit to a particular release cadence within the year. + + +Local version identifiers +========================= + +Public version identifiers are designed to support distribution via :term:`PyPI +`. Python packaging tools also support the notion +of a :ref:`local version identifier `, which can be +used to identify local development builds not intended for publication, or +modified variants of a release maintained by a redistributor. + +A local version identifier takes the form of a public version identifier, +followed by "+" and a local version label. For example, a package with +Fedora-specific patches applied could have the version "1.2.1+fedora.4". +Another example is versions computed by setuptools-scm_, a setuptools plugin +that reads the version from Git data. In a Git repository with some commits +since the latest release, setuptools-scm generates a version like +"0.5.dev1+gd00980f", or if the repository has untracked changes, like +"0.5.dev1+gd00980f.d20231217". + +.. _runtime-version-access: + +Accessing version information at runtime +======================================== + +Version information for all :term:`distribution packages ` +that are locally available in the current environment can be obtained at runtime +using the standard library's :func:`importlib.metadata.version` function:: + + >>> importlib.metadata.version("cryptography") + '41.0.7' + +Many projects also choose to version their top level +:term:`import packages ` by providing a package level +``__version__`` attribute:: + + >>> import cryptography + >>> cryptography.__version__ + '41.0.7' + +This technique can be particularly valuable for CLI applications which want +to ensure that version query invocations (such as ``pip -V``) run as quickly +as possible. + +Package publishers wishing to ensure their reported distribution package and +import package versions are consistent with each other can review the +:ref:`single-source-version` discussion for potential approaches to doing so. + +As import packages and modules are not *required* to publish runtime +version information in this way (see the withdrawn proposal in +:pep:`PEP 396 <396>`), the ``__version__`` attribute should either only be +queried with interfaces that are known to provide it (such as a project +querying its own version or the version of one of its direct dependencies), +or else the querying code should be designed to handle the case where the +attribute is missing [#fallback-to-dist-version]_. + +Some projects may need to publish version information for external APIs +that aren't the version of the module itself. Such projects should +define their own project-specific ways of obtaining the relevant information +at runtime. For example, the standard library's :mod:`ssl` module offers +multiple ways to access the underlying OpenSSL library version:: + + >>> ssl.OPENSSL_VERSION + 'OpenSSL 3.2.2 4 Jun 2024' + >>> ssl.OPENSSL_VERSION_INFO + (3, 2, 0, 2, 0) + >>> hex(ssl.OPENSSL_VERSION_NUMBER) + '0x30200020' + +-------------------------------------------------------------------------------- + +.. [#version-examples] Some more examples of unusual version numbers are + given in a `blog post `_ by Seth Larson. + +.. [#semver-strictness] For some personal viewpoints on this issue, see these + blog posts: `by Hynek Schlawak `_, `by Donald Stufft + `_, `by BernΓ‘t GΓ‘bor `_, `by + Brett Cannon `_. For a humoristic take, read about + ZeroVer_. + +.. [#fallback-to-dist-version] A full list mapping the top level names available + for import to the distribution packages that provide those import packages and + modules may be obtained through the standard library's + :func:`importlib.metadata.packages_distributions` function. This means that + even code that is attempting to infer a version to report for all importable + top-level names has a means to fall back to reporting the distribution + version information if no ``__version__`` attribute is defined. Only standard + library modules, and modules added via means other than Python package + installation would fail to have version information reported in that case. + + +.. _zerover: https://0ver.org +.. _calver: https://calver.org +.. _semver: https://semver.org +.. _semver-bernat-gabor: https://bernat.tech/posts/version-numbers/ +.. _semver-brett-cannon: https://snarky.ca/why-i-dont-like-semver/ +.. _semver-donald-stufft: https://caremad.io/posts/2016/02/versioning-software/ +.. _semver-hynek-schlawack: https://hynek.me/articles/semver-will-not-save-you/ +.. _setuptools-scm: https://setuptools-scm.readthedocs.io +.. _versions-seth-larson: https://sethmlarson.dev/pep-440 diff --git a/source/discussions/wheel-vs-egg.rst b/source/discussions/wheel-vs-egg.rst deleted file mode 100644 index 8368fd2c0..000000000 --- a/source/discussions/wheel-vs-egg.rst +++ /dev/null @@ -1,49 +0,0 @@ -.. _`Wheel vs Egg`: - -============ -Wheel vs Egg -============ - -:term:`Wheel` and :term:`Egg` are both packaging formats that aim to support the -use case of needing an install artifact that doesn't require building or -compilation, which can be costly in testing and production workflows. - -The :term:`Egg` format was introduced by :ref:`setuptools` in 2004, whereas the -:term:`Wheel` format was introduced by :pep:`427` in 2012. - -:term:`Wheel` is currently considered the standard for :term:`built ` and :term:`binary ` packaging for Python. - -Here's a breakdown of the important differences between :term:`Wheel` and :term:`Egg`. - - -* :term:`Wheel` has an :pep:`official PEP <427>`. :term:`Egg` did not. - -* :term:`Wheel` is a :term:`distribution ` format, i.e a packaging - format. [1]_ :term:`Egg` was both a distribution format and a runtime - installation format (if left zipped), and was designed to be importable. - -* :term:`Wheel` archives do not include .pyc files. Therefore, when the - distribution only contains Python files (i.e. no compiled extensions), and is - compatible with Python 2 and 3, it's possible for a wheel to be "universal", - similar to an :term:`sdist `. - -* :term:`Wheel` uses :pep:`PEP376-compliant <376>` ``.dist-info`` - directories. Egg used ``.egg-info``. - -* :term:`Wheel` has a :pep:`richer file naming convention <425>`. A single - wheel archive can indicate its compatibility with a number of Python language - versions and implementations, ABIs, and system architectures. - -* :term:`Wheel` is versioned. Every wheel file contains the version of the wheel - specification and the implementation that packaged it. - -* :term:`Wheel` is internally organized by `sysconfig path type - `_, - therefore making it easier to convert to other formats. - ----- - -.. [1] Circumstantially, in some cases, wheels can be used as an importable - runtime format, although :pep:`this is not officially supported at this time - <427#is-it-possible-to-import-python-code-directly-from-a-wheel-file>`. diff --git a/source/flow.rst b/source/flow.rst index 8a7d390b8..947c399db 100644 --- a/source/flow.rst +++ b/source/flow.rst @@ -82,9 +82,12 @@ For example, here is a table for using :ref:`hatch`: requires = ["hatchling"] build-backend = "hatchling.build" -With such a table in the :file:`pyproject.toml` file, a "frontend" tool like -:ref:`build` can run your chosen build tool's "backend" to create the build -artifacts. Your build tool may also provide its own frontend. An install tool +With such a table in the :file:`pyproject.toml` file, +a ":term:`frontend `" tool like +:ref:`build` can run your chosen +build tool's ":term:`backend `" +to create the build artifacts. +Your build tool may also provide its own frontend. An install tool like :ref:`pip` also acts as a frontend when it runs your build tool's backend to install from a source distribution. @@ -93,11 +96,13 @@ required in the :file:`pyproject.toml` file. For example, you might specify: * a ``[project]`` table containing project :doc:`Core Metadata ` - (name, version, author and so forth); see - :doc:`Declaring project metadata ` - for more detail + (name, version, author and so forth), + +* a ``[tool]`` table containing tool-specific configuration options. + +Refer to the :ref:`pyproject.toml guide ` for a +complete guide to ``pyproject.toml`` configuration. -* a ``[tool]`` table containing tool-specific configuration options Build artifacts =============== diff --git a/source/glossary.rst b/source/glossary.rst index 0d740bf39..40c041f4c 100644 --- a/source/glossary.rst +++ b/source/glossary.rst @@ -12,16 +12,79 @@ Glossary extensions. + Build Backend + + A library that takes a source tree + and builds a :term:`source distribution ` or + :term:`built distribution ` from it. + The build is delegated to the backend by a + :term:`frontend `. + All backends offer a standardized interface. + + Examples of build backends are + :ref:`flit's flit-core `, + :ref:`hatch's hatchling `, + :ref:`maturin`, + :ref:`meson-python`, + :ref:`scikit-build-core`, + and :ref:`setuptools`. + + + Build Frontend + + A tool that users might run + that takes arbitrary source trees or + :term:`source distributions ` + and builds source distributions or :term:`wheels ` from them. + The actual building is delegated to each source tree's + :term:`build backend `. + + Examples of build frontends are :ref:`pip` and :ref:`build`. + + Built Distribution A :term:`Distribution ` format containing files and metadata that only need to be moved to the correct location on the target system, to be installed. :term:`Wheel` is such a format, whereas - distutil's :term:`Source Distribution ` is not, in that it requires a build step before it can be installed. This format does not imply that Python files have to be precompiled (:term:`Wheel` intentionally does not include compiled - Python files). + Python files). See :ref:`package-formats` for more information. + + + Built Metadata + + The concrete form :term:`Core Metadata` takes + when included inside an installed :term:`Project` (``METADATA`` file) + or a :term:`Distribution Archive` + (``PKG-INFO`` in a + :term:`Sdist ` + and ``METADATA`` in a :term:`Wheel`). + + + Core Metadata + + The :ref:`specification ` + and the set of :term:`Core Metadata Field`\s it defines + that describe key static attributes of + a :term:`Distribution Package` or :term:`Installed Project`. + + + Core Metadata Field + + A single key-value pair + (or sequence of such with the same name, for multiple-use fields) + defined in the :term:`Core Metadata` spec + and stored in the :term:`Built Metadata`. + Notably, distinct from a :term:`Pyproject Metadata Key`. + + + Distribution Archive + + The physical distribution artifact (i.e. a file on disk) + for a :term:`Distribution Package`. Distribution Package @@ -37,14 +100,14 @@ Glossary :term:`Import Package` (which is also commonly called a "package") or another kind of distribution (e.g. a Linux distribution or the Python language distribution), which are often referred to with the single term - "distribution". + "distribution". See :ref:`distribution-package-vs-import-package` + for a breakdown of the differences. Egg A :term:`Built Distribution` format introduced by :ref:`setuptools`, - which is being replaced by :term:`Wheel`. For details, see ` - :doc:`The Internal Structure of Python Eggs ` and - `Python Eggs `_ + which has been replaced by :term:`Wheel`. For details, see + :ref:`egg-format`. Extension Module @@ -56,6 +119,25 @@ Glossary extensions. + Import Package + + A Python module which can contain other modules or recursively, other + packages. + + An import package is more commonly referred to with the single word + "package", but this guide will use the expanded term when more clarity + is needed to prevent confusion with a :term:`Distribution Package` which + is also commonly called a "package". See :ref:`distribution-package-vs-import-package` + for a breakdown of the differences. + + + Installed Project + + A :term:`Project` that is installed for use with + a Python interpreter or :term:`Virtual Environment`, + as described in the specification :ref:`recording-installed-packages`. + + Known Good Set (KGS) A set of distributions at specified versions which are compatible with @@ -65,15 +147,39 @@ Glossary multiple individual distributions. - Import Package + License Classifier - A Python module which can contain other modules or recursively, other - packages. + A PyPI Trove classifier + (as :ref:`described ` + in the :term:`Core Metadata` specification) + which begins with ``License ::``. + + + License Expression + SPDX Expression + + A string with valid SPDX license expression syntax, + including one or more SPDX :term:`License Identifier`\(s), + which describes a :term:`Distribution Archive`'s license(s) + and how they inter-relate. + Examples: + ``GPL-3.0-or-later``, + ``MIT AND (Apache-2.0 OR BSD-2-Clause)`` + + + License Identifier + SPDX Identifier + + A valid SPDX short-form license identifier, + originally specified in :pep:`639`. + This includes all valid SPDX identifiers and + the custom ``LicenseRef-[idstring]`` strings conforming to the + SPDX specification. + Examples: + ``MIT``, + ``GPL-3.0-only``, + ``LicenseRef-My-Custom-License`` - An import package is more commonly referred to with the single word - "package", but this guide will use the expanded term when more clarity - is needed to prevent confusion with a :term:`Distribution Package` which - is also commonly called a "package". Module @@ -118,19 +224,70 @@ Glossary 'bar'. + Project Root Directory + + The filesystem directory in which + a :term:`Project`'s :term:`source tree ` is located. + + + Project Source Tree + + The on-disk format of a :term:`Project` used for development, + containing its raw source code before being packaged + into a + :term:`Source Distribution ` + or :term:`Built Distribution`. + + + Project Source Metadata + + Metadata defined by the package author + in a :term:`Project`'s :term:`source tree `, + to be transformed into :term:`Core Metadata field`\s + in the :term:`Built Metadata` + by the project's :term:`build backend `. + Can be written as :term:`Pyproject Metadata`, + or in a tool-specific format + (under the ``[tool]`` table in ``pyproject.toml``, + or in a tool's own configuration file). + + Pure Module A :term:`Module` written in Python and contained in a single ``.py`` file (and possibly associated ``.pyc`` and/or ``.pyo`` files). + Pyproject Metadata + + The :term:`Project Source Metadata` format + defined by the :ref:`declaring-project-metadata` specification + and originally introduced in :pep:`621`, + stored as :term:`Pyproject Metadata Key`\s + under the ``[project]`` table of a :term:`pyproject.toml` file. + Notably, *not* a tool-specific source metadata format + under the ``[tool]`` table in ``pyproject.toml``. + + + Pyproject Metadata Key + + A top-level TOML key in the ``[project]`` table in ``pyproject.toml``; + part of the :term:`Pyproject Metadata`. + Notably, distinct from a :term:`Core Metadata Field`. + + + Pyproject Metadata Subkey + + A second-level TOML key under a table-valued + :term:`Pyproject Metadata Key`. + + Python Packaging Authority (PyPA) PyPA is a working group that maintains many of the relevant projects in Python packaging. They maintain a site at :doc:`pypa.io `, host projects on `GitHub - `_ and `Bitbucket - `_, and discuss issues on the + `_, and discuss issues on the `distutils-sig mailing list `_ and `the Python Discourse forum `__. @@ -177,10 +334,8 @@ Glossary Requirement Specifier A format used by :ref:`pip` to install packages from a :term:`Package - Index`. For an EBNF diagram of the format, see the - `pkg_resources.Requirement - `_ - entry in the :ref:`setuptools` docs. For example, "foo>=1.3" is a + Index`. For an EBNF diagram of the format, see :ref:`dependency-specifiers`. + For example, "foo>=1.3" is a requirement specifier, where "foo" is the project name, and the ">=1.3" portion is the :term:`Version Specifier` @@ -191,6 +346,23 @@ Glossary docs on :ref:`pip:Requirements Files`. + Root License Directory + License Directory + + The directory under which license files are stored in a + :term:`Project Source Tree`, :term:`Distribution Archive` + or :term:`Installed Project`. + For a :term:`Project Source Tree` or + :term:`Source Distribution (or "sdist")`, this is the + :term:`Project Root Directory`. + For a :term:`Built Distribution` or :term:`Installed Project`, + this is the :file:`.dist-info/licenses/` directory of + the wheel archive or project folder respectively. + Also, the root directory that paths + recorded in the ``License-File`` + :term:`Core Metadata Field` are relative to. + + setup.py setup.cfg @@ -207,10 +379,11 @@ Glossary Source Distribution (or "sdist") - A :term:`distribution ` format (usually generated - using ``python setup.py sdist``) that provides metadata and the + A :term:`distribution ` format (usually generated + using ``python -m build --sdist``) that provides metadata and the essential source files needed for installing by a tool like :ref:`pip`, - or for generating a :term:`Built Distribution`. + or for generating a :term:`Built Distribution`. See :ref:`package-formats` + for more information. System Package @@ -222,11 +395,10 @@ Glossary Version Specifier The version component of a :term:`Requirement Specifier`. For example, - the ">=1.3" portion of "foo>=1.3". :pep:`440` contains - a :pep:`full specification - <440#version-specifiers>` of the - specifiers that Python packaging currently supports. Support for PEP440 - was implemented in :ref:`setuptools` v8.0 and :ref:`pip` v6.0. + the ">=1.3" portion of "foo>=1.3". Read the :ref:`Version specifier specification + ` for a full description of the + specifiers that Python packaging currently supports. Support for this + specification was implemented in :ref:`setuptools` v8.0 and :ref:`pip` v6.0. Virtual Environment @@ -235,11 +407,22 @@ Glossary wide. For more information, see the section on :ref:`Creating and using Virtual Environments`. + + Wheel Format Wheel - A :term:`Built Distribution` format introduced by :pep:`427`, - which is intended to replace the :term:`Egg` format. Wheel is currently - supported by :ref:`pip`. + The standard :term:`Built Distribution` format + originally introduced in :pep:`427` + and defined by the :ref:`binary-distribution-format` specification. + See :ref:`package-formats` for more information. + Not to be confused with its reference implementation, + the :term:`Wheel Project`. + + + Wheel Project + + The PyPA reference implementation of the :term:`Wheel Format`; see :ref:`wheel`. + Working Set diff --git a/source/guides/analyzing-pypi-package-downloads.rst b/source/guides/analyzing-pypi-package-downloads.rst index 55bdf370b..2ad02fed5 100644 --- a/source/guides/analyzing-pypi-package-downloads.rst +++ b/source/guides/analyzing-pypi-package-downloads.rst @@ -1,3 +1,5 @@ +.. _analyzing-pypi-package-downloads: + ================================ Analyzing PyPI package downloads ================================ @@ -7,9 +9,6 @@ to learn more about downloads of a package (or packages) hosted on PyPI. For example, you can use it to discover the distribution of Python versions used to download a package. -.. contents:: Contents - :local: - Background ========== @@ -35,7 +34,7 @@ PyPI does not display download statistics for a number of reasons: [#]_ doesn't mean it's good; Similarly just because a project hasn't been downloaded a lot doesn't mean it's bad! -In short, because it's value is low for various reasons, and the tradeoffs +In short, because its value is low for various reasons, and the tradeoffs required to make it work are high, it has been not an effective use of limited resources. @@ -51,7 +50,7 @@ Getting set up In order to use `Google BigQuery`_ to query the `public PyPI download statistics dataset`_, you'll need a Google account and to enable the BigQuery -API on a Google Cloud Platform project. You can run the up to 1TB of queries +API on a Google Cloud Platform project. You can run up to 1TB of queries per month `using the BigQuery free tier without a credit card `__ @@ -94,7 +93,7 @@ Useful queries Run queries in the `BigQuery web UI`_ by clicking the "Compose query" button. -Note that the rows are stored in a partitioned, which helps +Note that the rows are stored in a partitioned table, which helps limit the cost of queries. These example queries analyze downloads from recent history by filtering on the ``timestamp`` column. @@ -104,7 +103,7 @@ Counting package downloads The following query counts the total number of downloads for the project "pytest". -:: +.. code-block:: sql #standardSQL SELECT COUNT(*) AS num_downloads @@ -121,10 +120,10 @@ The following query counts the total number of downloads for the project | 26190085 | +---------------+ -To only count downloads from pip, filter on the ``details.installer.name`` +To count downloads from pip only, filter on the ``details.installer.name`` column. -:: +.. code-block:: sql #standardSQL SELECT COUNT(*) AS num_downloads @@ -148,7 +147,7 @@ Package downloads over time To group by monthly downloads, use the ``TIMESTAMP_TRUNC`` function. Also filtering by this column reduces corresponding costs. -:: +.. code-block:: sql #standardSQL SELECT @@ -186,7 +185,7 @@ Python versions over time Extract the Python version from the ``details.python`` column. Warning: This query processes over 500 GB of data. -:: +.. code-block:: sql #standardSQL SELECT @@ -217,6 +216,43 @@ query processes over 500 GB of data. | 3.5 | 1894153540 | +--------+---------------+ + +Getting absolute links to artifacts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It's sometimes helpful to be able to get the absolute links to download +artifacts from PyPI based on their hashes, e.g. if a particular project or +release has been deleted from PyPI. The metadata table includes the ``path`` +column, which includes the hash and artifact filename. + +.. note:: + The URL generated here is not guaranteed to be stable, but currently aligns with the URL where PyPI artifacts are hosted. + +.. code-block:: sql + + SELECT + CONCAT('https://files.pythonhosted.org/packages', path) as url + FROM + `bigquery-public-data.pypi.distribution_metadata` + WHERE + filename LIKE 'sampleproject%' + + ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| url | ++===================================================================================================================================================================+ +| https://files.pythonhosted.org/packages/eb/45/79be82bdeafcecb9dca474cad4003e32ef8e4a0dec6abbd4145ccb02abe1/sampleproject-1.2.0.tar.gz | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| https://files.pythonhosted.org/packages/56/0a/178e8bbb585ec5b13af42dae48b1d7425d6575b3ff9b02e5ec475e38e1d6/sampleproject_nomura-1.2.0-py2.py3-none-any.whl | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| https://files.pythonhosted.org/packages/63/88/3200eeaf22571f18d2c41e288862502e33365ccbdc12b892db23f51f8e70/sampleproject_nomura-1.2.0.tar.gz | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| https://files.pythonhosted.org/packages/21/e9/2743311822e71c0756394b6c5ab15cb64ca66c78c6c6a5cd872c9ed33154/sampleproject_doubleyoung18-1.3.0-py2.py3-none-any.whl | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| https://files.pythonhosted.org/packages/6f/5b/2f3fe94e1c02816fe23c7ceee5292fb186912929e1972eee7fb729fa27af/sampleproject-1.3.1.tar.gz | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + Caveats ======= @@ -274,7 +310,7 @@ Install `pypinfo`_ using pip. .. code-block:: bash - python -m pip install pypinfo + python3 -m pip install pypinfo Usage: @@ -306,7 +342,7 @@ References .. _public PyPI download statistics dataset: https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=pypi&page=dataset .. _Google BigQuery: https://cloud.google.com/bigquery .. _BigQuery web UI: https://console.cloud.google.com/bigquery -.. _pypinfo: https://github.com/ofek/pypinfo/blob/master/README.rst +.. _pypinfo: https://github.com/ofek/pypinfo .. _google-cloud-bigquery: https://cloud.google.com/bigquery/docs/reference/libraries .. _pandas-gbq: https://pandas-gbq.readthedocs.io/en/latest/ .. _Pandas: https://pandas.pydata.org/ diff --git a/source/guides/creating-and-discovering-plugins.rst b/source/guides/creating-and-discovering-plugins.rst index 2928d22e2..601f2b4a6 100644 --- a/source/guides/creating-and-discovering-plugins.rst +++ b/source/guides/creating-and-discovering-plugins.rst @@ -41,18 +41,17 @@ then ``discovered_plugins`` would be: .. code-block:: python { - 'flask_sqlachemy': , + 'flask_sqlalchemy': , 'flask_talisman': , } -Using naming convention for plugins also allows you to query the -Python Package Index's `simple API`_ for all packages that conform to your -naming convention. +Using naming convention for plugins also allows you to query +the Python Package Index's :ref:`simple repository API ` +for all packages that conform to your naming convention. .. _Flask: https://pypi.org/project/Flask/ .. _Flask-SQLAlchemy: https://pypi.org/project/Flask-SQLAlchemy/ .. _Flask-Talisman: https://pypi.org/project/flask-talisman -.. _simple API: https://www.python.org/dev/peps/pep-0503/#specification Using namespace packages @@ -117,27 +116,25 @@ a list of packages to :func:`setup`'s ``packages`` argument instead of using :doc:`packaging-namespace-packages` documentation and clearly document which approach is preferred for plugins to your project. +.. _plugin-entry-points: + Using package metadata ====================== -:doc:`Setuptools ` provides :doc:`special support -` for plugins. By providing the -``entry_points`` argument to :func:`setup` in :file:`setup.py` plugins can -register themselves for discovery. +Packages can have metadata for plugins described in the :ref:`entry-points`. +By specifying them, a package announces that it contains a specific kind of plugin. +Another package supporting this kind of plugin can use the metadata to discover that plugin. For example if you have a package named ``myapp-plugin-a`` and it includes -in its :file:`setup.py`: +the following in its ``pyproject.toml``: -.. code-block:: python +.. code-block:: toml - setup( - ... - entry_points={'myapp.plugins': 'a = myapp_plugin_a'}, - ... - ) + [project.entry-points.'myapp.plugins'] + a = 'myapp_plugin_a' Then you can discover and load all of the registered entry points by using -:func:`importlib.metadata.entry_points` (or the `backport`_ +:func:`importlib.metadata.entry_points` (or the backport_ ``importlib_metadata >= 3.6`` for Python 3.6-3.9): .. code-block:: python diff --git a/source/guides/creating-command-line-tools.rst b/source/guides/creating-command-line-tools.rst new file mode 100644 index 000000000..cbe8b3bb0 --- /dev/null +++ b/source/guides/creating-command-line-tools.rst @@ -0,0 +1,183 @@ +.. _creating-command-line-tools: + +========================================= +Creating and packaging command-line tools +========================================= + +This guide will walk you through creating and packaging a standalone command-line application +that can be installed with :ref:`pipx`, a tool for creating and managing :term:`Python Virtual Environments ` +and exposing the executable scripts of packages (and available manual pages) for use on the command-line. + +Creating the package +==================== + +First of all, create a source tree for the :term:`project `. For the sake of an example, we'll +build a simple tool outputting a greeting (a string) for a person based on arguments given on the command-line. + +.. todo:: Advise on the optimal structure of a Python package in another guide or discussion and link to it here. + +This project will adhere to :ref:`src-layout ` and in the end be alike this file tree, +with the top-level folder and package name ``greetings``: + +:: + + . + β”œβ”€β”€ pyproject.toml + └── src + └── greetings + β”œβ”€β”€ cli.py + β”œβ”€β”€ greet.py + β”œβ”€β”€ __init__.py + └── __main__.py + +The actual code responsible for the tool's functionality will be stored in the file :file:`greet.py`, +named after the main module: + +.. code-block:: python + + import typer + from typing_extensions import Annotated + + + def greet( + name: Annotated[str, typer.Argument(help="The (last, if --title is given) name of the person to greet")] = "", + title: Annotated[str, typer.Option(help="The preferred title of the person to greet")] = "", + doctor: Annotated[bool, typer.Option(help="Whether the person is a doctor (MD or PhD)")] = False, + count: Annotated[int, typer.Option(help="Number of times to greet the person")] = 1 + ): + greeting = "Greetings, " + if doctor and not title: + title = "Dr." + if not name: + if title: + name = title.lower().rstrip(".") + else: + name = "friend" + if title: + greeting += f"{title} " + greeting += f"{name}!" + for i in range(0, count): + print(greeting) + +The above function receives several keyword arguments that determine how the greeting to output is constructed. +Now, construct the command-line interface to provision it with the same, which is done +in :file:`cli.py`: + +.. code-block:: python + + import typer + + from .greet import greet + + + app = typer.Typer() + app.command()(greet) + + + if __name__ == "__main__": + app() + +The command-line interface is built with typer_, an easy-to-use CLI parser based on Python type hints. It provides +auto-completion and nicely styled command-line help out of the box. Another option would be :py:mod:`argparse`, +a command-line parser which is included in Python's standard library. It is sufficient for most needs, but requires +a lot of code, usually in ``cli.py``, to function properly. Alternatively, docopt_ makes it possible to create CLI +interfaces based solely on docstrings; advanced users are encouraged to make use of click_ (on which ``typer`` is based). + +Now, add an empty :file:`__init__.py` file, to define the project as a regular :term:`import package `. + +The file :file:`__main__.py` marks the main entry point for the application when running it via :mod:`runpy` +(i.e. ``python -m greetings``, which works immediately with flat layout, but requires installation of the package with src layout), +so initialize the command-line interface here: + +.. code-block:: python + + if __name__ == "__main__": + from greetings.cli import app + app() + +.. note:: + + In order to enable calling the command-line interface directly from the :term:`source tree `, + i.e. as ``python src/greetings``, a certain hack could be placed in this file; read more at + :ref:`running-cli-from-source-src-layout`. + + +``pyproject.toml`` +------------------ + +The project's :term:`metadata ` is placed in :term:`pyproject.toml`. The :term:`pyproject metadata keys ` and the ``[build-system]`` table may be filled in as described in :ref:`writing-pyproject-toml`, adding a dependency +on ``typer`` (this tutorial uses version *0.12.3*). + +For the project to be recognised as a command-line tool, additionally a ``console_scripts`` :ref:`entry point ` (see :ref:`console_scripts`) needs to be added as a :term:`subkey `: + +.. code-block:: toml + + [project.scripts] + greet = "greetings.cli:app" + +Now, the project's source tree is ready to be transformed into a :term:`distribution package `, +which makes it installable. + + +Installing the package with ``pipx`` +==================================== + +After installing ``pipx`` as described in :ref:`installing-stand-alone-command-line-tools`, install your project: + +.. code-block:: console + + $ cd path/to/greetings/ + $ pipx install . + +This will expose the executable script we defined as an entry point and make the command ``greet`` available. +Let's test it: + +.. code-block:: console + + $ greet + Greetings, friend! + $ greet --doctor Brennan + Greetings, Dr. Brennan! + $ greet --title Ms. Parks + Greetings, Ms. Parks! + $ greet --title Mr. + Greetings, Mr. mr! + +Since this example uses ``typer``, you could now also get an overview of the program's usage by calling it with +the ``--help`` option, or configure completions via the ``--install-completion`` option. + +To just run the program without installing it permanently, use ``pipx run``, which will create a temporary +(but cached) virtual environment for it: + +.. code-block:: console + + $ pipx run --spec . greet --doctor + +This syntax is a bit impractical, however; as the name of the entry point we defined above does not match the package name, +we need to state explicitly which executable script to run (even though there is only one in existence). + +There is, however, a more practical solution to this problem, in the form of an entry point specific to ``pipx run``. +The same can be defined as follows in :file:`pyproject.toml`: + +.. code-block:: toml + + [project.entry-points."pipx.run"] + greetings = "greetings.cli:app" + + +Thanks to this entry point (which *must* match the package name), ``pipx`` will pick up the executable script as the +default one and run it, which makes this command possible: + +.. code-block:: console + + $ pipx run . --doctor + +Conclusion +========== + +You know by now how to package a command-line application written in Python. A further step could be to distribute your package, +meaning uploading it to a :term:`package index `, most commonly :term:`PyPI `. To do that, follow the instructions at :ref:`Packaging your project`. And once you're done, don't forget to :ref:`do some research ` on how your package is received! + +.. _click: https://click.palletsprojects.com/ +.. _docopt: https://docopt.readthedocs.io/en/latest/ +.. _typer: https://typer.tiangolo.com/ diff --git a/source/guides/distributing-packages-using-setuptools.rst b/source/guides/distributing-packages-using-setuptools.rst index 057393c0b..bf4227aae 100644 --- a/source/guides/distributing-packages-using-setuptools.rst +++ b/source/guides/distributing-packages-using-setuptools.rst @@ -4,6 +4,9 @@ Packaging and distributing projects =================================== +:Page Status: Outdated +:Last Reviewed: 2023-12-14 + This section covers some additional details on configuring, packaging and distributing Python projects with ``setuptools`` that aren't covered by the introductory tutorial in :doc:`/tutorials/packaging-projects`. It still assumes @@ -15,12 +18,10 @@ development as a whole. For example, it does not provide guidance or tool recommendations for version control, documentation, or testing. For more reference material, see :std:doc:`Building and Distributing -Packages ` in the :ref:`setuptools` docs, but note +Packages ` in the :ref:`setuptools` docs, but note that some advisory content there may be outdated. In the event of conflicts, prefer the advice in the Python Packaging User Guide. -.. contents:: Contents - :local: Requirements for packaging and distributing @@ -59,7 +60,7 @@ setup.py The most important file is :file:`setup.py` which exists at the root of your project directory. For an example, see the `setup.py -`_ in the `PyPA +`_ in the `PyPA sample project `_. :file:`setup.py` serves two primary functions: @@ -72,7 +73,7 @@ sample project `_. 2. It's the command line interface for running various commands that relate to packaging tasks. To get a listing of available commands, run - ``python setup.py --help-commands``. + ``python3 setup.py --help-commands``. setup.cfg @@ -80,7 +81,7 @@ setup.cfg :file:`setup.cfg` is an ini file that contains option defaults for :file:`setup.py` commands. For an example, see the `setup.cfg -`_ in the `PyPA +`_ in the `PyPA sample project `_. @@ -155,207 +156,21 @@ As mentioned above, the primary feature of :file:`setup.py` is that it contains a global ``setup()`` function. The keyword arguments to this function are how specific details of your project are defined. -The most relevant arguments are explained below. Most of the snippets given are +Some are temporarily explained below until their information is moved elsewhere. +The full list can be found :doc:`in the setuptools documentation +`. + +Most of the snippets given are taken from the `setup.py -`_ contained in the +`_ contained in the `PyPA sample project `_. -.. _`setup() name`: - -``name`` -~~~~~~~~ - -:: - - name='sample', - -This is the name of your project, determining how your project is listed on -:term:`PyPI `. Per :pep:`508`, valid project -names must: - -- Consist only of ASCII letters, digits, underscores (``_``), hyphens (``-``), - and/or periods (``.``), and -- Start & end with an ASCII letter or digit. - -Comparison of project names is case insensitive and treats arbitrarily-long -runs of underscores, hyphens, and/or periods as equal. For example, if you -register a project named ``cool-stuff``, users will be able to download it or -declare a dependency on it using any of the following spellings:: - - Cool-Stuff - cool.stuff - COOL_STUFF - CoOl__-.-__sTuFF - - -``version`` -~~~~~~~~~~~ - -:: - - version='1.2.0', - -This is the current version of your project, allowing your users to determine whether or not -they have the latest version, and to indicate which specific versions they've tested their own -software against. - -Versions are displayed on :term:`PyPI ` for each release if you -publish your project. See :ref:`Choosing a versioning scheme` for more information on ways to use versions to convey compatibility information to your users. -If the project code itself needs run-time access to the version, the simplest -way is to keep the version in both :file:`setup.py` and your code. If you'd -rather not duplicate the value, there are a few ways to manage this. See the -":ref:`Single sourcing the version`" Advanced Topics section. - -.. _`description`: - -``description`` -~~~~~~~~~~~~~~~ - -:: - - description='A sample Python project', - long_description=long_description, - long_description_content_type='text/x-rst', - -Give a short and long description for your project. - -These values will be displayed on :term:`PyPI ` -if you publish your project. On ``pypi.org``, the user interface displays -``description`` in the grey banner and ``long_description`` in the section -named "Project Description". - -``description`` is also displayed in lists of projects. For example, it's -visible in the search results pages such as https://pypi.org/search/?q=jupyter, -the front-page lists of trending projects and new releases, and the list of -projects you maintain within your account profile (such as -https://pypi.org/user/jaraco/). - -A `content type -`_ -can be specified with the ``long_description_content_type`` argument, which can -be one of ``text/plain``, ``text/x-rst``, or ``text/markdown``, corresponding -to no formatting, `reStructuredText (reST) -`_, -and the Github-flavored Markdown dialect of `Markdown -`_ respectively. - -``url`` -~~~~~~~ - -:: - - url='https://github.com/pypa/sampleproject', - - -Give a homepage URL for your project. - - -``author`` -~~~~~~~~~~ - -:: - - author='A. Random Developer', - author_email='author@example.com', - -Provide details about the author. - - -``license`` -~~~~~~~~~~~ - -:: - - license='MIT', - -The ``license`` argument doesn't have to indicate the license under -which your package is being released, although you may optionally do -so if you want. If you're using a standard, well-known license, then -your main indication can and should be via the ``classifiers`` -argument. Classifiers exist for all major open-source licenses. -The ``license`` argument is more typically used to indicate differences -from well-known licenses, or to include your own, unique license. As a -general rule, it's a good idea to use a standard, well-known license, -both to avoid confusion and because some organizations avoid software -whose license is unapproved. - - -``classifiers`` -~~~~~~~~~~~~~~~ - -:: - - classifiers=[ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - 'Development Status :: 3 - Alpha', - - # Indicate who your project is intended for - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Build Tools', - - # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: MIT License', - - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - -Provide a list of classifiers that categorize your project. For a full listing, -see https://pypi.org/classifiers/. - -Although the list of classifiers is often used to declare what Python versions -a project supports, this information is only used for searching & browsing -projects on PyPI, not for installing projects. To actually restrict what -Python versions a project can be installed on, use the :ref:`python_requires` -argument. - -To prevent a package from being uploaded to PyPI, use the special -``'Private :: Do Not Upload'`` classifier. PyPI will always reject packages with -classifiers beginning with ``"Private ::'``. - - -``keywords`` -~~~~~~~~~~~~ - -:: - - keywords='sample setuptools development', - -List keywords that describe your project. - - -``project_urls`` -~~~~~~~~~~~~~~~~ - -:: - - project_urls={ - 'Documentation': 'https://packaging.python.org/tutorials/distributing-packages/', - 'Funding': 'https://donate.pypi.org', - 'Say Thanks!': 'http://saythanks.io/to/example', - 'Source': 'https://github.com/pypa/sampleproject/', - 'Tracker': 'https://github.com/pypa/sampleproject/issues', - }, - -List additional relevant URLs about your project. This is the place to link to -bug trackers, source repositories, or where to support package development. -The string of the key is the exact text that will be displayed on PyPI. ``packages`` @@ -399,38 +214,6 @@ specification that is used to install its dependencies. For more on using "install_requires" see :ref:`install_requires vs Requirements files`. -.. _python_requires: - -``python_requires`` -~~~~~~~~~~~~~~~~~~~ - -If your project only runs on certain Python versions, setting the -``python_requires`` argument to the appropriate :pep:`440` version specifier -string will prevent :ref:`pip` from installing the project on other Python -versions. For example, if your package is for Python 3+ only, write:: - - python_requires='>=3', - -If your package is for Python 2.6, 2.7, and all versions of Python 3 starting -with 3.3, write:: - - python_requires='>=2.6, !=3.0.*, !=3.1.*, !=3.2.*', - -And so on. - -.. note:: - - Support for this feature is relatively recent. Your project's source - distributions and wheels (see :ref:`Packaging Your Project`) must be built - using at least version 24.2.0 of :ref:`setuptools` in order for the - ``python_requires`` argument to be recognized and the appropriate metadata - generated. - - In addition, only versions 9.0.0 and higher of :ref:`pip` recognize the - ``python_requires`` metadata. Users with earlier versions of pip will be - able to download & install projects on any Python version regardless of the - projects' ``python_requires`` values. - .. _`Package Data`: @@ -484,7 +267,7 @@ Each file name in ``files`` is interpreted relative to the :file:`setup.py` script at the top of the project source distribution. For more information see the distutils section on :ref:`Installing Additional Files -`. +`. .. note:: @@ -498,169 +281,17 @@ For more information see the distutils section on :ref:`Installing Additional Fi ~~~~~~~~~~~ Although ``setup()`` supports a :ref:`scripts -` +` keyword for pointing to pre-made scripts to install, the recommended approach to achieve cross-platform compatibility is to use :ref:`console_scripts` entry points (see below). -``entry_points`` -~~~~~~~~~~~~~~~~ - -:: - - entry_points={ - ... - }, - - -Use this keyword to specify any plugins that your project provides for any named -entry points that may be defined by your project or others that you depend on. - -For more information, see the section on -:ref:`Advertising Behavior ` -from the :ref:`setuptools` docs. - -The most commonly used entry point is "console_scripts" (see below). - -.. _`console_scripts`: - -``console_scripts`` -******************* - -:: - - entry_points={ - 'console_scripts': [ - 'sample=sample:main', - ], - }, - -Use ``console_script`` -:ref:`entry points ` -to register your script interfaces. You can then let the toolchain handle the -work of turning these interfaces into actual scripts [2]_. The scripts will be -generated during the install of your :term:`distribution `. - -For more information, see :doc:`Entry Points ` -from the :doc:`setuptools docs `. - -.. _`Choosing a versioning scheme`: Choosing a versioning scheme ---------------------------- -Standards compliance for interoperability -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Different Python projects may use different versioning schemes based on the needs of that -particular project, but all of them are required to comply with the flexible :pep:`public version -scheme <440#public-version-identifiers>` specified -in :pep:`440` in order to be supported in tools and libraries like ``pip`` -and ``setuptools``. - -Here are some examples of compliant version numbers:: - - 1.2.0.dev1 # Development release - 1.2.0a1 # Alpha Release - 1.2.0b1 # Beta Release - 1.2.0rc1 # Release Candidate - 1.2.0 # Final Release - 1.2.0.post1 # Post Release - 15.10 # Date based release - 23 # Serial release - -To further accommodate historical variations in approaches to version numbering, -:pep:`440` also defines a comprehensive technique for :pep:`version -normalisation <440#normalization>` that maps -variant spellings of different version numbers to a standardised canonical form. - -Scheme choices -~~~~~~~~~~~~~~ - -Semantic versioning (preferred) -******************************* - -For new projects, the recommended versioning scheme is based on `Semantic Versioning -`_, but adopts a different approach to handling pre-releases and -build metadata. - -The essence of semantic versioning is a 3-part MAJOR.MINOR.MAINTENANCE numbering scheme, -where the project author increments: - -1. MAJOR version when they make incompatible API changes, -2. MINOR version when they add functionality in a backwards-compatible manner, and -3. MAINTENANCE version when they make backwards-compatible bug fixes. - -Adopting this approach as a project author allows users to make use of :pep:`"compatible release" -<440#compatible-release>` specifiers, where -``name ~= X.Y`` requires at least release X.Y, but also allows any later release with -a matching MAJOR version. - -Python projects adopting semantic versioning should abide by clauses 1-8 of the -`Semantic Versioning 2.0.0 specification `_. - -Date based versioning -********************* - -Semantic versioning is not a suitable choice for all projects, such as those with a regular -time based release cadence and a deprecation process that provides warnings for a number of -releases prior to removal of a feature. - -A key advantage of date based versioning is that it is straightforward to tell how old the -base feature set of a particular release is given just the version number. - -Version numbers for date based projects typically take the form of YEAR.MONTH (for example, -``12.04``, ``15.10``). - -Serial versioning -***************** - -This is the simplest possible versioning scheme, and consists of a single number which is -incremented every release. - -While serial versioning is very easy to manage as a developer, it is the hardest to track -as an end user, as serial version numbers convey little or no information regarding API -backwards compatibility. - -Hybrid schemes -************** - -Combinations of the above schemes are possible. For example, a project may combine date -based versioning with serial versioning to create a YEAR.SERIAL numbering scheme that -readily conveys the approximate age of a release, but doesn't otherwise commit to a particular -release cadence within the year. - -Pre-release versioning -~~~~~~~~~~~~~~~~~~~~~~ - -Regardless of the base versioning scheme, pre-releases for a given final release may be -published as: - -* zero or more dev releases (denoted with a ".devN" suffix) -* zero or more alpha releases (denoted with a ".aN" suffix) -* zero or more beta releases (denoted with a ".bN" suffix) -* zero or more release candidates (denoted with a ".rcN" suffix) - -``pip`` and other modern Python package installers ignore pre-releases by default when -deciding which versions of dependencies to install. - - -Local version identifiers -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Public version identifiers are designed to support distribution via -:term:`PyPI `. Python's software distribution tools also support -the notion of a :pep:`local version identifier -<440#local-version-identifiers>`, which can be used to -identify local development builds not intended for publication, or modified variants of a release -maintained by a redistributor. - -A local version identifier takes the form ``+``. -For example:: - - 1.2.0.dev1+hg.5.b11e5e6f0b0b # 5th VCS commit since 1.2.0.dev1 release - 1.2.1+fedora.4 # Package with downstream Fedora patches applied +See :ref:`versioning` for information on common version schemes and how to +choose between them. Working in "development mode" @@ -669,7 +300,7 @@ Working in "development mode" You can install a project in "editable" or "develop" mode while you're working on it. When installed as editable, a project can be -edited in-place without reinstallation: +edited in-place without reinstallation: changes to Python source files in projects installed as editable will be reflected the next time an interpreter process is started. To install a Python package in "editable"/"development" mode @@ -677,7 +308,7 @@ Change directory to the root of the project directory and run: .. code-block:: bash - python -m pip install -e . + python3 -m pip install -e . The pip command-line flag ``-e`` is short for ``--editable``, and ``.`` refers @@ -693,7 +324,7 @@ you want "bar" installed from VCS in editable mode, then you could construct a requirements file like so:: -e . - -e git+https://somerepo/bar.git#egg=bar + -e bar @ git+https://somerepo/bar.git The first line says to install your project and any dependencies. The second line overrides the "bar" dependency, such that it's fulfilled from VCS, not @@ -712,12 +343,12 @@ Lastly, if you don't want to install any dependencies at all, you can run: .. code-block:: bash - python -m pip install -e . --no-deps + python3 -m pip install -e . --no-deps For more information, see the :doc:`Development Mode ` section -of the :doc:`setuptools docs `. +of the :ref:`setuptools` docs. .. _`Packaging your project`: @@ -812,7 +443,7 @@ To build the wheel: .. code-block:: bash - python -m build --wheel + python3 -m build --wheel .. tab:: Windows @@ -959,10 +590,3 @@ your project to appear on the site. access. :ref:`pip` is currently considering changing this by `making user installs the default behavior `_. - - -.. [2] Specifically, the "console_script" approach generates ``.exe`` files on - Windows, which are necessary because the OS special-cases ``.exe`` files. - Script-execution features like ``PATHEXT`` and the :pep:`Python Launcher for - Windows <397>` allow scripts to - be used in many cases, but not all. diff --git a/source/guides/dropping-older-python-versions.rst b/source/guides/dropping-older-python-versions.rst index 080f7282b..267d7b923 100644 --- a/source/guides/dropping-older-python-versions.rst +++ b/source/guides/dropping-older-python-versions.rst @@ -4,34 +4,27 @@ Dropping support for older Python versions ========================================== -Dropping support for older Python versions is supported by the standard :ref:`core-metadata` 1.2 specification via a "Requires-Python" attribute. +The ability to drop support for older Python versions is enabled by the standard :ref:`core-metadata` 1.2 specification via the :ref:`"Requires-Python" ` attribute. -Metadata 1.2+ clients, such as Pip 9.0+, will adhere to this specification by matching the current Python runtime and comparing it with the required version +Metadata 1.2+ installers, such as Pip, will adhere to this specification by matching the current Python runtime and comparing it with the required version in the package metadata. If they do not match, it will attempt to install the last package distribution that supported that Python runtime. -This mechanism can be used to drop support for older Python versions, by amending the "Requires-Python" attribute in the package metadata. - -This guide is specifically for users of :ref:`setuptools`, other packaging tools such as ``flit`` may offer similar functionality but users will need to consult relevant documentation. +This mechanism can be used to drop support for older Python versions, by amending the ``Requires-Python`` attribute in the package metadata. Requirements ------------ -This workflow requires that: - -1. The publisher is using the latest version of :ref:`setuptools`, -2. The latest version of :ref:`twine` is used to upload the package, -3. The user installing the package has at least Pip 9.0, or a client that supports the Metadata 1.2 specification. +This workflow requires that the user installing the package uses Pip [#]_, or another installer that supports the Metadata 1.2 specification. Dealing with the universal wheels --------------------------------- -Traditionally, projects providing Python code that is semantically +Traditionally, :ref:`setuptools` projects providing Python code that is semantically compatible with both Python 2 and Python 3, produce :term:`wheels ` that have a ``py2.py3`` tag in their names. When dropping support for Python 2, it is important not to forget to change this tag to just ``py3``. It is often configured within :file:`setup.cfg` under -the ``[bdist_wheel]`` section by setting ``universal = 1`` if they -use setuptools. +the ``[bdist_wheel]`` section by setting ``universal = 1``. If you use this method, either remove this option or section, or explicitly set ``universal`` to ``0``: @@ -41,69 +34,71 @@ explicitly set ``universal`` to ``0``: # setup.cfg [bdist_wheel] - universal = 0 # Make the generated wheels have `py3` tag + universal = 0 # Make the generated wheels have "py3" tag -.. tip:: +.. hint:: - Since it is possible to override the :file:`setup.cfg` settings via - CLI flags, make sure that your scripts don't have ``--universal`` in - your package creation scripts. + Regarding :ref:`deprecated ` direct ``setup.py`` invocations, + passing the ``--universal`` flag on the command line could override this setting. Defining the Python version required ------------------------------------ -1. Download the newest version of Setuptools -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Ensure that before you generate source distributions or binary distributions, you update Setuptools and install twine. +1. Install twine +~~~~~~~~~~~~~~~~ +Ensure that you have twine available at its latest version. Steps: .. tab:: Unix/macOS .. code-block:: bash - python3 -m pip installβ€Š --upgrade setuptools twine + python3 -m pip install --upgrade twine .. tab:: Windows .. code-block:: bat - py -m pip install β€Š--upgrade setuptools twine - -`setuptools` version should be above 24.0.0. + py -m pip install --upgrade twine 2. Specify the version ranges for supported Python distributions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can specify version ranges and exclusion rules, such as at least Python 3. Or, Python 2.7, 3.4 and beyond. +Set the version ranges declaring which Python distributions are supported +within your project's :file:`pyproject.toml`. The :ref:`requires-python` configuration field +corresponds to the :ref:`Requires-Python ` core metadata field: -Examples:: +.. code-block:: toml - Requires-Python: ">=3" - Requires-Python: ">2.7,!=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [build-system] + ... -The way to set those values is within the call to ``setup`` within your -:file:`setup.py` script. This will insert the ``Requires-Python`` -metadata values based on the argument you provide in ``python_requires``. + [project] + requires-python = ">= 3.8" # At least Python 3.8 -.. code-block:: python +You can specify version ranges and exclusion rules (complying with the :ref:`version-specifiers` specification), +such as at least Python 3.9. Or, at least Python 3.7 and beyond, skipping the 3.7.0 and 3.7.1 point releases: - from setuptools import setup +.. code-block:: toml + requires-python = ">= 3.9" + requires-python = ">= 3.7, != 3.7.0, != 3.7.1" - setup( - # Your setup arguments - python_requires='>=2.7', # Your supported Python ranges - ) + +If using the :ref:`setuptools` build backend, consult the `dependency-management`_ documentation for more options. + +.. caution:: + Avoid adding upper bounds to the version ranges, e. g. ``">= 3.8, < 3.10"``. Doing so can cause different errors + and version conflicts. See the `discourse-discussion`_ for more information. 3. Validating the Metadata before publishing ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Within a Python source package (the zip or the tar-gz file you download) is a text file called PKG-INFO. -This file is generated by Distutils or :ref:`setuptools` when it generates the source package. -The file contains a set of keys and values, the list of keys is part of the PyPa standard metadata format. +This file is generated by the :term:`build backend ` when it generates the source package. +The file contains a set of keys and values, the list of keys is part of the PyPA standard metadata format. You can see the contents of the generated file like this: @@ -113,24 +108,31 @@ You can see the contents of the generated file like this: Validate that the following is in place, before publishing the package: -- If you have upgraded correctly, the Metadata-Version value should be 1.2 or higher. -- The Requires-Python field is set and matches your specification in setup.py. +- If you have upgraded correctly, the ``Metadata-Version`` value should be 1.2 or higher. +- The ``Requires-Python`` field is set and matches your specification in the configuration file. -4. Using Twine to publish +4. Publishing the package ~~~~~~~~~~~~~~~~~~~~~~~~~ -Twine has a number of advantages, apart from being faster it is now the supported method for publishing packages. - -Make sure you are using the newest version of Twine, at least 1.9. +Proceed as suggested in :ref:`Uploading your Project to PyPI`. -Dropping a Python release +Dropping a Python version ------------------------- -Once you have published a package with the Requires-Python metadata, you can then make a further update removing that Python runtime from support. +In principle, at least metadata support for Python versions should be kept as long as possible, because +once that has been dropped, people still depending on a version will be forced to downgrade. +If however supporting a specific version becomes a blocker for a new feature or other issues occur, the metadata +``Requires-Python`` should be amended. Of course this also depends on whether the project needs to be stable and +well-covered for a wider range of users. + +Each version compatibility change should have its own release. + +.. tip:: -It must be done in this order for the automated fallback to work. + When dropping a Python version, it might also be rewarding to upgrade the project's code syntax generally, apart from updating the versions used in visible places (like the testing environment). Tools like pyupgrade_ or `ruff `_ can automate some of this work. -For example, you published the Requires-Python: ">=2.7" as version 1.0.0 of your package. +.. _discourse-discussion: https://discuss.python.org/t/requires-python-upper-limits/12663 +.. _pyupgrade: https://pypi.org/project/pyupgrade/ +.. _dependency-management: https://setuptools.pypa.io/en/latest/userguide/dependency_management.html#python-requirement -If you were then to update the version string to ">=3.5", and publish a new version 2.0.0 of your package, any users running Pip 9.0+ from version 2.7 will -have version 1.0.0 of the package installed, and any >=3.5 users will receive version 2.0.0. +.. [#] Support for the Metadata 1.2 specification has been added in Pip 9.0. diff --git a/source/guides/github-actions-ci-cd-sample/publish-to-pypi.yml b/source/guides/github-actions-ci-cd-sample/publish-to-pypi.yml new file mode 100644 index 000000000..155f82555 --- /dev/null +++ b/source/guides/github-actions-ci-cd-sample/publish-to-pypi.yml @@ -0,0 +1,76 @@ +name: Publish Python 🐍 distribution πŸ“¦ to PyPI and TestPyPI + +on: push + +jobs: + build: + name: Build distribution πŸ“¦ + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.x" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v5 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution πŸ“¦ to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/ # Replace with your PyPI project name + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v6 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution πŸ“¦ to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + publish-to-testpypi: + name: Publish Python 🐍 distribution πŸ“¦ to TestPyPI + needs: + - build + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/p/ + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v6 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution πŸ“¦ to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ diff --git a/source/guides/github-actions-ci-cd-sample/publish-to-test-pypi.yml b/source/guides/github-actions-ci-cd-sample/publish-to-test-pypi.yml deleted file mode 100644 index ebbc70c28..000000000 --- a/source/guides/github-actions-ci-cd-sample/publish-to-test-pypi.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Publish Python 🐍 distributions πŸ“¦ to PyPI and TestPyPI - -on: push - -jobs: - build-n-publish: - name: Build and publish Python 🐍 distributions πŸ“¦ to PyPI and TestPyPI - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@master - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install pypa/build - run: >- - python -m - pip install - build - --user - - name: Build a binary wheel and a source tarball - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - . - # Actually publish to PyPI/TestPyPI - - name: Publish distribution πŸ“¦ to Test PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ - - name: Publish distribution πŸ“¦ to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/source/guides/hosting-your-own-index.rst b/source/guides/hosting-your-own-index.rst index a4ff2b35c..ebcd228ee 100644 --- a/source/guides/hosting-your-own-index.rst +++ b/source/guides/hosting-your-own-index.rst @@ -6,7 +6,7 @@ Hosting your own simple repository If you wish to host your own simple repository [1]_, you can either use a -software package like :doc:`devpi ` or you can use simply create the proper +software package like :doc:`devpi ` or you can simply create the proper directory structure and use any web server that can serve static files and generate an autoindex. @@ -35,8 +35,7 @@ all repositories using a valid HTTPS setup. =================== The directory layout is fairly simple, within a root directory you need to -create a directory for each project. This directory should be the normalized -name (as defined by :pep:`503`) of the project. Within each of these directories +create a directory for each project. This directory should be the :ref:`normalized name ` of the project. Within each of these directories simply place each of the downloadable files. If you have the projects "Foo" (with the versions 1.0 and 2.0) and "bar" (with the version 0.1) You should end up with a structure that looks like:: @@ -53,10 +52,86 @@ directory with autoindex enabled. For an example using the built in Web server in `Twisted`_, you would simply run ``twistd -n web --path .`` and then instruct users to add the URL to their installer's configuration. + +Existing projects +================= + +.. list-table:: + :header-rows: 1 + + * - Project + - Package upload + - PyPI fall-through [2]_ + - Additional notes + + * - :ref:`devpi` + - βœ” + - βœ” + - multiple indexes with inheritance, with syncing, replication, fail-over; + mirroring + + * - :ref:`simpleindex` + - + - βœ” + - + + * - :ref:`pypiserver` + - βœ” + - + - + + * - :ref:`pypiprivate` + - + - + - + + * - :ref:`pypicloud` + - + - + - unmaintained; also cached proxying; authentication, authorisation + + * - :ref:`pywharf` + - + - + - unmaintained; serve files in GitHub + + * - :ref:`pulppython` + - βœ” + - + - also mirroring, proxying; plugin for Pulp + + * - :ref:`pip2pi` + - + - + - also mirroring; manual synchronisation + + * - :ref:`dumb-pypi` + - + - + - not a server, but a static file site generator + + * - :ref:`httpserver` + - + - + - standard-library + + * - `Apache `_ + - + - βœ” + - using + `mod_rewrite + `_ + and + `mod_cache_disk + `_, + you can cache requests to package indexes through an Apache server + ---- .. [1] For complete documentation of the simple repository protocol, see - :pep:`503`. + :ref:`simple repository API `. +.. [2] Can be configured to fall back to PyPI (or another package index) + if a requested package is missing. .. _Twisted: https://twistedmatrix.com/ diff --git a/source/guides/index-mirrors-and-caches.rst b/source/guides/index-mirrors-and-caches.rst index 18dc3463e..f3b7bd243 100644 --- a/source/guides/index-mirrors-and-caches.rst +++ b/source/guides/index-mirrors-and-caches.rst @@ -5,22 +5,24 @@ Package index mirrors and caches ================================ :Page Status: Incomplete -:Last Reviewed: 2014-12-24 +:Last Reviewed: 2023-11-08 -.. contents:: Contents - :local: +Mirroring or caching of PyPI (and other +:term:`package indexes `) can be used to speed up local +package installation, +allow offline work, handle corporate firewalls or just plain Internet flakiness. +There are multiple classes of options in this area: -Mirroring or caching of PyPI can be used to speed up local package installation, -allow offline work, handle corporate firewalls or just plain Internet flakiness. +1. local/hosted caching of package indexes. -Three options are available in this area: +2. local/hosted mirroring of a package index. A mirror is a (whole or + partial) copy of a package index, which can be used in place of the + original index. -1. pip provides local caching options, -2. devpi provides higher-level caching option, potentially shared amongst - many users or machines, and -3. bandersnatch provides a local complete mirror of all PyPI :term:`packages - `. +3. private package index with fall-through to public package indexes (for + example, to mitigate dependency confusion attacks), also known as a + proxy. Caching with pip @@ -33,34 +35,83 @@ cached copies of :term:`packages `: by downloading all the requirements for a project and then pointing pip at those downloaded files instead of going to PyPI. 2. A variation on the above which pre-builds the installation files for - the requirements using :ref:`python -m pip wheel `: + the requirements using :ref:`python3 -m pip wheel `: .. code-block:: bash - python -m pip wheel --wheel-dir=/tmp/wheelhouse SomeProject - python -m pip install --no-index --find-links=/tmp/wheelhouse SomeProject - - -Caching with devpi ------------------- - -devpi is a caching proxy server which you run on your laptop, or some other -machine you know will always be available to you. See the `devpi -documentation for getting started`__. - -__ https://devpi.net/docs/devpi/devpi/latest/+d/quickstart-pypimirror.html - - -Complete mirror with bandersnatch ----------------------------------- - -bandersnatch will set up a complete local mirror of all PyPI :term:`packages -` (externally-hosted packages are not mirrored). See -the `bandersnatch documentation for getting that going`__. - -__ https://github.com/pypa/bandersnatch/ - -A benefit of devpi is that it will create a mirror which includes -:term:`packages ` that are external to PyPI, unlike -bandersnatch which will only cache :term:`packages ` -hosted on PyPI. + python3 -m pip wheel --wheel-dir=/tmp/wheelhouse SomeProject + python3 -m pip install --no-index --find-links=/tmp/wheelhouse SomeProject + + +Existing projects +----------------- + +.. list-table:: + :header-rows: 1 + + * - Project + - Cache + - Mirror + - Proxy + - Additional notes + + * - :ref:`devpi` + - βœ” + - βœ” + - + - multiple indexes with inheritance; syncing, replication, fail-over; + package upload + + * - :ref:`bandersnatch` + - βœ” + - βœ” + - + - + + * - :ref:`simpleindex` + - + - + - βœ” + - custom plugin enables caching; re-routing to other package indexes + + * - :ref:`pypicloud` + - βœ” + - + - βœ” + - unmaintained; authentication, authorisation + + * - :ref:`pulppython` + - + - βœ” + - βœ” + - plugin for Pulp; multiple proxied indexes; package upload + + * - :ref:`proxpi` + - βœ” + - + - βœ” + - multiple proxied indexes + + * - :ref:`nginx_pypi_cache` + - βœ” + - + - βœ” + - multiple proxied indexes + + * - :ref:`flaskpypiproxy` + - βœ” + - + - βœ” + - unmaintained + + * - `Apache `_ + - βœ” + - + - βœ” + - using + `mod_rewrite + `_ + and + `mod_cache_disk + `_, + you can cache requests to package indexes through an Apache server diff --git a/source/guides/index.rst b/source/guides/index.rst index 10c189a17..b87d0b1a8 100644 --- a/source/guides/index.rst +++ b/source/guides/index.rst @@ -6,34 +6,10 @@ already familiar with the basics of Python packaging. If you're looking for an introduction to packaging, see :doc:`/tutorials/index`. .. toctree:: - :maxdepth: 1 - :caption: Installing Packages: - - installing-using-pip-and-virtual-environments - installing-stand-alone-command-line-tools - installing-using-linux-tools - installing-scientific-packages - index-mirrors-and-caches - hosting-your-own-index - -.. toctree:: - :maxdepth: 1 - :caption: Building and Publishing Projects: - - distributing-packages-using-setuptools - using-manifest-in - single-sourcing-package-version - dropping-older-python-versions - packaging-binary-extensions - packaging-namespace-packages - creating-and-discovering-plugins - using-testpypi - making-a-pypi-friendly-readme - publishing-package-distribution-releases-using-github-actions-ci-cd-workflows - -.. toctree:: - :maxdepth: 1 - :caption: Miscellaneous: + :titlesonly: + section-install + section-build-and-publish + section-hosting tool-recommendations analyzing-pypi-package-downloads diff --git a/source/guides/installing-scientific-packages.rst b/source/guides/installing-scientific-packages.rst index 04f555731..a1aeae567 100644 --- a/source/guides/installing-scientific-packages.rst +++ b/source/guides/installing-scientific-packages.rst @@ -4,9 +4,6 @@ Installing scientific packages ============================== -.. contents:: Contents - :local: - Scientific software tends to have more complex dependencies than most, and it will often have multiple build options to take advantage of different @@ -17,13 +14,13 @@ In particular, `NumPy `__, which provides the basis for most of the software in the `scientific Python stack `_ can be configured to interoperate with different FORTRAN libraries, and can take advantage -of different levels of vectorised instructions available in modern CPUs. +of different levels of vectorized instructions available in modern CPUs. Starting with version 1.10.4 of NumPy and version 1.0.0 of SciPy, pre-built 32-bit and 64-bit binaries in the ``wheel`` format are available for all major operating systems (Windows, macOS, and Linux) on PyPI. Note, however, that on Windows, NumPy binaries are linked against the `ATLAS -`__ BLAS/LAPACK library, restricted to SSE2 +`__ BLAS/LAPACK library, restricted to SSE2 instructions, so they may not provide optimal linear algebra performance. There are a number of alternative options for obtaining scientific Python @@ -64,12 +61,6 @@ up a suitable environment to build extensions locally. The extensions provided in these installers are typically compatible with the CPython Windows installers published on python.org. -For projects which don't provide their own Windows installers (and even -some which do), Christoph Gohlke at the University of California provides -a `collection of Windows installers -`__. Many Python users on -Windows have reported a positive experience with these prebuilt versions. - As with Linux system packages, the Windows installers will only install into a system Python installation - they do not support installation in virtual environments. Allowing access to distributions installed into the system Python @@ -98,7 +89,7 @@ SciPy distributions ------------------- The SciPy site lists `several distributions -`_ +`_ that provide the full SciPy stack to end users in an easy to use and update format. @@ -127,21 +118,22 @@ be loaded and unloaded from the user's environment. The conda cross-platform package manager ---------------------------------------- -`Anaconda `_ is a Python distribution -published by Anaconda, Inc. It is a stable collection of Open Source -packages for big data and scientific use. As of the 5.0 release of Anaconda, -about 200 packages are installed by default, and a total of 400-500 can be -installed and updated from the Anaconda repository. - ``conda`` is an open source (BSD licensed) package management system and -environment management system included in Anaconda that allows users to install +environment management system that allows users to install multiple versions of binary software packages and their dependencies, and easily switch between them. It is a cross-platform tool working on Windows, -macOS, and Linux. Conda can be used to package up and distribute all kinds of +MacOS, and Linux. Conda can be used to package up and distribute all kinds of packages, it is not limited to just Python packages. It has full support for native virtual environments. Conda makes environments first-class citizens, making it easy to create independent environments even for C libraries. It is written in Python, but is Python-agnostic. Conda manages Python itself as a package, so that :command:`conda update python` is possible, in contrast to -pip, which only manages Python packages. Conda is available in Anaconda and -Miniconda (an easy-to-install download with just Python and conda). +pip, which only manages Python packages. + +Anaconda `Anaconda `_ is a Python distribution published by Anaconda, Inc. It is a stable collection of Open Source packages for big data and scientific use, and a collection of Graphical Interface utilities for managing conda environments. + +In addition to the full distribution provided by Anaconda, the conda package manager itself is available in `miniconda `_, `miniforge `_, and `pixi `_. + + +Conda packages are available on multiple channels on Anaconda.org, including the +default channel supported by Anaconda, Inc, the community supported conda-forge channel, which provides a wide variety of pre-built packages, and some domain-specific package collections. diff --git a/source/guides/installing-stand-alone-command-line-tools.rst b/source/guides/installing-stand-alone-command-line-tools.rst index 958699d42..c078fd1e4 100644 --- a/source/guides/installing-stand-alone-command-line-tools.rst +++ b/source/guides/installing-stand-alone-command-line-tools.rst @@ -1,3 +1,5 @@ +.. _installing-stand-alone-command-line-tools: + Installing stand alone command line tools ========================================= @@ -54,11 +56,11 @@ For example: .. code-block:: console $ pipx install cowsay - installed package cowsay 2.0, Python 3.6.2+ - These binaries are now globally available + installed package cowsay 6.1, installed using Python 3.12.2 + These apps are now globally available - cowsay done! ✨ 🌟 ✨ - $ cowsay moo + $ cowsay -t moo ___ < moo > === @@ -66,8 +68,10 @@ For example: \ ^__^ (oo)\_______ - (__)\ )\/ ||----w | + (__)\ )\/ || || + ||----w | + To see a list of packages installed with pipx and which applications are available, use ``pipx list``: @@ -75,18 +79,21 @@ available, use ``pipx list``: .. code-block:: console $ pipx list - venvs are in /Users/user/.local/pipx/venvs - symlinks to binaries are in /Users/user/.local/bin - package black 18.9b0, Python 3.6.2+ + venvs are in /Users/user/Library/Application Support/pipx/venvs + apps are exposed on your $PATH at /Users/user/.local/bin + manual pages are exposed at /Users/user/.local/share/man + package black 24.2.0, installed using Python 3.12.2 - black - blackd - package cowsay 2.0, Python 3.6.2+ + package cowsay 6.1, installed using Python 3.12.2 - cowsay - package mypy 0.660, Python 3.6.2+ + package mypy 1.9.0, installed using Python 3.12.2 - dmypy - mypy + - mypyc - stubgen - package nox 2018.10.17, Python 3.6.2+ + - stubtest + package nox 2024.3.2, installed using Python 3.12.2 - nox - tox-to-nox @@ -103,14 +110,14 @@ pipx can be upgraded or uninstalled with pip: .. code-block:: bash - python3 -m pip install -U pipx + python3 -m pip install --upgrade pipx python3 -m pip uninstall pipx .. tab:: Windows .. code-block:: bat - py -m pip install -U pipx + py -m pip install --upgrade pipx py -m pip uninstall pipx pipx also allows you to install and run the latest version of an application @@ -118,7 +125,7 @@ in a temporary, ephemeral environment. For example: .. code-block:: bash - pipx run cowsay moooo + pipx run cowsay -t moooo To see the full list of commands pipx offers, run: @@ -126,4 +133,4 @@ To see the full list of commands pipx offers, run: pipx --help -You can learn more about pipx at https://pypa.github.io/pipx/. +You can learn more about pipx at https://pipx.pypa.io/. diff --git a/source/guides/installing-using-linux-tools.rst b/source/guides/installing-using-linux-tools.rst index c91ce7924..56647f3e9 100644 --- a/source/guides/installing-using-linux-tools.rst +++ b/source/guides/installing-using-linux-tools.rst @@ -7,9 +7,6 @@ Installing pip/setuptools/wheel with Linux Package Managers :Page Status: Incomplete :Last Reviewed: 2021-07-26 -.. contents:: Contents - :local: - This section covers how to install :ref:`pip`, :ref:`setuptools`, and :ref:`wheel` using Linux package managers. @@ -38,7 +35,7 @@ Fedora sudo dnf install python3-pip python3-wheel To learn more about Python in Fedora, please visit the `official Fedora docs`_, -`Python Classroom`_ or `Fedora Loves Python`_. +`Python Classroom`_ or `Fedora Loves Python`_. .. _official Fedora docs: https://developer.fedoraproject.org/tech/languages/python/python-installation.html .. _Python Classroom: https://labs.fedoraproject.org/en/python-classroom/ @@ -54,7 +51,7 @@ To install pip and wheel for the system Python, there are two options: 1. Enable the `EPEL repository `_ using `these instructions - `__. + `__. On EPEL 7, you can install pip and wheel like so: .. code-block:: bash @@ -90,7 +87,7 @@ To install pip, wheel, and setuptools, in a parallel, non-system environment * For Redhat, see here: https://developers.redhat.com/products/softwarecollections/overview - * For CentOS, see here: https://www.softwarecollections.org/en/ + * For CentOS, see here: https://github.com/sclorg Be aware that collections may not contain the most recent versions. @@ -110,12 +107,15 @@ To install pip, wheel, and setuptools, in a parallel, non-system environment openSUSE ~~~~~~~~ - .. code-block:: bash - - sudo zypper install python3-pip python3-setuptools python3-wheel +.. code-block:: bash + + sudo zypper install python3-pip python3-setuptools python3-wheel -Debian/Ubuntu -~~~~~~~~~~~~~ + +.. _debian-ubuntu: + +Debian/Ubuntu and derivatives +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Firstly, update and refresh repository lists by running this command: @@ -134,9 +134,9 @@ Firstly, update and refresh repository lists by running this command: Arch Linux ~~~~~~~~~~ - .. code-block:: bash +.. code-block:: bash - sudo pacman -S python-pip + sudo pacman -S python-pip ---- diff --git a/source/guides/installing-using-pip-and-virtual-environments.rst b/source/guides/installing-using-pip-and-virtual-environments.rst index 1ea79be5b..22d1840cc 100644 --- a/source/guides/installing-using-pip-and-virtual-environments.rst +++ b/source/guides/installing-using-pip-and-virtual-environments.rst @@ -1,205 +1,206 @@ -Installing packages using pip and virtual environments -====================================================== +Install packages in a virtual environment using pip and venv +============================================================ -This guide discusses how to install packages using :ref:`pip` and -a virtual environment manager: either :ref:`venv` for Python 3 or :ref:`virtualenv` -for Python 2. These are the lowest-level tools for managing Python -packages and are recommended if higher-level tools do not suit your needs. +This guide discusses how to create and activate a virtual environment using +the standard library's virtual environment tool :ref:`venv` and install packages. +The guide covers how to: -.. note:: This doc uses the term **package** to refer to a - :term:`Distribution Package` which is different from an :term:`Import - Package` that which is used to import modules in your Python source code. +* Create and activate a virtual environment +* Prepare pip +* Install packages into a virtual environment using the ``pip`` command +* Use and create a requirements file -Installing pip --------------- +.. note:: This guide applies to supported versions of Python, currently 3.8 + and higher. -:ref:`pip` is the reference Python package manager. It's used to install and -update packages. You'll need to make sure you have the latest version of pip -installed. +.. note:: This guide uses the term **package** to refer to a + :term:`Distribution Package`, which commonly is installed from an external + host. This differs from the term :term:`Import Package` which refers to + import modules in your Python source code. -.. tab:: Unix/macOS - Debian and most other distributions include a `python-pip`_ package; if you - want to use the Linux distribution-provided versions of pip, see - :doc:`/guides/installing-using-linux-tools`. +.. important:: + This guide has the prerequisite that you are using an official Python version obtained from + . If you are using your operating + system's package manager to install Python, please ensure that Python is + installed before proceeding with these steps. - You can also install pip yourself to ensure you have the latest version. It's - recommended to use the system pip to bootstrap a user installation of pip: - .. code-block:: bash +Create and Use Virtual Environments +----------------------------------- - python3 -m pip install --user --upgrade pip +Create a new virtual environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - python3 -m pip --version +:ref:`venv` (for Python 3) allows you to manage separate package installations for +different projects. It creates a "virtual" isolated Python installation. When +you switch projects, you can create a new virtual environment which is isolated +from other virtual environments. You benefit from the virtual environment +since packages can be installed confidently and will not interfere with +another project's environment. - Afterwards, you should have the latest version of pip installed in your - user site: +.. tip:: + It is recommended to use a virtual environment when working with third + party packages. - .. code-block:: text +To create a virtual environment, go to your project's directory and run the +following command. This will create a new virtual environment in a local folder +named ``.venv``: - pip 21.1.3 from $HOME/.local/lib/python3.9/site-packages (python 3.9) +.. tab:: Unix/macOS - .. _python-pip: https://packages.debian.org/stable/python/python3-pip + .. code-block:: bash -.. tab:: Windows + python3 -m venv .venv - The Python installers for Windows include pip. You can make sure that pip is - up-to-date by running: +.. tab:: Windows .. code-block:: bat - py -m pip install --upgrade pip - - py -m pip --version - - Afterwards, you should have the latest version of pip: - - .. code-block:: text - - pip 21.1.3 from c:\python39\lib\site-packages (Python 3.9.4) + py -m venv .venv +The second argument is the location to create the virtual environment. Generally, you +can just create this in your project and call it ``.venv``. +``venv`` will create a virtual Python installation in the ``.venv`` folder. -Installing virtualenv ---------------------- - -.. Note:: If you are using Python 3.3 or newer, the :mod:`venv` module is - the preferred way to create and manage virtual environments. - venv is included in the Python standard library and requires no additional installation. - If you are using venv, you may skip this section. +.. Note:: You should exclude your virtual environment directory from your version + control system using ``.gitignore`` or similar. -:ref:`virtualenv` is used to manage Python packages for different projects. -Using virtualenv allows you to avoid installing Python packages globally -which could break system tools or other projects. You can install virtualenv -using pip. +Activate a virtual environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before you can start installing or using packages in your virtual environment you'll +need to ``activate`` it. Activating a virtual environment will put the +virtual environment-specific ``python`` and ``pip`` executables into your +shell's ``PATH``. .. tab:: Unix/macOS .. code-block:: bash - python3 -m pip install --user virtualenv + source .venv/bin/activate .. tab:: Windows .. code-block:: bat - py -m pip install --user virtualenv - - - -Creating a virtual environment ------------------------------- + .venv\Scripts\activate -:ref:`venv` (for Python 3) and :ref:`virtualenv` (for Python 2) allow -you to manage separate package installations for -different projects. They essentially allow you to create a "virtual" isolated -Python installation and install packages into that virtual installation. When -you switch projects, you can simply create a new virtual environment and not -have to worry about breaking the packages installed in the other environments. -It is always recommended to use a virtual environment while developing Python -applications. - -To create a virtual environment, go to your project's directory and run -venv. If you are using Python 2, replace ``venv`` with ``virtualenv`` -in the below commands. +To confirm the virtual environment is activated, check the location of your +Python interpreter: .. tab:: Unix/macOS .. code-block:: bash - python3 -m venv env + which python .. tab:: Windows .. code-block:: bat - py -m venv env + where python -The second argument is the location to create the virtual environment. Generally, you -can just create this in your project and call it ``env``. +While the virtual environment is active, the above command will output a +filepath that includes the ``.venv`` directory, by ending with the following: -venv will create a virtual Python installation in the ``env`` folder. +.. tab:: Unix/macOS -.. Note:: You should exclude your virtual environment directory from your version - control system using ``.gitignore`` or similar. + .. code-block:: bash + .venv/bin/python -Activating a virtual environment --------------------------------- +.. tab:: Windows -Before you can start installing or using packages in your virtual environment you'll -need to *activate* it. Activating a virtual environment will put the -virtual environment-specific -``python`` and ``pip`` executables into your shell's ``PATH``. + .. code-block:: bat -.. tab:: Unix/macOS + .venv\Scripts\python - .. code-block:: bash - source env/bin/activate +While a virtual environment is activated, pip will install packages into that +specific environment. This enables you to import and use packages in your +Python application. -.. tab:: Windows - .. code-block:: bat +Deactivate a virtual environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - .\env\Scripts\activate +If you want to switch projects or leave your virtual environment, +``deactivate`` the environment: -You can confirm you're in the virtual environment by checking the location of your -Python interpreter: +.. code-block:: bash -.. tab:: Unix/macOS + deactivate - .. code-block:: bash +.. note:: + Closing your shell will deactivate the virtual environment. If + you open a new shell window and want to use the virtual environment, + reactivate it. - which python +Reactivate a virtual environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. tab:: Windows +If you want to reactivate an existing virtual environment, follow the same +instructions about activating a virtual environment. There's no need to create +a new virtual environment. - .. code-block:: bat - where python +Prepare pip +----------- + +:ref:`pip` is the reference Python package manager. +It's used to install and update packages into a virtual environment. -It should be in the ``env`` directory: .. tab:: Unix/macOS + The Python installers for macOS include pip. On Linux, you may have to install + an additional package such as ``python3-pip``. You can make sure that pip is + up-to-date by running: + .. code-block:: bash - .../env/bin/python + python3 -m pip install --upgrade pip + python3 -m pip --version -.. tab:: Windows + Afterwards, you should have the latest version of pip installed in your + user site: - .. code-block:: bat + .. code-block:: text - ...\env\Scripts\python.exe + pip 23.3.1 from .../.venv/lib/python3.9/site-packages (python 3.9) +.. tab:: Windows -As long as your virtual environment is activated pip will install packages into that -specific environment and you'll be able to import and use packages in your -Python application. + The Python installers for Windows include pip. You can make sure that pip is + up-to-date by running: + .. code-block:: bat -Leaving the virtual environment -------------------------------- + py -m pip install --upgrade pip + py -m pip --version -If you want to switch projects or otherwise leave your virtual environment, simply run: + Afterwards, you should have the latest version of pip: -.. code-block:: bash + .. code-block:: text + + pip 23.3.1 from .venv\lib\site-packages (Python 3.9.4) - deactivate -If you want to re-enter the virtual environment just follow the same instructions above -about activating a virtual environment. There's no need to re-create the virtual environment. +Install packages using pip +-------------------------- +When your virtual environment is activated, you can install packages. Use the +``pip install`` command to install packages. -Installing packages -------------------- +Install a package +~~~~~~~~~~~~~~~~~ -Now that you're in your virtual environment you can install packages. Let's install the +For example, let's install the `Requests`_ library from the :term:`Python Package Index (PyPI)`: .. tab:: Unix/macOS @@ -234,8 +235,8 @@ pip should download requests and all of its dependencies and install them: .. _Requests: https://pypi.org/project/requests/ -Installing specific versions ------------------------------ +Install a specific package version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ pip allows you to specify which version of a package to install using :term:`version specifiers `. For example, to install @@ -245,13 +246,13 @@ a specific version of ``requests``: .. code-block:: bash - python3 -m pip install requests==2.18.4 + python3 -m pip install 'requests==2.18.4' .. tab:: Windows .. code-block:: bat - py -m pip install requests==2.18.4 + py -m pip install "requests==2.18.4" To install the latest ``2.x`` release of requests: @@ -259,13 +260,13 @@ To install the latest ``2.x`` release of requests: .. code-block:: bash - python3 -m pip install requests>=2.0.0,<3.0.0 + python3 -m pip install 'requests>=2.0.0,<3.0.0' .. tab:: Windows .. code-block:: bat - py -m pip install requests>=2.0.0,<3.0.0 + py -m pip install "requests>=2.0.0,<3.0.0" To install pre-release versions of packages, use the ``--pre`` flag: @@ -282,8 +283,8 @@ To install pre-release versions of packages, use the ``--pre`` flag: py -m pip install --pre requests -Installing extras ------------------ +Install extras +~~~~~~~~~~~~~~ Some packages have optional `extras`_. You can tell pip to install these by specifying the extra in brackets: @@ -292,22 +293,23 @@ specifying the extra in brackets: .. code-block:: bash - python3 -m pip install requests[security] + python3 -m pip install 'requests[security]' .. tab:: Windows .. code-block:: bat - py -m pip install requests[security] + py -m pip install "requests[security]" .. _extras: https://setuptools.readthedocs.io/en/latest/userguide/dependency_management.html#optional-dependencies -Installing from source ----------------------- +Install a package from source +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -pip can install a package directly from source, for example: +pip can install a package directly from its source code. For example, to install +the source code in the ``google-auth`` directory: .. tab:: Unix/macOS @@ -341,22 +343,22 @@ installed package without needing to re-install: py -m pip install --editable . -Installing from version control systems ---------------------------------------- +Install from version control systems +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ pip can install packages directly from their version control system. For example, you can install directly from a git repository: .. code-block:: bash - git+https://github.com/GoogleCloudPlatform/google-auth-library-python.git#egg=google-auth + google-auth @ git+https://github.com/GoogleCloudPlatform/google-auth-library-python.git For more information on supported version control systems and syntax, see pip's documentation on :ref:`VCS Support `. -Installing from local archives ------------------------------- +Install from local archives +~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you have a local copy of a :term:`Distribution Package`'s archive (a zip, wheel, or tar file) you can install it directly with pip: @@ -394,8 +396,8 @@ connectivity or if you want to strictly control the origin of distribution packages. -Using other package indexes ---------------------------- +Install from other package indexes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you want to download packages from a different index than the :term:`Python Package Index (PyPI)`, you can use the ``--index-url`` flag: @@ -446,8 +448,8 @@ install the latest version of ``requests`` and all of its dependencies: py -m pip install --upgrade requests -Using requirements files ------------------------- +Using a requirements file +------------------------- Instead of installing packages individually, pip allows you to declare all dependencies in a :ref:`Requirements File `. For @@ -506,5 +508,5 @@ Which will output a list of package specifiers such as: six==1.11.0 urllib3==1.22 -This is useful for creating :ref:`pip:Requirements Files` that can re-create -the exact versions of all packages installed in an environment. +The ``pip freeze`` command is useful for creating :ref:`pip:Requirements Files` +that can re-create the exact versions of all packages installed in an environment. diff --git a/source/guides/installing-using-virtualenv.rst b/source/guides/installing-using-virtualenv.rst new file mode 100644 index 000000000..a584b89d5 --- /dev/null +++ b/source/guides/installing-using-virtualenv.rst @@ -0,0 +1,15 @@ +Installing packages using virtualenv +==================================== + +This guide discusses how to install packages using :ref:`pip` and +:ref:`virtualenv`, a tool to create isolated Python environments. + +.. important:: + This "how to" guide on installing packages and using :ref:`virtualenv` is + under development. Please refer to the :ref:`virtualenv` documentation for + details on installation and usage. + + +.. note:: This doc uses the term **package** to refer to a + :term:`Distribution Package` which is different from an :term:`Import + Package` that which is used to import modules in your Python source code. diff --git a/source/guides/licensing-examples-and-user-scenarios.rst b/source/guides/licensing-examples-and-user-scenarios.rst new file mode 100644 index 000000000..b6cdfe327 --- /dev/null +++ b/source/guides/licensing-examples-and-user-scenarios.rst @@ -0,0 +1,358 @@ +.. _licensing-examples-and-user-scenarios: + + +===================================== +Licensing examples and user scenarios +===================================== + + +:pep:`639` has specified the way to declare a :term:`Distribution Archive`'s +license and paths to license files and other legally required information. +This document aims to provide clear guidance how to migrate from the legacy +to the standardized way of declaring licenses. +Make sure your preferred build backend supports :pep:`639` before +trying to apply the newer guidelines. + + +Licensing Examples +================== + +.. _licensing-example-basic: + +Basic example +------------- + +The Setuptools project itself, as of `version 75.6.0 `__, +does not use the ``License`` field in its own project source metadata. +Further, it no longer explicitly specifies ``license_file``/``license_files`` +as it did previously, since Setuptools relies on its own automatic +inclusion of license-related files matching common patterns, +such as the :file:`LICENSE` file it uses. + +It includes the following license-related metadata in its +:file:`pyproject.toml`: + +.. code-block:: toml + + [project] + classifiers = [ + "License :: OSI Approved :: MIT License" + ] + +The simplest migration to PEP 639 would consist of using this instead: + +.. code-block:: toml + + [project] + license = "MIT" + +Or, if the project used :file:`setup.cfg`, in its ``[metadata]`` table: + +.. code-block:: ini + + [metadata] + license = MIT + +The output Core Metadata for the :term:`Distribution Package` would then be: + +.. code-block:: email + + License-Expression: MIT + License-File: LICENSE + +The :file:`LICENSE` file would be stored at :file:`/setuptools-{VERSION}/LICENSE` +in the sdist and :file:`/setuptools-{VERSION}.dist-info/licenses/LICENSE` +in the wheel, and unpacked from there into the site directory (e.g. +:file:`site-packages/`) on installation; :file:`/` is the root of the respective +archive and ``{VERSION}`` the version of the Setuptools release in the Core +Metadata. + + +.. _licensing-example-advanced: + +Advanced example +---------------- + +Suppose Setuptools were to include the licenses of the third-party projects +that are vendored in the :file:`setuptools/_vendor/` and :file:`pkg_resources/_vendor/` +directories; specifically: + +.. code-block:: text + + packaging==21.2 + pyparsing==2.2.1 + ordered-set==3.1.1 + more_itertools==8.8.0 + +The appropriate license expressions are: + +.. code-block:: text + + packaging: Apache-2.0 OR BSD-2-Clause + pyparsing: MIT + ordered-set: MIT + more_itertools: MIT + +A comprehensive license expression covering both Setuptools +proper and its vendored dependencies would contain these metadata, +combining all the license expressions into one. Such an expression might be: + +.. code-block:: text + + MIT AND (Apache-2.0 OR BSD-2-Clause) + +In addition, per the requirements of the licenses, the relevant license files +must be included in the package. Suppose the :file:`LICENSE` file contains the text +of the MIT license and the copyrights used by Setuptools, ``pyparsing``, +``more_itertools`` and ``ordered-set``; and the :file:`LICENSE*` files in the +:file:`setuptools/_vendor/packaging/` directory contain the Apache 2.0 and +2-clause BSD license text, and the Packaging copyright statement and +`license choice notice `__. + +Specifically, we assume the license files are located at the following +paths in the project source tree (relative to the project root and +:file:`pyproject.toml`): + +.. code-block:: text + + LICENSE + setuptools/_vendor/packaging/LICENSE + setuptools/_vendor/packaging/LICENSE.APACHE + setuptools/_vendor/packaging/LICENSE.BSD + +Putting it all together, our :file:`pyproject.toml` would be: + +.. code-block:: toml + + [project] + license = "MIT AND (Apache-2.0 OR BSD-2-Clause)" + license-files = [ + "LICENSE*", + "setuptools/_vendor/LICENSE*", + ] + +Or alternatively, the license files can be specified explicitly (paths will be +interpreted as glob patterns): + +.. code-block:: toml + + [project] + license = "MIT AND (Apache-2.0 OR BSD-2-Clause)" + license-files = [ + "LICENSE", + "setuptools/_vendor/LICENSE", + "setuptools/_vendor/LICENSE.APACHE", + "setuptools/_vendor/LICENSE.BSD", + ] + +If our project used :file:`setup.cfg`, we could define this in : + +.. code-block:: ini + + [metadata] + license = MIT AND (Apache-2.0 OR BSD-2-Clause) + license_files = + LICENSE + setuptools/_vendor/packaging/LICENSE + setuptools/_vendor/packaging/LICENSE.APACHE + setuptools/_vendor/packaging/LICENSE.BSD + +With either approach, the output Core Metadata in the distribution +would be: + +.. code-block:: email + + License-Expression: MIT AND (Apache-2.0 OR BSD-2-Clause) + License-File: LICENSE + License-File: setuptools/_vendor/packaging/LICENSE + License-File: setuptools/_vendor/packaging/LICENSE.APACHE + License-File: setuptools/_vendor/packaging/LICENSE.BSD + +In the resulting sdist, with :file:`/` as the root of the archive and ``{VERSION}`` +the version of the Setuptools release specified in the Core Metadata, +the license files would be located at the paths: + +.. code-block:: text + + /setuptools-{VERSION}/LICENSE + /setuptools-{VERSION}/setuptools/_vendor/packaging/LICENSE + /setuptools-{VERSION}/setuptools/_vendor/packaging/LICENSE.APACHE + /setuptools-{VERSION}/setuptools/_vendor/packaging/LICENSE.BSD + +In the built wheel, with :file:`/` being the root of the archive and +``{VERSION}`` as the previous, the license files would be stored at: + +.. code-block:: text + + /setuptools-{VERSION}.dist-info/licenses/LICENSE + /setuptools-{VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE + /setuptools-{VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE.APACHE + /setuptools-{VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE.BSD + +Finally, in the installed project, with :file:`site-packages/` being the site dir +and ``{VERSION}`` as the previous, the license files would be installed to: + +.. code-block:: text + + site-packages/setuptools-{VERSION}.dist-info/licenses/LICENSE + site-packages/setuptools-{VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE + site-packages/setuptools-{VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE.APACHE + site-packages/setuptools-{VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE.BSD + + +Expression examples +''''''''''''''''''' + +Some additional examples of valid ``License-Expression`` values: + +.. code-block:: email + + License-Expression: MIT + License-Expression: BSD-3-Clause + License-Expression: MIT AND (Apache-2.0 OR BSD-2-Clause) + License-Expression: MIT OR GPL-2.0-or-later OR (FSFUL AND BSD-2-Clause) + License-Expression: GPL-3.0-only WITH Classpath-Exception-2.0 OR BSD-3-Clause + License-Expression: LicenseRef-Public-Domain OR CC0-1.0 OR Unlicense + License-Expression: LicenseRef-Proprietary + License-Expression: LicenseRef-Custom-License + + +User Scenarios +============== + +The following covers the range of common use cases from a user perspective, +providing guidance for each. Do note that the following +should **not** be considered legal advice, and readers should consult a +licensed legal practitioner in their jurisdiction if they are unsure about +the specifics for their situation. + + +I have a private package that won't be distributed +-------------------------------------------------- + +If your package isn't shared publicly, i.e. outside your company, +organization or household, it *usually* isn't strictly necessary to include +a formal license, so you wouldn't necessarily have to do anything extra here. + +However, it is still a good idea to include ``LicenseRef-Proprietary`` +as a license expression in your package configuration, and/or a +copyright statement and any legal notices in a :file:`LICENSE.txt` file +in the root of your project directory, which will be automatically +included by packaging tools. + + +I just want to share my own work without legal restrictions +----------------------------------------------------------- + +While you aren't required to include a license, if you don't, no one has +`any permission to download, use or improve your work `__, +so that's probably the *opposite* of what you actually want. +The `MIT license `__ is a great choice instead, as it's simple, +widely used and allows anyone to do whatever they want with your work +(other than sue you, which you probably also don't want). + +To apply it, just paste `the text `__ into a file named +:file:`LICENSE.txt` at the root of your repo, and add the year and your name to +the copyright line. Then, just add ``license = "MIT"`` under +``[project]`` in your :file:`pyproject.toml` if your packaging tool supports it, +or in its config file/section. You're done! + + +I want to distribute my project under a specific license +-------------------------------------------------------- + +To use a particular license, simply paste its text into a :file:`LICENSE.txt` +file at the root of your repo, if you don't have it in a file starting with +:file:`LICENSE` or :file:`COPYING` already, and add +``license = "LICENSE-ID"`` under ``[project]`` in your +:file:`pyproject.toml` if your packaging tool supports it, or else in its +config file. You can find the ``LICENSE-ID`` +and copyable license text on sites like +`ChooseALicense `__ or `SPDX `__. + +Many popular code hosts, project templates and packaging tools can add the +license file for you, and may support the expression as well in the future. + + +I maintain an existing package that's already licensed +------------------------------------------------------ + +If you already have license files and metadata in your project, you +should only need to make a couple of tweaks to take advantage of the new +functionality. + +In your project config file, enter your license expression under +``license`` (``[project]`` table in :file:`pyproject.toml`), +or the equivalent for your packaging tool, +and make sure to remove any legacy ``license`` table subkeys or +``License ::`` classifiers. Your existing ``license`` value may already +be valid as one (e.g. ``MIT``, ``Apache-2.0 OR BSD-2-Clause``, etc); +otherwise, check the `SPDX license list `__ for the identifier +that matches the license used. + +Make sure to list your license files under ``license-files`` +under ``[project]`` in :file:`pyproject.toml` +or else in your tool's configuration file. + +See the :ref:`licensing-example-basic` for a simple but complete real-world demo +of how this works in practice. +See also the best-effort guidance on how to translate license classifiers +into license expression provided by the :pep:`639` authors: +`Mapping License Classifiers to SPDX Identifiers `__. +Packaging tools may support automatically converting legacy licensing +metadata; check your tool's documentation for more information. + + +My package includes other code under different licenses +------------------------------------------------------- + +If your project includes code from others covered by different licenses, +such as vendored dependencies or files copied from other open source +software, you can construct a license expression +to describe the licenses involved and the relationship +between them. + +In short, ``License-1 AND License-2`` mean that *both* licenses apply +(for example, you included a file under another license), and +``License-1 OR License-2`` means that *either* of the licenses can be used, at +the user's option (for example, you want to allow users a choice of multiple +licenses). You can use parenthesis (``()``) for grouping to form expressions +that cover even the most complex situations. + +In your project config file, enter your license expression under +``license`` (``[project]`` table of :file:`pyproject.toml`), +or the equivalent for your packaging tool, +and make sure to remove any legacy ``license`` table subkeys +or ``License ::`` classifiers. + +Also, make sure you add the full license text of all the licenses as files +somewhere in your project repository. List the +relative path or glob patterns to each of them under ``license-files`` +under ``[project]`` in :file:`pyproject.toml` +(if your tool supports it), or else in your tool's configuration file. + +As an example, if your project was licensed MIT but incorporated +a vendored dependency (say, ``packaging``) that was licensed under +either Apache 2.0 or the 2-clause BSD, your license expression would +be ``MIT AND (Apache-2.0 OR BSD-2-Clause)``. You might have a +:file:`LICENSE.txt` in your repo root, and a :file:`LICENSE-APACHE.txt` and +:file:`LICENSE-BSD.txt` in the :file:`_vendor/` subdirectory, so to include +all of them, you'd specify ``["LICENSE.txt", "_vendor/packaging/LICENSE*"]`` +as glob patterns, or +``["LICENSE.txt", "_vendor/LICENSE-APACHE.txt", "_vendor/LICENSE-BSD.txt"]`` +as literal file paths. + +See a fully worked out :ref:`licensing-example-advanced` for an end-to-end +application of this to a real-world complex project, with many technical +details, and consult a `tutorial `__ for more help and examples +using SPDX identifiers and expressions. + + +.. _chooseamitlicense: https://choosealicense.com/licenses/mit/ +.. _choosealicenselist: https://choosealicense.com/licenses/ +.. _dontchoosealicense: https://choosealicense.com/no-permission/ +.. _mappingclassifierstospdx: https://peps.python.org/pep-0639/appendix-mapping-classifiers/ +.. _packaginglicense: https://github.com/pypa/packaging/blob/21.2/LICENSE +.. _setuptools7560: https://github.com/pypa/setuptools/blob/v75.6.0/pyproject.toml +.. _spdxlist: https://spdx.org/licenses/ +.. _spdxtutorial: https://github.com/david-a-wheeler/spdx-tutorial diff --git a/source/guides/making-a-pypi-friendly-readme.rst b/source/guides/making-a-pypi-friendly-readme.rst index 5e6eef4c7..4a3a20670 100644 --- a/source/guides/making-a-pypi-friendly-readme.rst +++ b/source/guides/making-a-pypi-friendly-readme.rst @@ -58,10 +58,10 @@ such as ``text/plain``, ``text/x-rst`` (for reStructuredText), or ``text/markdow py -m pip install --user --upgrade setuptools wheel twine The minimum required versions of the respective tools are: - - - ``setuptools >= 38.6.0`` - - ``wheel >= 0.31.0`` - - ``twine >= 1.11.0`` + + - ``setuptools >= 38.6.0`` + - ``wheel >= 0.31.0`` + - ``twine >= 1.11.0`` It's recommended that you use ``twine`` to upload the project's distribution packages: diff --git a/source/guides/modernize-setup-py-project.rst b/source/guides/modernize-setup-py-project.rst new file mode 100644 index 000000000..1f71d1973 --- /dev/null +++ b/source/guides/modernize-setup-py-project.rst @@ -0,0 +1,248 @@ +.. _modernize-setup-py-project: + + +============================================== +How to modernize a ``setup.py`` based project? +============================================== + + +Should ``pyproject.toml`` be added? +=================================== + +A :term:`pyproject.toml` file is strongly recommended. +The presence of a :file:`pyproject.toml` file itself does not bring much. [#]_ +What is actually strongly recommended is the ``[build-system]`` table in :file:`pyproject.toml`. + +.. [#] Note that it has influence on the build isolation feature of pip, + see below. + + +Should ``setup.py`` be deleted? +=============================== + +No, :file:`setup.py` can exist in a modern :ref:`setuptools` based project. +The :term:`setup.py` file is a valid configuration file for setuptools +that happens to be written in Python. +However, the following commands are deprecated and **MUST NOT** be run anymore, +and their recommended replacement commands should be used instead: + ++---------------------------------+----------------------------------------+ +| Deprecated | Recommendation | ++=================================+========================================+ +| ``python setup.py install`` | ``python -m pip install .`` | ++---------------------------------+----------------------------------------+ +| ``python setup.py develop`` | ``python -m pip install --editable .`` | ++---------------------------------+----------------------------------------+ +| ``python setup.py sdist`` | ``python -m build`` | ++---------------------------------+ | +| ``python setup.py bdist_wheel`` | | ++---------------------------------+----------------------------------------+ + + +For more details: + +* :ref:`setup-py-deprecated` + + +Where to start? +=============== + +The :term:`project` must contain a :file:`pyproject.toml` file at the root of its source tree +that contains a ``[build-system]`` table like so: + +.. code:: toml + + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + + +This is the standardized method of letting :term:`build frontends ` know +that :ref:`setuptools` is the :term:`build backend ` for this project. + +Note that the presence of a :file:`pyproject.toml` file (even if empty) +triggers :ref:`pip` to change its default behavior to use *build isolation*. + +For more details: + +* :ref:`distributing-packages` +* :ref:`pyproject-build-system-table` +* :doc:`pip:reference/build-system` + + +How to handle additional build-time dependencies? +================================================= + +On top of setuptools itself, +if :file:`setup.py` depends on other third-party libraries (outside of Python's standard library), +those must be listed in the ``requires`` list of the ``[build-system]`` table, +so that the build frontend knows to install them +when building the :term:`distributions `. + +For example, a :file:`setup.py` file such as this: + +.. code:: python + + import setuptools + import some_build_toolkit # comes from the `some-build-toolkit` library + + def get_version(): + version = some_build_toolkit.compute_version() + return version + + setuptools.setup( + name="my-project", + version=get_version(), + ) + + +requires a :file:`pyproject.toml` file like this (:file:`setup.py` stays unchanged): + +.. code:: toml + + [build-system] + requires = [ + "setuptools", + "some-build-toolkit", + ] + build-backend = "setuptools.build_meta" + + +For more details: + +* :ref:`pyproject-build-system-table` + + +What is the build isolation feature? +==================================== + +Build frontends typically create an ephemeral virtual environment +where they install only the build dependencies (and their dependencies) +that are listed under ``build-system.requires`` +and trigger the build in that environment. + +For some projects this isolation is unwanted and it can be deactivated as follows: + +* ``python -m build --no-isolation`` +* ``python -m pip install --no-build-isolation`` + +For more details: + +* :doc:`pip:reference/build-system` + + +How to handle packaging metadata? +================================= + +All static metadata can optionally be moved to a ``[project]`` table in :file:`pyproject.toml`. + +For example, a :file:`setup.py` file such as this: + +.. code:: python + + import setuptools + + setuptools.setup( + name="my-project", + version="1.2.3", + ) + + +can be entirely replaced by a :file:`pyproject.toml` file like this: + +.. code:: toml + + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + + [project] + name = "my-project" + version = "1.2.3" + + +Read :ref:`pyproject-project-table` for the full specification +of the content allowed in the ``[project]`` table. + + +How to handle dynamic metadata? +=============================== + +If some packaging metadata fields are not static +they need to be listed as ``dynamic`` in this ``[project]`` table. + +For example, a :file:`setup.py` file such as this: + +.. code:: python + + import setuptools + import some_build_toolkit + + def get_version(): + version = some_build_toolkit.compute_version() + return version + + setuptools.setup( + name="my-project", + version=get_version(), + ) + + +can be modernized as follows: + +.. code:: toml + + [build-system] + requires = [ + "setuptools", + "some-build-toolkit", + ] + build-backend = "setuptools.build_meta" + + [project] + name = "my-project" + dynamic = ["version"] + + +.. code:: python + + import setuptools + import some_build_toolkit + + def get_version(): + version = some_build_toolkit.compute_version() + return version + + setuptools.setup( + version=get_version(), + ) + + +For more details: + +* :ref:`declaring-project-metadata-dynamic` + + +What if something that can not be changed expects a ``setup.py`` file? +====================================================================== + +For example, a process exists that can not be changed easily +and it needs to execute a command such as ``python setup.py --name``. + +It is perfectly fine to leave a :file:`setup.py` file in the project source tree +even after all its content has been moved to :file:`pyproject.toml`. +This file can be as minimalistic as this: + +.. code:: python + + import setuptools + + setuptools.setup() + + +Where to read more about this? +============================== + +* :ref:`pyproject-toml-spec` +* :doc:`pip:reference/build-system` +* :doc:`setuptools:build_meta` diff --git a/source/guides/multi-version-installs.rst b/source/guides/multi-version-installs.rst index 0c2929fbb..a09bc900a 100644 --- a/source/guides/multi-version-installs.rst +++ b/source/guides/multi-version-installs.rst @@ -35,7 +35,7 @@ This can be worked around by setting all dependencies in ``__main__.__requires__`` before importing ``pkg_resources`` for the first time, but that approach does mean that standard command line invocations of the affected tools can't be used - it's necessary to write a custom -wrapper script or use ``python -c ''`` to invoke the application's +wrapper script or use ``python3 -c ''`` to invoke the application's main entry point directly. Refer to the `pkg_resources documentation diff --git a/source/guides/packaging-binary-extensions.rst b/source/guides/packaging-binary-extensions.rst index f27eb4730..de8a9d2d6 100644 --- a/source/guides/packaging-binary-extensions.rst +++ b/source/guides/packaging-binary-extensions.rst @@ -13,8 +13,6 @@ C API for use by other software. One of the most common uses of this C API is to create importable C extensions that allow things which aren't always easy to achieve in pure Python code. -.. contents:: Contents - :local: An overview of binary extensions ================================ @@ -169,7 +167,7 @@ wrapper modules up to date. for automatic wrapping with Cython. It also supports performance-oriented Python implementations that provide a CPython-like C-API, such as PyPy and Pyston. - + * :doc:`pybind11 ` is a pure C++11 library that provides a clean C++ interface to the CPython (and PyPy) C API. It does not require a pre-processing step; it is written entirely in @@ -230,34 +228,82 @@ The CPython :doc:`Extending and Embedding ` guide includes an introduction to writing a :doc:`custom extension module in C `. -.. +FIXME: Elaborate that all this is one of the reasons why you probably +*don't* want to handcode your extension modules :) + + +Extension module lifecycle +-------------------------- + +FIXME: This section needs to be fleshed out. + + +Implications of shared static state and subinterpreters +------------------------------------------------------- + +FIXME: This section needs to be fleshed out. + + +Implications of the GIL +----------------------- + +FIXME: This section needs to be fleshed out. + - FIXME +Memory allocation APIs +---------------------- - * mention the stable ABI (3.2+, link to the CPython C API docs) - * mention the module lifecycle - * mention the challenges of shared static state and subinterpreters - * mention the implications of the GIL for extension modules - * mention the memory allocation APIs in 3.4+ +FIXME: This section needs to be fleshed out. - * mention again that all this is one of the reasons why you probably *don't* - want to handcode your extension modules :) + +.. _cpython-stable-abi: + +ABI Compatibility +----------------- + +The CPython C API does not guarantee ABI stability between minor releases +(3.2, 3.3, 3.4, etc.). This means that, typically, if you build an +extension module against one version of Python, it is only guaranteed to +work with the same minor version of Python and not with any other minor +versions. + +Python 3.2 introduced the Limited API, with is a well-defined subset of +Python's C API. The symbols needed for the Limited API form the +"Stable ABI" which is guaranteed to be compatible across all Python 3.x +versions. Wheels containing extensions built against the stable ABI use +the ``abi3`` ABI tag, to reflect that they're compatible with all Python +3.x versions. + +CPython's :doc:`C API stability` page provides +detailed information about the API / ABI stability guarantees, how to use +the Limited API and the exact contents of the "Limited API". Building binary extensions ========================== +FIXME: Cover the build-backends available for building extensions. + Building extensions for multiple platforms ------------------------------------------ If you plan to distribute your extension, you should provide -:term:`wheels ` for all the platforms you intend to support. For most -extensions, this is at least one package per Python version times the number of -OS and architectures you support. These are usually built on continuous -integration (CI) systems. There are tools to help you build highly -redistributable binaries from CI; these include :ref:`cibuildwheel` and -:ref:`multibuild`. +:term:`wheels ` for all the platforms you intend to support. These +are usually built on continuous integration (CI) systems. There are tools +to help you build highly redistributable binaries from CI; these include +:ref:`cibuildwheel` and :ref:`multibuild`. +For most extensions, you will need to build wheels for all the platforms +you intend to support. This means that the number of wheels you need to +build is the product of:: + + count(Python minor versions) * count(OS) * count(architectures) + +Using CPython's :ref:`Stable ABI ` can help significantly +reduce the number of wheels you need to provide, since a single wheel on a +platform can be used with all Python minor versions; eliminating one dimension +of the matrix. It also removes the need to generate new wheels for each new +minor version of Python. Binary extensions for Windows ----------------------------- @@ -310,20 +356,24 @@ Spinning Wheels wiki Publishing binary extensions ============================ -For interim guidance on this topic, see the discussion in -:issue:`this issue <284>`. +Publishing binary extensions through PyPI uses the same upload mechanisms as +publishing pure Python packages. You build a wheel file for your extension +using the build-backend and upload it to PyPI using +:doc:`twine `. -.. +Avoid binary-only releases +-------------------------- - FIXME +It is strongly recommended that you publish your binary extensions as +well as the source code that was used to build them. This allows users to +build the extension from source if they need to. Notably, this is required +for certain Linux distributions that build from source within their +own build systems for the distro package repositories. - * cover publishing as wheel files on PyPI or a custom index server - * cover creation of Windows and macOS installers - * cover weak linking - * mention the fact that Linux distros have a requirement to build from - source in their own build systems, so binary-only releases are strongly - discouraged +Weak linking +------------ +FIXME: This section needs to be fleshed out. Additional resources ==================== @@ -353,3 +403,15 @@ a Debian system, see the following articles: * `What are (c)python extension modules? `_ * `Releasing the gil `_ * `Writing cpython extension modules using C++ `_ + +Additional considerations for binary wheels +------------------------------------------- + +The `pypackaging-native `_ website has +additional coverage of packaging Python packages with native code. It aims to +provide an overview of the most important packaging issues for such projects, +with in-depth explanations and references. + +Examples of topics covered are non-Python compiled dependencies ("native +dependencies"), the importance of the ABI (Application Binary Interface) of +native code, dependency on SIMD code and cross compilation. diff --git a/source/guides/packaging-namespace-packages.rst b/source/guides/packaging-namespace-packages.rst index 64bce312c..3d929d527 100644 --- a/source/guides/packaging-namespace-packages.rst +++ b/source/guides/packaging-namespace-packages.rst @@ -1,3 +1,5 @@ +.. _packaging-namespace-packages: + ============================ Packaging namespace packages ============================ @@ -9,7 +11,7 @@ single :term:`package ` across multiple, separate have the following package structure: .. code-block:: text - + mynamespace/ __init__.py subpackage_a/ @@ -19,7 +21,7 @@ have the following package structure: __init__.py ... module_b.py - setup.py + pyproject.toml And you use this package in your code like so:: @@ -29,19 +31,21 @@ And you use this package in your code like so:: Then you can break these sub-packages into two separate distributions: .. code-block:: text - + mynamespace-subpackage-a/ - setup.py - mynamespace/ - subpackage_a/ - __init__.py + pyproject.toml + src/ + mynamespace/ + subpackage_a/ + __init__.py mynamespace-subpackage-b/ - setup.py - mynamespace/ - subpackage_b/ - __init__.py - module_b.py + pyproject.toml + src/ + mynamespace/ + subpackage_b/ + __init__.py + module_b.py Each sub-package can now be separately installed, used, and versioned. @@ -57,24 +61,15 @@ import object short). Creating a namespace package ============================ -There are currently three different approaches to creating namespace packages: +There are currently two different approaches to creating namespace packages, +from which the latter is discouraged: #. Use `native namespace packages`_. This type of namespace package is defined in :pep:`420` and is available in Python 3.3 and later. This is recommended if packages in your namespace only ever need to support Python 3 and installation via ``pip``. -#. Use `pkgutil-style namespace packages`_. This is recommended for new - packages that need to support Python 2 and 3 and installation via both - ``pip`` and ``python setup.py install``. -#. Use `pkg_resources-style namespace packages`_. This method is recommended if - you need compatibility with packages already using this method or if your - package needs to be zip-safe. - -.. warning:: While native namespace packages and pkgutil-style namespace - packages are largely compatible, pkg_resources-style namespace packages - are not compatible with the other methods. It's inadvisable to use - different methods in different distributions that provide packages to the - same namespace. +#. Use `legacy namespace packages`_. This comprises `pkgutil-style namespace packages`_ + and `pkg_resources-style namespace packages`_. Native namespace packages ------------------------- @@ -82,27 +77,56 @@ Native namespace packages Python 3.3 added **implicit** namespace packages from :pep:`420`. All that is required to create a native namespace package is that you just omit :file:`__init__.py` from the namespace package directory. An example file -structure: +structure (following :ref:`src-layout `): .. code-block:: text - setup.py - mynamespace/ - # No __init__.py here. - subpackage_a/ - # Sub-packages have __init__.py. - __init__.py - module.py + mynamespace-subpackage-a/ + pyproject.toml # AND/OR setup.py, setup.cfg + src/ + mynamespace/ # namespace package + # No __init__.py here. + subpackage_a/ + # Regular import packages have an __init__.py. + __init__.py + module.py It is extremely important that every distribution that uses the namespace package omits the :file:`__init__.py` or uses a pkgutil-style :file:`__init__.py`. If any distribution does not, it will cause the namespace logic to fail and the other sub-packages will not be importable. -Because ``mynamespace`` doesn't contain an :file:`__init__.py`, -:func:`setuptools.find_packages` won't find the sub-package. You must -use :func:`setuptools.find_namespace_packages` instead or explicitly -list all packages in your :file:`setup.py`. For example: +The ``src-layout`` directory structure allows automatic discovery of packages +by most :term:`build backends `. See :ref:`src-layout-vs-flat-layout` +for more information. If however you want to manage exclusions or inclusions of packages +yourself, this is possible to be configured in the top-level :file:`pyproject.toml`: + +.. code-block:: toml + + [build-system] + ... + + [tool.setuptools.packages.find] + where = ["src/"] + include = ["mynamespace.subpackage_a"] + + [project] + name = "mynamespace-subpackage-a" + ... + +The same can be accomplished with a :file:`setup.cfg`: + +.. code-block:: ini + + [options] + package_dir = + =src + packages = find_namespace: + + [options.packages.find] + where = src + +Or :file:`setup.py`: .. code-block:: python @@ -111,9 +135,13 @@ list all packages in your :file:`setup.py`. For example: setup( name='mynamespace-subpackage-a', ... - packages=find_namespace_packages(include=['mynamespace.*']) + packages=find_namespace_packages(where='src/', include=['mynamespace.subpackage_a']), + package_dir={'': 'src'}, ) +:ref:`setuptools` will search the directory structure for implicit namespace +packages by default. + A complete working example of two native namespace packages can be found in the `native namespace package example project`_. @@ -125,8 +153,25 @@ the `native namespace package example project`_. only support Python 3 and pkgutil-style namespace packages in the distributions that need to support Python 2 and 3. + +Legacy namespace packages +------------------------- + +These two methods, that were used to create namespace packages prior to :pep:`420`, +are now considered to be obsolete and should not be used unless you need compatibility +with packages already using this method. Also, :doc:`pkg_resources ` +has been deprecated. + +To migrate an existing package, all packages sharing the namespace must be migrated simultaneously. + +.. warning:: While native namespace packages and pkgutil-style namespace + packages are largely compatible, pkg_resources-style namespace packages + are not compatible with the other methods. It's inadvisable to use + different methods in different distributions that provide packages to the + same namespace. + pkgutil-style namespace packages --------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Python 2.3 introduced the :doc:`pkgutil ` module and the :py:func:`python:pkgutil.extend_path` function. This can be used to declare namespace @@ -138,22 +183,24 @@ To create a pkgutil-style namespace package, you need to provide an .. code-block:: text - setup.py - mynamespace/ - __init__.py # Namespace package __init__.py - subpackage_a/ - __init__.py # Sub-package __init__.py - module.py + mynamespace-subpackage-a/ + src/ + pyproject.toml # AND/OR setup.cfg, setup.py + mynamespace/ + __init__.py # Namespace package __init__.py + subpackage_a/ + __init__.py # Regular package __init__.py + module.py The :file:`__init__.py` file for the namespace package needs to contain -**only** the following: +the following: .. code-block:: python __path__ = __import__('pkgutil').extend_path(__path__, __name__) -**Every** distribution that uses the namespace package must include an -identical :file:`__init__.py`. If any distribution does not, it will cause the +**Every** distribution that uses the namespace package must include such +an :file:`__init__.py`. If any distribution does not, it will cause the namespace logic to fail and the other sub-packages will not be importable. Any additional code in :file:`__init__.py` will be inaccessible. @@ -167,7 +214,7 @@ in the `pkgutil namespace example project`_. pkg_resources-style namespace packages --------------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :doc:`Setuptools ` provides the `pkg_resources.declare_namespace`_ function and the ``namespace_packages`` argument to :func:`~setuptools.setup`. Together @@ -183,22 +230,24 @@ To create a pkg_resources-style namespace package, you need to provide an .. code-block:: text - setup.py - mynamespace/ - __init__.py # Namespace package __init__.py - subpackage_a/ - __init__.py # Sub-package __init__.py - module.py + mynamespace-subpackage-a/ + src/ + pyproject.toml # AND/OR setup.cfg, setup.py + mynamespace/ + __init__.py # Namespace package __init__.py + subpackage_a/ + __init__.py # Regular package __init__.py + module.py The :file:`__init__.py` file for the namespace package needs to contain -**only** the following: +the following: .. code-block:: python __import__('pkg_resources').declare_namespace(__name__) -**Every** distribution that uses the namespace package must include an -identical :file:`__init__.py`. If any distribution does not, it will cause the +**Every** distribution that uses the namespace package must include such an +:file:`__init__.py`. If any distribution does not, it will cause the namespace logic to fail and the other sub-packages will not be importable. Any additional code in :file:`__init__.py` will be inaccessible. diff --git a/source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst b/source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst index 730098587..a3d893c9f 100644 --- a/source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst +++ b/source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst @@ -1,3 +1,5 @@ +.. _trusted-publishing: + ============================================================================= Publishing package distribution releases using GitHub Actions CI/CD workflows ============================================================================= @@ -8,124 +10,189 @@ popular choice is having a workflow that's triggered by a ``push`` event. This guide shows you how to publish a Python distribution whenever a tagged commit is pushed. -It will use the `pypa/gh-action-pypi-publish GitHub Action`_. +It will use the `pypa/gh-action-pypi-publish GitHub Action`_ for +publishing. It also uses GitHub's `upload-artifact`_ and `download-artifact`_ actions +for temporarily storing and downloading the source packages. .. attention:: - This guide *assumes* that you already have a project that - you know how to build distributions for and *it lives on GitHub*. + This guide *assumes* that you already have a project that you know how to + build distributions for and *it lives on GitHub*. This guide also avoids + details of building platform specific projects. If you have binary + components, check out :ref:`cibuildwheel`'s GitHub Action examples. -Saving credentials on GitHub -============================ +Configuring Trusted Publishing +============================== -In this guide, we'll demonstrate uploading to both -PyPI and TestPyPI, meaning that we'll have two separate sets -of credentials. And we'll need to save them in the GitHub repository -settings. +This guide relies on PyPI's `Trusted Publishing`_ implementation to connect +to `GitHub Actions CI/CD`_. This is recommended for security reasons, since +the generated tokens are created for each of your projects +individually and expire automatically. Otherwise, you'll need to generate an +`API token`_ for both PyPI and TestPyPI. In case of publishing to third-party +indexes like :doc:`devpi `, you may need to provide a +username/password combination. + +Since this guide will demonstrate uploading to both +PyPI and TestPyPI, we'll need two trusted publishers configured. +The following steps will lead you through creating the "pending" publishers +for your new :term:`PyPI project `. +However it is also possible to add `Trusted Publishing`_ to any +pre-existing project, if you are its owner. -Let's begin! πŸš€ +.. attention:: -1. Go to https://pypi.org/manage/account/#api-tokens and - create a new `API token`_. If you have the project on PyPI - already, limit the token scope to just that project. - You can call it something like - ``GitHub Actions CI/CD β€” project-org/project-repo`` - in order for it to be easily distinguishable in the token - list. - **Don't close the page just yet β€” you won't see that token - again.** -2. In a separate browser tab or window, go to the ``Settings`` - tab of your target repository and then click on `Secrets`_ - in the left sidebar. -3. Create a new secret called ``PYPI_API_TOKEN`` and copy-paste - the token from the first step. -4. Now, go to https://test.pypi.org/manage/account/#api-tokens - and repeat the steps. Save that TestPyPI token on GitHub - as ``TEST_PYPI_API_TOKEN``. + If you followed earlier versions of this guide, you + have created the secrets ``PYPI_API_TOKEN`` and ``TEST_PYPI_API_TOKEN`` + for direct PyPI and TestPyPI access. These are obsolete now and + you should remove them from your GitHub repository and revoke + them in your PyPI and TestPyPI account settings in case you are replacing your old setup with the new one. - .. attention:: + +Let's begin! πŸš€ + +1. Go to https://pypi.org/manage/account/publishing/. +2. Fill in the name you wish to publish your new + :term:`PyPI project ` under + (the ``name`` value in your ``setup.cfg`` or ``pyproject.toml``), + the GitHub repository owner's name (org or user), + and repository name, and the name of the release workflow file under + the ``.github/`` folder, see :ref:`workflow-definition`. + Finally, add the name of the GitHub Environment + (``pypi``) we're going set up under your repository. + Register the trusted publisher. +3. Now, go to https://test.pypi.org/manage/account/publishing/ and repeat + the second step, but this time, enter ``testpypi`` as the name of the + GitHub Environment. +4. Your "pending" publishers are now ready for their first use and will + create your projects automatically once you use them + for the first time. + + .. note:: If you don't have a TestPyPI account, you'll need to create it. It's not the same as a regular PyPI account. + .. attention:: + + For security reasons, you must require `manual approval `_ + on each run for the ``pypi`` environment. + + +.. _workflow-definition: + Creating a workflow definition ============================== GitHub CI/CD workflows are declared in YAML files stored in the ``.github/workflows/`` directory of your repository. -Let's create a ``.github/workflows/publish-to-test-pypi.yml`` +Let's create a ``.github/workflows/publish-to-pypi.yml`` file. Start it with a meaningful name and define the event that should make GitHub run this workflow: -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml :end-before: jobs: +Checking out the project and building distributions +=================================================== + +We will have to define two jobs to publish to PyPI +and TestPyPI respectively, and an additional job to +build the distribution packages. + +First, we'll define the job for building the dist packages of +your project and storing them for later use: + +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml + :language: yaml + :start-at: jobs: + :end-before: Install pypa/build + +This will download your repository into the CI runner and then +install and activate the newest available Python 3 release. + +And now we can build the dists from source and store them. +In this example, we'll use the ``build`` package. +So add this to the steps list: + +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml + :language: yaml + :start-at: Install pypa/build + :end-before: publish-to-pypi Defining a workflow job environment =================================== -Now, let's add initial setup for our job. It's a process that -will execute commands that we'll define later. +Now, let's add initial setup for our job that will publish to PyPI. +It's a process that will execute commands that we'll define later. In this guide, we'll use the latest stable Ubuntu LTS version -provided by GitHub Actions: +provided by GitHub Actions. This also defines a GitHub Environment +for the job to run in its context and a URL to be displayed in GitHub's +UI nicely. Additionally, it allows acquiring an OpenID Connect token +that the ``pypi-publish`` actions needs to implement secretless +Trusted Publishing to PyPI. -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml - :start-after: on: + :start-after: path: dist/ :end-before: steps: +This will also ensure that the PyPI publishing workflow is only triggered +if the current commit is tagged. -Checking out the project and building distributions -=================================================== +Publishing the distribution to PyPI +=================================== -Then, add the following under the ``build-n-publish`` section: +Finally, add the following steps at the end: -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml - :start-after: runs-on: - :end-before: Install pypa/build + :start-after: id-token: write + :end-before: publish-to-testpypi: -This will download your repository into the CI runner and then -install and activate Python 3.10. - -And now we can build dists from source. In this example, we'll -use ``build`` package, assuming that your project has a -``pyproject.toml`` properly set up (see -:pep:`517`/:pep:`518`). +This step uses the `pypa/gh-action-pypi-publish`_ GitHub +Action: after the stored distribution package has been +downloaded by the `download-artifact`_ action, it uploads +the contents of the ``dist/`` folder into PyPI unconditionally. .. tip:: - You can use any other method for building distributions as long as - it produces ready-to-upload artifacts saved into the - ``dist/`` folder. + Starting with version + `v1.11.0 `_, + `pypa/gh-action-pypi-publish`_ generates and uploads :pep:`740`-compatible + attestations for each distribution by default. No additional manual + signing steps are required. -So add this to the steps list: -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml +Separate workflow for publishing to TestPyPI +============================================ + +Now, repeat these steps and create another job for +publishing to the TestPyPI package index under the ``jobs`` +section: + +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml - :start-after: version: "3.10" - :end-before: Actually publish to PyPI/TestPyPI + :start-at: publish-to-testpypi + +.. tip:: + Requiring manual approvals in the ``testpypi`` GitHub Environment is typically unnecessary as it's designed to run on each commit to the main branch and is often used to indicate a healthy release publishing pipeline. -Publishing the distribution to PyPI and TestPyPI -================================================ -Finally, add the following steps at the end: +The whole CI/CD workflow +======================== -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml - :language: yaml - :start-after: Actually publish to PyPI/TestPyPI +This paragraph showcases the whole workflow after following the above guide. -These two steps use the `pypa/gh-action-pypi-publish`_ GitHub -Action: the first one uploads contents of the ``dist/`` folder -into TestPyPI unconditionally and the second does that to -PyPI, but only if the current commit is tagged. +.. collapse:: Click here to display the entire GitHub Actions CI/CD workflow definition + .. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml + :language: yaml That's all, folks! ================== @@ -134,7 +201,20 @@ Now, whenever you push a tagged commit to your Git repository remote on GitHub, this workflow will publish it to PyPI. And it'll publish any push to TestPyPI which is useful for providing test builds to your alpha users as well as making -sure that your release pipeline remains healthy! +sure that your release pipeline remains healthy! + +.. attention:: + + If your repository has frequent commit activity and every push is uploaded + to TestPyPI as described, the project might exceed the + `PyPI project size limit `_. + The limit could be increased, but a better solution may constitute to + use a PyPI-compatible server like :ref:`pypiserver` in the CI for testing purposes. + +.. note:: + + It is recommended to keep the integrated GitHub Actions at their latest + versions, updating them frequently. .. _API token: https://pypi.org/help/#apitoken @@ -144,5 +224,10 @@ sure that your release pipeline remains healthy! https://github.com/pypa/gh-action-pypi-publish .. _`pypa/gh-action-pypi-publish GitHub Action`: https://github.com/marketplace/actions/pypi-publish +.. _`download-artifact`: + https://github.com/actions/download-artifact +.. _`upload-artifact`: + https://github.com/actions/upload-artifact .. _Secrets: https://docs.github.com/en/actions/reference/encrypted-secrets +.. _Trusted Publishing: https://docs.pypi.org/trusted-publishers/ diff --git a/source/guides/section-build-and-publish.rst b/source/guides/section-build-and-publish.rst new file mode 100644 index 000000000..52f827553 --- /dev/null +++ b/source/guides/section-build-and-publish.rst @@ -0,0 +1,19 @@ +======================= +Building and Publishing +======================= + +.. toctree:: + :titlesonly: + + writing-pyproject-toml + distributing-packages-using-setuptools + dropping-older-python-versions + packaging-binary-extensions + packaging-namespace-packages + creating-command-line-tools + creating-and-discovering-plugins + using-testpypi + making-a-pypi-friendly-readme + publishing-package-distribution-releases-using-github-actions-ci-cd-workflows + modernize-setup-py-project + licensing-examples-and-user-scenarios diff --git a/source/guides/section-hosting.rst b/source/guides/section-hosting.rst new file mode 100644 index 000000000..f8f708c18 --- /dev/null +++ b/source/guides/section-hosting.rst @@ -0,0 +1,9 @@ +======= +Hosting +======= + +.. toctree:: + :titlesonly: + + index-mirrors-and-caches + hosting-your-own-index diff --git a/source/guides/section-install.rst b/source/guides/section-install.rst new file mode 100644 index 000000000..515092b4d --- /dev/null +++ b/source/guides/section-install.rst @@ -0,0 +1,12 @@ +============ +Installation +============ + +.. toctree:: + :titlesonly: + + installing-using-pip-and-virtual-environments + installing-using-virtualenv + installing-stand-alone-command-line-tools + installing-using-linux-tools + installing-scientific-packages diff --git a/source/guides/single-sourcing-package-version.rst b/source/guides/single-sourcing-package-version.rst index 1f6caa70e..7ed3d87da 100644 --- a/source/guides/single-sourcing-package-version.rst +++ b/source/guides/single-sourcing-package-version.rst @@ -1,163 +1,8 @@ -.. _`Single sourcing the version`: +:orphan: -=================================== -Single-sourcing the package version -=================================== +.. meta:: + :http-equiv=refresh: 0; url=../../discussions/single-source-version/ +Redirecting stale single-source package version link... -There are many techniques to maintain a single source of truth for the version -number of your project: - -#. Read the file in :file:`setup.py` and get the version. Example (from `pip setup.py - `_):: - - import codecs - import os.path - - def read(rel_path): - here = os.path.abspath(os.path.dirname(__file__)) - with codecs.open(os.path.join(here, rel_path), 'r') as fp: - return fp.read() - - def get_version(rel_path): - for line in read(rel_path).splitlines(): - if line.startswith('__version__'): - delim = '"' if '"' in line else "'" - return line.split(delim)[1] - else: - raise RuntimeError("Unable to find version string.") - - setup( - ... - version=get_version("package/__init__.py") - ... - ) - - .. note:: - - As of the release of setuptools 46.4.0, one can accomplish the same - thing by instead placing the following in the project's ``setup.cfg`` - file (replacing "package" with the import name of the package): - - .. code-block:: ini - - [metadata] - version = attr: package.__version__ - - Earlier versions of setuptools implemented the ``attr:`` directive by - importing the module, but setuptools 46.4.0 added rudimentary AST - analysis so that ``attr:`` can function without having to import any of - the package's dependencies. - - Also, please be aware that declarative config indicators, including the - ``attr:`` directive, are not supported in parameters to ``setup.py``. - -#. Use an external build tool that either manages updating both locations, or - offers an API that both locations can use. - - Few tools you could use, in no particular order, and not necessarily complete: - `bump2version `_, - `changes `_, - `commitizen `_, - `zest.releaser `_. - - -#. Set the value to a ``__version__`` global variable in a dedicated module in - your project (e.g. :file:`version.py`), then have :file:`setup.py` read and - ``exec`` the value into a variable. - - :: - - version = {} - with open("...sample/version.py") as fp: - exec(fp.read(), version) - # later on we use: version['__version__'] - - Example using this technique: `warehouse `_. - -#. Place the value in a simple ``VERSION`` text file and have both - :file:`setup.py` and the project code read it. - - :: - - with open(os.path.join(mypackage_root_dir, 'VERSION')) as version_file: - version = version_file.read().strip() - - An advantage with this technique is that it's not specific to Python. Any - tool can read the version. - - .. warning:: - - With this approach you must make sure that the ``VERSION`` file is included in - all your source and binary distributions (e.g. add ``include VERSION`` to your - :file:`MANIFEST.in`). - -#. Set the value in :file:`setup.py`, and have the project code use the - ``importlib.metadata`` API to fetch the value at runtime. - (``importlib.metadata`` was introduced in Python 3.8 and is available to - older versions as the ``importlib-metadata`` project.) An installed - project's version can be fetched with the API as follows:: - - import sys - - if sys.version_info >= (3, 8): - from importlib import metadata - else: - import importlib_metadata as metadata - - assert metadata.version('pip') == '1.2.0' - - Be aware that the ``importlib.metadata`` API only knows about what's in the - installation metadata, which is not necessarily the code that's currently - imported. - - If a project uses this method to fetch its version at runtime, then its - ``install_requires`` value needs to be edited to install - ``importlib-metadata`` on pre-3.8 versions of Python like so:: - - setup( - ... - install_requires=[ - ... - 'importlib-metadata >= 1.0 ; python_version < "3.8"', - ... - ], - ... - ) - - An older (and less efficient) alternative to ``importlib.metadata`` is the - ``pkg_resources`` API provided by ``setuptools``:: - - import pkg_resources - assert pkg_resources.get_distribution('pip').version == '1.2.0' - - If a project uses ``pkg_resources`` to fetch its own version at runtime, - then ``setuptools`` must be added to the project's ``install_requires`` - list. - - Example using this technique: `setuptools `_. - - -#. Set the value to ``__version__`` in ``sample/__init__.py`` and import - ``sample`` in :file:`setup.py`. - - :: - - import sample - setup( - ... - version=sample.__version__ - ... - ) - - .. warning:: - - Although this technique is common, beware that it will fail if - ``sample/__init__.py`` imports packages from ``install_requires`` - dependencies, which will very likely not be installed yet when - :file:`setup.py` is run. - - -#. Keep the version number in the tags of a version control system (Git, Mercurial, etc) - instead of in the code, and automatically extract it from there using - `setuptools_scm `_. +If the page doesn't automatically refresh, see :ref:`single-source-version`. diff --git a/source/guides/supporting-multiple-python-versions.rst b/source/guides/supporting-multiple-python-versions.rst index 23c1f51c0..7e945aa53 100644 --- a/source/guides/supporting-multiple-python-versions.rst +++ b/source/guides/supporting-multiple-python-versions.rst @@ -9,9 +9,6 @@ Supporting multiple Python versions :Page Status: Obsolete :Last Reviewed: 2014-12-24 -.. contents:: Contents - :local: - :: @@ -52,7 +49,7 @@ Automated testing and continuous integration Several hosted services for automated testing are available. These services will typically monitor your source code repository (e.g. at -`Github `_ or `Bitbucket `_) +`GitHub `_ or `Bitbucket `_) and run your project's test suite every time a new commit is made. These services also offer facilities to run your project's test suite on @@ -65,7 +62,7 @@ of many continuous-integration systems. There are two hosted services which when used in conjunction provide automated testing across Linux, Mac and Windows: - - `Travis CI `_ provides both a Linux and a macOS + - `Travis CI `_ provides both a Linux and a macOS environment. The Linux environment is Ubuntu 12.04 LTS Server Edition 64 bit while the macOS is 10.9.2 at the time of writing. - `Appveyor `_ provides a Windows environment diff --git a/source/guides/supporting-windows-using-appveyor.rst b/source/guides/supporting-windows-using-appveyor.rst index 79242510c..e884dd976 100644 --- a/source/guides/supporting-windows-using-appveyor.rst +++ b/source/guides/supporting-windows-using-appveyor.rst @@ -12,9 +12,6 @@ service to provide Windows support for your project. This includes testing the code on Windows, and building Windows-targeted binaries for projects that use C extensions. -.. contents:: Contents - :local: - Background ========== @@ -25,7 +22,7 @@ non-trivial, and may require buying software licenses. The Appveyor service is a continuous integration service, much like the better-known `Travis`_ service that is commonly used for testing by projects -hosted on `Github`_. However, unlike Travis, the build workers on Appveyor are +hosted on `GitHub`_. However, unlike Travis, the build workers on Appveyor are Windows hosts and have the necessary compilers installed to build Python extensions. @@ -44,7 +41,7 @@ have an account on the service. Instructions on setting up an account are given in `the Appveyor documentation `__. The free tier of account is perfectly adequate for open source projects. -Appveyor provides integration with `Github`_ and `Bitbucket`_, so as long as +Appveyor provides integration with `GitHub`_ and `Bitbucket`_, so as long as your project is hosted on one of those two services, setting up Appveyor integration is straightforward. @@ -240,6 +237,6 @@ For reference, the SDK setup support script is listed here: :linenos: .. _Appveyor: https://www.appveyor.com/ -.. _Travis: https://travis-ci.org/ -.. _Github: https://github.com +.. _Travis: https://travis-ci.com/ +.. _GitHub: https://github.com .. _Bitbucket: https://bitbucket.org/ diff --git a/source/guides/tool-recommendations.rst b/source/guides/tool-recommendations.rst index fc0afd1df..bf8d93d5a 100644 --- a/source/guides/tool-recommendations.rst +++ b/source/guides/tool-recommendations.rst @@ -4,111 +4,187 @@ Tool recommendations ==================== -If you're familiar with Python packaging and installation, and just want to know -what tools are currently recommended, then here it is. +The Python packaging landscape consists of many different tools. For many tasks, +the :term:`Python Packaging Authority ` +(PyPA, the working group which encompasses many packaging tools and +maintains this guide) purposefully does not make a blanket recommendation; for +example, the reason there are many build backends is that the landscape was +opened up in order to enable the development of new backends serving certain users' +needs better than the previously unique backend, setuptools. This guide does +point to some tools that are widely recognized, and also makes some +recommendations of tools that you should *not* use because they are deprecated +or insecure. + + +Virtual environments +==================== + +The standard tools to create and use virtual environments manually are +:ref:`virtualenv` (PyPA project) and :doc:`venv ` (part of +the Python standard library, though missing some features of virtualenv). + + +Installing packages +=================== + +:ref:`Pip` is the standard tool to install packages from :term:`PyPI `. You may want to read pip's recommendations for +:doc:`secure installs `. Pip is available by default +in most Python installations through the standard library package +:doc:`ensurepip `. + +Alternatively, consider :ref:`pipx` for the specific use case of installing Python +applications that are distributed through PyPI and run from the command line. +Pipx is a wrapper around pip and venv that installs each +application into a dedicated virtual environment. This avoids conflicts between +the dependencies of different applications, and also with system-wide applications +making use of the same Python interpreter (especially on Linux). + +For scientific software specifically, consider :ref:`Conda` or :ref:`Spack`. + +.. todo:: Write a "pip vs. Conda" comparison, here or in a new discussion. + +Do **not** use ``easy_install`` (part of :ref:`setuptools`), which is deprecated +in favor of pip (see :ref:`pip vs easy_install` for details). Likewise, do +**not** use ``python setup.py install`` or ``python setup.py develop``, which +are also deprecated (see :ref:`setup-py-deprecated` for background and +:ref:`modernize-setup-py-project` for migration advice). + + +Lock files +========== + +:ref:`pip-tools` and :ref:`Pipenv` are two recognized tools to create lock +files, which contain the exact versions of all packages installed into an +environment, for reproducibility purposes. + + +Build backends +============== + +.. important:: + + Please, remember: this document does not seek to steer the reader towards + a particular tool, only to enumerate common tools. Different use cases often + need specialized workflows. + +Popular :term:`build backends ` for pure-Python packages include, +in alphabetical order: + +- :doc:`Flit-core ` -- developed with but separate from :ref:`Flit`. + A minimal and opinionated build backend. It does not support plugins. + +- Hatchling_ -- developed with but separate from :ref:`Hatch`. Supports plugins. + +- PDM-backend_ -- developed with but separate from :ref:`PDM`. Supports plugins. +- Poetry-core_ -- developed with but separate from :ref:`Poetry`. Supports + plugins. -Application dependency management -================================= + Unlike other backends on this list, Poetry-core does not support the standard + :ref:`[project] table ` (it uses a different format, + in the ``[tool.poetry]`` table). -* Use :ref:`pip` in a `secure manner`_ to install a Python application and its - dependencies during deployment. +- :ref:`setuptools`, which used to be the only build backend. Supports plugins. -* Use :ref:`virtualenv` or :doc:`venv ` to isolate - application-specific dependencies from a shared Python installation. [4]_ + .. caution:: -* Use `pip-tools`_, :ref:`pipenv`, or `poetry`_ to generate the fully-specified - application-specific dependencies, when developing Python applications. + If you use setuptools, please be aware that some features that predate + standardisation efforts are now deprecated and only *temporarily kept* + for compatibility. -.. _secure manner: https://pip.pypa.io/en/latest/topics/secure-installs/ -.. _pip-tools: https://github.com/jazzband/pip-tools -.. _Poetry: https://python-poetry.org/ + In particular, do **not** use direct ``python setup.py`` invocations. On the + other hand, configuring setuptools with a :file:`setup.py` file is still fully + supported, although it is recommended to use the modern :ref:`[project] table + in pyproject.toml ` (or :file:`setup.cfg`) whenever possible and keep + :file:`setup.py` only if programmatic configuration is needed. See + :ref:`setup-py-deprecated`. -Installation tool recommendations -================================= + Other examples of deprecated features you should **not** use include the + ``setup_requires`` argument to ``setup()`` (use the :ref:`[build-system] table + ` in :file:`pyproject.toml` instead), and + the ``easy_install`` command (cf. :ref:`pip vs easy_install`). -* Use :ref:`pip` to install Python :term:`packages ` from - :term:`PyPI `. [1]_ [2]_ Depending on how :ref:`pip` - is installed, you may need to also install :ref:`wheel` to get the benefit - of wheel caching. [3]_ +Do **not** use :ref:`distutils`, which is deprecated, and has been removed from +the standard library in Python 3.12, although it still remains available from +setuptools. -* Use :ref:`virtualenv` or :doc:`venv ` to isolate - project-specific dependencies from a shared Python installation. [4]_ +.. _extension-module-tool-recommendations: -* If you're looking for management of fully integrated cross-platform software - stacks, consider: +Build backends for extension modules +------------------------------------ - * :ref:`buildout`: primarily focused on the web development community +For packages with :term:`extension modules `, it is best to use +a build system with dedicated support for the language the extension is written in, +for example: - * :ref:`spack`, :ref:`hashdist`, or :ref:`conda`: primarily focused - on the scientific community. +- :ref:`setuptools` -- natively supports C and C++ (with third-party plugins for Go and Rust), +- :ref:`meson-python` -- C, C++, Fortran, Rust, and other languages supported by Meson, +- :ref:`scikit-build-core` -- C, C++, Fortran, and other languages supported by CMake, +- :ref:`maturin` -- Rust, via Cargo. -Packaging tool recommendations -============================== +Building distributions +====================== -* Use :ref:`setuptools` to define projects. [5]_ [6]_ +The standard tool to build :term:`source distributions ` and :term:`wheels ` for uploading to PyPI is :ref:`build`. It +will invoke whichever build backend you :ref:`declared +` in :file:`pyproject.toml`. -* Use :ref:`build` to create :term:`Source Distributions - ` and :term:`wheels `. +Do **not** use ``python setup.py sdist`` and ``python setup.py bdist_wheel`` for +this task. All direct invocations of :file:`setup.py` are :ref:`deprecated +`. -If you have binary extensions and want to distribute wheels for multiple -platforms, use :ref:`cibuildwheel` as part of your CI setup to build -distributable wheels. +If you have :term:`extension modules ` and want to distribute +wheels for multiple platforms, use :ref:`cibuildwheel` as part of your CI setup +to build distributable wheels. -* Use `twine `_ for uploading distributions - to :term:`PyPI `. +Uploading to PyPI +================= -Publishing platform migration -============================= +For projects hosted on or published via supported CI/CD platforms, it is +recommended to use the :ref:`Trusted Publishing `, which +allows the package to be securely uploaded to PyPI from a CI/CD workflow +without a manually configured API token. -The original Python Package Index implementation (previously hosted at -`pypi.python.org `_) has been phased out in favour -of an updated implementation hosted at `pypi.org `_. +As of November 2024, PyPI supports the following platforms as Trusted Publishing +providers: -See :ref:`Migrating to PyPI.org` for more information on the status of the -migration, and what settings to change in your clients. +* GitHub Actions (on ``https://github.com``) +* GitLab CI/CD (on ``https://gitlab.com``) +* ActiveState +* Google Cloud ----- +The other available method is to upload the package manually using :ref:`twine`. -.. [1] There are some cases where you might choose to use ``easy_install`` (from - :ref:`setuptools`), e.g. if you need to install from :term:`Eggs ` - (which pip doesn't support). For a detailed breakdown, see :ref:`pip vs - easy_install`. +.. danger:: -.. [2] The acceptance of :pep:`453` means that :ref:`pip` - will be available by default in most installations of Python 3.4 or - later. See the :pep:`rationale section <453#rationale>` from :pep:`453` - as for why pip was chosen. + **Never** use ``python setup.py upload`` for this task. In addition to being + :ref:`deprecated `, it is insecure. -.. [3] `get-pip.py `_ and - :ref:`virtualenv` install - :ref:`wheel`, whereas :ref:`ensurepip` and :ref:`venv ` do not - currently. Also, the common "python-pip" package that's found in various - linux distros, does not depend on "python-wheel" currently. -.. [4] Beginning with Python 3.4, ``venv`` will create virtualenv environments - with ``pip`` installed, thereby making it an equal alternative to - :ref:`virtualenv`. However, using :ref:`virtualenv` will still be - recommended for users that need cross-version consistency. +Workflow tools +============== -.. [5] Although you can use pure ``distutils`` for many projects, it does not - support defining dependencies on other projects and is missing several - convenience utilities for automatically populating distribution metadata - correctly that are provided by ``setuptools``. Being outside the - standard library, ``setuptools`` also offers a more consistent feature - set across different versions of Python, and (unlike ``distutils``), - recent versions of ``setuptools`` support all of the modern metadata - fields described in :ref:`core-metadata`. +These tools are environment managers that automatically manage virtual +environments for a project. They also act as "task runners", allowing you to +define and invoke tasks such as running tests, compiling documentation, +regenerating some files, etc. Some of them provide shortcuts for building +distributions and uploading to PyPI, and some support lock files for +applications. They often call the tools mentioned above under the hood. In +alphabetical order: - Even for projects that do choose to use ``distutils``, when :ref:`pip` - installs such projects directly from source (rather than installing - from a prebuilt :term:`wheel ` file), it will actually build - your project using :ref:`setuptools` instead. +- :ref:`Flit`, +- :ref:`Hatch`, +- :doc:`nox `, +- :ref:`PDM`, +- :ref:`Pipenv`, +- :ref:`Poetry`, +- :doc:`tox `. -.. [6] `distribute`_ (a fork of setuptools) was merged back into - :ref:`setuptools` in June 2013, thereby making setuptools the default - choice for packaging. -.. _distribute: https://pypi.org/project/distribute +.. _hatchling: https://pypi.org/project/hatchling/ +.. _pdm-backend: https://backend.pdm-project.org +.. _poetry-core: https://pypi.org/project/poetry-core/ diff --git a/source/guides/using-manifest-in.rst b/source/guides/using-manifest-in.rst index 1eaf42c18..24bf154a1 100644 --- a/source/guides/using-manifest-in.rst +++ b/source/guides/using-manifest-in.rst @@ -1,111 +1,9 @@ -.. _`Using MANIFEST.in`: +:orphan: ============================================================ Including files in source distributions with ``MANIFEST.in`` ============================================================ -When building a :term:`source distribution ` -for your package, by default only a minimal set of files are included. You may -find yourself wanting to include extra files in the source distribution, such -as an authors/contributors file, a :file:`docs/` directory, or a directory of -data files used for testing purposes. There may even be extra files that you -*need* to include; for example, if your :file:`setup.py` computes your -project's ``long_description`` by reading from both a README and a changelog -file, you'll need to include both those files in the sdist so that people that -build or install from the sdist get the correct results. - -Adding & removing files to & from the source distribution is done by writing a -:file:`MANIFEST.in` file at the project root. - - -How files are included in an sdist -================================== - -The following files are included in a source distribution by default: - -- all Python source files implied by the ``py_modules`` and ``packages`` - ``setup()`` arguments -- all C source files mentioned in the ``ext_modules`` or ``libraries`` - ``setup()`` arguments -- scripts specified by the ``scripts`` ``setup()`` argument -- all files specified by the ``package_data`` and ``data_files`` ``setup()`` - arguments -- the file specified by the ``license_file`` option in :file:`setup.cfg` - (setuptools 40.8.0+) -- all files specified by the ``license_files`` option in :file:`setup.cfg` - (setuptools 42.0.0+) -- all files matching the pattern :file:`test/test*.py` -- :file:`setup.py` (or whatever you called your setup script) -- :file:`setup.cfg` -- :file:`README` -- :file:`README.txt` -- :file:`README.rst` (Python 3.7+ or setuptools 0.6.27+) -- :file:`README.md` (setuptools 36.4.0+) -- :file:`pyproject.toml` (setuptools 43.0.0+) -- :file:`MANIFEST.in` - -After adding the above files to the sdist, the commands in :file:`MANIFEST.in` -(if such a file exists) are executed in order to add and remove further files -to and from the sdist. Default files can even be removed from the sdist with the -appropriate :file:`MANIFEST.in` command. - -After processing the :file:`MANIFEST.in` file, setuptools removes the -:file:`build/` directory as well as any directories named :file:`RCS`, -:file:`CVS`, or :file:`.svn` from the sdist, and it adds a :file:`PKG-INFO` -file and an :file:`*.egg-info` directory. This behavior cannot be changed with -:file:`MANIFEST.in`. - - -:file:`MANIFEST.in` commands -============================ - -A :file:`MANIFEST.in` file consists of commands, one per line, instructing -setuptools to add or remove some set of files from the sdist. The commands -are: - -========================================================= ================================================================================================== -Command Description -========================================================= ================================================================================================== -:samp:`include {pat1} {pat2} ...` Add all files matching any of the listed patterns - (Files must be given as paths relative to the root of the project) -:samp:`exclude {pat1} {pat2} ...` Remove all files matching any of the listed patterns - (Files must be given as paths relative to the root of the project) -:samp:`recursive-include {dir-pattern} {pat1} {pat2} ...` Add all files under directories matching ``dir-pattern`` that match any of the listed patterns -:samp:`recursive-exclude {dir-pattern} {pat1} {pat2} ...` Remove all files under directories matching ``dir-pattern`` that match any of the listed patterns -:samp:`global-include {pat1} {pat2} ...` Add all files anywhere in the source tree matching any of the listed patterns -:samp:`global-exclude {pat1} {pat2} ...` Remove all files anywhere in the source tree matching any of the listed patterns -:samp:`graft {dir-pattern}` Add all files under directories matching ``dir-pattern`` -:samp:`prune {dir-pattern}` Remove all files under directories matching ``dir-pattern`` -========================================================= ================================================================================================== - -The patterns here are glob-style patterns: ``*`` matches zero or more regular -filename characters (on Unix, everything except forward slash; on Windows, -everything except backslash and colon); ``?`` matches a single regular filename -character, and ``[chars]`` matches any one of the characters between the square -brackets (which may contain character ranges, e.g., ``[a-z]`` or -``[a-fA-F0-9]``). Setuptools also has undocumented support for ``**`` matching -zero or more characters including forward slash, backslash, and colon. - -Directory patterns are relative to the root of the project directory; e.g., -``graft example*`` will include a directory named :file:`examples` in the -project root but will not include :file:`docs/examples/`. - -File & directory names in :file:`MANIFEST.in` should be ``/``-separated; -setuptools will automatically convert the slashes to the local platform's -appropriate directory separator. - -Commands are processed in the order they appear in the :file:`MANIFEST.in` -file. For example, given the commands: - -.. code-block:: bash - - graft tests - global-exclude *.py[cod] - -the contents of the directory tree :file:`tests` will first be added to the -sdist, and then after that all files in the sdist with a ``.pyc``, ``.pyo``, or -``.pyd`` extension will be removed from the sdist. If the commands were in the -opposite order, then ``*.pyc`` files etc. would be only be removed from what -was already in the sdist before adding :file:`tests`, and if :file:`tests` -happened to contain any ``*.pyc`` files, they would end up included in the -sdist because the exclusion happened before they were included. +The information on this page has moved to +:doc:`setuptools:userguide/miscellaneous` in the setuptools +documentation. diff --git a/source/guides/using-testpypi.rst b/source/guides/using-testpypi.rst index 1d466e4ea..1dbc82130 100644 --- a/source/guides/using-testpypi.rst +++ b/source/guides/using-testpypi.rst @@ -72,12 +72,13 @@ you're testing has dependencies: Setting up TestPyPI in :file:`.pypirc` -------------------------------------- -If you want to avoid entering your username, you can configure TestPyPI in -your :file:`$HOME/.pypirc`: +If you want to avoid being prompted for your username and password every time, +you can configure TestPyPI in your :file:`$HOME/.pypirc`: -.. code:: +.. code:: ini [testpypi] - username = + username = __token__ + password = For more details, see the :ref:`specification ` for :file:`.pypirc`. diff --git a/source/guides/writing-pyproject-toml.rst b/source/guides/writing-pyproject-toml.rst new file mode 100644 index 000000000..a1a595a13 --- /dev/null +++ b/source/guides/writing-pyproject-toml.rst @@ -0,0 +1,595 @@ +.. _writing-pyproject-toml: + +=============================== +Writing your ``pyproject.toml`` +=============================== + +``pyproject.toml`` is a configuration file used by packaging tools, as +well as other tools such as linters, type checkers, etc. There are +three possible TOML tables in this file. + +- The ``[build-system]`` table is **strongly recommended**. It allows + you to declare which :term:`build backend` you use and which other + dependencies are needed to build your project. + +- The ``[project]`` table is the format that most build backends use to specify + your project's basic metadata, such as the dependencies, your name, etc. + +- The ``[tool]`` table has tool-specific subtables, e.g., ``[tool.hatch]``, + ``[tool.black]``, ``[tool.mypy]``. We only touch upon this table here because + its contents are defined by each tool. Consult the particular tool's + documentation to know what it can contain. + +.. note:: + + The ``[build-system]`` table should always be present, + regardless of which build backend you use (``[build-system]`` *defines* the + build tool you use). + + On the other hand, the ``[project]`` table is understood by *most* build + backends, but some build backends use a different format. + + A notable exception is Poetry_, which before version 2.0 (released January + 5, 2025) did not use the ``[project]`` table, it used the ``[tool.poetry]`` + table instead. With version 2.0, it supports both. + Also, the setuptools_ build backend supports both the ``[project]`` table, + and the older format in ``setup.cfg`` or ``setup.py``. + + For new projects, use the ``[project]`` table, and keep ``setup.py`` only if + some programmatic configuration is needed (such as building C extensions), + but the ``setup.cfg`` and ``setup.py`` formats are still valid. See + :ref:`setup-py-deprecated`. + + +.. _pyproject-guide-build-system-table: + +Declaring the build backend +=========================== + +The ``[build-system]`` table contains a ``build-backend`` key, which specifies +the build backend to be used. It also contains a ``requires`` key, which is a +list of dependencies needed to build the project -- this is typically just the +build backend package, but it may also contain additional dependencies. You can +also constrain the versions, e.g., ``requires = ["setuptools >= 61.0"]``. + +Usually, you'll just copy what your build backend's documentation +suggests (after :ref:`choosing your build backend `). +Here are the values for some common build backends: + +.. include:: ../shared/build-backend-tabs.rst + + +Static vs. dynamic metadata +=========================== + +The rest of this guide is devoted to the ``[project]`` table. + +Most of the time, you will directly write the value of a ``[project]`` +field. For example: ``requires-python = ">= 3.8"``, or ``version = +"1.0"``. + +However, in some cases, it is useful to let your build backend compute +the metadata for you. For example: many build backends can read the +version from a ``__version__`` attribute in your code, a Git tag, or +similar. In such cases, you should mark the field as dynamic using, e.g., + +.. code-block:: toml + + [project] + dynamic = ["version"] + + +When a field is dynamic, it is the build backend's responsibility to +fill it. Consult your build backend's documentation to learn how it +does it. + + +Basic information +================= + +.. _`setup() name`: + +``name`` +-------- + +Put the name of your project on PyPI. This field is required and is the +only field that cannot be marked as dynamic. + +.. code-block:: toml + + [project] + name = "spam-eggs" + +The project name must consist of ASCII letters, digits, underscores "``_``", +hyphens "``-``" and periods "``.``". It must not start or end with an +underscore, hyphen or period. + +Comparison of project names is case insensitive and treats arbitrarily long runs +of underscores, hyphens, and/or periods as equal. For example, if you register +a project named ``cool-stuff``, users will be able to download it or declare a +dependency on it using any of the following spellings: ``Cool-Stuff``, +``cool.stuff``, ``COOL_STUFF``, ``CoOl__-.-__sTuFF``. + + +``version`` +----------- + +Put the version of your project. + +.. code-block:: toml + + [project] + version = "2020.0.0" + +Some more complicated version specifiers like ``2020.0.0a1`` (for an alpha +release) are possible; see the :ref:`specification ` +for full details. + +This field is required, although it is often marked as dynamic using + +.. code-block:: toml + + [project] + dynamic = ["version"] + +This allows use cases such as filling the version from a ``__version__`` +attribute or a Git tag. Consult the :ref:`single-source-version` +discussion for more details. + + +Dependencies and requirements +============================= + +``dependencies``/``optional-dependencies`` +------------------------------------------ + +If your project has dependencies, list them like this: + +.. code-block:: toml + + [project] + dependencies = [ + "httpx", + "gidgethub[httpx]>4.0.0", + "django>2.1; os_name != 'nt'", + "django>2.0; os_name == 'nt'", + ] + +See :ref:`Dependency specifiers ` for the full +syntax you can use to constrain versions. + +You may want to make some of your dependencies optional, if they are +only needed for a specific feature of your package. In that case, put +them in ``optional-dependencies``. + +.. code-block:: toml + + [project.optional-dependencies] + gui = ["PyQt5"] + cli = [ + "rich", + "click", + ] + +Each of the keys defines a "packaging extra". In the example above, one +could use, e.g., ``pip install your-project-name[gui]`` to install your +project with GUI support, adding the PyQt5 dependency. + + +.. _requires-python: +.. _python_requires: + +``requires-python`` +------------------- + +This lets you declare the minimum version of Python that you support +[#requires-python-upper-bounds]_. + +.. code-block:: toml + + [project] + requires-python = ">= 3.8" + + +.. _`console_scripts`: + +Creating executable scripts +=========================== + +To install a command as part of your package, declare it in the +``[project.scripts]`` table. + +.. code-block:: toml + + [project.scripts] + spam-cli = "spam:main_cli" + +In this example, after installing your project, a ``spam-cli`` command +will be available. Executing this command will do the equivalent of +``import sys; from spam import main_cli; sys.exit(main_cli())``. + +On Windows, scripts packaged this way need a terminal, so if you launch +them from within a graphical application, they will make a terminal pop +up. To prevent this from happening, use the ``[project.gui-scripts]`` +table instead of ``[project.scripts]``. + +.. code-block:: toml + + [project.gui-scripts] + spam-gui = "spam:main_gui" + +In that case, launching your script from the command line will give back +control immediately, leaving the script to run in the background. + +The difference between ``[project.scripts]`` and +``[project.gui-scripts]`` is only relevant on Windows. + + + +About your project +================== + +``authors``/``maintainers`` +--------------------------- + +Both of these fields contain lists of people identified by a name and/or +an email address. + +.. code-block:: toml + + [project] + authors = [ + {name = "Pradyun Gedam", email = "pradyun@example.com"}, + {name = "Tzu-Ping Chung", email = "tzu-ping@example.com"}, + {name = "Another person"}, + {email = "different.person@example.com"}, + ] + maintainers = [ + {name = "Brett Cannon", email = "brett@example.com"} + ] + + +.. _description: + +``description`` +--------------- + +This should be a one-line description of your project, to show as the "headline" +of your project page on PyPI (`example `_), and other places such as +lists of search results (`example `_). + +.. code-block:: toml + + [project] + description = "Lovely Spam! Wonderful Spam!" + + +``readme`` +---------- + +This is a longer description of your project, to display on your project +page on PyPI. Typically, your project will have a ``README.md`` or +``README.rst`` file and you just put its file name here. + +.. code-block:: toml + + [project] + readme = "README.md" + +The README's format is auto-detected from the extension: + +- ``README.md`` β†’ `GitHub-flavored Markdown `_, +- ``README.rst`` β†’ `reStructuredText `_ (without Sphinx extensions). + +You can also specify the format explicitly, like this: + +.. code-block:: toml + + [project] + readme = {file = "README.txt", content-type = "text/markdown"} + # or + readme = {file = "README.txt", content-type = "text/x-rst"} + + +.. _license-and-license-files: + +``license`` and ``license-files`` +--------------------------------- + +As per :pep:`639`, licenses should be declared with two fields: + +- ``license`` is an :term:`SPDX license expression ` + consisting of one or more :term:`license identifiers `. +- ``license-files`` is a list of license file glob patterns. + +A previous PEP had specified ``license`` to be a table with a ``file`` or a +``text`` key, this format is now deprecated. Most :term:`build backends` now support the new format as shown in the following table. + +.. list-table:: build backend versions that introduced :pep:`639` support + :header-rows: 1 + + * - hatchling + - setuptools + - flit-core [#flit-core-pep639]_ + - pdm-backend + - poetry-core + - uv-build + * - 1.27.0 + - 77.0.3 + - 3.12 + - 2.4.0 + - 2.2.0 + - 0.7.19 + + +.. _license: + +``license`` +''''''''''' + +The new format for ``license`` is a valid :term:`SPDX license expression ` +consisting of one or more :term:`license identifiers `. +The full license list is available at the +`SPDX license list page `_. The supported list version is +3.17 or any later compatible one. + +.. code-block:: toml + + [project] + license = "GPL-3.0-or-later" + # or + license = "MIT AND (Apache-2.0 OR BSD-2-Clause)" + +.. note:: If you get a build error that ``license`` should be a dict/table, + your build backend doesn't yet support the new format. See the + `above section `_ for more context. + The now deprecated format is `described in PEP 621 `__. + +As a general rule, it is a good idea to use a standard, well-known +license, both to avoid confusion and because some organizations avoid software +whose license is unapproved. + +If your :term:`Distribution Archive` is licensed with a license that doesn't +have an existing SPDX identifier, you can create a custom one in format +``LicenseRef-[idstring]``. The custom identifiers must follow the SPDX +specification, `clause 10.1 `_ of the version 2.2 or any later +compatible one. + +.. code-block:: toml + + [project] + license = "LicenseRef-My-Custom-License" + + +.. _license-files: + +``license-files`` +''''''''''''''''' + +This is a list of license files and files containing other legal +information you want to distribute with your package. + +.. code-block:: toml + + [project] + license-files = ["LICEN[CS]E*", "vendored/licenses/*.txt", "AUTHORS.md"] + +The glob patterns must follow the specification: + +- Alphanumeric characters, underscores (``_``), hyphens (``-``) and dots (``.``) + will be matched verbatim. +- Special characters: ``*``, ``?``, ``**`` and character ranges: [] are supported. +- Path delimiters must be the forward slash character (``/``). +- Patterns are relative to the directory containing :file:`pyproject.toml`, and + thus may not start with a slash character. +- Parent directory indicators (``..``) must not be used. +- Each glob must match at least one file. + +Literal paths are valid globs. +Any characters or character sequences not covered by this specification are +invalid. + + +``keywords`` +------------ + +This will help PyPI's search box to suggest your project when people +search for these keywords. + +.. code-block:: toml + + [project] + keywords = ["egg", "bacon", "sausage", "tomatoes", "Lobster Thermidor"] + + +.. _classifiers: + +``classifiers`` +--------------- + +A list of PyPI classifiers that apply to your project. Check the +`full list of possibilities `_. + +.. code-block:: toml + + classifiers = [ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 4 - Beta", + + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + + # Specify the Python versions you support here. + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + ] + +Although the list of classifiers is often used to declare what Python versions a +project supports, this information is only used for searching and browsing +projects on PyPI, not for installing projects. To actually restrict what Python +versions a project can be installed on, use the :ref:`requires-python` argument. + +To prevent a package from being uploaded to PyPI, use the special ``Private :: +Do Not Upload`` classifier. PyPI will always reject packages with classifiers +beginning with ``Private ::``. + +.. _writing-pyproject-toml-urls: + +``urls`` +-------- + +A list of URLs associated with your project, displayed on the left +sidebar of your PyPI project page. + +.. note:: + + See :ref:`well-known-labels` for a listing + of labels that PyPI and other packaging tools are specifically aware of, + and `PyPI's project metadata docs `_ + for PyPI-specific URL processing. + +.. code-block:: toml + + [project.urls] + Homepage = "https://example.com" + Documentation = "https://readthedocs.org" + Repository = "https://github.com/me/spam.git" + Issues = "https://github.com/me/spam/issues" + Changelog = "https://github.com/me/spam/blob/master/CHANGELOG.md" + +Note that if the label contains spaces, it needs to be quoted, e.g., +``Website = "https://example.com"`` but +``"Official Website" = "https://example.com"``. + +Users are advised to use :ref:`well-known-labels` for their project URLs +where appropriate, since consumers of metadata (like package indices) can +specialize their presentation. + +For example in the following metadata, neither ``MyHomepage`` nor +``"Download Link"`` is a well-known label, so they will be rendered verbatim: + +.. code-block:: toml + + [project.urls] + MyHomepage = "https://example.com" + "Download Link" = "https://example.com/abc.tar.gz" + + +Whereas in this metadata ``HomePage`` and ``DOWNLOAD`` both have +well-known equivalents (``homepage`` and ``download``), and can be presented +with those semantics in mind (the project's home page and its external +download location, respectively). + +.. code-block:: toml + + [project.urls] + HomePage = "https://example.com" + DOWNLOAD = "https://example.com/abc.tar.gz" + +Advanced plugins +================ + +Some packages can be extended through plugins. Examples include Pytest_ +and Pygments_. To create such a plugin, you need to declare it in a subtable +of ``[project.entry-points]`` like this: + +.. code-block:: toml + + [project.entry-points."spam.magical"] + tomatoes = "spam:main_tomatoes" + +See the :ref:`Plugin guide ` for more information. + + + +A full example +============== + +.. code-block:: toml + + [build-system] + requires = ["hatchling"] + build-backend = "hatchling.build" + + [project] + name = "spam-eggs" + version = "2020.0.0" + dependencies = [ + "httpx", + "gidgethub[httpx]>4.0.0", + "django>2.1; os_name != 'nt'", + "django>2.0; os_name == 'nt'", + ] + requires-python = ">=3.8" + authors = [ + {name = "Pradyun Gedam", email = "pradyun@example.com"}, + {name = "Tzu-Ping Chung", email = "tzu-ping@example.com"}, + {name = "Another person"}, + {email = "different.person@example.com"}, + ] + maintainers = [ + {name = "Brett Cannon", email = "brett@example.com"} + ] + description = "Lovely Spam! Wonderful Spam!" + readme = "README.rst" + license = "MIT" + license-files = ["LICEN[CS]E.*"] + keywords = ["egg", "bacon", "sausage", "tomatoes", "Lobster Thermidor"] + classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python" + ] + + [project.optional-dependencies] + gui = ["PyQt5"] + cli = [ + "rich", + "click", + ] + + [project.urls] + Homepage = "https://example.com" + Documentation = "https://readthedocs.org" + Repository = "https://github.com/me/spam.git" + "Bug Tracker" = "https://github.com/me/spam/issues" + Changelog = "https://github.com/me/spam/blob/master/CHANGELOG.md" + + [project.scripts] + spam-cli = "spam:main_cli" + + [project.gui-scripts] + spam-gui = "spam:main_gui" + + [project.entry-points."spam.magical"] + tomatoes = "spam:main_tomatoes" + + +------------------ + +.. [#requires-python-upper-bounds] Think twice before applying an upper bound + like ``requires-python = "<= 3.10"`` here. `This blog post `_ + contains some information regarding possible problems. + +.. [#flit-core-pep639] flit-core `does not yet `_ support WITH in SPDX license expressions. + +.. _flit-issue-735: https://github.com/pypa/flit/issues/735 +.. _gfm: https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax +.. _setuptools: https://setuptools.pypa.io +.. _poetry: https://python-poetry.org +.. _pypi-pip: https://pypi.org/project/pip +.. _pypi-search-pip: https://pypi.org/search?q=pip +.. _classifier-list: https://pypi.org/classifiers +.. _requires-python-blog-post: https://iscinumpy.dev/post/bound-version-constraints/#pinning-the-python-version-is-special +.. _pytest: https://pytest.org +.. _pygments: https://pygments.org +.. _rest: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html +.. _spdxcustomids: https://spdx.github.io/spdx-spec/v2.2.2/other-licensing-information-detected/ +.. _spdxlicenselist: https://spdx.org/licenses/ diff --git a/source/index.rst b/source/index.rst index 4cd8c8425..aa522c8c5 100644 --- a/source/index.rst +++ b/source/index.rst @@ -27,39 +27,67 @@ references to help you distribute and install Python packages with modern tools. This guide is maintained on `GitHub`_ by the :doc:`Python Packaging Authority `. We -happily accept any :doc:`contributions and feedback `. 😊 +happily accept :doc:`contributions and feedback `. 😊 .. _GitHub: https://github.com/pypa/packaging.python.org -Get started -=========== +Overview and Flow +================= -Essential tools and concepts for working within the Python -development ecosystem are covered in our :doc:`tutorials/index` section: +.. note:: -* To get an overview of the flow used to publish your code, see the - :doc:`packaging flow ` -* To learn how to install packages, see the - :doc:`tutorial on installing packages ` -* To learn how to manage dependencies in a version controlled project, see the - :doc:`tutorial on managing application dependencies ` -* To learn how to package and distribute your projects, see the - :doc:`tutorial on packaging and distributing ` -* To get an overview of packaging options for Python libraries and - applications, see the :doc:`Overview of Python Packaging ` + Building your understanding of Python packaging is a journey. Patience and + continuous improvement are key to success. The overview and flow sections + provide a starting point for understanding the Python packaging ecosystem. +The :doc:`overview` explains Python packaging +and its use when preparing and distributing projects. +This section helps you build understanding about selecting the tools and +processes that are most suitable for your use case. +It includes what packaging is, the problems that it solves, and +key considerations. -Learn more -========== +To get an overview of the workflow used to publish your code, see +:doc:`packaging flow `. -Beyond our :doc:`tutorials/index`, this guide has several other resources: +Tutorials +========= -* The :doc:`guides/index` section for walk throughs, such as - :doc:`guides/installing-using-linux-tools` or :doc:`guides/packaging-binary-extensions`. -* The :doc:`discussions/index` section for in-depth references on topics such as - :doc:`discussions/deploying-python-applications` or :doc:`discussions/pip-vs-easy-install`. -* The :doc:`specifications/index` section for packaging interoperability specifications. +Tutorials walk through the steps needed to complete a project for the first time. +Tutorials aim to help you succeed and provide a starting point for future +exploration. +The :doc:`tutorials/index` section includes: + +* A :doc:`tutorial on installing packages ` +* A :doc:`tutorial on managing application dependencies ` + in a version controlled project +* A :doc:`tutorial on packaging and distributing ` + your project + +Guides +====== + +Guides provide steps to perform a specific task. Guides are more focused on +users who are already familiar with Python packaging and are looking for +specific information. + +The :doc:`guides/index` section provides "how to" instructions in three major +areas: package installation; building and distributing packages; miscellaneous +topics. -Additionally, there is a list of :doc:`other projects ` maintained -by members of the Python Packaging Authority. +Explanations and Discussions +============================ + +The :doc:`discussions/index` section provides in-depth explanations and discussion +about topics, such as: + +* :doc:`discussions/deploying-python-applications` +* :doc:`discussions/pip-vs-easy-install` + +Reference +========= + +* The :doc:`specifications/index` section for packaging interoperability specifications. +* The list of :doc:`other projects ` maintained by members of the Python Packaging Authority. +* The :doc:`glossary` for definitions of terms used in Python packaging. diff --git a/source/key_projects.rst b/source/key_projects.rst index ba4461792..e4501fe0e 100644 --- a/source/key_projects.rst +++ b/source/key_projects.rst @@ -18,6 +18,7 @@ PyPA Projects bandersnatch ============ +`Docs `__ | `Issues `__ | `GitHub `__ | `PyPI `__ @@ -27,6 +28,7 @@ create a complete mirror of the contents of PyPI. Organizations thus save bandwidth and latency on package downloads (especially in the context of automated tests) and to prevent heavily loading PyPI's Content Delivery Network (CDN). +Files can be served from a local directory or `AWS S3`_. .. _build: @@ -34,7 +36,7 @@ Content Delivery Network (CDN). build ===== -`Docs ` | +:any:`Docs ` | `Issues `__ | `GitHub `__ | `PyPI `__ @@ -64,8 +66,8 @@ distlib ======= :doc:`Docs ` | -`Issues `__ | -`Bitbucket `__ | +`Issues `__ | +`GitHub `__ | `PyPI `__ ``distlib`` is a library which implements low-level functions that @@ -84,6 +86,30 @@ behaviours when asked to handle legacy packages and metadata that predate the modern interoperability standards and fall into the subset of packages that are incompatible with those standards. + +.. _distutils: + +distutils +========= + +The original Python packaging system, added to the standard library in +Python 2.0 and removed in 3.12. + +Due to the challenges of maintaining a packaging system +where feature updates are tightly coupled to language runtime updates, +direct usage of :ref:`distutils` has been actively discouraged, with +:ref:`Setuptools` being the preferred replacement. :ref:`Setuptools` +not only provides features that plain :ref:`distutils` doesn't offer +(such as dependency declarations and entry point declarations), it +also provides a consistent build interface and feature set across all +supported Python versions. + +Consequently, :ref:`distutils` was deprecated in Python 3.10 by :pep:`632` and +has been :doc:`removed ` from the standard library in +Python 3.12. Setuptools bundles the standalone copy of distutils, and it is +injected even on Python < 3.12 if you import setuptools first or use pip. + + .. _flit: flit @@ -103,6 +129,11 @@ such as :ref:`setuptools` to build distributions, or :ref:`twine` to upload them to PyPI. Flit requires Python 3, but you can use it to distribute modules for Python 2, so long as they can be imported on Python 3. +The flit package is lifted by `Matthias Bussonnier +`__ since October 2023 on the `tidelift platform +`__, and funds sent to the PSF and +earmarked for PyPA usage. + .. _flit-rationale: https://flit.readthedocs.io/en/latest/rationale.html .. _hatch: @@ -116,7 +147,7 @@ hatch Hatch is a unified command-line tool meant to conveniently manage dependencies and environment isolation for Python developers. Python -package developers use Hatch and its build backend Hatchling to +package developers use Hatch and its :term:`build backend ` Hatchling to configure, version, specify dependencies for, and publish packages to PyPI. Its plugin system allows for easily extending functionality. @@ -180,11 +211,11 @@ Pipenv Pipenv is a project that aims to bring the best of all packaging worlds to the Python world. It harnesses :ref:`Pipfile`, :ref:`pip`, and :ref:`virtualenv` into one single toolchain. It can autoimport ``requirements.txt`` and also -check for CVEs in `Pipfile` using `safety `_. +check for CVEs in `Pipfile`_ using `safety `_. Pipenv aims to help users manage environments, dependencies, and imported packages on the command line. It also works well on Windows -(which other tools often underserve), makes and checkes file hashes, +(which other tools often underserve), makes and checks file hashes, to ensure compliance with hash-locked dependency specifiers, and eases uninstallation of packages and dependencies. @@ -204,7 +235,7 @@ application-centric alternative to :ref:`pip`'s lower-level pipx ==== -`Docs `__ | +`Docs `__ | `GitHub `__ | `PyPI `__ @@ -215,7 +246,7 @@ causing dependency conflicts with other packages installed on the system. Python Packaging User Guide =========================== -:doc:`Docs ` | +:doc:`Docs ` | `Issues `__ | `GitHub `__ @@ -239,7 +270,7 @@ display on PyPI. .. _setuptools: .. _easy_install: -setuptools +Setuptools ========== `Docs `__ | @@ -247,15 +278,11 @@ setuptools `GitHub `__ | `PyPI `__ -setuptools (which includes ``easy_install``) is a collection of +Setuptools (which includes ``easy_install``) is a collection of enhancements to the Python distutils that allow you to more easily build and distribute Python :term:`distributions `, especially ones that have dependencies on other packages. -`distribute`_ was a fork of setuptools that was merged back into setuptools (in -v0.7), thereby making setuptools the primary choice for Python packaging. - - .. _trove-classifiers: trove-classifiers @@ -267,8 +294,7 @@ trove-classifiers trove-classifiers is the canonical source for `classifiers on PyPI `_, which project maintainers use to -`systematically describe their projects -`_ +:ref:`systematically describe their projects ` so that users can better find projects that match their needs on the PyPI. The trove-classifiers package contains a list of valid classifiers and @@ -304,18 +330,17 @@ it's fast and secure, it's maintained, and it reliably works. virtualenv ========== -:doc:`Docs ` | +`Docs `__ | `Issues `__ | `GitHub `__ | `PyPI `__ -virtualenv is a tool which uses the command-line path environment -variable to create isolated Python :term:`Virtual Environments -`, much as :ref:`venv` does. virtualenv provides -additional functionality, compared to :ref:`venv`, by supporting Python -2.7 and by providing convenient features for configuring, maintaining, -duplicating, and troubleshooting the virtual environments. For more -information, see the section on :ref:`Creating and using Virtual +virtualenv is a tool for creating isolated Python :term:`Virtual Environments +`, like :ref:`venv`. Unlike :ref:`venv`, virtualenv can +create virtual environments for other versions of Python, which it locates +using the PATH environment variable. It also provides convenient features for +configuring, maintaining, duplicating, and troubleshooting virtual environments. +For more information, see the section on :ref:`Creating and using Virtual Environments`. @@ -379,26 +404,30 @@ conda :doc:`Docs ` -conda is the package management tool for `Anaconda -`__ Python installations. -Anaconda Python is a distribution from `Anaconda, Inc -`__ specifically aimed at the scientific -community, and in particular on Windows where the installation of binary -extensions is often difficult. +Conda is a package, dependency, and environment management system for any language β€” Python, R, +Ruby, C/C++, Fortran, and more. It is written in Python and +widely used in the Python scientific computing community, due to its support for non-Python +compiled libraries and extensions. It is used as the basis of the `Anaconda +`__ Python distribution from Anaconda, Inc. It was originally +aimed at the scientific community, but can also be used on its own, or with the +:doc:`miniconda `, `miniforge `_ or +`pixi `_ systems. It is available for Windows, Mac and Linux systems. Conda is a completely separate tool from :ref:`pip`, virtualenv and wheel, but provides -many of their combined features in terms of package management, virtual environment -management and deployment of binary extensions. - -Conda does not install packages from PyPI and can install only from -the official Anaconda repositories, or anaconda.org (a place for -user-contributed *conda* packages), or a local (e.g. intranet) package -server. However, note that :ref:`pip` can be installed into, and work -side-by-side with conda for managing :term:`distributions -` from PyPI. Also, `conda skeleton -`__ -is a tool to make Python packages installable by conda by first -fetching them from PyPI and modifying their metadata. +many of their combined features, such as package management, virtual environment +management and deployment of binary extensions and other binary code. + +Conda does not install packages from PyPI -- it can only manage packages built specifically +for conda, which can be made available on a "conda channel", such as those hosted on +`anaconda.org `__, or a local (e.g. intranet) package server. +In addition to the "default" channels managed by `Anaconda, Inc. `__, there are a wide variety of packages from the community supported +`conda-forge project `__ + +Note that :ref:`pip` can be installed into, and work side-by-side with conda +for managing :term:`distributions ` from PyPI. It is also possible +to build conda packages from Python source packages using tools such as +`conda skeleton +`__: a tool to automatically make conda packages from Python packages available on PyPI. .. _devpi: @@ -413,6 +442,21 @@ devpi features a powerful PyPI-compatible server and PyPI proxy cache with a complementary command line tool to drive packaging, testing and release activities with Python. devpi also provides a browsable and searchable web interface. +devpi supports mirroring PyPI, multiple +:term:`package indexes ` with inheritance, syncing between +these indexes, index replication and fail-over, and package upload. + +.. _dumb-pypi: + +dumb-pypi +========= + +`GitHub `__ | +`PyPI `__ + +dumb-pypi is a simple :term:`package index ` static file site +generator, which then must be hosted by a static file webserver to become the +package index. It supports serving the hash, core-metadata, and yank-status. .. _enscons: @@ -435,6 +479,20 @@ enscons. .. _SCons: https://scons.org/ +.. _flaskpypiproxy: + +Flask-Pypi-Proxy +================ + +`Docs `__ | +:gh:`GitHub ` | +`PyPI `__ + +.. warning:: Not maintained, project archived + +Flask-Pypi-Proxy is a :term:`package index ` as a cached +proxy for PyPI. + .. _hashdist: Hashdist @@ -452,16 +510,54 @@ problem of installing scientific software, and making package distribution stateless, cached, and branchable. It is used by some researchers but has been lacking in maintenance since 2016. +.. _maturin: + +Maturin +======= + +`Docs `__ | +`GitHub `__ + +Maturin is a build backend for Rust extension modules, also written in +Rust. It supports building wheels for python 3.7+ on Windows, Linux, macOS and +FreeBSD, can upload them to PyPI and has basic PyPy and GraalPy support. + + +.. _meson-python: + +meson-python +============ + +`Docs `__ | +`GitHub `__ + +``meson-python`` is a build backend that uses the Meson_ build system. It enables +Python package authors to use Meson_ as the build system for their package. It +supports a wide variety of languages, including C, and is able to fill the needs +of most complex build configurations. + +.. _Meson: https://github.com/mesonbuild/meson + .. _multibuild: multibuild ========== -`GitHub `__ +`GitHub `__ Multibuild is a set of CI scripts for building and testing Python :term:`wheels ` for Linux, macOS, and (less flexibly) Windows. Also see :ref:`cibuildwheel`. +.. _nginx_pypi_cache: + +nginx_pypi_cache +================ + +:gh:`GitHub ` + +nginx_pypi_cache is a :term:`package index ` caching proxy +using `nginx `_. + .. _pdm: pdm @@ -471,9 +567,7 @@ pdm `GitHub `__ | `PyPI `__ -PDM is a modern Python package manager with :pep:`582` support. It installs and -manages packages in a similar way to ``npm`` that doesn't need to create a -:term:`virtual environment` at all. It also uses :term:`pyproject.toml` to store +PDM is a modern Python package manager. It uses :term:`pyproject.toml` to store project metadata as defined in :pep:`621`. .. _pex: @@ -485,18 +579,22 @@ pex `GitHub `__ | `PyPI `__ -pex is both a library and tool for generating :file:`.pex` (Python EXecutable) +Pex is a tool for generating :file:`.pex` (Python EXecutable) files, standalone Python environments in the spirit of :ref:`virtualenv`. -:file:`.pex` files are just carefully constructed zip files with a -``#!/usr/bin/env python`` and special :file:`__main__.py`, and are designed to -make deployment of Python applications as simple as ``cp``. +PEX files are :doc:`zipapps ` that +make deployment of Python applications as simple as ``cp``. A single PEX +file can support multiple target platforms and can be created from standard +:ref:`pip`-resolvable requirements, a lockfile generated with ``pex3 lock ...`` +or even another PEX. PEX files can optionally have tools embedded that support +turning the PEX file into a standard venv, graphing dependencies and more. .. _pip-tools: pip-tools ========= -`GitHub and Docs `__ | +`Docs `__ | +`GitHub `__ | `PyPI `__ pip-tools is a suite of tools meant for Python system administrators @@ -508,6 +606,17 @@ requirements from information in other parts of their program, update all dependencies (a feature :ref:`pip` currently does not provide), and create layers of constraints for the program to obey. +.. _pip2pi: + +pip2pi +========= + +:gh:`GitHub ` | +`PyPI `__ + +pip2pi is a :term:`package index ` server where specific +packages are manually synchronised. + .. _piwheels: piwheels @@ -539,17 +648,69 @@ functionality within :ref:`pip`, provides its own dependency resolver. It attempts to speed users' experience of installation and dependency resolution by locally caching metadata about dependencies. +.. _proxpi: + +proxpi +====== + +:gh:`GitHub ` | +`PyPI `__ + +proxpi is a simple :term:`package index ` which proxies PyPI +and other indexes with caching. + +.. _pulppython: + +Pulp-python +=========== + +`Docs `__ | +:gh:`GitHub ` | +`PyPI `__ + +Pulp-python is the Python :term:`package index ` plugin for +`Pulp `_. Pulp-python supports mirrors backed by +local or `AWS S3`_, package upload, and proxying to multiple package +indexes. + +.. _pypicloud: + +PyPI Cloud +========== + +`Docs `__ | +:gh:`GitHub ` | +`PyPI `__ + +.. warning:: Not maintained, project archived + +PyPI Cloud is a :term:`package index ` server, backed by +`AWS S3`_ or another cloud storage service, or local files. PyPI Cloud +supports redirect/cached proxying for PyPI, as well as authentication and +authorisation. + +.. _pypiprivate: + +pypiprivate +=========== + +:gh:`GitHub ` | +`PyPI `__ + +pypiprivate serves a local (or `AWS S3`_-hosted) directory of packages as a +:term:`package index `. + .. _pypiserver: pypiserver ========== -`Docs `__ | `GitHub `__ | `PyPI `__ pypiserver is a minimalist application that serves as a private Python -package index within organizations, implementing a simple API and +:term:`package index ` (from a local directory) within +organizations, implementing a simple API and browser interface. You can upload private packages using standard upload tools, and users can download and install them with :ref:`pip`, without publishing them publicly. Organizations who use pypiserver @@ -572,6 +733,18 @@ environment so developers can start coding right away. PyScaffold can also be used with existing projects to make packaging easier. +.. _pywharf: + +pywharf +======= + +:gh:`GitHub ` | +`PyPI `__ + +.. warning:: Not maintained, project archived + +pywharf is a :term:`package index ` server, serving files +locally or from `GitHub `_. .. _scikit-build: @@ -582,14 +755,29 @@ scikit-build `GitHub `__ | `PyPI `__ -Scikit-build is an improved build system generator for CPython -C/C++/Fortran/Cython extensions that integrates with :ref:`setuptools`, :ref:`wheel` -and :ref:`pip`. It internally uses `cmake `__ (available -on PyPI) to provide better support for additional compilers, build systems, -cross compilation, and locating dependencies and their associated -build requirements. To speed up and parallelize the build of large projects, -the user can install `ninja `__ (also available -on PyPI). +Scikit-build is a :ref:`setuptools` wrapper for CPython that builds +C/C++/Fortran/Cython extensions It uses +`cmake `__ (available on PyPI) to provide +better support for additional compilers, build systems, cross compilation, and +locating dependencies and their associated build requirements. To speed up and +parallelize the build of large projects, the user can install `ninja +`__ (also available on PyPI). + +.. _scikit-build-core: + +scikit-build-core +================= + +`Docs `__ | +`GitHub `__ | +`PyPI `__ + +Scikit-build-core is a build backend for CPython C/C++/Fortran/Cython +extensions. It enables users to write extensions with `cmake +`__ (available on PyPI) to provide better +support for additional compilers, build systems, cross compilation, and +locating dependencies and their associated build requirements. CMake/Ninja +are automatically downloaded from PyPI if not available on the system. .. _shiv: @@ -605,6 +793,19 @@ Python zipapps as outlined in :pep:`441`, but with all their dependencies included. Its primary goal is making distributing Python applications and command line tools fast & easy. +.. _simpleindex: + +simpleindex +=========== + +:gh:`GitHub ` | +`PyPI `__ + +simpleindex is a :term:`package index ` which routes URLs to +multiple package indexes (including PyPI), serves local (or cloud-hosted, +for example `AWS S3`_, with a custom plugin) directories of packages, and +supports custom plugins. + .. _spack: Spack @@ -657,27 +858,17 @@ A package in the Python Standard Library that provides support for bootstrapping cases, end users won't use this module, but rather it will be used during the build of the Python distribution. +.. _httpserver: -.. _distutils: +http.server +=========== -distutils -========= - -`Docs `__ | -`Issues `__ - -The original Python packaging system, added to the standard library in -Python 2.0. - -Due to the challenges of maintaining a packaging system -where feature updates are tightly coupled to language runtime updates, -direct usage of :ref:`distutils` is now actively discouraged, with -:ref:`Setuptools` being the preferred replacement. :ref:`Setuptools` -not only provides features that plain :ref:`distutils` doesn't offer -(such as dependency declarations and entry point declarations), it -also provides a consistent build interface and feature set across all -supported Python versions. +:doc:`Docs ` | +:gh:`Issues ` +A package and command-line interface which can host a directory as a +website, for example as a :term:`package index ` (see +:ref:`Hosting your Own Simple Repository`). .. _venv: @@ -694,6 +885,6 @@ information, see the section on :ref:`Creating and using Virtual Environments`. ---- -.. _distribute: https://pypi.org/project/distribute .. _Sphinx: https://www.sphinx-doc.org/en/master/ .. _pytest: https://docs.pytest.org/en/stable/ +.. _`AWS S3`: https://aws.amazon.com/s3/ diff --git a/source/news.rst b/source/news.rst index f6bdbbf27..a8c70dc1b 100644 --- a/source/news.rst +++ b/source/news.rst @@ -1,6 +1,10 @@ News ==== +.. note:: This document is not currently updated. Previously, the document + highlighted changes in Python packaging. + + September 2019 -------------- - Added a guide about publishing dists via GitHub Actions. (:pr:`647`) @@ -118,7 +122,7 @@ March 2018 - Updated "installing scientific packages". (:pr:`455`) - Added :file:`long_description_content_type` to follow PEP 556. (:pr:`457`) - Clarified a long description classifier on pypi.org. (:pr:`456`) -- Updated Core Metadata spec to follw PEP 556. (:pr:`412`) +- Updated Core Metadata spec to follow PEP 556. (:pr:`412`) February 2018 ------------- @@ -214,7 +218,7 @@ April 2017 - Added documentation explaining prominently how to install ``pip`` in ``/usr/local``. (:pr:`230`) - Updated development mode documentation to mention that order of local packages matters. (:pr:`208`) - Convert readthedocs link for their ``.org`` -> ``.io`` migration for hosted projects (:pr:`239`) -- Swaped order of :file:`setup.py` arguments for the upload command, as order +- Swapped order of :file:`setup.py` arguments for the upload command, as order is significant. (:pr:`260`) - Explained how to install from unsupported sources using a helper application. (:pr:`289`) diff --git a/source/overview.rst b/source/overview.rst index 7031327b2..70ef2d058 100644 --- a/source/overview.rst +++ b/source/overview.rst @@ -1,6 +1,6 @@ -=================================== -An Overview of Packaging for Python -=================================== +============================ +Overview of Python Packaging +============================ .. Editors, see notes at the bottom of the document for maintenance info. @@ -19,9 +19,6 @@ This overview provides a general-purpose decision tree for reasoning about Python's plethora of packaging options. Read on to choose the best technology for your next project. -.. contents:: Contents - :local: - Thinking about deployment ------------------------- @@ -66,8 +63,8 @@ library. This is great for sharing simple scripts and snippets between people who both have compatible Python versions (such as via email, StackOverflow, or GitHub gists). There are even some entire Python -libraries that offer this as an option, such as `bottle.py -`_ and :doc:`boltons +libraries that offer this as an option, such as +:doc:`bottle.py` and :doc:`boltons `. However, this pattern won't scale for projects that consist of @@ -95,7 +92,7 @@ you can use Python's native packaging tools to create a *source* Python's *sdists* are compressed archives (``.tar.gz`` files) containing one or more packages or modules. If your code is pure-Python, and you only depend on other Python packages, you can -:doc:`go here to learn more `. +go to the :ref:`source-distribution-format` specification to learn more. If you rely on any non-Python code, or non-Python packages (such as `libxml2 `_ in the case of @@ -167,7 +164,7 @@ Python's native packaging is mostly built for distributing reusable code, called libraries, between developers. You can piggyback **tools**, or basic applications for developers, on top of Python's library packaging, using technologies like -:doc:`setuptools entry_points `. +:doc:`setuptools entry_points `. Libraries are building blocks, not complete applications. For distributing applications, there's a whole new world of technologies @@ -198,20 +195,12 @@ hood, you can always read the sections beyond. Service platforms ***************** -If you're developing for a "Platform-as-a-Service" or "PaaS" like -Heroku or Google App Engine, you are going to want to follow their -respective packaging guides. - -* `Heroku `_ -* `Google App Engine `_ -* `PythonAnywhere `_ -* `OpenShift `_ -* "Serverless" frameworks like :gh:`Zappa ` - -In all these setups, the platform takes care of packaging and -deployment, as long as you follow their patterns. Most software does -not fit one of these templates, hence the existence of all the other -options below. +If you're developing for a +"`Platform-as-a-Service `_" +or "PaaS", you are going to want to follow their respective packaging +guides. These types of platforms take care of packaging and deployment, +as long as you follow their patterns. Most software does not fit one of +these templates, hence the existence of all the other options below. If you're developing software that will be deployed to machines you own, users' personal computers, or any other arrangement, read on. @@ -300,7 +289,7 @@ Bringing your own Python executable Computing as we know it is defined by the ability to execute programs. Every operating system natively supports one or more formats -of program they can natively execute. +of programs they can natively execute. There are many techniques and technologies which turn your Python program into one of these formats, most of which involve embedding the @@ -350,18 +339,22 @@ originated and where the technologies below work best: Bringing your own kernel ^^^^^^^^^^^^^^^^^^^^^^^^ -Most operating systems support some form of classical virtualization, +Most desktop operating systems support some form of classical virtualization, running applications packaged as images containing a full operating system of their own. Running these virtual machines, or VMs, is a mature approach, widespread in data center environments. These techniques are mostly reserved for larger scale deployments in data centers, though certain complex applications can benefit from -this packaging. Technologies are Python agnostic, and include: - -* `Vagrant `_ -* `VHD `_, `AMI `_, and :doc:`other formats ` -* `OpenStack `_ - A cloud management system in Python, with extensive VM support +this packaging. The technologies are Python agnostic, and include: + +* KVM on Linux +* Hyper-V on Windows +* `VHD `_, + `AMI `_, + and :doc:`other formats ` +* `OpenStack `_ - + A cloud management system written in Python, with extensive VM support Bringing your own hardware ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/source/shared/build-backend-tabs.rst b/source/shared/build-backend-tabs.rst new file mode 100644 index 000000000..64ef0bf2f --- /dev/null +++ b/source/shared/build-backend-tabs.rst @@ -0,0 +1,42 @@ +.. (comment) This file is included in guides/writing-pyproject-toml.rst and tutorials/packaging-projects.rst. +.. The minimum versions here are the versions that introduced support for PEP 639. + +.. tab:: Hatchling + + .. code-block:: toml + + [build-system] + requires = ["hatchling >= 1.26"] + build-backend = "hatchling.build" + +.. tab:: setuptools + + .. code-block:: toml + + [build-system] + requires = ["setuptools >= 77.0.3"] + build-backend = "setuptools.build_meta" + +.. tab:: Flit + + .. code-block:: toml + + [build-system] + requires = ["flit_core >= 3.12.0, <4"] + build-backend = "flit_core.buildapi" + +.. tab:: PDM + + .. code-block:: toml + + [build-system] + requires = ["pdm-backend >= 2.4.0"] + build-backend = "pdm.backend" + +.. tab:: uv-build + + .. code-block:: toml + + [build-system] + requires = ["uv_build >= 0.9.21, <0.10.0"] + build-backend = "uv_build" diff --git a/source/specifications/binary-distribution-format.rst b/source/specifications/binary-distribution-format.rst index 1f4bd89fa..8bb41ab40 100644 --- a/source/specifications/binary-distribution-format.rst +++ b/source/specifications/binary-distribution-format.rst @@ -1,3 +1,4 @@ +.. highlight:: text .. _binary-distribution-format: @@ -5,14 +6,8 @@ Binary distribution format ========================== -The binary distribution format (:term:`wheel `) was originally defined -in :pep:`427`. The current version of the specification is here. - - -Abstract -======== - -This PEP describes a built-package format for Python called "wheel". +This page specifies the binary distribution format for Python packages, +also called the wheel format. A wheel is a ZIP-format archive with a specially formatted file name and the ``.whl`` extension. It contains a single distribution nearly as it @@ -23,31 +18,6 @@ may be installed by simply unpacking into site-packages with the standard out onto their final paths at any later time. -PEP Acceptance -============== - -This PEP was accepted, and the defined wheel version updated to 1.0, by -Nick Coghlan on 16th February, 2013 [1]_ - - -Rationale -========= - -Python needs a package format that is easier to install than sdist. -Python's sdist packages are defined by and require the distutils and -setuptools build systems, running arbitrary code to build-and-install, -and re-compile, code just so it can be installed into a new -virtualenv. This system of conflating build-install is slow, hard to -maintain, and hinders innovation in both build systems and installers. - -Wheel attempts to remedy these problems by providing a simpler -interface between the build system and the installer. The wheel -binary package format frees installers from having to know about the -build system, saves time by amortizing compile time over many -installations, and removes the need to install a build system in the -target environment. - - Details ======= @@ -73,8 +43,8 @@ Wheel installation notionally consists of two phases: destination path. Each subdirectory of ``distribution-1.0.data/`` is a key into a dict of destination directories, such as ``distribution-1.0.data/(purelib|platlib|headers|scripts|data)``. - The initially supported paths are taken from - ``distutils.command.install``. + These subdirectories are :ref:`installation paths defined by sysconfig + `. c. If applicable, update scripts starting with ``#!python`` to point to the correct interpreter. d. Update ``distribution-1.0.dist-info/RECORD`` with the installed @@ -114,6 +84,8 @@ Place ``.dist-info`` at the end of the archive. File Format ----------- +.. _wheel-file-name-spec: + File name convention '''''''''''''''''''' @@ -134,6 +106,23 @@ build tag the first item being the initial digits as an ``int``, and the second item being the remainder of the tag as a ``str``. + A common use-case for build numbers is rebuilding a binary + distribution due to a change in the build environment, + like when using the manylinux image to build + distributions using pre-release CPython versions. + + .. warning:: + + Build numbers are not a part of the distribution version and thus are difficult + to reference externally, especially so outside the Python ecosystem of tools and standards. + A common case where a distribution would need to referenced externally is when + resolving a security vulnerability. + + Due to this limitation, new distributions which need to be referenced externally + **should not** use build numbers when building the new distribution. + Instead a **new distribution version** should be created for such cases. + + language implementation and version tag E.g. 'py27', 'py2', 'py3'. @@ -161,12 +150,12 @@ this character cannot appear within any component. This is handled as follows: - In distribution names, any run of ``-_.`` characters (HYPHEN-MINUS, LOW LINE and FULL STOP) should be replaced with ``_`` (LOW LINE), and uppercase characters should be replaced with corresponding lowercase ones. This is - equivalent to :pep:`503` normalisation followed by replacing ``-`` with ``_``. - Tools consuming wheels must be prepared to accept ``.`` (FULL STOP) and - uppercase letters, however, as these were allowed by an earlier version of - this specification. -- Version numbers should be normalised according to :pep:`440`. Normalised - version numbers cannot contain ``-``. + equivalent to regular :ref:`name normalization ` followed + by replacing ``-`` with ``_``. Tools consuming wheels must be prepared to accept + ``.`` (FULL STOP) and uppercase letters, however, as these were allowed by an earlier + version of this specification. +- Version numbers should be normalised according to the :ref:`Version specifier + specification `. Normalised version numbers cannot contain ``-``. - The remaining components may not contain ``-`` characters, so no escaping is necessary. @@ -186,13 +175,17 @@ File contents ''''''''''''' The contents of a wheel file, where {distribution} is replaced with the -name of the package, e.g. ``beaglevote`` and {version} is replaced with -its version, e.g. ``1.0.0``, consist of: +:ref:`normalized name ` of the package, e.g. +``beaglevote`` and {version} is replaced +with its :ref:`normalized version `, +e.g. ``1.0.0``, (with dash/``-`` characters replaced with underscore/``_`` characters +in both fields) consist of: #. ``/``, the root of the archive, contains all files to be installed in ``purelib`` or ``platlib`` as specified in ``WHEEL``. ``purelib`` and ``platlib`` are usually both ``site-packages``. #. ``{distribution}-{version}.dist-info/`` contains metadata. +#. :file:`{distribution}-{version}.dist-info/licenses/` contains license files. #. ``{distribution}-{version}.data/`` contains one subdirectory for each non-empty install scheme key not already covered, where the subdirectory name is an index into a dictionary of install paths @@ -200,7 +193,7 @@ its version, e.g. ``1.0.0``, consist of: #. Python scripts must appear in ``scripts`` and begin with exactly ``b'#!python'`` in order to enjoy script wrapper generation and ``#!python`` rewriting at install time. They may have any or no - extension. + extension. The ``scripts`` directory may only contain regular files. #. ``{distribution}-{version}.dist-info/METADATA`` is Metadata version 1.1 or greater format metadata. #. ``{distribution}-{version}.dist-info/WHEEL`` is metadata about the archive @@ -260,6 +253,36 @@ The .dist-info directory installation will fail if any file in the archive is not both mentioned and correctly hashed in RECORD. +Subdirectories in :file:`.dist-info/` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Subdirectories under :file:`.dist-info/` are reserved for future use. +The following subdirectory names under :file:`.dist-info/` are reserved for specific usage: + +================= ============== +Subdirectory name PEP / Standard +================= ============== +``licenses`` :pep:`639` +``license_files`` :pep:`639` +``LICENSES`` `REUSE licensing framework `__ +``sboms`` :pep:`770` +================= ============== + +The :file:`.dist-info/licenses/` directory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If the metadata version is 2.4 or greater and one or more ``License-File`` +fields is specified, the :file:`.dist-info/` directory MUST contain a +:file:`licenses/` subdirectory, which MUST contain the files listed in the +``License-File`` fields in the :file:`METADATA` file at their respective paths +relative to the :file:`licenses/` directory. + +The :file:`.dist-info/sboms/` directory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +All files contained within the :file:`.dist-info/sboms/` directory MUST +be Software Bill-of-Materials (SBOM) files that describe software contained +within the distribution archive. The .data directory ^^^^^^^^^^^^^^^^^^^ @@ -299,7 +322,9 @@ or the installation will fail. If JSON web signatures are used, one or more JSON Web Signature JSON Serialization (JWS-JS) signatures is stored in a file RECORD.jws adjacent to RECORD. JWS is used to sign RECORD by including the SHA-256 hash of -RECORD as the signature's JSON payload:: +RECORD as the signature's JSON payload: + +.. code-block:: json { "hash": "sha256=ADD-r2urObZHcxBW3Cr-vDCu5RJwT4CaRTHiFmbcIYY" } @@ -321,33 +346,6 @@ See - https://datatracker.ietf.org/doc/html/draft-jones-jose-json-private-key-01 -Comparison to .egg ------------------- - -#. Wheel is an installation format; egg is importable. Wheel archives - do not need to include .pyc and are less tied to a specific Python - version or implementation. Wheel can install (pure Python) packages - built with previous versions of Python so you don't always have to - wait for the packager to catch up. -#. Wheel uses .dist-info directories; egg uses .egg-info. Wheel is - compatible with the new world of Python packaging and the new - concepts it brings. -#. Wheel has a richer file naming convention for today's - multi-implementation world. A single wheel archive can indicate - its compatibility with a number of Python language versions and - implementations, ABIs, and system architectures. Historically the - ABI has been specific to a CPython release, wheel is ready for the - stable ABI. -#. Wheel is lossless. The first wheel implementation bdist_wheel - always generates egg-info, and then converts it to a .whl. It is - also possible to convert existing eggs and bdist_wininst - distributions. -#. Wheel is versioned. Every wheel file contains the version of the - wheel specification and the implementation that packaged it. - Hopefully the next migration can simply be to Wheel 2.0. -#. Wheel is a reference to the other Python. - - FAQ === @@ -409,6 +407,8 @@ What's the deal with "purelib" vs. "platlib"? be at the root with the appropriate setting given for "Root-is-purelib". +.. _binary-distribution-format-import-wheel: + Is it possible to import Python code directly from a wheel file? ---------------------------------------------------------------- @@ -450,20 +450,21 @@ Is it possible to import Python code directly from a wheel file? aware that many projects will require a failure to be reproduced with a fully installed package before accepting it as a genuine bug. -Changes -======= - -Since :pep:`427`, this specification has changed as follows: - -- The rules on escaping in wheel filenames were revised, to bring them into line - with what popular tools actually do (February 2021). - -References -========== +History +======= -.. [1] PEP acceptance - (https://mail.python.org/pipermail/python-dev/2013-February/124103.html) +- February 2013: This specification was approved through :pep:`427`. +- February 2021: The rules on escaping in wheel filenames were revised, to bring + them into line with what popular tools actually do. +- December 2024: Clarified that the ``scripts`` folder should only contain + regular files (the expected behaviour of consuming tools when encountering + symlinks or subdirectories in this folder is not formally defined, and hence + may vary between tools). +- December 2024: The :file:`.dist-info/licenses/` directory was specified through + :pep:`639`. +- January 2025: Clarified that name and version needs to be normalized for + ``.dist-info`` and ``.data`` directories. Appendix @@ -480,9 +481,3 @@ Example urlsafe-base64-nopad implementation:: def urlsafe_b64decode_nopad(data): pad = b'=' * (4 - (len(data) & 3)) return base64.urlsafe_b64decode(data + pad) - - -Copyright -========= - -This document has been placed into the public domain. diff --git a/source/specifications/build-details/examples/build-details-v1.0.json b/source/specifications/build-details/examples/build-details-v1.0.json new file mode 100644 index 000000000..dd08b230f --- /dev/null +++ b/source/specifications/build-details/examples/build-details-v1.0.json @@ -0,0 +1,51 @@ +{ + "schema_version": "1.0", + "base_prefix": "/usr", + "base_interpreter": "/usr/bin/python", + "platform": "linux-x86_64", + "language": { + "version": "3.14", + "version_info": { + "major": 3, + "minor": 14, + "micro": 0, + "releaselevel": "alpha", + "serial": 0 + } + }, + "implementation": { + "name": "cpython", + "version": { + "major": 3, + "minor": 14, + "micro": 0, + "releaselevel": "alpha", + "serial": 0 + }, + "hexversion": 51249312, + "cache_tag": "cpython-314", + "_multiarch": "x86_64-linux-gnu" + }, + "abi": { + "flags": ["t", "d"], + "extension_suffix": ".cpython-314-x86_64-linux-gnu.so", + "stable_abi_suffix": ".abi3.so" + }, + "suffixes": { + "source": [".py"], + "bytecode": [".pyc"], + "optimized_bytecode": [".pyc"], + "debug_bytecode": [".pyc"], + "extensions": [".cpython-314-x86_64-linux-gnu.so", ".abi3.so", ".so"] + }, + "libpython": { + "dynamic": "/usr/lib/libpython3.14.so.1.0", + "dynamic_stableabi": "/usr/lib/libpython3.so", + "static": "/usr/lib/python3.14/config-3.14-x86_64-linux-gnu/libpython3.14.a", + "link_extensions": true + }, + "c_api": { + "headers": "/usr/include/python3.14", + "pkgconfig_path": "/usr/lib/pkgconfig" + } +} diff --git a/source/specifications/build-details/index.rst b/source/specifications/build-details/index.rst new file mode 100644 index 000000000..0cd5b5fe5 --- /dev/null +++ b/source/specifications/build-details/index.rst @@ -0,0 +1,52 @@ +.. _build-details: + +========================== +:file:`build-details.json` +========================== + +.. toctree:: + :hidden: + + v1.0 + + +The ``build-details.json`` file is a standardized file format that provides +build-specfic information of a Python installation, such as its version, +extension ABI details, and other information that is specific to that particular +build of Python. + +Starting from Python 3.14, a ``build-details.json`` file is installed in the +platform-independent standard library directory (``stdlib``, e.g. +``/usr/lib/python3.14/build-details.json``). + +Please refer to the :ref:`latest version ` for its +specification. + +.. + Update to point to the latest version! + +.. literalinclude:: examples/build-details-v1.0.json + :caption: Example + :language: json + :linenos: + + +Changelog +--------- + +.. + Order in decreasing order. + +v1.0 +~~~~ + +.. list-table:: + + * - Specification + - :ref:`build-details-v1.0` + + * - Schema + - https://packaging.python.org/en/latest/specifications/schemas/build-details-v1.0.schema.json + + +- Initial version, introduced by :pep:`739`. diff --git a/source/specifications/build-details/v1.0.rst b/source/specifications/build-details/v1.0.rst new file mode 100644 index 000000000..3a8cfe277 --- /dev/null +++ b/source/specifications/build-details/v1.0.rst @@ -0,0 +1,18 @@ +.. _build-details-v1.0: + +=========================== +``build-details.json`` v1.0 +=========================== + + +Specification +------------- + +.. jsonschema:: ../../../extra/specifications/schemas/build-details-v1.0.schema.json + :lift_title: false + + +Example +------- + +.. literalinclude:: examples/build-details-v1.0.json diff --git a/source/specifications/core-metadata.rst b/source/specifications/core-metadata.rst index 2e85d0909..eb9a03ff6 100644 --- a/source/specifications/core-metadata.rst +++ b/source/specifications/core-metadata.rst @@ -1,9 +1,13 @@ +.. highlight:: text + .. _`core-metadata`: ============================ Core metadata specifications ============================ +This page describes version 2.5, approved in September 2025. + Fields defined in the following specification should be considered valid, complete and not subject to change. The required fields are: @@ -37,8 +41,8 @@ to a new format. more relaxed formatting rules even for metadata files that are nominally less than version 2.1. -.. contents:: Contents - :local: + +.. _core-metadata-metadata-version: Metadata-Version ================ @@ -46,13 +50,14 @@ Metadata-Version .. versionadded:: 1.0 Version of the file format; legal values are "1.0", "1.1", "1.2", "2.1", -"2.2", and "2.3". +"2.2", "2.3", "2.4", and "2.5". -Automated tools consuming metadata SHOULD warn if ``metadata_version`` is +Automated tools consuming metadata SHOULD warn if ``metadata-version`` is greater than the highest version they support, and MUST fail if -``metadata_version`` has a greater major version than the highest -version they support (as described in :pep:`440`, the major version is the -value before the first dot). +``metadata-version`` has a greater major version than the highest +version they support (as described in the +:ref:`Version specifier specification `, +the major version is the value before the first dot). For broader compatibility, build tools MAY choose to produce distribution metadata using the lowest metadata version that includes @@ -60,7 +65,7 @@ all of the needed fields. Example:: - Metadata-Version: 2.3 + Metadata-Version: 2.4 .. _core-metadata-name: @@ -70,26 +75,17 @@ Name .. versionadded:: 1.0 .. versionchanged:: 2.1 - Added additional restrictions on format from :pep:`508` + Added restrictions on format from the :ref:`name format `. The name of the distribution. The name field is the primary identifier for a -distribution. A valid name consists only of ASCII letters and numbers, period, -underscore and hyphen. It must start and end with a letter or number. -Distribution names are limited to those which match the following -regex (run with ``re.IGNORECASE``):: - - ^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$ +distribution. It must conform to the :ref:`name format specification +`. Example:: Name: BeagleVote -To normalize a distribution name for comparison purposes, it should be -lowercased with all runs of the characters ``.``, ``-``, or ``_`` replaced with -a single ``-`` character. This can be done using the following snippet of code -(as specified in :pep:`503`):: - - re.sub(r"[-_.]+", "-", name).lower() +For comparison purposes, the names should be :ref:`normalized ` before comparing. .. _core-metadata-version: @@ -99,13 +95,16 @@ Version .. versionadded:: 1.0 A string containing the distribution's version number. This -field must be in the format specified in :pep:`440`. +field must be in the format specified in the +:ref:`Version specifier specification `. Example:: Version: 1.0a2 +.. _core-metadata-dynamic: + Dynamic (multiple use) ====================== @@ -134,8 +133,19 @@ only, and indicates that the field value was calculated at wheel build time, and may not be the same as the value in the sdist or in other wheels for the project. +Note in particular that if you have obtained a prebuilt wheel, you cannot +assume that a field which is not marked as ``Dynamic`` will have the same value +in other wheels, as some wheels are not built directly from the sdist, but are +modified from existing wheels (the ``auditwheel`` tool does this, for example, +and it's commonly used when building wheels for PyPI). Such modifications +*could* include changing metadata (even non-dynamic metadata). Similarly, if +you have a sdist and a wheel which you didn't build from that sdist, you cannot +assume that the wheel's metadata matches that of the sdist, even if the field +is not marked as ``Dynamic``. + Full details of the semantics of ``Dynamic`` are described in :pep:`643`. +.. _core-metadata-platform: Platform (multiple use) ======================= @@ -151,6 +161,7 @@ Examples:: Platform: ObscureUnix Platform: RareDOS +.. _core-metadata-supported-platform: Supported-Platform (multiple use) ================================= @@ -284,7 +295,7 @@ Other parameters might be specific to the chosen subtype. For example, for the specifying the variant of Markdown in use (defaults to ``GFM`` if not specified). Currently, two variants are recognized: -- ``GFM`` for :rfc:`Github-flavored Markdown <7764#section-3.2>` +- ``GFM`` for :rfc:`GitHub-flavored Markdown <7764#section-3.2>` - ``CommonMark`` for :rfc:`CommonMark <7764#section-3.5>` Example:: @@ -342,30 +353,6 @@ Example:: These tools have been very widely used for many years, so it was easier to update the specification to match the de facto standard. -.. _home-page-optional: - -Home-page -========= - -.. versionadded:: 1.0 - -A string containing the URL for the distribution's home page. - -Example:: - - Home-page: http://www.example.com/~cschultz/bvote/ - - -Download-URL -============ - -.. versionadded:: 1.1 - -A string containing the URL from which this version of the distribution -can be downloaded. (This means that the URL can't be something like -".../BeagleVote-latest.tgz", but instead must be ".../BeagleVote-0.45.tgz".) - - .. _author-optional: .. _core-metadata-author: @@ -459,6 +446,14 @@ License ======= .. versionadded:: 1.0 +.. deprecated:: 2.4 + in favour of ``License-Expression``. + +.. warning:: + As of Metadata 2.4, ``License`` and ``License-Expression`` are mutually + exclusive. If both are specified, tools which parse metadata will disregard + ``License`` and PyPI will reject uploads. + See `PEP 639 `__. Text indicating the license covering the distribution where the license is not a selection from the "License" Trove classifiers. See @@ -476,6 +471,56 @@ Examples:: License: GPL version 3, excluding DRM provisions +.. _license-expression-optional: +.. _core-metadata-license-expression: + +License-Expression +================== + +.. versionadded:: 2.4 + +Text string that is a valid SPDX +:term:`license expression `, +as specified in :doc:`/specifications/license-expression`. + +Note that the expression in this field only applies to the +:term:`Distribution Archive` containing the metadata with this field (e.g., +:term:`Source Distribution ` or :term:`Wheel`), +not the project overall or other files related to the project (including other +distribution archives). + +Examples:: + + License-Expression: MIT + License-Expression: BSD-3-Clause + License-Expression: MIT AND (Apache-2.0 OR BSD-2-Clause) + License-Expression: MIT OR GPL-2.0-or-later OR (FSFUL AND BSD-2-Clause) + License-Expression: GPL-3.0-only WITH Classpath-Exception-2.0 OR BSD-3-Clause + License-Expression: LicenseRef-Special-License OR CC0-1.0 OR Unlicense + License-Expression: LicenseRef-Proprietary + + +.. _license-file-optional: +.. _core-metadata-license-file: + +License-File (multiple use) +=========================== + +.. versionadded:: 2.4 + +Each entry is a string representation of the path of a license-related file. +The path is located within the project source tree, relative to the project +root directory. For details see :pep:`639`. + +Examples:: + + License-File: LICENSE + License-File: AUTHORS + License-File: LICENSE.txt + License-File: licenses/LICENSE.MIT + License-File: licenses/LICENSE.CC0 + + .. _metadata-classifier: .. _core-metadata-classifier: @@ -489,6 +534,11 @@ for the distribution. Classifiers are described in :pep:`301`, and the Python Package Index publishes a dynamic list of `currently defined classifiers `__. +.. note:: + The use of ``License ::`` classifiers is deprecated as of Metadata 2.4, + use ``License-Expression`` instead. See + `PEP 639 `_. + This field may be followed by an environment marker after a semicolon. Examples:: @@ -548,19 +598,19 @@ Requires-Python .. versionadded:: 1.2 This field specifies the Python version(s) that the distribution is -guaranteed to be compatible with. Installation tools may look at this when +compatible with. Installation tools may look at this when picking which version of a project to install. The value must be in the format specified in :doc:`version-specifiers`. -This field cannot be followed by an environment marker. +For example, if a distribution uses :ref:`f-strings ` +then it may prevent installation on Python < 3.6 by specifying:: -Examples:: + Requires-Python: >=3.6 - Requires-Python: >=3 - Requires-Python: >2.6,!=3.0.*,!=3.1.* - Requires-Python: ~=2.6 +This field cannot be followed by an environment marker. +.. _core-metadata-requires-external: Requires-External (multiple use) ================================ @@ -583,8 +633,8 @@ This field may be followed by an environment marker after a semicolon. Because they refer to non-Python software releases, version numbers for this field are **not** required to conform to the format -specified in :pep:`440`: they should correspond to the -version scheme used by the external dependency. +specified in the :ref:`Version specifier specification `: +they should correspond to the version scheme used by the external dependency. Notice that there is no particular rule on the strings to be used. @@ -611,6 +661,10 @@ Example:: The label is free text limited to 32 characters. +Starting with :pep:`753`, project metadata consumers (such as the Python +Package Index) can use a standard normalization process to discover "well-known" +labels, which can then be given special presentations when being rendered +for human consumption. See :ref:`well-known-project-urls`. .. _metadata_provides_extra: .. _core-metadata-provides-extra: @@ -630,7 +684,7 @@ of lowercase ASCII letters, ASCII numbers, and hyphen. It must start and end with a letter or number. Hyphens cannot be followed by another hyphen. Names are limited to those which match the following regex (which guarantees unambiguity):: - ^([a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9])$ + ^[a-z0-9]+(-[a-z0-9]+)*$ The specified name may be used to make a dependency conditional on whether the @@ -670,6 +724,101 @@ user SHOULD be warned and the value ignored to avoid ambiguity. Tools MAY choose to raise an error when reading an invalid name for older metadata versions. +.. _core-metadata-import-name: + +Import-Name (multiple use) +========================== + +.. versionadded:: 2.5 + +A string containing an import name that the project exclusively provides when +installed. The specified import name MUST be a valid Python identifier or can +be empty. The import names listed in this field MUST be importable when the +project is installed on *some* platform for the same version of the project. +This implies that the metadata MUST be consistent across all sdists and wheels +for a project release. + +An import name MAY be followed by a semicolon and the term "private" +(e.g. ``; private``) with any amount of whitespace surrounding the semicolon. +This signals to tools that the import name is not part of the public API for +the project. + +Projects SHOULD list all the shortest import names that are exclusively provided +by the project. If any of the shortest names are dotted names, all intervening +names from that name to the top-level name SHOULD also be listed appropriately +in ``Import-Name`` and/or ``Import-Namespace``. + +If a project lists the same name in both ``Import-Name`` and +``Import-Namespace``, tools MUST raise an error due to ambiguity. + +Tools SHOULD raise an error when two projects that are about to be installed +list names that overlap in each other's ``Import-Name`` entries, or when a +project has an entry in ``Import-Name`` that overlaps with another project's +``Import-Namespace`` entries. This is to avoid projects unexpectedly shadowing +another project's code. Tools MAY warn or raise an error when installing a +project into a preexisting environment where there is import name overlap with +a project that is already installed. + +Projects MAY have an empty ``Import-Name`` field in their metadata to represent +a project with no import names (i.e. there are no Python modules of any kind in +the distribution file). + +Since projects MAY have no ``Import-Name`` metadata (either because the +project uses an older metadata version, or because it didn't specify any), then +tools have no information about what names the project provides. However, in +practice the majority of projects have their project name match what their +import name would be. As such, it is a reasonable assumption to make that a +project name that is normalized in some way to an import name +(e.g. ``packaging.utils.canonicalize_name(name, validate=True).replace("-", "_")``) +can be used if some answer is needed. + +Examples:: + + Import-Name: PIL + Import-Name: _private_module ; private + Import-Name: zope.interface + Import-Name: + + +.. _core-metadata-import-namespace: + +Import-Namespace (multiple use) +=============================== + +.. versionadded:: 2.5 + +A string containing an import name that the project provides when installed, but +not exclusively. The specified import name MUST be a valid Python identifier. +This field is used for namespace packages where multiple projects can contribute +to the same import namespace. Projects all listing the same import name in +``Import-Namespace`` can be installed together without shadowing each other. + +An import name MAY be followed by a semicolon and the term "private" (e.g. +``; private``) with any amount of whitespace surrounding the semicolon. This +signals to tools that the import name is not part of the public API for the +project. + +Projects SHOULD list all the shortest import names that are exclusively provided +by the project. If any of the shortest names are dotted names, all intervening +names from that name to the top-level name SHOULD also be listed appropriately +in ``Import-Name`` and/or ``Import-Namespace``. + +The import names listed in this field MUST be importable when the project is +installed on *some* platform for the same version of the project. This implies +that the metadata MUST be consistent across all sdists and wheels for a project +release. + +If a project lists the same name in both ``Import-Name`` and +``Import-Namespace``, tools MUST raise an error due to ambiguity. + +Note that ``Import-Namespace`` CANNOT be empty like ``Import-Name``. + +Examples:: + + Import-Namespace: zope + Import-Name: _private_module ; private + + Rarely Used Fields ================== @@ -685,6 +834,7 @@ as they're still potentially useful for informational purposes, and can also be used for their originally intended purpose in combination with a curated package repository. +.. _core-metadata-provides-dist: Provides-Dist (multiple use) ---------------------------- @@ -723,9 +873,10 @@ This field may be followed by an environment marker after a semicolon. Examples:: Provides-Dist: OtherProject - Provides-Dist: AnotherProject (3.4) + Provides-Dist: AnotherProject==3.4 Provides-Dist: virtual_package; python_version >= "3.4" +.. _core-metadata-obsoletes-dist: Obsoletes-Dist (multiple use) ----------------------------- @@ -755,6 +906,171 @@ Examples:: Obsoletes-Dist: OtherProject (<3.0) Obsoletes-Dist: Foo; os_name == "posix" + +Deprecated Fields +================= + +Deprecated fields should be avoided, but they are valid metadata fields. They +may be removed in future versions of the core metadata standard (at which point +they will only be valid in files that specify a metadata version prior to the +removal). Tools SHOULD warn users when deprecated fields are used. + +.. _home-page-optional: +.. _core-metadata-home-page: + +Home-page +--------- + +.. versionadded:: 1.0 + +.. deprecated:: 1.2 + + Per :pep:`753`, use :ref:`core-metadata-project-url` instead. + +A string containing the URL for the distribution's home page. + +Example:: + + Home-page: http://www.example.com/~cschultz/bvote/ + +.. _core-metadata-download-url: + +Download-URL +------------ + +.. versionadded:: 1.1 + +.. deprecated:: 1.2 + + Per :pep:`753`, use :ref:`core-metadata-project-url` instead. + +A string containing the URL from which this version of the distribution +can be downloaded. (This means that the URL can't be something like +"``.../BeagleVote-latest.tgz``", but instead must be +"``.../BeagleVote-0.45.tgz``".) + +Requires +-------- + +.. versionadded:: 1.1 +.. deprecated:: 1.2 + in favour of ``Requires-Dist`` + +Each entry contains a string describing some other module or package required +by this package. + +The format of a requirement string is identical to that of a module or package +name usable with the ``import`` statement, optionally followed by a version +declaration within parentheses. + +A version declaration is a series of conditional operators and version numbers, +separated by commas. Conditional operators must be one of "<", ">"', "<=", +">=", "==", and "!=". Version numbers must be in the format accepted by the +``distutils.version.StrictVersion`` class: two or three dot-separated numeric +components, with an optional "pre-release" tag on the end consisting of the +letter 'a' or 'b' followed by a number. Example version numbers are "1.0", +"2.3a2", "1.3.99", + +Any number of conditional operators can be specified, e.g. the string ">1.0, +!=1.3.4, <2.0" is a legal version declaration. + +All of the following are possible requirement strings: "rfc822", "zlib +(>=1.1.4)", "zope". + +There’s no canonical list of what strings should be used; the Python community +is left to choose its own standards. + +Examples:: + + Requires: re + Requires: sys + Requires: zlib + Requires: xml.parsers.expat (>1.0) + Requires: psycopg + + +Provides +-------- + +.. versionadded:: 1.1 +.. deprecated:: 1.2 + in favour of ``Provides-Dist`` + +Each entry contains a string describing a package or module that will be +provided by this package once it is installed. These strings should match the +ones used in Requirements fields. A version declaration may be supplied +(without a comparison operator); the package’s version number will be implied +if none is specified. + +Examples:: + + Provides: xml + Provides: xml.utils + Provides: xml.utils.iso8601 + Provides: xml.dom + Provides: xmltools (1.3) + + +Obsoletes +--------- + +.. versionadded:: 1.1 +.. deprecated:: 1.2 + in favour of ``Obsoletes-Dist`` + +Each entry contains a string describing a package or module that this package +renders obsolete, meaning that the two packages should not be installed at the +same time. Version declarations can be supplied. + +The most common use of this field will be in case a package name changes, e.g. +Gorgon 2.3 gets subsumed into Torqued Python 1.0. When you install Torqued +Python, the Gorgon package should be removed. + +Example:: + + Obsoletes: Gorgon + + +History +======= + +- March 2001: Core metadata 1.0 was approved through :pep:`241`. + +- April 2003: Core metadata 1.1 was approved through :pep:`314`. + +- February 2010: Core metadata 1.2 was approved through :pep:`345`. + +- February 2018: Core metadata 2.1 was approved through :pep:`566`. + + - Added ``Description-Content-Type`` and ``Provides-Extra``. + - Added canonical method for transforming metadata to JSON. + - Restricted the grammar of the ``Name`` field. + +- October 2020: Core metadata 2.2 was approved through :pep:`643`. + + - Added the ``Dynamic`` field. + +- March 2022: Core metadata 2.3 was approved through :pep:`685`. + + - Restricted extra names to be normalized. + +- August 2024: Core metadata 2.4 was approved through :pep:`639`. + + - Added the ``License-Expression`` field. + - Added the ``License-File`` field. + +- August 2025: Clarified that ``Dynamic`` only affects how fields + must be treated when building a wheel from a sdist, not when modifying + a wheel. + +- September 2025: Core metadata 2.5 was approved through :pep:`794`. + + - Added the ``Import-Name`` field. + - Added the ``Import-Namespace`` field. + +- October 2025: Clarified that ``License-Expression`` applies to the containing + distribution file and not the project itself. + ---- .. [1] reStructuredText markup: diff --git a/source/specifications/declaring-build-dependencies.rst b/source/specifications/declaring-build-dependencies.rst deleted file mode 100644 index 74f9c3ca2..000000000 --- a/source/specifications/declaring-build-dependencies.rst +++ /dev/null @@ -1,10 +0,0 @@ - -.. _declaring-build-dependencies: - -=================================== -Declaring build system dependencies -=================================== - -`pyproject.toml` is a build system independent file format defined in :pep:`518` -that projects may provide in order to declare any Python level dependencies that -must be installed in order to run the project's build system successfully. diff --git a/source/specifications/declaring-project-metadata.rst b/source/specifications/declaring-project-metadata.rst deleted file mode 100644 index 193f32f19..000000000 --- a/source/specifications/declaring-project-metadata.rst +++ /dev/null @@ -1,307 +0,0 @@ -.. _declaring-project-metadata: - -========================== -Declaring project metadata -========================== - -:pep:`621` specifies how to write a project's -:ref:`core metadata ` in a ``pyproject.toml`` file for -packaging-related tools to consume. It defines the following -specification as the canonical source for the format used. - -There are two kinds of metadata: *static* and *dynamic*. Static -metadata is specified in the ``pyproject.toml`` file directly and -cannot be specified or changed by a tool. Dynamic metadata is listed -via the ``dynamic`` field (defined later in this specification) and -represents metadata that a tool will later provide. - -The fields defined in this specification MUST be in a table named -``[project]`` in ``pyproject.toml``. No tools may add fields to this -table which are not defined by this specification. For tools wishing -to store their own settings in ``pyproject.toml``, they may use the -``[tool]`` table as defined in the -:ref:`build dependency declaration specification `. -The lack of a ``[project]`` table implicitly means the build back-end -will dynamically provide all fields. - -The only fields required to be statically defined are: - -- ``name`` - -The fields which are required but may be specified *either* statically -or listed as dynamic are: - -- ``version`` - -All other fields are considered optional and may be specified -statically, listed as dynamic, or left unspecified. - - -``name`` -======== - -- TOML_ type: string -- Corresponding :ref:`core metadata ` field: - :ref:`Name ` - -The name of the project. - -Tools SHOULD normalize this name, as specified by :pep:`503`, as soon -as it is read for internal consistency. - - -``version`` -=========== - -- TOML_ type: string -- Corresponding :ref:`core metadata ` field: - :ref:`Version ` - -The version of the project as supported by :pep:`440`. - -Users SHOULD prefer to specify already-normalized versions. - - -``description`` -=============== - -- TOML_ type: string -- Corresponding :ref:`core metadata ` field: - :ref:`Summary ` - -The summary description of the project. - - -``readme`` -========== - -- TOML_ type: string or table -- Corresponding :ref:`core metadata ` field: - :ref:`Description ` and - :ref:`Description-Content-Type ` - -The full description of the project (i.e. the README). - -The field accepts either a string or a table. If it is a string then -it is a path relative to ``pyproject.toml`` to a text file containing -the full description. Tools MUST assume the file's encoding is UTF-8. -If the file path ends in a case-insensitive ``.md`` suffix, then tools -MUST assume the content-type is ``text/markdown``. If the file path -ends in a case-insensitive ``.rst``, then tools MUST assume the -content-type is ``text/x-rst``. If a tool recognizes more extensions -than this PEP, they MAY infer the content-type for the user without -specifying this field as ``dynamic``. For all unrecognized suffixes -when a content-type is not provided, tools MUST raise an error. - -The ``readme`` field may also take a table. The ``file`` key has a -string value representing a path relative to ``pyproject.toml`` to a -file containing the full description. The ``text`` key has a string -value which is the full description. These keys are -mutually-exclusive, thus tools MUST raise an error if the metadata -specifies both keys. - -A table specified in the ``readme`` field also has a ``content-type`` -field which takes a string specifying the content-type of the full -description. A tool MUST raise an error if the metadata does not -specify this field in the table. If the metadata does not specify the -``charset`` parameter, then it is assumed to be UTF-8. Tools MAY -support other encodings if they choose to. Tools MAY support -alternative content-types which they can transform to a content-type -as supported by the :ref:`core metadata `. Otherwise -tools MUST raise an error for unsupported content-types. - - -``requires-python`` -=================== - -- TOML_ type: string -- Corresponding :ref:`core metadata ` field: - :ref:`Requires-Python ` - -The Python version requirements of the project. - - -``license`` -=========== - -- TOML_ type: table -- Corresponding :ref:`core metadata ` field: - :ref:`License ` - -The table may have one of two keys. The ``file`` key has a string -value that is a file path relative to ``pyproject.toml`` to the file -which contains the license for the project. Tools MUST assume the -file's encoding is UTF-8. The ``text`` key has a string value which is -the license of the project. These keys are mutually exclusive, so a -tool MUST raise an error if the metadata specifies both keys. - - -``authors``/``maintainers`` -=========================== - -- TOML_ type: Array of inline tables with string keys and values -- Corresponding :ref:`core metadata ` field: - :ref:`Author `, - :ref:`Author-email `, - :ref:`Maintainer `, and - :ref:`Maintainer-email ` - -The people or organizations considered to be the "authors" of the -project. The exact meaning is open to interpretation β€” it may list the -original or primary authors, current maintainers, or owners of the -package. - -The "maintainers" field is similar to "authors" in that its exact -meaning is open to interpretation. - -These fields accept an array of tables with 2 keys: ``name`` and -``email``. Both values must be strings. The ``name`` value MUST be a -valid email name (i.e. whatever can be put as a name, before an email, -in :rfc:`822`) and not contain commas. The ``email`` value MUST be a -valid email address. Both keys are optional, but at least one of the -keys must be specified in the table. - -Using the data to fill in :ref:`core metadata ` is as -follows: - -1. If only ``name`` is provided, the value goes in - :ref:`Author ` or - :ref:`Maintainer ` as appropriate. -2. If only ``email`` is provided, the value goes in - :ref:`Author-email ` or - :ref:`Maintainer-email ` - as appropriate. -3. If both ``email`` and ``name`` are provided, the value goes in - :ref:`Author-email ` or - :ref:`Maintainer-email ` - as appropriate, with the format ``{name} <{email}>``. -4. Multiple values should be separated by commas. - - -``keywords`` -============ - -- TOML_ type: array of strings -- Corresponding :ref:`core metadata ` field: - :ref:`Keywords ` - -The keywords for the project. - - -``classifiers`` -=============== - -- TOML_ type: array of strings -- Corresponding :ref:`core metadata ` field: - :ref:`Classifier ` - -Trove classifiers which apply to the project. - - -``urls`` -======== - -- TOML_ type: table with keys and values of strings -- Corresponding :ref:`core metadata ` field: - :ref:`Project-URL ` - -A table of URLs where the key is the URL label and the value is the -URL itself. - - -Entry points -============ - -- TOML_ type: table (``[project.scripts]``, ``[project.gui-scripts]``, - and ``[project.entry-points]``) -- :ref:`Entry points specification ` - -There are three tables related to entry points. The -``[project.scripts]`` table corresponds to the ``console_scripts`` -group in the :ref:`entry points specification `. The key -of the table is the name of the entry point and the value is the -object reference. - -The ``[project.gui-scripts]`` table corresponds to the ``gui_scripts`` -group in the :ref:`entry points specification `. Its -format is the same as ``[project.scripts]``. - -The ``[project.entry-points]`` table is a collection of tables. Each -sub-table's name is an entry point group. The key and value semantics -are the same as ``[project.scripts]``. Users MUST NOT create -nested sub-tables but instead keep the entry point groups to only one -level deep. - -Build back-ends MUST raise an error if the metadata defines a -``[project.entry-points.console_scripts]`` or -``[project.entry-points.gui_scripts]`` table, as they would -be ambiguous in the face of ``[project.scripts]`` and -``[project.gui-scripts]``, respectively. - - -``dependencies``/``optional-dependencies`` -========================================== - -- TOML_ type: Array of :pep:`508` strings (``dependencies``), and a - table with values of arrays of :pep:`508` strings - (``optional-dependencies``) -- Corresponding :ref:`core metadata ` field: - :ref:`Requires-Dist ` and - :ref:`Provides-Extra ` - -The (optional) dependencies of the project. - -For ``dependencies``, it is a key whose value is an array of strings. -Each string represents a dependency of the project and MUST be -formatted as a valid :pep:`508` string. Each string maps directly to -a :ref:`Requires-Dist ` entry. - -For ``optional-dependencies``, it is a table where each key specifies -an extra and whose value is an array of strings. The strings of the -arrays must be valid :pep:`508` strings. The keys MUST be valid values -for :ref:`Provides-Extra `. Each value -in the array thus becomes a corresponding -:ref:`Requires-Dist ` entry for the -matching :ref:`Provides-Extra ` -metadata. - - -``dynamic`` -=========== - -- TOML_ type: array of string -- A corresponding :ref:`core metadata ` field does not - exist - -Specifies which fields listed by this PEP were intentionally -unspecified so another tool can/will provide such metadata -dynamically. This clearly delineates which metadata is purposefully -unspecified and expected to stay unspecified compared to being -provided via tooling later on. - -- A build back-end MUST honour statically-specified metadata (which - means the metadata did not list the field in ``dynamic``). -- A build back-end MUST raise an error if the metadata specifies - ``name`` in ``dynamic``. -- If the :ref:`core metadata ` specification lists a - field as "Required", then the metadata MUST specify the field - statically or list it in ``dynamic`` (build back-ends MUST raise an - error otherwise, i.e. it should not be possible for a required field - to not be listed somehow in the ``[project]`` table). -- If the :ref:`core metadata ` specification lists a - field as "Optional", the metadata MAY list it in ``dynamic`` if the - expectation is a build back-end will provide the data for the field - later. -- Build back-ends MUST raise an error if the metadata specifies a - field statically as well as being listed in ``dynamic``. -- If the metadata does not list a field in ``dynamic``, then a build - back-end CANNOT fill in the requisite metadata on behalf of the user - (i.e. ``dynamic`` is the only way to allow a tool to fill in - metadata and the user must opt into the filling in). -- Build back-ends MUST raise an error if the metadata specifies a - field in ``dynamic`` but the build back-end was unable to determine - the data for it (omitting the data, if determined to be the accurate - value, is acceptable). - - -.. _TOML: https://toml.io diff --git a/source/specifications/dependency-groups.rst b/source/specifications/dependency-groups.rst new file mode 100644 index 000000000..a35afb475 --- /dev/null +++ b/source/specifications/dependency-groups.rst @@ -0,0 +1,252 @@ +.. _dependency-groups: + +================= +Dependency Groups +================= + +This specification defines dependency groups, a mechanism for storing package +requirements in ``pyproject.toml`` files such that they are not included in +project metadata when it is built. + +Dependency groups are suitable for internal development use-cases like linting +and testing, as well as for projects which are not built for distribution, like +collections of related scripts. + +Fundamentally, dependency groups should be thought of as being a standardized +subset of the capabilities of ``requirements.txt`` files (which are +``pip``-specific). + +Specification +============= + +Examples +-------- + +This is a simple table which shows ``docs`` and ``test`` groups:: + + [dependency-groups] + docs = ["sphinx"] + test = ["pytest>7", "coverage"] + +and a similar table which defines ``docs``, ``test``, and ``coverage`` groups:: + + [dependency-groups] + docs = ["sphinx"] + coverage = ["coverage[toml]"] + test = ["pytest>7", {include-group = "coverage"}] + +The ``[dependency-groups]`` Table +--------------------------------- + +Dependency groups are defined as a table in ``pyproject.toml`` named +``dependency-groups``. The ``dependency-groups`` table contains an arbitrary +number of user-defined keys, each of which has, as its value, a list of +requirements. + +``[dependency-groups]`` keys, sometimes also called "group names", must be +:ref:`valid non-normalized names `. Tools which handle Dependency +Groups MUST :ref:`normalize ` these names before +comparisons. + +Tools SHOULD prefer to present the original, non-normalized name to users, and +if duplicate names are detected after normalization, tools SHOULD emit an +error. + +Requirement lists, the values in ``[dependency-groups]``, may contain strings, +tables (``dict`` in Python), or a mix of strings and tables. Strings must be +valid :ref:`dependency specifiers `, and tables must be +valid Dependency Group Includes. + +Dependency Group Include +------------------------ + +A Dependency Group Include includes another Dependency Group in the current +group. + +An include is a table with exactly one key, ``"include-group"``, whose value is +a string, the name of another Dependency Group. + +Includes are defined to be exactly equivalent to the contents of the named +Dependency Group, inserted into the current group at the location of the include. +For example, if ``foo = ["a", "b"]`` is one group, and +``bar = ["c", {include-group = "foo"}, "d"]`` is another, then ``bar`` should +evaluate to ``["c", "a", "b", "d"]`` when Dependency Group Includes are expanded. + +Dependency Group Includes may specify the same package multiple times. +Tools SHOULD NOT deduplicate or otherwise alter the list contents produced by the +include. For example, given the following table: + +.. code-block:: toml + + [dependency-groups] + group-a = ["foo"] + group-b = ["foo>1.0"] + group-c = ["foo<1.0"] + all = [ + "foo", + {include-group = "group-a"}, + {include-group = "group-b"}, + {include-group = "group-c"}, + ] + +The resolved value of ``all`` SHOULD be ``["foo", "foo", "foo>1.0", "foo<1.0"]``. +Tools should handle such a list exactly as they would handle any other case in +which they are asked to process the same requirement multiple times with +different version constraints. + +Dependency Group Includes may include groups containing Dependency Group Includes, +in which case those includes should be expanded as well. Dependency Group Includes +MUST NOT include cycles, and tools SHOULD report an error if they detect a cycle. + +Package Building +---------------- + +Build backends MUST NOT include Dependency Group data in built distributions as +package metadata. This means that sdist ``PKG-INFO`` and wheel ``METADATA`` +files should not include referenceable fields containing dependency groups. + +It is, however, valid to use dependency groups in the evaluation of dynamic +metadata, and ``pyproject.toml`` files included in sdists will still contain +``[dependency-groups]``. However, the table's contents are not part of a built +package's interfaces. + +Installing Dependency Groups & Extras +------------------------------------- + +There is no syntax or specification-defined interface for installing or +referring to dependency groups. Tools are expected to provide dedicated +interfaces for this purpose. + +Tools MAY choose to provide the same or similar interfaces for interacting +with dependency groups as they do for managing extras. Tools authors are +advised that the specification does not forbid having an extra whose name +matches a Dependency Group. Separately, users are advised to avoid creating +dependency groups whose names match extras, and tools MAY treat such matching +as an error. + +Validation and Compatibility +---------------------------- + +Tools supporting dependency groups may want to validate data before using it. +When implementing such validation, authors should be aware of the possibility +of future extensions to the specification, so that they do not unnecessarily +emit errors or warnings. + +Tools SHOULD error when evaluating or processing unrecognized data in +dependency groups. + +Tools SHOULD NOT eagerly validate the contents of *all* dependency groups +unless they have a need to do so. + +This means that in the presence of the following data, most tools should allow +the ``foo`` group to be used and only error if the ``bar`` group is used: + +.. code-block:: toml + + [dependency-groups] + foo = ["pyparsing"] + bar = [{set-phasers-to = "stun"}] + +.. note:: + + There are several known cases of tools which have good cause to be + stricter. Linters and validators are an example, as their purpose is to + validate the contents of all dependency groups. + +Reference Implementation +======================== + +The following Reference Implementation prints the contents of a Dependency +Group to stdout, newline delimited. +The output is therefore valid ``requirements.txt`` data. + +.. code-block:: python + + import re + import sys + import tomllib + from collections import defaultdict + + from packaging.requirements import Requirement + + + def _normalize_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + + def _normalize_group_names(dependency_groups: dict) -> dict: + original_names = defaultdict(list) + normalized_groups = {} + + for group_name, value in dependency_groups.items(): + normed_group_name = _normalize_name(group_name) + original_names[normed_group_name].append(group_name) + normalized_groups[normed_group_name] = value + + errors = [] + for normed_name, names in original_names.items(): + if len(names) > 1: + errors.append(f"{normed_name} ({', '.join(names)})") + if errors: + raise ValueError(f"Duplicate dependency group names: {', '.join(errors)}") + + return normalized_groups + + + def _resolve_dependency_group( + dependency_groups: dict, group: str, past_groups: tuple[str, ...] = () + ) -> list[str]: + if group in past_groups: + raise ValueError(f"Cyclic dependency group include: {group} -> {past_groups}") + + if group not in dependency_groups: + raise LookupError(f"Dependency group '{group}' not found") + + raw_group = dependency_groups[group] + if not isinstance(raw_group, list): + raise ValueError(f"Dependency group '{group}' is not a list") + + realized_group = [] + for item in raw_group: + if isinstance(item, str): + # packaging.requirements.Requirement parsing ensures that this is a valid + # PEP 508 Dependency Specifier + # raises InvalidRequirement on failure + Requirement(item) + realized_group.append(item) + elif isinstance(item, dict): + if tuple(item.keys()) != ("include-group",): + raise ValueError(f"Invalid dependency group item: {item}") + + include_group = _normalize_name(next(iter(item.values()))) + realized_group.extend( + _resolve_dependency_group( + dependency_groups, include_group, past_groups + (group,) + ) + ) + else: + raise ValueError(f"Invalid dependency group item: {item}") + + return realized_group + + + def resolve(dependency_groups: dict, group: str) -> list[str]: + if not isinstance(dependency_groups, dict): + raise TypeError("Dependency Groups table is not a dict") + if not isinstance(group, str): + raise TypeError("Dependency group name is not a str") + return _resolve_dependency_group(dependency_groups, group) + + + if __name__ == "__main__": + with open("pyproject.toml", "rb") as fp: + pyproject = tomllib.load(fp) + + dependency_groups_raw = pyproject["dependency-groups"] + dependency_groups = _normalize_group_names(dependency_groups_raw) + print("\n".join(resolve(pyproject["dependency-groups"], sys.argv[1]))) + +History +======= + +- October 2024: This specification was approved through :pep:`735`. diff --git a/source/specifications/dependency-specifiers.rst b/source/specifications/dependency-specifiers.rst index ab33de726..99886563c 100644 --- a/source/specifications/dependency-specifiers.rst +++ b/source/specifications/dependency-specifiers.rst @@ -1,3 +1,4 @@ +.. highlight:: text .. _dependency-specifiers: @@ -5,10 +6,546 @@ Dependency specifiers ===================== -The dependency specifier format used to declare a dependency on another -component is defined in :pep:`508`. +This document describes the dependency specifiers format as originally specified +in :pep:`508`. -The environment markers section in this PEP supersedes the environment markers -section in :pep:`345`. +The job of a dependency is to enable tools like pip [#pip]_ to find the right +package to install. Sometimes this is very loose - just specifying a name, and +sometimes very specific - referring to a specific file to install. Sometimes +dependencies are only relevant in one platform, or only some versions are +acceptable, so the language permits describing all these cases. + +The language defined is a compact line based format which is already in +widespread use in pip requirements files, though we do not specify the command +line option handling that those files permit. There is one caveat - the +URL reference form, specified in :ref:`Versioning specifier specification ` +is not actually implemented in pip, but we use that format rather +than pip's current native format. + +Specification +============= + +Examples +-------- + +All features of the language shown with a name based lookup:: + + requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < "2.7" + +A minimal URL based lookup:: + + pip @ https://github.com/pypa/pip/archive/1.3.1.zip#sha1=da9234ee9982d4bbb3c72346a6de940a148ea686 + +Concepts +-------- + +A dependency specification always specifies a distribution name. It may +include extras, which expand the dependencies of the named distribution to +enable optional features. The version installed can be controlled using +version limits, or giving the URL to a specific artifact to install. Finally +the dependency can be made conditional using environment markers. + +Grammar +------- + +We first cover the grammar briefly and then drill into the semantics of each +section later. + +A distribution specification is written in ASCII text. We use a parsley +[#parsley]_ grammar to provide a precise grammar. It is expected that the +specification will be embedded into a larger system which offers framing such +as comments, multiple line support via continuations, or other such features. + +The full grammar including annotations to build a useful parse tree is +included at the end of this document. + +Versions may be specified according to the rules of the +:ref:`Version specifier specification `. (Note: +URI is defined in :rfc:`std-66 <3986>`):: + + version_cmp = wsp* '<=' | '<' | '!=' | '===' | '==' | '>=' | '>' | '~=' + version = wsp* ( letterOrDigit | '-' | '_' | '.' | '*' | '+' | '!' )+ + version_one = version_cmp version wsp* + version_many = version_one (',' version_one)* (',' wsp*)? + versionspec = ( '(' version_many ')' ) | version_many + urlspec = '@' wsp* + +Environment markers allow making a specification only take effect in some +environments:: + + marker_op = version_cmp | (wsp+ 'in' wsp+) | (wsp+ 'not' wsp+ 'in' wsp+) + python_str_c = (wsp | letter | digit | '(' | ')' | '.' | '{' | '}' | + '-' | '_' | '*' | '#' | ':' | ';' | ',' | '/' | '?' | + '[' | ']' | '!' | '~' | '`' | '@' | '$' | '%' | '^' | + '&' | '=' | '+' | '|' | '<' | '>' ) + dquote = '"' + squote = '\\'' + python_str = (squote (python_str_c | dquote)* squote | + dquote (python_str_c | squote)* dquote) + env_var = ('python_version' | 'python_full_version' | + 'os_name' | 'sys_platform' | 'platform_release' | + 'platform_system' | 'platform_version' | + 'platform_machine' | 'platform_python_implementation' | + 'implementation_name' | 'implementation_version' | + 'extra' | 'extras' | 'dependency_groups' # ONLY when defined by a containing layer + ) + marker_var = wsp* (env_var | python_str) + marker_expr = marker_var marker_op marker_var + | wsp* '(' marker wsp* ')' + marker_and = marker_expr wsp* 'and' marker_expr + | marker_expr + marker_or = marker_and wsp* 'or' marker_and + | marker_and + marker = marker_or + quoted_marker = ';' wsp* marker + +Optional components of a distribution may be specified using the extras +field:: + + identifier_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit) + identifier = letterOrDigit identifier_end* + name = identifier + extras_list = identifier (wsp* ',' wsp* identifier)* + extras = '[' wsp* extras_list? wsp* ']' Restrictions on names for extras is defined in :pep:`685`. + +Giving us a rule for name based requirements:: + + name_req = name wsp* extras? wsp* versionspec? wsp* quoted_marker? + +And a rule for direct reference specifications:: + + url_req = name wsp* extras? wsp* urlspec (wsp+ quoted_marker?)? + +Leading to the unified rule that can specify a dependency.:: + + specification = wsp* ( url_req | name_req ) wsp* + +Whitespace +---------- + +Non line-breaking whitespace is mostly optional with no semantic meaning. The +sole exception is detecting the end of a URL requirement. + +.. _dependency-specifiers-names: + +Names +----- + +Python distribution names are currently defined in :pep:`345`. Names +act as the primary identifier for distributions. They are present in all +dependency specifications, and are sufficient to be a specification on their +own. However, PyPI places strict restrictions on names - they must match a +case insensitive regex or they won't be accepted. Accordingly, in this +document we limit the acceptable values for identifiers to that regex. A full +redefinition of name may take place in a future metadata PEP. The regex (run +with re.IGNORECASE) is:: + + ^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])\Z + +.. _dependency-specifiers-extras: + +Extras +------ + +An extra is an optional part of a distribution. Distributions can specify as +many extras as they wish, and each extra results in the declaration of +additional dependencies of the distribution **when** the extra is used in a +dependency specification. For instance:: + + requests[security,tests] + +Extras union in the dependencies they define with the dependencies of the +distribution they are attached to. The example above would result in requests +being installed, and requests own dependencies, and also any dependencies that +are listed in the "security" extra of requests. + +If multiple extras are listed, all the dependencies are unioned together. + +.. _dependency-specifiers-versions: + +Versions +-------- + +See the :ref:`Version specifier specification ` for +more detail on both version numbers and version comparisons. Version +specifications limit the versions of a distribution that can be +used. They only apply to distributions looked up by name, rather than +via a URL. Version comparison are also used in the markers feature. The +optional brackets around a version are present for compatibility with +:pep:`345` but should not be generated, only accepted. + +.. _dependency-specifiers-environment-markers: + +Environment Markers +------------------- + +Environment markers allow a dependency specification to provide a rule that +describes when the dependency should be used. For instance, consider a package +that needs argparse. In Python 2.7 argparse is always present. On older Python +versions it has to be installed as a dependency. This can be expressed as so:: + + argparse;python_version<"2.7" + +A marker expression evaluates to either True or False. When it evaluates to +False, the dependency specification should be ignored. + +The marker language is inspired by Python itself, chosen for the ability to +safely evaluate it without running arbitrary code that could become a security +vulnerability. Markers were first standardised in :pep:`345`. This document +fixes some issues that were observed in the design described in :pep:`426`. + +Comparisons in marker expressions are typed by the comparison operator and the +type of the marker value. The operators that are not in + perform the same as they do for strings or sets in Python based on +whether the marker value is a string or set itself. The operators +use the version comparison rules of the +:ref:`Version specifier specification ` when those are +defined (that is when both sides have a valid version specifier). If there is no +defined behaviour of this specification and the operator exists in Python, then +the operator falls back to the Python behaviour for the types involved. +Otherwise an error should be raised. e.g. the following will result in errors:: + + "dog" ~= "fred" + python_version ~= "surprise" + +User supplied constants are always encoded as strings with either ``'`` or +``"`` quote marks. Note that backslash escapes are not defined, but existing +implementations do support them. They are not included in this +specification because they add complexity and there is no observable need for +them today. Similarly we do not define non-ASCII character support: all the +runtime variables we are referencing are expected to be ASCII-only. + +The variables in the marker grammar such as "os_name" resolve to values looked +up in the Python runtime. With the exception of "extra" all values are defined +on all Python versions today - it is an error in the implementation of markers +if a value is not defined. + +Unknown variables must raise an error rather than resulting in a comparison +that evaluates to True or False. + +Variables whose value cannot be calculated on a given Python implementation +should evaluate to ``0`` for versions, and an empty string for all other +variables. + +The "extra" variable is special. It is used by wheels to signal which +specifications apply to a given extra in the wheel ``METADATA`` file, but +since the ``METADATA`` file is based on a draft version of :pep:`426`, there is +no current specification for this. Regardless, outside of a context where this +special handling is taking place, the "extra" variable should result in an +error like all other unknown variables. + +The "extras" and "dependency_groups" variables are also special. They are used +to specify any requested extras or dependency groups when installing from a lock +file. Outside of the context of lock files, these two variables should result in +an error like all other unknown variables. + +.. list-table:: + :header-rows: 1 + + * - Marker + - Python equivalent + - Type + - Sample values + * - ``os_name`` + - :py:data:`os.name` + - String + - ``posix``, ``java`` + * - ``sys_platform`` + - :py:data:`sys.platform` + - String + - ``linux``, ``linux2``, ``darwin``, ``java1.8.0_51`` (note that "linux" + is from Python3 and "linux2" from Python2) + * - ``platform_machine`` + - :py:func:`platform.machine()` + - String + - ``x86_64`` + * - ``platform_python_implementation`` + - :py:func:`platform.python_implementation()` + - String + - ``CPython``, ``Jython`` + * - ``platform_release`` + - :py:func:`platform.release()` + - String + - ``3.14.1-x86_64-linode39``, ``14.5.0``, ``1.8.0_51`` + * - ``platform_system`` + - :py:func:`platform.system()` + - String + - ``Linux``, ``Windows``, ``Java`` + * - ``platform_version`` + - :py:func:`platform.version()` + - String + - ``#1 SMP Fri Apr 25 13:07:35 EDT 2014`` + ``Java HotSpot(TM) 64-Bit Server VM, 25.51-b03, Oracle Corporation`` + ``Darwin Kernel Version 14.5.0: Wed Jul 29 02:18:53 PDT 2015; root:xnu-2782.40.9~2/RELEASE_X86_64`` + * - ``python_version`` + - ``'.'.join(platform.python_version_tuple()[:2])`` + - :ref:`Version ` + - ``3.4``, ``2.7`` + * - ``python_full_version`` + - :py:func:`platform.python_version()` + - :ref:`Version ` + - ``3.4.0``, ``3.5.0b1`` + * - ``implementation_name`` + - :py:data:`sys.implementation.name ` + - String + - ``cpython`` + * - ``implementation_version`` + - see definition below + - :ref:`Version ` + - ``3.4.0``, ``3.5.0b1`` + * - ``extra`` + - An error except when defined by the context interpreting the + specification. + - String + - ``toml`` + * - ``extras`` + - An error except when defined by the context interpreting the + specification. + - Set of strings + - ``{"toml"}`` + * - ``dependency_groups`` + - An error except when defined by the context interpreting the + specification. + - Set of strings + - ``{"test"}`` + +The ``implementation_version`` marker variable is derived from +:py:data:`sys.implementation.version `: + +.. code-block:: python + + def format_full_version(info): + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version + + if hasattr(sys, 'implementation'): + implementation_version = format_full_version(sys.implementation.version) + else: + implementation_version = "0" + +This environment markers section, initially defined through :pep:`508`, supersedes the environment markers +section in :pep:`345`. + +.. _dependency-specifiers-grammar: + +Complete Grammar +================ + +The complete parsley grammar:: + + wsp = ' ' | '\t' + version_cmp = wsp* <'<=' | '<' | '!=' | '===' | '==' | '>=' | '>' | '~='> + version = wsp* <( letterOrDigit | '-' | '_' | '.' | '*' | '+' | '!' )+> + version_one = version_cmp:op version:v wsp* -> (op, v) + version_many = version_one:v1 (',' version_one)*:v2 (',' wsp*)? -> [v1] + v2 + versionspec = ('(' version_many:v ')' ->v) | version_many + urlspec = '@' wsp* + marker_op = version_cmp | (wsp* 'in') | (wsp* 'not' wsp+ 'in') + python_str_c = (wsp | letter | digit | '(' | ')' | '.' | '{' | '}' | + '-' | '_' | '*' | '#' | ':' | ';' | ',' | '/' | '?' | + '[' | ']' | '!' | '~' | '`' | '@' | '$' | '%' | '^' | + '&' | '=' | '+' | '|' | '<' | '>' ) + dquote = '"' + squote = '\\'' + python_str = (squote <(python_str_c | dquote)*>:s squote | + dquote <(python_str_c | squote)*>:s dquote) -> s + env_var = ('python_version' | 'python_full_version' | + 'os_name' | 'sys_platform' | 'platform_release' | + 'platform_system' | 'platform_version' | + 'platform_machine' | 'platform_python_implementation' | + 'implementation_name' | 'implementation_version' | + 'extra' | 'extras' | 'dependency_groups' # ONLY when defined by a containing layer + ):varname -> lookup(varname) + marker_var = wsp* (env_var | python_str) + marker_expr = marker_var:l marker_op:o marker_var:r -> (o, l, r) + | wsp* '(' marker:m wsp* ')' -> m + marker_and = marker_expr:l wsp* 'and' marker_expr:r -> ('and', l, r) + | marker_expr:m -> m + marker_or = marker_and:l wsp* 'or' marker_and:r -> ('or', l, r) + | marker_and:m -> m + marker = marker_or + quoted_marker = ';' wsp* marker + identifier_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit) + identifier = < letterOrDigit identifier_end* > + name = identifier + extras_list = identifier:i (wsp* ',' wsp* identifier)*:ids -> [i] + ids + extras = '[' wsp* extras_list?:e wsp* ']' -> e + name_req = (name:n wsp* extras?:e wsp* versionspec?:v wsp* quoted_marker?:m + -> (n, e or [], v or [], m)) + url_req = (name:n wsp* extras?:e wsp* urlspec:v (wsp+ | end) quoted_marker?:m + -> (n, e or [], v, m)) + specification = wsp* ( url_req | name_req ):s wsp* -> s + # The result is a tuple - name, list-of-extras, + # list-of-version-constraints-or-a-url, marker-ast or None + + + URI_reference = + URI = scheme ':' hier_part ('?' query )? ( '#' fragment)? + hier_part = ('//' authority path_abempty) | path_absolute | path_rootless | path_empty + absolute_URI = scheme ':' hier_part ( '?' query )? + relative_ref = relative_part ( '?' query )? ( '#' fragment )? + relative_part = '//' authority path_abempty | path_absolute | path_noscheme | path_empty + scheme = letter ( letter | digit | '+' | '-' | '.')* + authority = ( userinfo '@' )? host ( ':' port )? + userinfo = ( unreserved | pct_encoded | sub_delims | ':')* + host = IP_literal | IPv4address | reg_name + port = digit* + IP_literal = '[' ( IPv6address | IPvFuture) ']' + IPvFuture = 'v' hexdig+ '.' ( unreserved | sub_delims | ':')+ + IPv6address = ( + ( h16 ':'){6} ls32 + | '::' ( h16 ':'){5} ls32 + | ( h16 )? '::' ( h16 ':'){4} ls32 + | ( ( h16 ':')? h16 )? '::' ( h16 ':'){3} ls32 + | ( ( h16 ':'){0,2} h16 )? '::' ( h16 ':'){2} ls32 + | ( ( h16 ':'){0,3} h16 )? '::' h16 ':' ls32 + | ( ( h16 ':'){0,4} h16 )? '::' ls32 + | ( ( h16 ':'){0,5} h16 )? '::' h16 + | ( ( h16 ':'){0,6} h16 )? '::' ) + h16 = hexdig{1,4} + ls32 = ( h16 ':' h16) | IPv4address + IPv4address = dec_octet '.' dec_octet '.' dec_octet '.' dec_octet + nz = ~'0' digit + dec_octet = ( + digit # 0-9 + | nz digit # 10-99 + | '1' digit{2} # 100-199 + | '2' ('0' | '1' | '2' | '3' | '4') digit # 200-249 + | '25' ('0' | '1' | '2' | '3' | '4' | '5') )# %250-255 + reg_name = ( unreserved | pct_encoded | sub_delims)* + path = ( + path_abempty # begins with '/' or is empty + | path_absolute # begins with '/' but not '//' + | path_noscheme # begins with a non-colon segment + | path_rootless # begins with a segment + | path_empty ) # zero characters + path_abempty = ( '/' segment)* + path_absolute = '/' ( segment_nz ( '/' segment)* )? + path_noscheme = segment_nz_nc ( '/' segment)* + path_rootless = segment_nz ( '/' segment)* + path_empty = pchar{0} + segment = pchar* + segment_nz = pchar+ + segment_nz_nc = ( unreserved | pct_encoded | sub_delims | '@')+ + # non-zero-length segment without any colon ':' + pchar = unreserved | pct_encoded | sub_delims | ':' | '@' + query = ( pchar | '/' | '?')* + fragment = ( pchar | '/' | '?')* + pct_encoded = '%' hexdig + unreserved = letter | digit | '-' | '.' | '_' | '~' + reserved = gen_delims | sub_delims + gen_delims = ':' | '/' | '?' | '#' | '(' | ')?' | '@' + sub_delims = '!' | '$' | '&' | '\\'' | '(' | ')' | '*' | '+' | ',' | ';' | '=' + hexdig = digit | 'a' | 'A' | 'b' | 'B' | 'c' | 'C' | 'd' | 'D' | 'e' | 'E' | 'f' | 'F' + +A test program - if the grammar is in a string ``grammar``: + +.. code-block:: python + + import os + import sys + import platform + + from parsley import makeGrammar + + grammar = """ + wsp ... + """ + tests = [ + "A", + "A.B-C_D", + "aa", + "name", + "name<=1", + "name>=3", + "name>=3,", + "name>=3,<2", + "name@http://foo.com", + "name [fred,bar] @ http://foo.com ; python_version=='2.7'", + "name[quux, strange];python_version<'2.7' and platform_version=='2'", + "name; os_name=='a' or os_name=='b'", + # Should parse as (a and b) or c + "name; os_name=='a' and os_name=='b' or os_name=='c'", + # Overriding precedence -> a and (b or c) + "name; os_name=='a' and (os_name=='b' or os_name=='c')", + # should parse as a or (b and c) + "name; os_name=='a' or os_name=='b' and os_name=='c'", + # Overriding precedence -> (a or b) and c + "name; (os_name=='a' or os_name=='b') and os_name=='c'", + ] + + def format_full_version(info): + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version + + if hasattr(sys, 'implementation'): + implementation_version = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name + else: + implementation_version = '0' + implementation_name = '' + bindings = { + 'implementation_name': implementation_name, + 'implementation_version': implementation_version, + 'os_name': os.name, + 'platform_machine': platform.machine(), + 'platform_python_implementation': platform.python_implementation(), + 'platform_release': platform.release(), + 'platform_system': platform.system(), + 'platform_version': platform.version(), + 'python_full_version': platform.python_version(), + 'python_version': '.'.join(platform.python_version_tuple()[:2]), + 'sys_platform': sys.platform, + } + + compiled = makeGrammar(grammar, {'lookup': bindings.__getitem__}) + for test in tests: + parsed = compiled(test).specification() + print("%s -> %s" % (test, parsed)) + + +History +======= + +- November 2015: This specification was approved through :pep:`508`. +- July 2019: The definition of ``python_version`` was `changed + `_ from ``platform.python_version()[:3]`` to + ``'.'.join(platform.python_version_tuple()[:2])``, to accommodate potential + future versions of Python with 2-digit major and minor versions + (e.g. 3.10). [#future_versions]_ +- June 2024: The definition of ``version_many`` was changed to allow trailing + commas, matching with the behavior of the Python implementation that has been + in use since late 2022. +- April 2025: Added ``extras`` and ``dependency_groups`` for + :ref:`lock-file-spec` as approved through :pep:`751`. +- August 2025: The suggested name validation regex was fixed to match the field + specification (it previously finished with ``$`` instead of ``\Z``, + incorrectly permitting trailing newlines) +- December 2025: Ensure ``===`` before ``==`` in grammar, to allow arbitrary + equality comparisons to be parsed. + + +References +========== + +.. [#pip] pip, the recommended installer for Python packages + (http://pip.readthedocs.org/en/stable/) + +.. [#parsley] The parsley PEG library. + (https://pypi.python.org/pypi/parsley/) + +.. [#future_versions] Future Python versions might be problematic with the + definition of Environment Marker Variable ``python_version`` + (https://github.com/python/peps/issues/560) + + + +.. _python-version-change: https://mail.python.org/pipermail/distutils-sig/2018-January/031920.html diff --git a/source/specifications/direct-url-data-structure.rst b/source/specifications/direct-url-data-structure.rst new file mode 100644 index 000000000..a82537f0a --- /dev/null +++ b/source/specifications/direct-url-data-structure.rst @@ -0,0 +1,303 @@ +.. highlight:: json + +.. _direct-url-data-structure: + +========================= +Direct URL Data Structure +========================= + +This document specifies a JSON-serializable abstract data structure that can represent +URLs to python projects and distribution artifacts such as VCS source trees, local +source trees, source distributions and wheels. + +At time of writing, it is not formally specified how to merge the parts of this +data structure into a single URL that can be passed to tools. A common representation is the +pip URL format (`VCS Support `_), other examples are provided in the +:ref:`Version specifier specification `. + +Specification +============= + +The Direct URL Data Structure MUST be a dictionary, serializable to JSON according to +:rfc:`8259`. + +It MUST contain at least two fields. The first one is ``url``, with +type ``string``. Its content must be a valid URL according to the +`WHATWG URL Standard `_. + +Depending on what ``url`` refers to, the second field MUST be one of ``vcs_info`` +(if ``url`` is a VCS reference), ``archive_info`` (if +``url`` is a source archive or a wheel), or ``dir_info`` (if ``url`` is a +local directory). These info fields have a (possibly empty) subdictionary as +value, with the possible keys defined below. + +Security Considerations +----------------------- + +When persisted, ``url`` MUST be stripped of any sensitive authentication information, +for security reasons. + +The user:password section of the URL MAY however +be composed of environment variables, matching the following regular +expression: + +.. code-block:: text + + \$\{[A-Za-z0-9-_]+\}(:\$\{[A-Za-z0-9-_]+\})? + +Additionally, the user:password section of the URL MAY be a +well-known, non security sensitive string. A typical example is ``git`` +in the case of a URL such as ``ssh://git@gitlab.com/user/repo``. + +.. _direct-url-data-structure-vcs: + +VCS URLs +-------- + +When ``url`` refers to a VCS repository, the ``vcs_info`` key MUST be present +as a dictionary with the following keys: + +- A ``vcs`` key (type ``string``) MUST be present, containing the name of the VCS + (i.e. one of ``git``, ``hg``, ``bzr``, ``svn``). Other VCS's SHOULD be registered by + writing a PEP to amend this specification. + The ``url`` value MUST be compatible with the corresponding VCS, + so an installer can hand it off without transformation to a + checkout/download command of the VCS. +- A ``requested_revision`` key (type ``string``) MAY be present naming a + branch/tag/ref/commit/revision/etc (in a format compatible with the VCS). This field + MUST match the revision requested by the user and MUST NOT exist when the user did + not select a specific revision. +- A ``commit_id`` key (type ``string``) MUST be present, containing the + exact commit/revision number that was/is to be installed. + If the VCS supports commit-hash + based revision identifiers, such commit-hash MUST be used as + ``commit_id`` in order to reference an immutable + version of the source code. + +.. _direct-url-data-structure-archive: + +Archive URLs +------------ + +When ``url`` refers to a source archive or a wheel, the ``archive_info`` key +MUST be present as a dictionary with the following keys: + +- A ``hashes`` key SHOULD be present as a dictionary mapping a hash name to a hex + encoded digest of the file. + + Multiple hashes can be included, and it is up to the consumer to decide what to do + with multiple hashes (it may validate all of them or a subset of them, or nothing at + all). + + These hash names SHOULD always be normalized to be lowercase. + + Any hash algorithm available via :py:mod:`hashlib` (specifically any that can be passed to + :py:func:`hashlib.new()` and do not require additional parameters) can be used as a key for + the hashes dictionary. At least one secure algorithm from + :py:data:`hashlib.algorithms_guaranteed` SHOULD always be included. At time of writing, + ``sha256`` specifically is recommended. + +- A deprecated ``hash`` key (type ``string``) MAY be present for backwards compatibility + purposes, with value ``=``. + +Producers of the data structure SHOULD emit the ``hashes`` key whether one or multiple +hashes are available. Producers SHOULD continue to emit the ``hash`` key in contexts +where they did so before, so as to keep backwards compatibility for existing clients. + +When both the ``hash`` and ``hashes`` keys are present, the hash represented in the +``hash`` key MUST also be present in the ``hashes`` dictionary, so consumers can +consider the ``hashes`` key only if it is present, and fall back to ``hash`` otherwise. + +.. _direct-url-data-structure-local-directory: + +Local directories +----------------- + +When ``url`` refers to a local directory, the ``dir_info`` key MUST be +present as a dictionary with the following key: + +- ``editable`` (type: ``boolean``): ``true`` if the distribution was/is to be installed + in editable mode, ``false`` otherwise. If absent, default to ``false``. + +When ``url`` refers to a local directory, it MUST have the ``file`` scheme and +be compliant with :rfc:`8089`. In +particular, the path component must be absolute. Symbolic links SHOULD be +preserved when making relative paths absolute. + +.. _direct-url-data-structure-subdirectories: + +Projects in subdirectories +-------------------------- + +A top-level ``subdirectory`` field MAY be present containing a directory path, +relative to the root of the VCS repository, source archive or local directory, +to specify where ``pyproject.toml`` or ``setup.py`` is located. + +.. _direct-url-data-structure-registered-vcs: + +Registered VCS +============== + +This section lists the registered VCS's; expanded, VCS-specific information +on how to use the ``vcs``, ``requested_revision``, and other fields of +``vcs_info``; and in +some cases additional VCS-specific fields. +Tools MAY support other VCS's although it is RECOMMENDED to register +them by writing a PEP to amend this specification. The ``vcs`` field SHOULD be the command name +(lowercased). Additional fields that would be necessary to +support such VCS SHOULD be prefixed with the VCS command name. + +Git +--- + +Home page + https://git-scm.com/ + +vcs command + git + +``vcs`` field + git + +``requested_revision`` field + A tag name, branch name, Git ref, commit hash, shortened commit hash, + or other commit-ish. + +``commit_id`` field + A commit hash (40 hexadecimal characters sha1). + +.. note:: + + Tools can use the ``git show-ref`` and ``git symbolic-ref`` commands + to determine if the ``requested_revision`` corresponds to a Git ref. + In turn, a ref beginning with ``refs/tags/`` corresponds to a tag, and + a ref beginning with ``refs/remotes/origin/`` after cloning corresponds + to a branch. + +Mercurial +--------- + +Home page + https://www.mercurial-scm.org/ + +vcs command + hg + +``vcs`` field + hg + +``requested_revision`` field + A tag name, branch name, changeset ID, shortened changeset ID. + +``commit_id`` field + A changeset ID (40 hexadecimal characters). + +Bazaar +------ + +Home page + https://www.breezy-vcs.org/ + +vcs command + bzr + +``vcs`` field + bzr + +``requested_revision`` field + A tag name, branch name, revision id. + +``commit_id`` field + A revision id. + +Subversion +---------- + +Home page + https://subversion.apache.org/ + +vcs command + svn + +``vcs`` field + svn + +``requested_revision`` field + ``requested_revision`` must be compatible with ``svn checkout`` ``--revision`` option. + In Subversion, branch or tag is part of ``url``. + +``commit_id`` field + Since Subversion does not support globally unique identifiers, + this field is the Subversion revision number in the corresponding + repository. + +JSON Schema +=========== + +The following JSON Schema can be used to validate the contents of ``direct_url.json``: + +.. literalinclude:: ../../extra/specifications/schemas/direct-url.schema.json + +Examples +======== + +Source archive: + +.. code:: + + { + "url": "https://github.com/pypa/pip/archive/1.3.1.zip", + "archive_info": { + "hashes": { + "sha256": "2dc6b5a470a1bde68946f263f1af1515a2574a150a30d6ce02c6ff742fcc0db8" + } + } + } + +Git URL with tag and commit-hash: + +.. code:: + + { + "url": "https://github.com/pypa/pip.git", + "vcs_info": { + "vcs": "git", + "requested_revision": "1.3.1", + "commit_id": "7921be1537eac1e97bc40179a57f0349c2aee67d" + } + } + +Local directory: + +.. code:: + + { + "url": "file:///home/user/project", + "dir_info": {} + } + +Local directory in editable mode: + +.. code:: + + { + "url": "file:///home/user/project", + "dir_info": { + "editable": true + } + } + + +History +======= + +- March 2020: This specification was approved through :pep:`610`, defining + the ``direct_url.json`` metadata file. +- January 2023: Added the ``archive_info.hashes`` key (`discussion + `_). + + + +.. _archive-info-hashes: https://discuss.python.org/t/22299 +.. _pip-vcs-support: https://pip.pypa.io/en/stable/topics/vcs-support/ +.. _whatwg-url-standard: https://url.spec.whatwg.org/ diff --git a/source/specifications/direct-url.rst b/source/specifications/direct-url.rst index 9b8180c47..babecd842 100644 --- a/source/specifications/direct-url.rst +++ b/source/specifications/direct-url.rst @@ -6,12 +6,10 @@ Recording the Direct URL Origin of installed distributions ========================================================== This document specifies a :file:`direct_url.json` file in the -:file:`*.dist-info` directory of an installed distribution, to record the -Direct URL Origin of the distribution. The layout of this file was originally -specified in :pep:`610` and is formally documented here. +``*.dist-info`` directory of an installed distribution, to record the +Direct URL Origin of the distribution. The general structure and usage of +``*.dist-info`` directories is described in :ref:`recording-installed-packages`. -.. contents:: Contents - :local: Specification ============= @@ -23,67 +21,8 @@ specifying a direct URL reference (including a VCS URL). This file MUST NOT be created when installing a distribution from an other type of requirement (i.e. name plus version specifier). -This JSON file MUST be a dictionary, compliant with :rfc:`8259` and UTF-8 encoded. - -If present, it MUST contain at least two fields. The first one is ``url``, with -type ``string``. Depending on what ``url`` refers to, the second field MUST be -one of ``vcs_info`` (if ``url`` is a VCS reference), ``archive_info`` (if -``url`` is a source archives or a wheel), or ``dir_info`` (if ``url`` is a -local directory). These info fields have a (possibly empty) subdictionary as -value, with the possible keys defined below. - -``url`` MUST be stripped of any sensitive authentication information, -for security reasons. - -The user:password section of the URL MAY however -be composed of environment variables, matching the following regular -expression:: - - \$\{[A-Za-z0-9-_]+\}(:\$\{[A-Za-z0-9-_]+\})? - -Additionally, the user:password section of the URL MAY be a -well-known, non security sensitive string. A typical example is ``git`` -in the case of an URL such as ``ssh://git@gitlab.com/user/repo``. - -When ``url`` refers to a VCS repository, the ``vcs_info`` key MUST be present -as a dictionary with the following keys: - -- A ``vcs`` key (type ``string``) MUST be present, containing the name of the VCS - (i.e. one of ``git``, ``hg``, ``bzr``, ``svn``). Other VCS's SHOULD be registered by - writing a PEP to amend this specification. - The ``url`` value MUST be compatible with the corresponding VCS, - so an installer can hand it off without transformation to a - checkout/download command of the VCS. -- A ``requested_revision`` key (type ``string``) MAY be present naming a - branch/tag/ref/commit/revision/etc (in a format compatible with the VCS) - to install. -- A ``commit_id`` key (type ``string``) MUST be present, containing the - exact commit/revision number that was installed. - If the VCS supports commit-hash - based revision identifiers, such commit-hash MUST be used as - ``commit_id`` in order to reference the immutable - version of the source code that was installed. - -When ``url`` refers to a source archive or a wheel, the ``archive_info`` key -MUST be present as a dictionary with the following key: - -- A ``hash`` key (type ``string``) SHOULD be present, with value - ``=``. - It is RECOMMENDED that only hashes which are unconditionally provided by - the latest version of the standard library's ``hashlib`` module be used for - source archive hashes. At time of writing, that list consists of 'md5', - 'sha1', 'sha224', 'sha256', 'sha384', and 'sha512'. - -When ``url`` refers to a local directory, the ``dir_info`` key MUST be -present as a dictionary with the following key: - -- ``editable`` (type: ``boolean``): ``true`` if the distribution was installed - in editable mode, ``false`` otherwise. If absent, default to ``false``. - -When ``url`` refers to a local directory, it MUST have the ``file`` sheme and -be compliant with :rfc:`8089`. In -particular, the path component must be absolute. Symbolic links SHOULD be -preserved when making relative paths absolute. +This JSON file MUST be a UTF-8 encoded, :rfc:`8259` compliant, serialization of the +:doc:`direct-url-data-structure`. .. note:: @@ -92,10 +31,6 @@ preserved when making relative paths absolute. therefore MUST NOT output any VCS related information (such as ``vcs_info``) in :file:`direct_url.json`. -A top-level ``subdirectory`` field MAY be present containing a directory path, -relative to the root of the VCS repository, source archive or local directory, -to specify where ``pyproject.toml`` or ``setup.py`` is located. - .. note:: As a general rule, installers should as much as possible preserve the @@ -105,184 +40,18 @@ to specify where ``pyproject.toml`` or ``setup.py`` is located. provided in the requested URL as faithfully as possible. This information is however *enriched* with more precise data, such as ``commit_id``. -Registered VCS -============== - -This section lists the registered VCS's; expanded, VCS-specific information -on how to use the ``vcs``, ``requested_revision``, and other fields of -``vcs_info``; and in -some cases additional VCS-specific fields. -Tools MAY support other VCS's although it is RECOMMENDED to register -them by writing a PEP to amend this specification. The ``vcs`` field SHOULD be the command name -(lowercased). Additional fields that would be necessary to -support such VCS SHOULD be prefixed with the VCS command name. - -Git ---- - -Home page - - https://git-scm.com/ - -vcs command - - git - -``vcs`` field - - git - -``requested_revision`` field - - A tag name, branch name, Git ref, commit hash, shortened commit hash, - or other commit-ish. - -``commit_id`` field - - A commit hash (40 hexadecimal characters sha1). - -.. note:: - - Installers can use the ``git show-ref`` and ``git symbolic-ref`` commands - to determine if the ``requested_revision`` corresponds to a Git ref. - In turn, a ref beginning with ``refs/tags/`` corresponds to a tag, and - a ref beginning with ``refs/remotes/origin/`` after cloning corresponds - to a branch. - -Mercurial ---------- - -Home page - - https://www.mercurial-scm.org/ - -vcs command - - hg - -``vcs`` field - - hg - -``requested_revision`` field - - A tag name, branch name, changeset ID, shortened changeset ID. - -``commit_id`` field - - A changeset ID (40 hexadecimal characters). - -Bazaar ------- - -Home page - - https://bazaar.canonical.com - -vcs command - - bzr - -``vcs`` field - - bzr - -``requested_revision`` field - - A tag name, branch name, revision id. - -``commit_id`` field - - A revision id. - -Subversion ----------- - -Home page - - https://subversion.apache.org/ - -vcs command - - svn - -``vcs`` field - - svn - -``requested_revision`` field - - ``requested_revision`` must be compatible with ``svn checkout`` ``--revision`` option. - In Subversion, branch or tag is part of ``url``. - -``commit_id`` field - - Since Subversion does not support globally unique identifiers, - this field is the Subversion revision number in the corresponding - repository. - -Examples -======== - -Example direct_url.json ------------------------ - -Source archive: - -.. code:: - - { - "url": "https://github.com/pypa/pip/archive/1.3.1.zip", - "archive_info": { - "hash": "sha256=2dc6b5a470a1bde68946f263f1af1515a2574a150a30d6ce02c6ff742fcc0db8" - } - } - -Git URL with tag and commit-hash: - -.. code:: - - { - "url": "https://github.com/pypa/pip.git", - "vcs_info": { - "vcs": "git", - "requested_revision": "1.3.1", - "commit_id": "7921be1537eac1e97bc40179a57f0349c2aee67d" - } - } - -Local directory: - -.. code:: - - { - "url": "file:///home/user/project", - "dir_info": {} - } - -Local directory installed in editable mode: - -.. code:: - - { - "url": "file:///home/user/project", - "dir_info": { - "editable": true - } - } - Example pip commands and their effect on direct_url.json --------------------------------------------------------- +======================================================== Commands that generate a ``direct_url.json``: * ``pip install https://example.com/app-1.0.tgz`` * ``pip install https://example.com/app-1.0.whl`` -* ``pip install "git+https://example.com/repo/app.git#egg=app&subdirectory=setup"`` +* ``pip install "app @ git+https://example.com/repo/app.git#subdirectory=setup"`` * ``pip install ./app`` * ``pip install file:///home/user/app`` -* ``pip install --editable "git+https://example.com/repo/app.git#egg=app&subdirectory=setup"`` +* ``pip install --editable "app @ git+https://example.com/repo/app.git#subdirectory=setup"`` (in which case, ``url`` will be the local directory where the git repository has been cloned to, and ``dir_info`` will be present with ``"editable": true`` and no ``vcs_info`` will be set) @@ -292,3 +61,9 @@ Commands that *do not* generate a ``direct_url.json`` * ``pip install app`` * ``pip install app --no-index --find-links https://example.com/`` + + +History +======= + +- March 2020: This specification was approved through :pep:`610`. diff --git a/source/specifications/entry-points.rst b/source/specifications/entry-points.rst index 6c94a25fb..dea039492 100644 --- a/source/specifications/entry-points.rst +++ b/source/specifications/entry-points.rst @@ -18,10 +18,10 @@ example: The entry point file format was originally developed to allow packages built with setuptools to provide integration point metadata that would be read at -runtime with ``importlib.metadata``. It is now defined as a PyPA interoperability -specification in order to allow build tools other than setuptools to publish -``importlib.metadata`` compatible entry point metadata, and runtime libraries other -than ``importlib.metadata`` to portably read published entry point metadata +runtime with :py:mod:`importlib.metadata`. It is now defined as a PyPA interoperability +specification in order to allow build tools other than ``setuptools`` to publish +:py:mod:`importlib.metadata` compatible entry point metadata, and runtime libraries other +than :py:mod:`importlib.metadata` to portably read published entry point metadata (potentially with different caching and conflict resolution strategies). Data model @@ -81,8 +81,9 @@ File format Entry points are defined in a file called :file:`entry_points.txt` in the :file:`*.dist-info` directory of the distribution. This is the directory -described in :pep:`376` for installed distributions, and in :pep:`427` for -wheels. The file uses the UTF-8 character encoding. +described in :ref:`recording-installed-packages` for installed distributions, +and in :ref:`binary-distribution-format` for wheels. +The file uses the UTF-8 character encoding. The file contents are in INI format, as read by Python's :mod:`configparser` module. However, configparser treats names as case-insensitive by default, @@ -110,7 +111,9 @@ restrictions on values specified in :pep:`685`. For tools writing the file, it is recommended only to insert a space between the object reference and the left square bracket. -For example:: +For example: + +.. code-block:: ini [console_scripts] foo = foomod:main @@ -141,11 +144,11 @@ For instance, the entry point ``mycmd = mymod:main`` would create a command The difference between ``console_scripts`` and ``gui_scripts`` only affects Windows systems. ``console_scripts`` are wrapped in a console executable, -so they are attached to a console and can use ``sys.stdin``, ``sys.stdout`` and -``sys.stderr`` for input and output. ``gui_scripts`` are wrapped in a GUI -executable, so they can be started without a console, but cannot use standard -streams unless application code redirects them. Other platforms do not have the -same distinction. +so they are attached to a console and can use :py:data:`sys.stdin`, +:py:data:`sys.stdout` and :py:data:`sys.stderr` for input and output. +``gui_scripts`` are wrapped in a GUI executable, so they can be started without +a console, but cannot use standard streams unless application code redirects them. +Other platforms do not have the same distinction. Install tools are expected to set up wrappers for both ``console_scripts`` and ``gui_scripts`` in the scripts directory of the install scheme. They are not @@ -155,3 +158,14 @@ which defines where command-line tools are found. As files are created from the names, and some filesystems are case-insensitive, packages should avoid using names in these groups which differ only in case. The behaviour of install tools when names differ only in case is undefined. + + +History +======= + +- October 2017: This specification was written to formalize the existing + entry points feature of setuptools (discussion_). + + + +.. _discussion: https://mail.python.org/pipermail/distutils-sig/2017-October/031585.html diff --git a/source/specifications/externally-managed-environments.rst b/source/specifications/externally-managed-environments.rst new file mode 100644 index 000000000..65fc14a62 --- /dev/null +++ b/source/specifications/externally-managed-environments.rst @@ -0,0 +1,474 @@ + +.. _externally-managed-environments: + +=============================== +Externally Managed Environments +=============================== + +While some Python installations are entirely managed by the user that installed +Python, others may be provided and managed by another means (such as the +operating system package manager in a Linux distribution, or as a bundled +Python environment in an application with a dedicated installer). + +Attempting to use conventional Python packaging tools to manipulate such +environments can be confusing at best and outright break the entire underlying +operating system at worst. Documentation and interoperability guides only go +so far in resolving such problems. + +This specification defines an ``EXTERNALLY-MANAGED`` marker file that allows a +Python installation to indicate to Python-specific tools such as ``pip`` that they +neither install nor remove packages into the interpreter’s default installation +environment, and should instead guide the end user towards using +:ref:`virtual-environments`. + +It also standardizes an interpretation of the ``sysconfig`` schemes so +that, if a Python-specific package manager is about to install a +package in an interpreter-wide context, it can do so in a manner that +will avoid conflicting with the external package manager and reduces +the risk of breaking software shipped by the external package manager. + + +Terminology +=========== + +A few terms used in this specification have multiple meanings in the +contexts that it spans. For clarity, this specification uses the following +terms in specific ways: + +distro + Short for "distribution," a collection of various sorts of + software, ideally designed to work properly together, including + (in contexts relevant to this document) the Python interpreter + itself, software written in Python, and software written in other + languages. That is, this is the sense used in phrases such as + "Linux distro" or "Berkeley Software Distribution." + + A distro can be an operating system (OS) of its own, such as + Debian, Fedora, or FreeBSD. It can also be an overlay distribution + that installs on top of an existing OS, such as Homebrew or + MacPorts. + + This document uses the short term "distro," because the term + "distribution" has another meaning in Python packaging contexts: a + source or binary distribution package of a single piece of Python + language software, that is, in the sense of + ``setuptools.dist.Distribution`` or "sdist". To avoid confusion, + this document does not use the plain term "distribution" at all. + In the Python packaging sense, it uses the full phrase + "distribution package" or just "package" (see below). + + The provider of a distro - the team or company that collects and + publishes the software and makes any needed modifications - is its + **distributor**. +package + A unit of software that can be installed and used within Python. + That is, this refers to what Python-specific packaging tools tend + to call a :term:`distribution package` or simply a "distribution"; + the colloquial abbreviation "package" is used in the sense of the + Python Package Index. + + This document does not use "package" in the sense of an importable + name that contains Python modules, though in many cases, a + distribution package consists of a single importable package of + the same name. + + This document generally does not use the term "package" to refer + to units of installation by a distro's package manager (such as + ``.deb`` or ``.rpm`` files). When needed, it uses phrasing such as + "a distro's package." (Again, in many cases, a Python package is + shipped inside a distro's package named something like ``python-`` + plus the Python package name.) +Python-specific package manager + A tool for installing, upgrading, and/or removing Python packages + in a manner that conforms to Python packaging standards. + The most popular Python-specific package + manager is pip_; other examples include the old `Easy + Install command `_ as well as direct usage of a + ``setup.py`` command. + + .. _pip: https://pip.pypa.io/en/stable/ + .. _easy-install: https://setuptools.readthedocs.io/en/latest/deprecated/easy_install.html + + (Note that the ``easy_install`` command was removed in + setuptools version 52, released 23 January 2021.) + + + (Conda_ is a bit of a special case, as the ``conda`` + command can install much more than just Python packages, making it + more like a distro package manager in some senses. Since the + ``conda`` command generally only operates on Conda-created + environments, most of the concerns in this document do not apply + to ``conda`` when acting as a Python-specific package manager.) + + .. _conda: https://conda.io +distro package manager + A tool for installing, upgrading, and/or removing a distro's + packages in an installed instance of that distro, which is capable + of installing Python packages as well as non-Python packages, and + therefore generally has its own database of installed software + unrelated to the :ref:`database of installed distributions + `. Examples include ``apt``, ``dpkg``, + ``dnf``, ``rpm``, ``pacman``, and ``brew``. The salient feature is + that if a package was installed by a distro package manager, removing or + upgrading it in a way that would satisfy a Python-specific package + manager will generally leave a distro package manager in an + inconsistent state. + + This document also uses phrases like "external package manager" or + "system's package manager" to refer to a distro package manager in + certain contexts. +shadow + To shadow an installed Python package is to cause some other + package to be preferred for imports without removing any files + from the shadowed package. This requires multiple entries on + ``sys.path``: if package A 2.0 installs module ``a.py`` in one + ``sys.path`` entry, and package A 1.0 installs module ``a.py`` in + a later ``sys.path`` entry, then ``import a`` returns the module + from the former, and we say that A 2.0 shadows A 1.0. + +Overview +======== + +This specification is twofold. + +First, it describes **a way for distributors of a Python interpreter to +mark that interpreter as having its packages managed by means external +to Python**, such that Python-specific tools like pip should not +change the installed packages in the interpreter's global ``sys.path`` +in any way (add, upgrade/downgrade, or remove) unless specifically +overridden. It also provides a means for the distributor to indicate +how to use a virtual environment as an alternative. + +This is an opt-in mechanism: by default, the Python interpreter +compiled from upstream sources will not be so marked, and so running +``pip install`` with a self-compiled interpreter, or with a distro +that has not explicitly marked its interpreter, will work as it always +has worked. + +Second, it sets the rule that when installing packages to an +interpreter's global context (either to an unmarked interpreter, or if +overriding the marking), **Python-specific package managers should +modify or delete files only within the directories of the sysconfig +scheme in which they would create files**. This permits a distributor +of a Python interpreter to set up two directories, one for its own +managed packages, and one for unmanaged packages installed by the end +user, and ensure that installing unmanaged packages will not delete +(or overwrite) files owned by the external package manager. + + +Marking an interpreter as using an external package manager +=========================================================== + +Before a Python-specific package installer (that is, a tool such as +pip - not an external tool such as apt) installs a package into a +certain Python context, it should make the following checks by +default: + +1. Is it running outside of a virtual environment? It can determine + this by whether ``sys.prefix == sys.base_prefix``. + +2. Is there an ``EXTERNALLY-MANAGED`` file in the directory identified + by ``sysconfig.get_path("stdlib", sysconfig.get_default_scheme())``? + +If both of these conditions are true, the installer should exit with +an error message indicating that package installation into this Python +interpreter's directory are disabled outside of a virtual environment. + +The installer should have a way for the user to override these rules, +such as a command-line flag ``--break-system-packages``. This option +should not be enabled by default and should carry some connotation +that its use is risky. + +The ``EXTERNALLY-MANAGED`` file is an INI-style metadata file intended +to be parsable by the standard library configparser_ module. If the +file can be parsed by +``configparser.ConfigParser(interpolation=None)`` using the UTF-8 +encoding, and it contains a section ``[externally-managed]``, then the +installer should look for an error message specified in the file and +output it as part of its error. If the first element of the tuple +returned by ``locale.getlocale(locale.LC_MESSAGES)``, i.e., the +language code, is not ``None``, it should look for the error message +as the value of a key named ``Error-`` followed by the language code. +If that key does not exist, and if the language code contains +underscore or hyphen, it should look for a key named ``Error-`` +followed by the portion of the language code before the underscore or +hyphen. If it cannot find either of those, or if the language code is +``None``, it should look for a key simply named ``Error``. + +.. _configparser: https://docs.python.org/3/library/configparser.html + +If the installer cannot find an error message in the file (either +because the file cannot be parsed or because no suitable error key +exists), then the installer should just use a pre-defined error +message of its own, which should suggest that the user create a +virtual environment to install packages. + +Software distributors who have a non-Python-specific package manager +that manages libraries in the ``sys.path`` of their Python package +should, in general, ship an ``EXTERNALLY-MANAGED`` file in their +standard library directory. For instance, Debian may ship a file in +``/usr/lib/python3.9/EXTERNALLY-MANAGED`` consisting of something like + +.. code-block:: ini + + [externally-managed] + Error=To install Python packages system-wide, try apt install + python3-xyz, where xyz is the package you are trying to + install. + + If you wish to install a non-Debian-packaged Python package, + create a virtual environment using python3 -m venv path/to/venv. + Then use path/to/venv/bin/python and path/to/venv/bin/pip. Make + sure you have python3-full installed. + + If you wish to install a non-Debian packaged Python application, + it may be easiest to use pipx install xyz, which will manage a + virtual environment for you. Make sure you have pipx installed. + + See /usr/share/doc/python3.9/README.venv for more information. + +which provides useful and distro-relevant information +to a user trying to install a package. Optionally, +translations can be provided in the same file: + +.. code-block:: ini + + Error-de_DE=Wenn ist das NunstΓΌck git und Slotermeyer? + + Ja! Beiherhund das Oder die Virtualenvironment gersput! + +In certain contexts, such as single-application container images that +aren't updated after creation, a distributor may choose not to ship an +``EXTERNALLY-MANAGED`` file, so that users can install whatever they +like (as they can today) without having to manually override this +rule. + +Writing to only the target ``sysconfig`` scheme +=============================================== + +Usually, a Python package installer installs to directories in a +scheme returned by the ``sysconfig`` standard library package. +Ordinarily, this is the scheme returned by +``sysconfig.get_default_scheme()``, but based on configuration (e.g. +``pip install --user``), it may use a different scheme. + +Whenever the installer is installing to a ``sysconfig`` scheme, this +specification declares that the installer should never modify or delete files +outside of that scheme. For instance, if it's upgrading a package, and +the package is already installed in a directory outside that scheme +(perhaps in a directory from another scheme), it should leave the +existing files alone. + +If the installer does end up shadowing an existing installation during +an upgrade, we recommend that it produces a warning at the end of its +run. + +If the installer is installing to a location outside of a +``sysconfig`` scheme (e.g., ``pip install --target``), then this +subsection does not apply. + +Recommendations for distros +=========================== + +This section is non-normative. It provides best practices we believe +distros should follow unless they have a specific reason otherwise. + +Mark the installation as externally managed +------------------------------------------- + +Distros should create an ``EXTERNALLY-MANAGED`` file in their +``stdlib`` directory. + +Guide users towards virtual environments +---------------------------------------- + +The file should contain a useful and distro-relevant error message +indicating both how to install system-wide packages via the distro's +package manager and how to set up a virtual environment. If your +distro is often used by users in a state where the ``python3`` command +is available (and especially where ``pip`` or ``get-pip`` is +available) but ``python3 -m venv`` does not work, the message should +indicate clearly how to make ``python3 -m venv`` work properly. + +Consider packaging pipx_, a tool for installing Python-language +applications, and suggesting it in the error. pipx automatically +creates a virtual environment for that application alone, which is a +much better default for end users who want to install some +Python-language software (which isn't available in the distro) but are +not themselves Python users. Packaging pipx in the distro avoids the +irony of instructing users to ``pip install --user +--break-system-packages pipx`` to *avoid* breaking system packages. +Consider arranging things so your distro's package / environment for +Python for end users (e.g., ``python3`` on Fedora or ``python3-full`` +on Debian) depends on pipx. + +.. _pipx: https://github.com/pypa/pipx + +Keep the marker file in container images +---------------------------------------- + +Distros that produce official images for single-application containers +(e.g., Docker container images) should keep the +``EXTERNALLY-MANAGED`` file, preferably in a way that makes it not +go away if a user of that image installs package updates inside +their image (think ``RUN apt-get dist-upgrade``). + +Create separate distro and local directories +-------------------------------------------- + +Distros should place two separate paths on the system interpreter's +``sys.path``, one for distro-installed packages and one for packages +installed by the local system administrator, and configure +``sysconfig.get_default_scheme()`` to point at the latter path. This +ensures that tools like pip will not modify distro-installed packages. +The path for the local system administrator should come before the +distro path on ``sys.path`` so that local installs take preference +over distro packages. + +For example, Fedora and Debian (and their derivatives) both implement +this split by using ``/usr/local`` for locally-installed packages and +``/usr`` for distro-installed packages. Fedora uses +``/usr/local/lib/python3.x/site-packages`` vs. +``/usr/lib/python3.x/site-packages``. (Debian uses +``/usr/local/lib/python3/dist-packages`` vs. +``/usr/lib/python3/dist-packages`` as an additional layer of +separation from a locally-compiled Python interpreter: if you build +and install upstream CPython in ``/usr/local/bin``, it will look at +``/usr/local/lib/python3/site-packages``, and Debian wishes to make +sure that packages installed via the locally-built interpreter don't +show up on ``sys.path`` for the distro interpreter.) + +Note that the ``/usr/local`` vs. ``/usr`` split is analogous to how +the ``PATH`` environment variable typically includes +``/usr/local/bin:/usr/bin`` and non-distro software installs to +``/usr/local`` by default. This split is `recommended by the +Filesystem Hierarchy Standard`__. + +.. __: https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch04s09.html + +There are two ways you could do this. One is, if you are building and +packaging Python libraries directly (e.g., your packaging helpers +unpack a wheel or call ``setup.py install``), arrange +for those tools to use a directory that is not in a ``sysconfig`` +scheme but is still on ``sys.path``. + +The other is to arrange for the default ``sysconfig`` scheme to change +when running inside a package build versus when running on an +installed system. The ``sysconfig`` customization hooks from +bpo-43976_ should make this easy (once accepted and implemented): +make your packaging tool set an +environment variable or some other detectable configuration, and +define a ``get_preferred_schemes`` function to return a different +scheme when called from inside a package build. Then you can use ``pip +install`` as part of your distro packaging. + +.. _bpo-43976: https://bugs.python.org/issue43976 + +We propose adding a ``--scheme=...`` option to instruct pip to run +against a specific scheme. (See `Implementation Notes`_ below for how +pip currently determines schemes.) Once that's available, for local +testing and possibly for actual packaging, you would be able to run +something like ``pip install --scheme=posix_distro`` to explicitly +install a package into your distro's location (bypassing +``get_preferred_schemes``). One could also, if absolutely needed, use +``pip uninstall --scheme=posix_distro`` to use pip to remove packages +from the system-managed directory. + +To install packages with pip, you would also need to either suppress +the ``EXTERNALLY-MANAGED`` marker file to allow pip to run or to +override it on the command line. You may want to use the same means +for suppressing the marker file in build chroots as you do in +container images. + +The advantage of setting these up to be automatic (suppressing the +marker file in your build environment and having +``get_preferred_schemes`` automatically return your distro's scheme) +is that an unadorned ``pip install`` will work inside a package build, +which generally means that an unmodified upstream build script that +happens to internally call ``pip install`` will do the right thing. +You can, of course, just ensure that your packaging process always +calls ``pip install --scheme=posix_distro --break-system-packages``, +which would work too. + +The best approach here depends a lot on your distro's conventions and +mechanisms for packaging. + +Similarly, the ``sysconfig`` paths that are not for importable Python +code - that is, ``include``, ``platinclude``, ``scripts``, and +``data`` - should also have two variants, one for use by +distro-packaged software and one for use for locally-installed +software, and the distro should be set up such that both are usable. +For instance, a typical FHS-compliant distro will use +``/usr/local/include`` for the default scheme's ``include`` and +``/usr/include`` for distro-packaged headers and place both on the +compiler's search path, and it will use ``/usr/local/bin`` for the +default scheme's ``scripts`` and ``/usr/bin`` for distro-packaged +entry points and place both on ``$PATH``. + + +Implementation Notes +==================== + +This section is non-normative and contains notes relevant to both the +specification and potential implementations. + +Currently (as of May 2021), pip does not directly expose a way to choose +a target ``sysconfig`` scheme, but it has three ways of looking up schemes +when installing: + +``pip install`` + Calls ``sysconfig.get_default_scheme()``, which is usually (in + upstream CPython and most current distros) the same as + ``get_preferred_scheme('prefix')``. + +``pip install --prefix=/some/path`` + Calls ``sysconfig.get_preferred_scheme('prefix')``. + +``pip install --user`` + Calls ``sysconfig.get_preferred_scheme('user')``. + +Finally, ``pip install --target=/some/path`` writes directly to +``/some/path`` without looking up any schemes. + +Debian currently carries a `patch to change the default install +location inside a virtual environment`__, using a few heuristics +(including checking for the ``VIRTUAL_ENV`` environment variable), +largely so that the directory used in a virtual environment remains +``site-packages`` and not ``dist-packages``. This does not +particularly affect this proposal, because the implementation of that +patch does not actually change the default ``sysconfig`` scheme, and +notably does not change the result of +``sysconfig.get_path("stdlib")``. + +.. __: https://sources.debian.org/src/python3.7/3.7.3-2+deb10u3/debian/patches/distutils-install-layout.diff/ + +Fedora currently carries a `patch to change the default install +location when not running inside rpmbuild`__, which they use to +implement the two-system-wide-directories approach. This is +conceptually the sort of hook envisioned by bpo-43976_, except +implemented as a code patch to ``distutils`` instead of as a changed +``sysconfig`` scheme. + +.. __: https://src.fedoraproject.org/rpms/python3.9/blob/f34/f/00251-change-user-install-location.patch + +The implementation of ``is_virtual_environment`` above, as well as the +logic to load the ``EXTERNALLY-MANAGED`` file and find the error +message from it, may as well get added to the standard library +(``sys`` and ``sysconfig``, respectively), to centralize their +implementations, but they don't need to be added yet. + + + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +History +======= + +- June 2022: This specification was approved through :pep:`668`. diff --git a/source/specifications/file-yanking.rst b/source/specifications/file-yanking.rst new file mode 100644 index 000000000..4ab8cd5cc --- /dev/null +++ b/source/specifications/file-yanking.rst @@ -0,0 +1,92 @@ +.. _file-yanking: + +============ +File Yanking +============ + +.. note:: + + This specification was originally defined in + :pep:`592`. + +.. note:: + + :pep:`592` includes changes to the HTML and JSON index APIs. + These changes are documented in the :ref:`simple-repository-api` + under :ref:`HTML - Project Detail ` + and :ref:`JSON - Project Detail `. + +Specification +============= + +Links in the simple repository **MAY** have a ``data-yanked`` attribute +which may have no value, or may have an arbitrary string as a value. The +presence of a ``data-yanked`` attribute **SHOULD** be interpreted as +indicating that the file pointed to by this particular link has been +"Yanked", and should not generally be selected by an installer, except +under specific scenarios. + +The value of the ``data-yanked`` attribute, if present, is an arbitrary +string that represents the reason for why the file has been yanked. Tools +that process the simple repository API **MAY** surface this string to +end users. + +The yanked attribute is not immutable once set, and may be rescinded in +the future (and once rescinded, may be reset as well). Thus API users +**MUST** be able to cope with a yanked file being "unyanked" (and even +yanked again). + +Installers +---------- + +The desirable experience for users is that once a file is yanked, when +a human being is currently trying to directly install a yanked file, that +it fails as if that file had been deleted. However, when a human did that +awhile ago, and now a computer is just continuing to mechanically follow +the original order to install the now yanked file, then it acts as if it +had not been yanked. + +An installer **MUST** ignore yanked releases, if the selection constraints +can be satisfied with a non-yanked version, and **MAY** refuse to use a +yanked release even if it means that the request cannot be satisfied at all. +An implementation **SHOULD** choose a policy that follows the spirit of the +intention above, and that prevents "new" dependencies on yanked +releases/files. + +What this means is left up to the specific installer, to decide how to best +fit into the overall usage of their installer. However, there are two +suggested approaches to take: + +1. Yanked files are always ignored, unless they are the only file that + matches a version specifier that "pins" to an exact version using + either ``==`` (without any modifiers that make it a range, such as + ``.*``) or ``===``. Matching this version specifier should otherwise + be done as per :ref:`the version specifiers specification + ` for things like local versions, zero padding, + etc. +2. Yanked files are always ignored, unless they are the only file that + matches what a lock file (such as ``Pipfile.lock`` or ``poetry.lock``) + specifies to be installed. In this case, a yanked file **SHOULD** not + be used when creating or updating a lock file from some input file or + command. + +Regardless of the specific strategy that an installer chooses for deciding +when to install yanked files, an installer **SHOULD** emit a warning when +it does decide to install a yanked file. That warning **MAY** utilize the +value of the ``data-yanked`` attribute (if it has a value) to provide more +specific feedback to the user about why that file had been yanked. + + +Mirrors +------- + +Mirrors can generally treat yanked files one of two ways: + +1. They may choose to omit them from their simple repository API completely, + providing a view over the repository that shows only "active", unyanked + files. +2. They may choose to include yanked files, and additionally mirror the + ``data-yanked`` attribute as well. + +Mirrors **MUST NOT** mirror a yanked file without also mirroring the +``data-yanked`` attribute for it. diff --git a/source/specifications/glob-patterns.rst b/source/specifications/glob-patterns.rst new file mode 100644 index 000000000..abdb15b0f --- /dev/null +++ b/source/specifications/glob-patterns.rst @@ -0,0 +1,115 @@ +================= +``glob`` patterns +================= + +Some PyPA specifications, e.g. :ref:`pyproject.toml's license-files +`, accept certain types of *glob patterns* +to match a given string containing wildcards and character ranges against +files and directories. This specification defines which patterns are acceptable +and how they should be handled. + + +Valid glob patterns +=================== + +For PyPA purposes, a *valid glob pattern* MUST be a string matched against +filesystem entries as specified below: + +- Alphanumeric characters, underscores (``_``), hyphens (``-``) and dots (``.``) + MUST be matched verbatim. + +- Special glob characters: ``*``, ``?``, ``**`` and character ranges: ``[]`` + containing only the verbatim matched characters MUST be supported. + Within ``[...]``, the hyphen indicates a locale-agnostic range (e.g. ``a-z``, + order based on Unicode code points). + Hyphens at the start or end are matched literally. + +- Path delimiters MUST be the forward slash character (``/``). + +- Patterns always refer to *relative paths*, + e.g., when used in :file:`pyproject.toml`, patterns should always be + relative to the directory containing that file. + Therefore the leading slash character MUST NOT be used. + +- Parent directory indicators (``..``) MUST NOT be used. + +Any characters or character sequences not covered by this specification are +invalid. Projects MUST NOT use such values. +Tools consuming glob patterns SHOULD reject invalid values with an error. + +Literal paths (e.g. :file:`LICENSE`) are valid globs which means they +can also be defined. + +Tools consuming glob patterns: + +- MUST treat each value as a glob pattern, and MUST raise an error if the + pattern contains invalid glob syntax. +- MUST raise an error if any individual user-specified pattern does not match + at least one file. + +Examples of valid glob patterns: + +.. code-block:: python + + "LICEN[CS]E*" + "AUTHORS*" + "licenses/LICENSE.MIT" + "licenses/LICENSE.CC0" + "LICENSE.txt" + "licenses/*" + +Examples of invalid glob patterns: + +.. code-block:: python + + "..\LICENSE.MIT" + # .. must not be used. + # \ is an invalid path delimiter, / must be used. + + "LICEN{CSE*" + # the { character is not allowed + + +Reference implementation in Python +================================== + +It is possible to defer the majority of the pattern matching against the file +system to the :mod:`glob` module in Python's standard library. It is necessary +however to perform additional validations. + +The code below is as a simple reference implementation: + +.. code-block:: python + + import os + import re + from glob import glob + + + def find_pattern(pattern: str) -> list[str]: + """ + >>> find_pattern("/LICENSE.MIT") + Traceback (most recent call last): + ... + ValueError: Pattern '/LICENSE.MIT' should be relative... + >>> find_pattern("../LICENSE.MIT") + Traceback (most recent call last): + ... + ValueError: Pattern '../LICENSE.MIT' cannot contain '..'... + >>> find_pattern("LICEN{CSE*") + Traceback (most recent call last): + ... + ValueError: Pattern 'LICEN{CSE*' contains invalid characters... + """ + if ".." in pattern: + raise ValueError(f"Pattern {pattern!r} cannot contain '..'") + if pattern.startswith((os.sep, "/")) or ":\\" in pattern: + raise ValueError( + f"Pattern {pattern!r} should be relative and must not start with '/'" + ) + if re.match(r'^[\w\-\.\/\*\?\[\]]+$', pattern) is None: + raise ValueError(f"Pattern '{pattern}' contains invalid characters.") + found = glob(pattern, recursive=True) + if not found: + raise ValueError(f"Pattern '{pattern}' did not match any files.") + return found diff --git a/source/specifications/index-hosted-attestations.rst b/source/specifications/index-hosted-attestations.rst new file mode 100644 index 000000000..d078e87bd --- /dev/null +++ b/source/specifications/index-hosted-attestations.rst @@ -0,0 +1,368 @@ + +.. _index-hosted-attestations: + +========================= +Index hosted attestations +========================= + +.. note:: This specification was originally defined in :pep:`740`. + +.. note:: + + :pep:`740` includes changes to the HTML and JSON index APIs. + These changes are documented in the :ref:`simple-repository-api` + under :ref:`simple-repository-api-base` and :ref:`json-serialization`. + +Specification +============= + +.. _upload-endpoint: + +Upload endpoint changes +----------------------- + +.. important:: + + The "legacy" upload API is not standardized. + See `PyPI's Upload API documentation `_ + for how attestations are uploaded. + +.. _attestation-object: + +Attestation objects +------------------- + +An attestation object is a JSON object with several required keys; applications +or signers may include additional keys so long as all explicitly +listed keys are provided. The required layout of an attestation +object is provided as pseudocode below. + +.. code-block:: python + + @dataclass + class Attestation: + version: Literal[1] + """ + The attestation object's version, which is always 1. + """ + + verification_material: VerificationMaterial + """ + Cryptographic materials used to verify `envelope`. + """ + + envelope: Envelope + """ + The enveloped attestation statement and signature. + """ + + + @dataclass + class Envelope: + statement: bytes + """ + The attestation statement. + + This is represented as opaque bytes on the wire (encoded as base64), + but it MUST be an JSON in-toto v1 Statement. + """ + + signature: bytes + """ + A signature for the above statement, encoded as base64. + """ + + @dataclass + class VerificationMaterial: + certificate: str + """ + The signing certificate, as `base64(DER(cert))`. + """ + + transparency_entries: list[object] + """ + One or more transparency log entries for this attestation's signature + and certificate. + """ + +A full data model for each object in ``transparency_entries`` is provided in +:ref:`appendix`. Attestation objects **SHOULD** include one or more +transparency log entries, and **MAY** include additional keys for other +sources of signed time (such as an :rfc:`3161` Time Stamping Authority or a +`Roughtime `__ server). + +Attestation objects are versioned; this PEP specifies version 1. Each version +is tied to a single cryptographic suite to minimize unnecessary cryptographic +agility. In version 1, the suite is as follows: + +* Certificates are specified as X.509 certificates, and comply with the + profile in :rfc:`5280`. +* The message signature algorithm is ECDSA, with the P-256 curve for public keys + and SHA-256 as the cryptographic digest function. + +Future PEPs may change this suite (and the overall shape of the attestation +object) by selecting a new version number. + +.. _payload-and-signature-generation: + +Attestation statement and signature generation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The *attestation statement* is the actual claim that is cryptographically signed +over within the attestation object (i.e., the ``envelope.statement``). + +The attestation statement is encoded as a +`v1 in-toto Statement object `__, +in JSON form. When serialized the statement is treated as an opaque binary blob, +avoiding the need for canonicalization. + +In addition to being a v1 in-toto Statement, the attestation statement is constrained +in the following ways: + +* The in-toto ``subject`` **MUST** contain only a single subject. +* ``subject[0].name`` is the distribution's filename, which **MUST** be + a valid :ref:`source distribution ` or + :ref:`wheel distribution ` filename. +* ``subject[0].digest`` **MUST** contain a SHA-256 digest. Other digests + **MAY** be present. The digests **MUST** be represented as hexadecimal strings. +* The following ``predicateType`` values are supported: + + * `SLSA Provenance `__: ``https://slsa.dev/provenance/v1`` + * `PyPI Publish Attestation `__: ``https://docs.pypi.org/attestations/publish/v1`` + +The signature over this statement is constructed using the +`v1 DSSE signature protocol `__, +with a ``PAYLOAD_TYPE`` of ``application/vnd.in-toto+json`` and a ``PAYLOAD_BODY`` of the JSON-encoded +statement above. No other ``PAYLOAD_TYPE`` is permitted. + +.. _provenance-object: + +Provenance objects +------------------ + +The index will serve uploaded attestations along with metadata that can assist +in verifying them in the form of JSON serialized objects. + +These *provenance objects* will be available via both the Simple Index +and JSON-based Simple API as described above, and will have the following layout: + +.. code-block:: json + + { + "version": 1, + "attestation_bundles": [ + { + "publisher": { + "kind": "important-ci-service", + "claims": {}, + "vendor-property": "foo", + "another-property": 123 + }, + "attestations": [ + { /* attestation 1 ... */ }, + { /* attestation 2 ... */ } + ] + } + ] + } + +or, as pseudocode: + +.. code-block:: python + + @dataclass + class Publisher: + kind: string + """ + The kind of Trusted Publisher. + """ + + claims: object | None + """ + Any context-specific claims retained by the index during Trusted Publisher + authentication. + """ + + _rest: object + """ + Each publisher object is open-ended, meaning that it MAY contain additional + fields beyond the ones specified explicitly above. This field signals that, + but is not itself present. + """ + + @dataclass + class AttestationBundle: + publisher: Publisher + """ + The publisher associated with this set of attestations. + """ + + attestations: list[Attestation] + """ + The set of attestations included in this bundle. + """ + + @dataclass + class Provenance: + version: Literal[1] + """ + The provenance object's version, which is always 1. + """ + + attestation_bundles: list[AttestationBundle] + """ + One or more attestation "bundles". + """ + +* ``version`` is ``1``. Like attestation objects, provenance objects are + versioned, and this PEP only defines version ``1``. +* ``attestation_bundles`` is a **required** JSON array, containing one + or more "bundles" of attestations. Each bundle corresponds to a + signing identity (such as a Trusted Publishing identity), and contains + one or more attestation objects. + + As noted in the ``Publisher`` model, + each ``AttestationBundle.publisher`` object is specific to its Trusted Publisher + but must include at minimum: + + * A ``kind`` key, which **MUST** be a JSON string that uniquely identifies the + kind of Trusted Publisher. + * A ``claims`` key, which **MUST** be a JSON object containing any context-specific + claims retained by the index during Trusted Publisher authentication. + + All other keys in the publisher object are publisher-specific. + + Each array of attestation objects is a superset of the ``attestations`` + array supplied by the uploaded through the ``attestations`` field at upload + time, as described in :ref:`upload-endpoint` and + :ref:`changes-to-provenance-objects`. + +.. _changes-to-provenance-objects: + +Changes to provenance objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Provenance objects are *not* immutable, and may change over time. Reasons +for changes to the provenance object include but are not limited to: + +* Addition of new attestations for a pre-existing signing identity: the index + **MAY** choose to allow additional attestations by pre-existing signing + identities, such as newer attestation versions for already uploaded + files. + +* Addition of new signing identities and associated attestations: the index + **MAY** choose to support attestations from sources other than the file's + uploader, such as third-party auditors or the index itself. These attestations + may be performed asynchronously, requiring the index to insert them into + the provenance object *post facto*. + +.. _attestation-verification: + +Attestation verification +------------------------ + +Verifying an attestation object against a distribution file requires verification of each of the +following: + +* ``version`` is ``1``. The verifier **MUST** reject any other version. +* ``verification_material.certificate`` is a valid signing certificate, as + issued by an *a priori* trusted authority (such as a root of trust already + present within the verifying client). +* ``verification_material.certificate`` identifies an appropriate signing + subject, such as the machine identity of the Trusted Publisher that published + the package. +* ``envelope.statement`` is a valid in-toto v1 Statement, with a subject + and digest that **MUST** match the distribution's filename and contents. + For the distribution's filename, matching **MUST** be performed by parsing + using the appropriate source distribution or wheel filename format, as + the statement's subject may be equivalent but normalized. +* ``envelope.signature`` is a valid signature for ``envelope.statement`` + corresponding to ``verification_material.certificate``, + as reconstituted via the + `v1 DSSE signature protocol `__. + +In addition to the above required steps, a verifier **MAY** additionally verify +``verification_material.transparency_entries`` on a policy basis, e.g. requiring +at least one transparency log entry or a threshold of entries. When verifying +transparency entries, the verifier **MUST** confirm that the inclusion time for +each entry lies within the signing certificate's validity period. + +.. _appendix: + +Appendix: Data models for Transparency Log Entries +==================================================== + +This appendix contains pseudocoded data models for transparency log entries +in attestation objects. Each transparency log entry serves as a source +of signed inclusion time, and can be verified either online or offline. + +.. code-block:: python + + @dataclass + class TransparencyLogEntry: + log_index: int + """ + The global index of the log entry, used when querying the log. + """ + + log_id: str + """ + An opaque, unique identifier for the log. + """ + + entry_kind: str + """ + The kind (type) of log entry. + """ + + entry_version: str + """ + The version of the log entry's submitted format. + """ + + integrated_time: int + """ + The UNIX timestamp from the log from when the entry was persisted. + """ + + inclusion_proof: InclusionProof + """ + The actual inclusion proof of the log entry. + """ + + + @dataclass + class InclusionProof: + log_index: int + """ + The index of the entry in the tree it was written to. + """ + + root_hash: str + """ + The digest stored at the root of the Merkle tree at the time of proof + generation. + """ + + tree_size: int + """ + The size of the Merkle tree at the time of proof generation. + """ + + hashes: list[str] + """ + A list of hashes required to complete the inclusion proof, sorted + in order from leaf to root. The leaf and root hashes are not themselves + included in this list; the root is supplied via `root_hash` and the client + must calculate the leaf hash. + """ + + checkpoint: str + """ + The signed tree head's signature, at the time of proof generation. + """ + + cosigned_checkpoints: list[str] + """ + Cosigned checkpoints from zero or more log witnesses. + """ diff --git a/source/specifications/index.rst b/source/specifications/index.rst index 6a282b243..c375654a2 100644 --- a/source/specifications/index.rst +++ b/source/specifications/index.rst @@ -8,39 +8,13 @@ by the Python Packaging Authority. The process for updating these standards, and for proposing new ones, is documented on `pypa.io `__. - -Package Distribution Metadata ------------------------------ - -.. toctree:: - :maxdepth: 1 - - core-metadata - version-specifiers - dependency-specifiers - declaring-build-dependencies - declaring-project-metadata - platform-compatibility-tags - recording-installed-packages - entry-points - direct-url - - -Package Distribution File Formats ---------------------------------- - .. toctree:: - :maxdepth: 1 - - source-distribution-format - binary-distribution-format - - -Package Index Interfaces ------------------------- - -.. toctree:: - :maxdepth: 1 - - pypirc - simple-repository-api + :titlesonly: + + section-distribution-metadata + section-installation-metadata + section-distribution-formats + section-package-indices + section-python-description-formats + section-reproducible-environments + schemas/index.rst diff --git a/source/specifications/inline-script-metadata.rst b/source/specifications/inline-script-metadata.rst new file mode 100644 index 000000000..6fa832a3e --- /dev/null +++ b/source/specifications/inline-script-metadata.rst @@ -0,0 +1,219 @@ +.. _inline-script-metadata: + +====================== +Inline script metadata +====================== + +This specification defines a metadata format that can be embedded in single-file +Python scripts to assist launchers, IDEs and other external tools which may need +to interact with such scripts. + + +Specification +============= + +This specification defines a metadata comment block format (loosely inspired by +`reStructuredText Directives`__). + +__ https://docutils.sourceforge.io/docs/ref/rst/directives.html + +Any Python script may have top-level comment blocks that MUST start with the +line ``# /// TYPE`` where ``TYPE`` determines how to process the content. That +is: a single ``#``, followed by a single space, followed by three forward +slashes, followed by a single space, followed by the type of metadata. Block +MUST end with the line ``# ///``. That is: a single ``#``, followed by a single +space, followed by three forward slashes. The ``TYPE`` MUST only consist of +ASCII letters, numbers and hyphens. + +Every line between these two lines (``# /// TYPE`` and ``# ///``) MUST be a +comment starting with ``#``. If there are characters after the ``#`` then the +first character MUST be a space. The embedded content is formed by taking away +the first two characters of each line if the second character is a space, +otherwise just the first character (which means the line consists of only a +single ``#``). + +Precedence for an ending line ``# ///`` is given when the next line is not +a valid embedded content line as described above. For example, the following +is a single fully valid block: + +.. code:: python + + # /// some-toml + # embedded-csharp = """ + # /// + # /// text + # /// + # /// + # public class MyClass { } + # """ + # /// + +A starting line MUST NOT be placed between another starting line and its ending +line. In such cases tools MAY produce an error. Unclosed blocks MUST be ignored. + +When there are multiple comment blocks of the same ``TYPE`` defined, tools MUST +produce an error. + +Tools reading embedded metadata MAY respect the standard Python encoding +declaration. If they choose not to do so, they MUST process the file as UTF-8. + +This is the canonical regular expression that MAY be used to parse the +metadata: + +.. code:: text + + (?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$ + +In circumstances where there is a discrepancy between the text specification +and the regular expression, the text specification takes precedence. + +Tools MUST NOT read from metadata blocks with types that have not been +standardized by this specification. + +script type +----------- + +The first type of metadata block is named ``script``, which contains +script metadata (dependency data and tool configuration). + +This document MAY include the top-level fields ``dependencies`` and ``requires-python``, +and MAY optionally include a ``[tool]`` table. + +The ``[tool]`` table MAY be used by any tool, script runner or otherwise, to configure +behavior. It has the same semantics as the :ref:`[tool] table in pyproject.toml +`. + +The top-level fields are: + +* ``dependencies``: A list of strings that specifies the runtime dependencies + of the script. Each entry MUST be a valid + :ref:`dependency specifier `. +* ``requires-python``: A string that specifies the Python version(s) with which + the script is compatible. The value of this field MUST be a valid + :ref:`version specifier `. + +Script runners MUST error if the specified ``dependencies`` cannot be provided. +Script runners SHOULD error if no version of Python that satisfies the specified +``requires-python`` can be provided. + +Example +------- + +The following is an example of a script with embedded metadata: + +.. code:: python + + # /// script + # requires-python = ">=3.11" + # dependencies = [ + # "requests<3", + # "rich", + # ] + # /// + + import requests + from rich.pretty import pprint + + resp = requests.get("https://peps.python.org/api/peps.json") + data = resp.json() + pprint([(k, v["title"]) for k, v in data.items()][:10]) + + +Reference Implementation +======================== + +The following is an example of how to read the metadata on Python 3.11 or +higher. + +.. code:: python + + import re + import tomllib + + REGEX = r'(?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$' + + def read(script: str) -> dict | None: + name = 'script' + matches = list( + filter(lambda m: m.group('type') == name, re.finditer(REGEX, script)) + ) + if len(matches) > 1: + raise ValueError(f'Multiple {name} blocks found') + elif len(matches) == 1: + content = ''.join( + line[2:] if line.startswith('# ') else line[1:] + for line in matches[0].group('content').splitlines(keepends=True) + ) + return tomllib.loads(content) + else: + return None + +Often tools will edit dependencies like package managers or dependency update +automation in CI. The following is a crude example of modifying the content +using the ``tomlkit`` library__. + +__ https://tomlkit.readthedocs.io/en/latest/ + +.. code:: python + + import re + + import tomlkit + + REGEX = r'(?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$' + + def add(script: str, dependency: str) -> str: + match = re.search(REGEX, script) + content = ''.join( + line[2:] if line.startswith('# ') else line[1:] + for line in match.group('content').splitlines(keepends=True) + ) + + config = tomlkit.parse(content) + config['dependencies'].append(dependency) + new_content = ''.join( + f'# {line}' if line.strip() else f'#{line}' + for line in tomlkit.dumps(config).splitlines(keepends=True) + ) + + start, end = match.span('content') + return script[:start] + new_content + script[end:] + +Note that this example used a library that preserves TOML formatting. This is +not a requirement for editing by any means but rather is a "nice to have" +feature. + +The following is an example of how to read a stream of arbitrary metadata +blocks. + +.. code:: python + + import re + from typing import Iterator + + REGEX = r'(?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$' + + def stream(script: str) -> Iterator[tuple[str, str]]: + for match in re.finditer(REGEX, script): + yield match.group('type'), ''.join( + line[2:] if line.startswith('# ') else line[1:] + for line in match.group('content').splitlines(keepends=True) + ) + + +Recommendations +=============== + +Tools that support managing different versions of Python should attempt to use +the highest available version of Python that is compatible with the script's +``requires-python`` metadata, if defined. + + +History +======= + +- October 2023: This specification was conditionally approved through :pep:`723`. +- January 2024: Through amendments to :pep:`723`, the ``pyproject`` metadata + block type was renamed to ``script``, and the ``[run]`` table was dropped, + making the ``dependencies`` and ``requires-python`` keys + top-level. Additionally, the specification is no longer provisional. diff --git a/source/specifications/license-expression.rst b/source/specifications/license-expression.rst new file mode 100644 index 000000000..50860b7aa --- /dev/null +++ b/source/specifications/license-expression.rst @@ -0,0 +1,56 @@ +================== +License Expression +================== + +:pep:`639` defined a new :ref:`pyproject.toml's license ` +value and added a corresponding :ref:`core metadata License-Expression field +`. +This specification defines which license expressions are acceptable. + + +Specification +============= + +License can be defined as a text string that is a valid SPDX +:term:`license expression `, +as documented in the `SPDX specification `__, +either Version 2.2 or a later compatible version. + +A license expression can use the following license identifiers: + +- Any SPDX-listed license short-form identifiers that are published in + the `SPDX License List `__, + version 3.17 or any later compatible version. + +- The custom ``LicenseRef-[idstring]`` string(s), where ``[idstring]`` is + a unique string containing letters, numbers, ``.`` and/or ``-``, + to identify licenses that are not included in the SPDX license list. + The custom identifiers must follow the SPDX specification, + `clause 10.1 `__ of the given specification version. + + +Examples of valid license expressions: + +.. code-block:: yaml + + MIT + BSD-3-Clause + MIT AND (Apache-2.0 OR BSD-2-Clause) + MIT OR GPL-2.0-or-later OR (FSFUL AND BSD-2-Clause) + GPL-3.0-only WITH Classpath-Exception-2.0 OR BSD-3-Clause + LicenseRef-Special-License OR CC0-1.0 OR Unlicense + LicenseRef-Proprietary + + +Examples of invalid license expressions: + +.. code-block:: yaml + + Use-it-after-midnight # No `LicenseRef` prefix + Apache-2.0 OR 2-BSD-Clause # 2-BSD-Clause is not a valid SPDX identifier + LicenseRef-License with spaces # spaces are not allowed + LicenseRef-License_with_underscores # underscore are not allowed + +.. _spdxcustom: https://spdx.github.io/spdx-spec/v2.2.2/other-licensing-information-detected/ +.. _spdxlist: https://spdx.org/licenses/ +.. _spdxpression: https://spdx.github.io/spdx-spec/v2.2.2/SPDX-license-expressions/ diff --git a/source/specifications/name-normalization.rst b/source/specifications/name-normalization.rst new file mode 100644 index 000000000..560d956b5 --- /dev/null +++ b/source/specifications/name-normalization.rst @@ -0,0 +1,58 @@ +======================= +Names and normalization +======================= + +This specification defines the format that names for packages and extras are +required to follow. It also describes how to normalize them, which should be +done before lookups and comparisons. + + +.. _name-format: + +Name format +=========== + +A valid name consists only of ASCII letters and numbers, period, +underscore and hyphen. It must start and end with a letter or number. +This means that valid project names are limited to those which match the +following regex (run with :py:data:`re.IGNORECASE`):: + + ^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])\Z + + +.. _name-normalization: + +Name normalization +================== + +The name should be lowercased with all runs of the characters ``.``, ``-``, or +``_`` replaced with a single ``-`` character. This can be implemented in Python +with the re module: + +.. code-block:: python + + import re + + def normalize(name): + return re.sub(r"[-_.]+", "-", name).lower() + +This means that the following names are all equivalent: + +* ``friendly-bard`` (normalized form) +* ``Friendly-Bard`` +* ``FRIENDLY-BARD`` +* ``friendly.bard`` +* ``friendly_bard`` +* ``friendly--bard`` +* ``FrIeNdLy-._.-bArD`` (a *terrible* way to write a name, but it is valid) + +History +======= + +- September 2015: The specification of name normalized was approved through + :pep:`503 <503#normalized-names>`. +- November 2015: The specification of valid names was approved through + :pep:`508 <508#names>`. +- August 2025: The suggested name validation regex was fixed to match the field + specification (it previously finished with ``$`` instead of ``\Z``, + incorrectly permitting trailing newlines) diff --git a/source/specifications/platform-compatibility-tags.rst b/source/specifications/platform-compatibility-tags.rst index 5020d2cd9..b4c14a4c0 100644 --- a/source/specifications/platform-compatibility-tags.rst +++ b/source/specifications/platform-compatibility-tags.rst @@ -9,52 +9,123 @@ Platform compatibility tags allow build tools to mark distributions as being compatible with specific platforms, and allows installers to understand which distributions are compatible with the system they are running on. -The platform compatibility tagging model used for the ``wheel`` distribution -format is defined in :pep:`425`. -Platform tags for Windows -------------------------- +Overview +======== -The scheme defined in :pep:`425` covers public distribution of wheel files to -systems running Windows. +The tag format is ``{python tag}-{abi tag}-{platform tag}``. -Platform tags for macOS (Mac OS X) ----------------------------------- +python tag + 'py27', 'cp33' +abi tag + 'cp32dmu', 'none' +platform tag + 'linux_x86_64', 'any' -The scheme defined in :pep:`425` covers public distribution of wheel files to -systems running macOS (previously known as Mac OS X). +For example, the tag ``py27-none-any`` indicates compatibility with Python 2.7 +(any Python 2.7 implementation) with no abi requirement, on any platform. + +The ``wheel`` built package format includes these tags in its filenames, +of the form +``{distribution}-{version}(-{build tag})?-{python tag}-{abitag}-{platform tag}.whl``. +Other package formats may have their own conventions. + +Any potential spaces in any tag should be replaced with ``_``. + + +Python Tag +========== + +The Python tag indicates the implementation and version required by +a distribution. Major implementations have abbreviated codes, initially: + +* py: Generic Python (does not require implementation-specific features) +* cp: CPython +* ip: IronPython +* pp: PyPy +* jy: Jython + +Other Python implementations should use :py:data:`sys.implementation.name `. + +The version is ``py_version_nodot``. CPython gets away with no dot, +but if one is needed the underscore ``_`` is used instead. PyPy should +probably use its own versions here ``pp18``, ``pp19``. + +The version can be just the major version ``2`` or ``3`` ``py2``, ``py3`` for +many pure-Python distributions. + +Importantly, major-version-only tags like ``py2`` and ``py3`` are not +shorthand for ``py20`` and ``py30``. Instead, these tags mean the packager +intentionally released a cross-version-compatible distribution. + +A single-source Python 2/3 compatible distribution can use the compound +tag ``py2.py3``. See `Compressed Tag Sets`_, below. + + +ABI Tag +======= + +The ABI tag indicates which Python ABI is required by any included +extension modules. For implementation-specific ABIs, the implementation +is abbreviated in the same way as the Python Tag, e.g. ``cp33d`` would be +the CPython 3.3 ABI with debugging. + +The CPython stable ABI is ``abi3`` as in the shared library suffix. + +Implementations with a very unstable ABI may use the first 6 bytes (as +8 base64-encoded characters) of the SHA-256 hash of their source code +revision and compiler flags, etc, but will probably not have a great need +to distribute binary distributions. Each implementation's community may +decide how to best use the ABI tag. + + +Platform Tag +============ + +.. important:: + Platform tags are dependent on the versioning of the operating system or + platform they represent and may change over time as the underlying platform + changes its versioning. + +Basic platform tags +------------------- + +In its simplest form, the platform tag is :py:func:`sysconfig.get_platform()` with +all hyphens ``-`` and periods ``.`` replaced with underscore ``_``. +Until the removal of :ref:`distutils` in Python 3.12, this +was ``distutils.util.get_platform()``. For example: + +* win32 +* linux_i386 +* linux_x86_64 -Platform tags for common Linux distributions --------------------------------------------- .. _manylinux: -The scheme defined in :pep:`425` is insufficient for public distribution of -wheel files (and \*nix wheel files in general) to Linux platforms, due to the -large ecosystem of Linux platforms and subtle differences between them. +``manylinux`` +------------- + +The simple scheme above is insufficient for public distribution of wheel files +to Linux platforms, due to the large ecosystem of Linux platforms and subtle +differences between them. -Instead, :pep:`600` defines the ``manylinux`` standard, which represents a -common subset of Linux platforms, and allows building wheels tagged with the +Instead, for those platforms, the ``manylinux`` standard represents a common +subset of Linux platforms, and allows building wheels tagged with the ``manylinux`` platform tag which can be used across most common Linux distributions. -There were multiple iterations of the ``manylinux`` specification, each -representing the common subset of Linux platforms at a given point in time: +The current standard is the future-proof :file:`manylinux_{x}_{y}` standard. It defines +tags of the form :file:`manylinux_{x}_{y}_{arch}`, where ``x`` and ``y`` are glibc major +and minor versions supported (e.g. ``manylinux_2_24_xxx`` should work on any +distro using glibc 2.24+), and ``arch`` is the architecture, matching the value +of :py:func:`sysconfig.get_platform()` on the system as in the "simple" form above. -* ``manylinux1`` (:pep:`513`) supports ``x86_64`` and ``i686`` - architectures, and is based on a compatible Linux platform from 2007. -* ``manylinux2010`` (:pep:`571`) supports ``x86_64`` and ``i686`` - architectures. and updates the previous specification to be based on a - compatible Linux platform from 2010 instead. -* ``manylinux2014`` (:pep:`599`) adds support for a number of - additional architectures (``aarch64``, ``armv7l``, ``ppc64``, ``ppc64le``, - and ``s390x``) and updates the base platform to a compatible Linux platform - from 2014. +The following older tags are still supported for backward compatibility: -``manylinux_x_y`` (:pep:`600`) supersedes all previous PEPs to define a -future-proof standard. It defines ``x`` and ``y`` as glibc major an minor -versions supported (e.g. ``manylinux_2_24`` should work on any distro using -glibc 2.24+). Previous tags are still supported for backward compatibility. +* ``manylinux1`` supports glibc 2.5 on ``x86_64`` and ``i686`` architectures. +* ``manylinux2010`` supports glibc 2.12 on ``x86_64`` and ``i686``. +* ``manylinux2014`` supports glibc 2.17 on ``x86_64``, ``i686``, ``aarch64``, + ``armv7l``, ``ppc64``, ``ppc64le``, and ``s390x``. In general, distributions built for older versions of the specification are forwards-compatible (meaning that ``manylinux1`` distributions should continue @@ -67,15 +138,6 @@ possible, with the caveat that the provided build environment for ``manylinux1`` and ``manylinux2010`` have reached end-of-life meaning that these images will no longer receive security updates. -Manylinux compatibility support -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. Note:: - * The ``manylinux2014`` specification is relatively new and is not yet widely - recognised by install tools. - * The ``manylinux_x_y`` specification is relatively new and is not yet widely - recognised by install tools. - The following table shows the minimum versions of relevant projects to support the various ``manylinux`` standards: @@ -86,13 +148,309 @@ pip ``>=8.1.0`` ``>=19.0`` ``>=19.3`` ``>=20.3`` auditwheel ``>=1.0.0`` ``>=2.0.0`` ``>=3.0.0`` ``>=3.3.0`` [#]_ ========== ============== ================= ================= ================= -Platform tags for other \*nix platforms ---------------------------------------- +.. [#] Only support for ``manylinux_2_24`` has been added in auditwheel 3.3.0 -The scheme defined in :pep:`425` is not generally sufficient for public -distribution of wheel files to other \*nix platforms. Efforts are currently -(albeit intermittently) under way to define improved compatibility tagging -schemes for AIX and for Alpine Linux. +``musllinux`` +------------- -.. [#] Only support for ``manylinux_2_24`` has been added in auditwheel 3.3.0 +The ``musllinux`` family of tags is similar to ``manylinux``, but for Linux +platforms that use the musl_ libc rather than glibc (a prime example being Alpine +Linux). The schema is :file:`musllinux_{x}_{y}_{arch}`, supporting musl ``x.y`` and higher +on the architecture ``arch``. + +The musl version values can be obtained by executing the musl libc shared +library the Python interpreter is currently running on, and parsing the output: + +.. code-block:: python + + import re + import subprocess + + def get_musl_major_minor(so: str) -> tuple[int, int] | None: + """Detect musl runtime version. + + Returns a two-tuple ``(major, minor)`` that indicates musl + library's version, or ``None`` if the given libc .so does not + output expected information. + + The libc library should output something like this to stderr:: + + musl libc (x86_64) + Version 1.2.2 + Dynamic Program Loader + """ + proc = subprocess.run([so], stderr=subprocess.PIPE, text=True) + lines = (line.strip() for line in proc.stderr.splitlines()) + lines = [line for line in lines if line] + if len(lines) < 2 or lines[0][:4] != "musl": + return None + match = re.match(r"Version (\d+)\.(\d+)", lines[1]) + if match: + return (int(match.group(1)), int(match.group(2))) + return None + +There are currently two possible ways to find the musl library’s location that a +Python interpreter is running on, either with the system ldd_ command, or by +parsing the ``PT_INTERP`` section’s value from the executable’s ELF_ header. + +.. _macos: + +macOS +----- + +macOS uses the ``macosx`` family of tags (the ``x`` suffix is a historical +artefact of Apple's official macOS naming scheme). The schema for compatibility +tags is :file:`macosx_{x}_{y}_{arch}`, indicating that the wheel is compatible +with macOS ``x.y`` or later on the architecture ``arch``. + +For macOS 10, the tag is :file:`macosx_10_{y}_{arch}`, where ``y`` corresponds +to the minor version number of the macOS release. For macOS 11 and higher, the +tag is :file:`macosx_{x}_0_{arch}`, where ``x`` corresponds to the major +version number of the macOS release. Following the published macOS major +versions, the ``x`` value is either ``10 <= x <= 15``, or ``>=26`` and +corresponding to the year of the macOS release. For example, +``macosx_11_0_arm64`` indicates compatibility with macOS 11 or later. + +macOS binaries can be compiled for a single architecture, or can include support +for multiple architectures in the same binary (sometimes called "fat" binaries). +To indicate support for a single architecture, the value of ``arch`` must match +the value of :py:func:`platform.machine()` on the system. To indicate +support multiple architectures, the ``arch`` tag should be an identifier from +the following list that describes the set of supported architectures: + +============== ======================================== +``arch`` Architectures supported +============== ======================================== +``universal2`` ``arm64``, ``x86_64`` +``universal`` ``i386``, ``ppc``, ``ppc64``, ``x86_64`` +``intel`` ``i386``, ``x86_64`` +``fat`` ``i386``, ``ppc`` +``fat3`` ``i386``, ``ppc``, ``x86_64`` +``fat64`` ``ppc64``, ``x86_64`` +============== ======================================== + +The minimum supported macOS version may also be constrained by architecture. For +example, macOS 11 (Big Sur) was the first release to support arm64. These +additional constraints are enforced transparently by the macOS compilation +toolchain when building binaries that support multiple architectures. + +.. _android: + +Android +------- + +Android uses the schema :file:`android_{apilevel}_{abi}`, indicating +compatibility with the given Android API level or later, on the given ABI. For +example, ``android_27_arm64_v8a`` indicates support for API level 27 or later, +on ``arm64_v8a`` devices. Android makes no distinction between physical devices +and emulated devices. + +The API level should be a positive integer. This is *not* the same thing as +the user-facing Android version. For example, the release known as Android +12 (code named "Snow Cone") uses API level 31 or 32, depending on the specific +Android version in use. Android's release documentation contains the `full list +of Android versions and their corresponding API levels +`__. + +There are 4 `supported ABIs `__. +Normalized according to the rules above, they are: + +* ``armeabi_v7a`` +* ``arm64_v8a`` +* ``x86`` +* ``x86_64`` + +Virtually all current physical devices use one of the ARM architectures. ``x86`` +and ``x86_64`` are supported for use in the emulator. ``x86`` has not been +supported as a development platform since 2020, and no new emulator images have +been released since then. + +.. _ios: + +iOS +--- + +iOS uses the schema :file:`ios_{x}_{y}_{arch}_{sdk}`, indicating compatibility with +iOS ``x.y`` or later, on the ``arch`` architecture, using the ``sdk`` SDK. + +The value of ``x`` and ``y`` correspond to the major and minor version number of +the iOS release, respectively. They must both be positive integers. The version +number always includes a major *and* minor version, even if Apple's official +version numbering only refers to the major value. For example, a +``ios_13_0_arm64_iphonesimulator`` indicates compatibility with iOS 13 or later. + +The value of ``arch`` must match the value of :py:func:`platform.machine()` on +the system. + +The value of ``sdk`` must be either ``iphoneos`` (for physical devices), or +``iphonesimulator`` (for device simulators). These SDKs have the same API +surface, but are incompatible at the binary level, even if they are running on +the same CPU architecture. Code compiled for an arm64 simulator will not run on +an arm64 device. + +The combination of :file:`{arch}_{sdk}` is referred to as the "multiarch". There +are three possible values for multiarch: + +* ``arm64_iphoneos``, for physical iPhone/iPad devices. This includes every + iOS device manufactured since ~2015; +* ``arm64_iphonesimulator``, for simulators running on Apple Silicon macOS + hardware; and +* ``x86_64_iphonesimulator``, for simulators running on x86_64 hardware. + +Use +=== + +The tags are used by installers to decide which built distribution +(if any) to download from a list of potential built distributions. +The installer maintains a list of (pyver, abi, arch) tuples that it +will support. If the built distribution's tag is ``in`` the list, then +it can be installed. + +It is recommended that installers try to choose the most feature complete +built distribution available (the one most specific to the installation +environment) by default before falling back to pure Python versions +published for older Python releases. Installers are also recommended to +provide a way to configure and re-order the list of allowed compatibility +tags; for example, a user might accept only the ``*-none-any`` tags to only +download built packages that advertise themselves as being pure Python. + +Another desirable installer feature might be to include "re-compile from +source if possible" as more preferable than some of the compatible but +legacy pre-built options. + +This example list is for an installer running under CPython 3.3 on a +linux_x86_64 system. It is in order from most-preferred (a distribution +with a compiled extension module, built for the current version of +Python) to least-preferred (a pure-Python distribution built with an +older version of Python): + +1. cp33-cp33m-linux_x86_64 +2. cp33-abi3-linux_x86_64 +3. cp3-abi3-linux_x86_64 +4. cp33-none-linux_x86_64* +5. cp3-none-linux_x86_64* +6. py33-none-linux_x86_64* +7. py3-none-linux_x86_64* +8. cp33-none-any +9. cp3-none-any +10. py33-none-any +11. py3-none-any +12. py32-none-any +13. py31-none-any +14. py30-none-any + +* Built distributions may be platform specific for reasons other than C + extensions, such as by including a native executable invoked as + a subprocess. + +Sometimes there will be more than one supported built distribution for a +particular version of a package. For example, a packager could release +a package tagged ``cp33-abi3-linux_x86_64`` that contains an optional C +extension and the same distribution tagged ``py3-none-any`` that does not. +The index of the tag in the supported tags list breaks the tie, and the +package with the C extension is installed in preference to the package +without because that tag appears first in the list. + +Compressed Tag Sets +=================== + +To allow for compact filenames of bdists that work with more than +one compatibility tag triple, each tag in a filename can instead be a +'.'-separated, sorted, collection of tags. For example, pip, a pure-Python +package that is written to run under Python 2 and 3 with the same source +code, could distribute a bdist with the tag ``py2.py3-none-any``. +The full list of simple tags is:: + + for x in pytag.split('.'): + for y in abitag.split('.'): + for z in platformtag.split('.'): + yield '-'.join((x, y, z)) + +A bdist format that implements this scheme should include the expanded +tags in bdist-specific metadata. This compression scheme can generate +large numbers of unsupported tags and "impossible" tags that are supported +by no Python implementation e.g. "cp33-cp31u-win64", so use it sparingly. + +FAQ +=== + +What tags are used by default? + Tools should use the most-preferred architecture dependent tag + e.g. ``cp33-cp33m-win32`` or the most-preferred pure python tag + e.g. ``py33-none-any`` by default. If the packager overrides the + default it indicates that they intended to provide cross-Python + compatibility. + +What tag do I use if my distribution uses a feature exclusive to the newest version of Python? + Compatibility tags aid installers in selecting the *most compatible* + build of a *single version* of a distribution. For example, when + there is no Python 3.3 compatible build of ``beaglevote-1.2.0`` + (it uses a Python 3.4 exclusive feature) it may still use the + ``py3-none-any`` tag instead of the ``py34-none-any`` tag. A Python + 3.3 user must combine other qualifiers, such as a requirement for the + older release ``beaglevote-1.1.0`` that does not use the new feature, + to get a compatible build. + +Why isn't there a ``.`` in the Python version number? + CPython has lasted 20+ years without a 3-digit major release. This + should continue for some time. Other implementations may use _ as + a delimiter, since both - and . delimit the surrounding filename. + +Why normalise hyphens and other non-alphanumeric characters to underscores? + To avoid conflicting with the ``.`` and ``-`` characters that separate + components of the filename, and for better compatibility with the + widest range of filesystem limitations for filenames (including + being usable in URL paths without quoting). + +Why not use special character rather than ``.`` or ``-``? + Either because that character is inconvenient or potentially confusing + in some contexts (for example, ``+`` must be quoted in URLs, ``~`` is + used to denote the user's home directory in POSIX), or because the + advantages weren't sufficiently compelling to justify changing the + existing reference implementation for the wheel format defined in :pep:`427` + (for example, using ``,`` rather than ``.`` to separate components + in a compressed tag). + +Who will maintain the registry of abbreviated implementations? + New two-letter abbreviations can be requested on the python-dev + mailing list. As a rule of thumb, abbreviations are reserved for + the current 4 most prominent implementations. + +Does the compatibility tag go into METADATA or PKG-INFO? + No. The compatibility tag is part of the built distribution's + metadata. METADATA / PKG-INFO should be valid for an entire + distribution, not a single build of that distribution. + +Why didn't you mention my favorite Python implementation? + The abbreviated tags facilitate sharing compiled Python code in a + public index. Your Python implementation can use this specification + too, but with longer tags. + Recall that all "pure Python" built distributions just use ``py``. + +Why is the ABI tag (the second tag) sometimes "none" in the reference implementation? + Since Python 2 does not have an easy way to get to the SOABI + (the concept comes from newer versions of Python 3) the reference + implementation at the time of writing guesses "none". Ideally it + would detect "py27(d|m|u)" analogous to newer versions of Python, + but in the meantime "none" is a good enough way to say "don't know". + + +History +======= + +- February 2013: The original version of this specification was approved through + :pep:`425`. +- January 2016: The ``manylinux1`` tag was approved through :pep:`513`. +- April 2018: The ``manylinux2010`` tag was approved through :pep:`571`. +- July 2019: The ``manylinux2014`` tag was approved through :pep:`599`. +- November 2019: The ``manylinux_x_y`` perennial tag was approved through + :pep:`600`. +- April 2021: The ``musllinux_x_y`` tag was approved through :pep:`656`. +- December 2023: The tags for iOS were approved through :pep:`730`. +- March 2024: The tags for Android were approved through :pep:`738`. + + +.. _musl: https://musl.libc.org +.. _ldd: https://www.man7.org/linux/man-pages/man1/ldd.1.html +.. _elf: https://refspecs.linuxfoundation.org/elf/elf.pdf diff --git a/source/specifications/project-status-markers.rst b/source/specifications/project-status-markers.rst new file mode 100644 index 000000000..90df74441 --- /dev/null +++ b/source/specifications/project-status-markers.rst @@ -0,0 +1,89 @@ +.. _project-status-markers: + +====================== +Project Status Markers +====================== + +.. note:: + + This specification was originally defined in + :pep:`792`. + +.. note:: + + :pep:`792` includes changes to the HTML and JSON index APIs. + These changes are documented in the :ref:`simple-repository-api` + under :ref:`HTML - Project Detail ` + and :ref:`JSON - Project Detail `. + +Specification +============= + +A project always has exactly one status. If no status is explicitly noted, +then the project is considered to be in the ``active`` state. + +Indices **MAY** implement any subset of the status markers specified, +as applicable to their needs. + +This standard does not prescribe *which* principals (i.e. project maintainers, +index administrators, etc.) are allowed to set and unset which statuses. + +``active`` +---------- + +Description: The project is active. This is the default status for a project. + +Index semantics: + +* The index hosting the project **MUST** allow uploads of new distributions to + the project. +* The index **MUST** offer existing distributions of the project for download. + +Installer semantics: none. + +``archived`` +------------ + +Description: The project does not expect to be updated in the future. + +Index semantics: + +* The index hosting the project **MUST NOT** allow uploads of new distributions to + the project. +* The index **MUST** offer existing distributions of the project for download. + +Installer semantics: + +* Installers **MAY** produce warnings about a project's archival. + +``quarantined`` +--------------- + +Description: The project is considered generally unsafe for use, e.g. due to +malware. + +Index semantics: + +* The index hosting the project **MUST NOT** allow uploads of new distributions to + the project. +* The index **MUST NOT** offer any distributions of the project for download. + +Installer semantics: + +* Installers **MAY** produce warnings about a project's quarantine, although + doing so is effectively moot (as the index will not offer any distributions + for installation). + +``deprecated`` +-------------- + +Description: The project is considered obsolete, and may have been superseded +by another project. + +Index semantics: + +* This status shares the same semantics as ``active``. + +Installer semantics: + +* Installers **MAY** produce warnings about a project's deprecation. diff --git a/source/specifications/pylock-toml.rst b/source/specifications/pylock-toml.rst new file mode 100644 index 000000000..342e608c5 --- /dev/null +++ b/source/specifications/pylock-toml.rst @@ -0,0 +1,842 @@ +.. _pylock-toml-spec: +.. _lock-file-spec: + +============================= +``pylock.toml`` Specification +============================= + +The ``pylock.toml`` file format is for specifying dependencies to enable +reproducible installation in a Python environment. + +.. note:: This specification was originally defined in :pep:`751`. + + +--------- +File Name +--------- + +A lock file MUST be named :file:`pylock.toml` or match the regular expression +``r"^pylock\.([^.]+)\.toml$"`` if a name for the lock file is desired or if +multiple lock files exist (i.e. the regular expression +``r"^pylock\.([^.]+\.)?toml$"`` for any file name). The prefix and suffix of a +named file MUST be lowercase when possible, for easy detection and removal, +e.g.: + +.. code-block:: Python + + if len(filename) > 11 and filename.startswith("pylock.") and filename.endswith(".toml"): + name = filename.removeprefix("pylock.").removesuffix(".toml") + +The expectation is that services that automatically install from lock files will +search for: + +1. The lock file with the service's name and doing the default install +2. A multi-use :file:`pylock.toml` with a dependency group with the name of the service +3. The default install of :file:`pylock.toml` + +E.g. a cloud host service named "spam" would first look for +:file:`pylock.spam.toml` to install from, and if that file didn't exist then install +from :file:`pylock.toml` and look for a dependency group named "spam" to use if +present. + +The lock file(s) SHOULD be located in the directory as appropriate for the scope +of the lock file. Locking against a single :file:`pyproject.toml`, for instance, +would place the :file:`pylock.toml` in the same directory. If the lock file covered +multiple projects in a monorepo, then the expectation is the :file:`pylock.toml` +file would be in the directory that held all the projects being locked. + + +----------- +File Format +----------- + +The format of the file is TOML_. + +Tools SHOULD write their lock files in a consistent way to minimize noise in +diff output. Keys in tables -- including the top-level table -- SHOULD be +recorded in a consistent order (if inspiration is desired, this specification has tried to +write down keys in a logical order). As well, tools SHOULD sort arrays in +consistent order. Usage of inline tables SHOULD also be kept consistent. + + +.. _pylock-lock-version: + +``lock-version`` +================ + +- **Type**: string; value of ``"1.0"`` +- **Required?**: yes +- **Inspiration**: :ref:`core-metadata-metadata-version` +- Record the file format version that the file adheres to. +- This PEP specifies the initial version -- and only valid value until future + updates to the standard change it -- as ``"1.0"``. +- If a tool supports the major version but not the minor version, a tool + SHOULD warn when an unknown key is seen. +- If a tool doesn't support a major version, it MUST raise an error. + + +.. _pylock-environments: + +``environments`` +================ + +- **Type**: Array of strings +- **Required?**: no +- **Inspiration**: uv_ +- A list of :ref:`dependency-specifiers-environment-markers` for + which the lock file is considered compatible with. +- Tools SHOULD write exclusive/non-overlapping environment markers to ease in + understanding. + + +.. _pylock-requires-python: + +``requires-python`` +=================== + +- **Type**: string +- **Required?**: no +- **Inspiration**: PDM_, Poetry_, uv_ +- Specifies the :ref:`core-metadata-requires-python` for the minimum + Python version compatible for any environment supported by the lock file + (i.e. the minimum viable Python version for the lock file). + + +.. _pylock-extras: + +``extras`` +========== + +- **Type**: Array of strings +- **Required?**: no; defaults to ``[]`` +- **Inspiration**: :ref:`core-metadata-provides-extra` +- The list of :ref:`extras ` supported + by this lock file. +- Lockers MAY choose to not support writing lock files that support extras and + dependency groups (i.e. tools may only support exporting a single-use lock + file). +- Tools supporting extras MUST also support dependency groups. +- Tools should explicitly set this key to an empty array to signal that the + inputs used to generate the lock file had no extras (e.g. a + :ref:`pyproject.toml ` file had no + :ref:`[project.optional-dependencies] ` + table), signalling that the lock file is, in effect, multi-use even if it only + looks to be single-use. + + +.. _pylock-dependency-groups: + +``dependency-groups`` +===================== + +- **Type**: Array of strings +- **Required?**: no; defaults to ``[]`` +- **Inspiration**: :ref:`pyproject-tool-table` +- The list of :ref:`dependency-groups` publicly supported by this lock + file (i.e. dependency groups users are expected to be able to specify via a + tool's UI). +- Lockers MAY choose to not support writing lock files that support extras and + dependency groups (i.e. tools may only support exporting a single-use lock + file). +- Tools supporting dependency groups MUST also support extras. +- Tools SHOULD explicitly set this key to an empty array to signal that the + inputs used to generate the lock file had no dependency groups (e.g. a + :ref:`pyproject.toml ` file had no + :ref:`[dependency-groups] ` table), signalling that the + lock file is, in effect, multi-use even if it only looks to be single-use. + + +.. _pylock-default-groups: + +``default-groups`` +================== + +- **Type**: Array of strings +- **Required?**: no; defaults to ``[]`` +- **Inspiration**: Poetry_, PDM_ +- The name of synthetic dependency groups to represent what should be installed + by default (e.g. what + :ref:`[project.dependencies] ` implicitly + represents). +- Meant to be used in situations where :ref:`pylock-packages-marker` + necessitates such a group to exist. +- The groups listed by this key SHOULD NOT be listed in + :ref:`pylock-dependency-groups` as the groups are not meant to be directly + exposed to users by name but instead via an installer's UI. + + +.. _pylock-created-by: + +``created-by`` +============== + +- **Type**: string +- **Required?**: yes +- **Inspiration**: Tools with their name in their lock file name +- Records the name of the tool used to create the lock file. +- Tools MAY use the :ref:`pylock-tool` table to record enough details that it + can be inferred what inputs were used to create the lock file. +- Tools SHOULD record the normalized name of the tool if it is available as a + Python package to facilitate finding the tool. + + +.. _pylock-packages: + +``[[packages]]`` +================ + +- **Type**: array of tables +- **Required?**: yes +- **Inspiration**: PDM_, Poetry_, uv_ +- An array containing all packages that *may* be installed. +- Packages MAY be listed multiple times with varying data, but all packages to + be installed MUST narrow down to a single entry at install time. + + +.. _pylock-packages-name: + +``packages.name`` +----------------- + +- **Type**: string +- **Required?**: yes +- **Inspiration**: :ref:`core-metadata-name` +- The name of the package :ref:`normalized `. + + +.. _pylock-packages-version: + +``packages.version`` +-------------------- + +- **Type**: string +- **Required?**: no +- **Inspiration**: :ref:`core-metadata-version` +- The version of the package. +- The version SHOULD be specified when the version is known to be stable + (i.e. when an :ref:`sdist ` or + :ref:`wheels ` are specified). +- The version MUST NOT be included when it cannot be guaranteed to be consistent + with the code used (i.e. when a + :ref:`source tree ` is used). + + +.. _pylock-packages-marker: + +``packages.marker`` +------------------- + +- **Type**: string +- **Required?**: no +- **Inspiration**: PDM_ +- The + :ref:`environment marker ` + which specify when the package should be installed. + + +.. _pylock-packages-requires-python: + +``packages.requires-python`` +---------------------------- + +- **Type**: string +- **Required?**: no +- **Inspiration**: :ref:`core-metadata-requires-python` +- Holds the :ref:`version-specifiers` for Python version compatibility + for the package. + + +.. _pylock-packages-dependencies: + +``[[packages.dependencies]]`` +----------------------------- + +- **Type**: array of tables +- **Required?**: no +- **Inspiration**: PDM_, Poetry_, uv_ +- Records the other entries in :ref:`pylock-packages` which are direct + dependencies of this package. +- Each entry is a table which contains the minimum information required to tell + which other package entry it corresponds to where doing a key-by-key + comparison would find the appropriate package with no ambiguity (e.g. if there + are two entries for the ``spam`` package, then you can include the version + number like ``{name = "spam", version = "1.0.0"}``, or by source like + ``{name = "spam", vcs = { url = "..."}``). +- Tools MUST NOT use this information when doing installation; it is purely + informational for auditing purposes. + + +.. _pylock-packages-vcs: + +``[packages.vcs]`` +------------------ + +- **Type**: table +- **Required?**: no; mutually-exclusive with :ref:`pylock-packages-directory`, + :ref:`pylock-packages-archive`, :ref:`pylock-packages-sdist`, and + :ref:`pylock-packages-wheels` +- **Inspiration**: :ref:`direct-url-data-structure` +- Record the version control system details for the + :ref:`source tree ` it + contains. +- Tools MAY choose to not support version control systems, both from a locking + and/or installation perspective. +- Tools MAY choose to only support a subset of the available VCS types. +- Tools SHOULD provide a way for users to opt in/out of using version control + systems. +- Installation from a version control system is considered originating from a + :ref:`direct URL reference `. + + +.. _pylock-packages-vcs-type: + +``packages.vcs.type`` +''''''''''''''''''''' + +- **Type**: string; supported values specified in + :ref:`direct-url-data-structure-registered-vcs` +- **Required?**: yes +- **Inspiration**: :ref:`direct-url-data-structure-vcs` +- The type of version control system used. + + +.. _pylock-packages-vcs-url: + +``packages.vcs.url`` +'''''''''''''''''''' + +- **Type**: string +- **Required?**: if :ref:`pylock-packages-vcs-path` is not specified +- **Inspiration**: :ref:`direct-url-data-structure-vcs` +- The URL_ to the source tree. + + +.. _pylock-packages-vcs-path: + +``packages.vcs.path`` +''''''''''''''''''''' + +- **Type**: string +- **Required?**: if :ref:`pylock-packages-vcs-url` is not specified +- **Inspiration**: :ref:`direct-url-data-structure-vcs` +- The path to the local directory of the source tree. +- If a relative path is used it MUST be relative to the location of this file. +- If the path is relative it MAY use POSIX-style path separators explicitly + for portability. + + +.. _pylock-packages-vcs-requested-revision: + +``packages.vcs.requested-revision`` +''''''''''''''''''''''''''''''''''' + +- **Type**: string +- **Required?**: no +- **Inspiration**: :ref:`direct-url-data-structure-vcs` +- The branch/tag/ref/commit/revision/etc. that the user requested. +- This is purely informational and to facilitate writing the + :ref:`direct-url-data-structure`; it MUST NOT be used to checkout + the repository. + + +.. _pylock-packages-vcs-commit-id: + +``packages.vcs.commit-id`` +'''''''''''''''''''''''''' + +- **Type**: string +- **Required?**: yes +- **Inspiration**: :ref:`direct-url-data-structure-vcs` +- The exact commit/revision number that is to be installed. +- If the VCS supports commit-hash based revision identifiers, such a + commit-hash, it MUST be used as the commit ID in order to reference an + immutable version of the source code. + + +.. _pylock-packages-vcs-subdirectory: + +``packages.vcs.subdirectory`` +''''''''''''''''''''''''''''' + +- **Type**: string +- **Required?**: no +- **Inspiration**: :ref:`direct-url-data-structure-subdirectories` +- The subdirectory within the + :ref:`source tree ` where + the project root of the project is (e.g. the location of the + :ref:`pyproject.toml ` file). +- The path MUST be relative to the root of the source tree structure. + + +.. _pylock-packages-directory: + +``[packages.directory]`` +------------------------ + +- **Type**: table +- **Required?**: no; mutually-exclusive with :ref:`pylock-packages-vcs`, + :ref:`pylock-packages-archive`, :ref:`pylock-packages-sdist`, and + :ref:`pylock-packages-wheels` +- **Inspiration**: :ref:`direct-url-data-structure-local-directory` +- Record the local directory details for the + :ref:`source tree ` it + contains. +- Tools MAY choose to not support local directories, both from a locking + and/or installation perspective. +- Tools SHOULD provide a way for users to opt in/out of using local directories. +- Installation from a directory is considered originating from a + :ref:`direct URL reference `. + + +.. _pylock-packages-directory-path: + +``packages.directory.path`` +''''''''''''''''''''''''''' + +- **Type**: string +- **Required?**: yes +- **Inspiration**: :ref:`direct-url-data-structure-local-directory` +- The local directory where the source tree is. +- If the path is relative it MUST be relative to the location of the lock file. +- If the path is relative it MAY use POSIX-style path separators for + portability. + + +.. _pylock-packages-directory-editable: + +``packages.directory.editable`` +''''''''''''''''''''''''''''''' + +- **Type**: boolean +- **Required?**: no; defaults to ``false`` +- **Inspiration**: :ref:`direct-url-data-structure-local-directory` +- A flag representing whether the source tree was an editable install at lock + time. +- An installer MAY choose to ignore this flag if user actions or context would + make an editable install unnecessary or undesirable (e.g. a container image + that will not be mounted for development purposes but instead deployed to + production where it would be treated at read-only). + + +.. _pylock-packages-directory-subdirectory: + +``packages.directory.subdirectory`` +''''''''''''''''''''''''''''''''''' + +See :ref:`pylock-packages-vcs-subdirectory`. + + +.. _pylock-packages-archive: + +``[packages.archive]`` +---------------------- + +- **Type**: table +- **Required?**: no +- **Inspiration**: :ref:`direct-url-data-structure-archive` +- A direct reference to an archive file to install from + (this can include wheels and sdists, as well as other archive formats + containing a source tree). +- Tools MAY choose to not support archive files, both from a locking + and/or installation perspective. +- Tools SHOULD provide a way for users to opt in/out of using archive files. +- Installation from an archive file is considered originating from a + :ref:`direct URL reference `. + + +.. _pylock-packages-archive-url: + +``packages.archive.url`` +'''''''''''''''''''''''' + +See :ref:`pylock-packages-vcs-url`. + + +.. _pylock-packages-archive-path: + +``packages.archive.path`` +''''''''''''''''''''''''' + +See :ref:`pylock-packages-vcs-path`. + + +.. _pylock-packages-archive-size: + +``packages.archive.size`` +''''''''''''''''''''''''' + +- **Type**: integer +- **Required?**: no +- **Inspiration**: uv_, :ref:`simple-repository-api` +- The size of the archive file. +- Tools SHOULD provide the file size when reasonably possible (e.g. the file + size is available via the Content-Length_ header from a HEAD_ HTTP request). + + +.. _pylock-packages-archive-upload-time: + +``packages.archive.upload-time`` +'''''''''''''''''''''''''''''''' + +- **Type**: datetime +- **Required?**: no +- **Inspiration**: :ref:`simple-repository-api` +- The time the file was uploaded. +- The date and time MUST be recorded in UTC. + + +.. _pylock-packages-archive-hashes: + +``[packages.archive.hashes]`` +''''''''''''''''''''''''''''' + +- **Type**: Table of strings +- **Required?**: yes +- **Inspiration**: PDM_, Poetry_, uv_, :ref:`simple-repository-api` +- A table listing known hash values of the file where the key is the hash + algorithm and the value is the hash value. +- The table MUST contain at least one entry. +- Hash algorithm keys SHOULD be lowercase. +- At least one secure algorithm from :py:data:`hashlib.algorithms_guaranteed` + SHOULD always be included (at time of writing, sha256 specifically is + recommended. + + +.. _pylock-packages-archive-subdirectory: + +``packages.archive.subdirectory`` +'''''''''''''''''''''''''''''''''' + +See :ref:`pylock-packages-vcs-subdirectory`. + + +.. _pylock-packages-index: + +``packages.index`` +------------------ + +- **Type**: string +- **Required?**: no +- **Inspiration**: uv_ +- The base URL for the package index from :ref:`simple-repository-api` + where the sdist and/or wheels were found (e.g. ``https://pypi.org/simple/``). +- When possible, this SHOULD be specified to assist with generating + `software bill of materials`_ -- aka SBOMs -- and to assist in finding a file + if a URL ceases to be valid. +- Tools MAY support installing from an index if the URL recorded for a specific + file is no longer valid (e.g. returns a 404 HTTP error code). + + +.. _pylock-packages-sdist: + +``[packages.sdist]`` +-------------------- + +- **Type**: table +- **Required?**: no; mutually-exclusive with :ref:`pylock-packages-vcs`, + :ref:`pylock-packages-directory`, and :ref:`pylock-packages-archive` +- **Inspiration**: uv_ +- Details of a :ref:`source-distribution-format-sdist` for the + package. +- Tools MAY choose to not support sdist files, both from a locking + and/or installation perspective. +- Tools SHOULD provide a way for users to opt in/out of using sdist files. + + +.. _pylock-packages-sdist-name: + +``packages.sdist.name`` +''''''''''''''''''''''' + +- **Type**: string +- **Required?**: no, not when the last component of + :ref:`pylock-packages-sdist-path`/ :ref:`pylock-packages-sdist-url` would be + the same value +- **Inspiration**: PDM_, Poetry_, uv_ +- The file name of the :ref:`source-distribution-format-sdist` file. + + +.. _pylock-packages-sdist-upload-time: + +``packages.sdist.upload-time`` +'''''''''''''''''''''''''''''' + +See :ref:`pylock-packages-archive-upload-time`. + + +.. _pylock-packages-sdist-url: + +``packages.sdist.url`` +'''''''''''''''''''''' + +See :ref:`pylock-packages-archive-url`. + + +.. _pylock-packages-sdist-path: + +``packages.sdist.path`` +''''''''''''''''''''''' + +See :ref:`pylock-packages-archive-path`. + + +.. _pylock-packages-sdist-size: + +``packages.sdist.size`` +''''''''''''''''''''''' + +See :ref:`pylock-packages-archive-size`. + + +.. _pylock-packages-sdist-hashes: + +``packages.sdist.hashes`` +''''''''''''''''''''''''' + +See :ref:`pylock-packages-archive-hashes`. + + + +.. _pylock-packages-wheels: + +``[[packages.wheels]]`` +----------------------- + +- **Type**: array of tables +- **Required?**: no; mutually-exclusive with :ref:`pylock-packages-vcs`, + :ref:`pylock-packages-directory`, and :ref:`pylock-packages-archive` +- **Inspiration**: PDM_, Poetry_, uv_ +- For recording the wheel files as specified by + :ref:`binary-distribution-format` for the package. +- Tools MUST support wheel files, both from a locking and installation + perspective. + + +.. _pylock-packages-wheels-name: + +``packages.wheels.name`` +'''''''''''''''''''''''' + +- **Type**: string +- **Required?**: no, not when the last component of + :ref:`pylock-packages-wheels-path`/ :ref:`pylock-packages-wheels-url` would be + the same value +- **Inspiration**: PDM_, Poetry_, uv_ +- The file name of the :ref:`binary-distribution-format` file. + + +.. _pylock-packages-wheels-upload-time: + +``packages.wheels.upload-time`` +''''''''''''''''''''''''''''''' + +See :ref:`pylock-packages-archive-upload-time`. + + +.. _pylock-packages-wheels-url: + +``packages.wheels.url`` +''''''''''''''''''''''' + +See :ref:`pylock-packages-archive-url`. + + +.. _pylock-packages-wheels-path: + +``packages.wheels.path`` +'''''''''''''''''''''''' + +See :ref:`pylock-packages-archive-path`. + + +.. _pylock-packages-wheels-size: + +``packages.wheels.size`` +'''''''''''''''''''''''' + +See :ref:`pylock-packages-archive-size`. + + +.. _pylock-packages-wheels-hashes: + +``packages.wheels.hashes`` +'''''''''''''''''''''''''' + +See :ref:`pylock-packages-archive-hashes`. + + +.. _pylock-packages-attestation-identities: + +``[[packages.attestation-identities]]`` +--------------------------------------- + +- **Type**: array of tables +- **Required?**: no +- **Inspiration**: :ref:`provenance-object` +- A recording of the attestations for **any** file recorded for this package. +- If available, tools SHOULD include the attestation identities found. +- Publisher-specific keys are to be included in the table as-is + (i.e. top-level), following the spec at + :ref:`index-hosted-attestations`. + + +.. _pylock-packages-attestation-identities-kind: + +``packages.attestation-identities.kind`` +'''''''''''''''''''''''''''''''''''''''' + +- **Type**: string +- **Required?**: yes +- **Inspiration**: :ref:`provenance-object` +- The unique identity of the Trusted Publisher. + + +.. _pylock-packages-tool: + +``[packages.tool]`` +------------------- + +- **Type**: table +- **Required?**: no +- **Inspiration**: :ref:`pyproject-tool-table` +- Similar usage as that of the :ref:`pylock-tool` table from the + :ref:`pyproject-toml-spec`, but at the package version level instead + of at the lock file level (which is also available via :ref:`pylock-tool`). +- Data recorded in the table MUST be disposable (i.e. it MUST NOT affect + installation). + + +.. _pylock-tool: + +``[tool]`` +========== + +- **Type**: table +- **Required?**: no +- **Inspiration**: :ref:`pyproject-tool-table` +- See :ref:`pylock-packages-tool`. + + +------- +Example +------- + +.. literalinclude:: pylock-toml/pylock.example.toml + + +------------ +Installation +------------ + +The following outlines the steps to be taken to install from a lock file +(while the requirements are prescriptive, the general steps and order are +a suggestion): + +#. Gather the extras and dependency groups to install and set ``extras`` and + ``dependency_groups`` for marker evaluation, respectively. + + #. ``extras`` SHOULD be set to the empty set by default. + #. ``dependency_groups`` SHOULD be the set created from + :ref:`pylock-default-groups` by default. + +#. Check if the metadata version specified by :ref:`pylock-lock-version` is + supported; an error or warning MUST be raised as appropriate. +#. If :ref:`pylock-requires-python` is specified, check that the environment + being installed for meets the requirement; an error MUST be raised if it is + not met. +#. If :ref:`pylock-environments` is specified, check that at least one of the + environment marker expressions is satisfied; an error MUST be raised if no + expression is satisfied. +#. For each package listed in :ref:`pylock-packages`: + + #. If :ref:`pylock-packages-marker` is specified, check if it is satisfied; + if it isn't, skip to the next package. + #. If :ref:`pylock-packages-requires-python` is specified, check if it is + satisfied; an error MUST be raised if it isn't. + #. Check that no other conflicting instance of the package has been slated to + be installed; an error about the ambiguity MUST be raised otherwise. + #. Check that the source of the package is specified appropriately (i.e. + there are no conflicting sources in the package entry); + an error MUST be raised if any issues are found. + #. Add the package to the set of packages to install. + +#. For each package to be installed: + + - If :ref:`pylock-packages-vcs` is set: + + #. Clone the repository to the commit ID specified in + :ref:`pylock-packages-vcs-commit-id`. + #. :ref:`Build ` the package, + respecting :ref:`pylock-packages-vcs-subdirectory`. + #. :ref:`Install `. + + - Else if :ref:`pylock-packages-directory` is set: + + #. :ref:`Build ` the package, + respecting :ref:`pylock-packages-directory-subdirectory`. + #. :ref:`Install `. + + - Else if :ref:`pylock-packages-archive` is set: + + #. Get the file. + #. Validate using :ref:`pylock-packages-archive-size` and + :ref:`pylock-packages-archive-hashes`. + #. :ref:`Build ` the package, + respecting :ref:`pylock-packages-archive-subdirectory`. + #. :ref:`Install `. + + - Else if there are entries for :ref:`pylock-packages-wheels`: + + #. Look for the appropriate wheel file based on + :ref:`pylock-packages-wheels-name`; if one is not found then move on to + :ref:`pylock-packages-sdist` or an error MUST be raised about a + lack of source for the project. + #. Get the file: + + - If :ref:`pylock-packages-wheels-path` is set, use it. + - Else if :ref:`pylock-packages-wheels-url` is set, try to use it; + optionally tools MAY use :ref:`pylock-packages-index` or some + tool-specific mechanism to download the selected wheel file (tools + MUST NOT try to change what wheel file to download based on what's + available; what file to install should be determined in an offline + fashion for reproducibility). + + #. Validate using :ref:`pylock-packages-wheels-size` and + :ref:`pylock-packages-wheels-hashes`. + #. :ref:`Install `. + + - Else if no :ref:`pylock-packages-wheels` file is found or + :ref:`pylock-packages-sdist` is solely set: + + #. Get the file. + + - If :ref:`pylock-packages-sdist-path` is set, use it. + - Else if :ref:`pylock-packages-sdist-url` is set, try to use it; tools + MAY use :ref:`pylock-packages-index` or some tool-specific mechanism + to download the file. + + #. Validate using :ref:`pylock-packages-sdist-size` and + :ref:`pylock-packages-sdist-hashes`. + #. :ref:`Build ` the package. + #. :ref:`Install `. + + +------- +History +------- + +- April 2025: Initial version, approved via :pep:`751`. + + +.. _Content-Length: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Length +.. _Dependabot: https://docs.github.com/en/code-security/dependabot +.. _HEAD: https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD +.. _PDM: https://pypi.org/project/pdm/ +.. _pip-tools: https://pypi.org/project/pip-tools/ +.. _Poetry: https://pypi.org/project/poetry/ +.. _requirements file: +.. _requirements files: https://pip.pypa.io/en/stable/reference/requirements-file-format/ +.. _software bill of materials: https://www.cisa.gov/sbom +.. _TOML: https://toml.io/ +.. _uv: https://pypi.org/project/uv/ +.. _URL: https://url.spec.whatwg.org/ diff --git a/source/specifications/pylock-toml/pylock.example.toml b/source/specifications/pylock-toml/pylock.example.toml new file mode 100644 index 000000000..45e8731b2 --- /dev/null +++ b/source/specifications/pylock-toml/pylock.example.toml @@ -0,0 +1,60 @@ +lock-version = '1.0' +environments = ["sys_platform == 'win32'", "sys_platform == 'linux'"] +requires-python = '== 3.12' +created-by = 'mousebender' + +[[packages]] +name = 'attrs' +version = '25.1.0' +requires-python = '>= 3.8' + + [[packages.wheels]] + name = 'attrs-25.1.0-py3-none-any.whl' + upload-time = 2025-01-25T11:30:10.164985+00:00 + url = 'https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl' + size = 63152 + hashes = {sha256 = 'c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a'} + + [[packages.attestation-identities]] + environment = 'release-pypi' + kind = 'GitHub' + repository = 'python-attrs/attrs' + workflow = 'pypi-package.yml' + +[[packages]] +name = 'cattrs' +version = '24.1.2' +requires-python = '>= 3.8' +dependencies = [ + {name = 'attrs'}, +] + + [[packages.wheels]] + name = 'cattrs-24.1.2-py3-none-any.whl' + upload-time = 2024-09-22T14:58:34.812643+00:00 + url = 'https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl' + size = 66446 + hashes = {sha256 = '67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0'} + +[[packages]] +name = 'numpy' +version = '2.2.3' +requires-python = '>= 3.10' + + [[packages.wheels]] + name = 'numpy-2.2.3-cp312-cp312-win_amd64.whl' + upload-time = 2025-02-13T16:51:21.821880+00:00 + url = 'https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl' + size = 12626357 + hashes = {sha256 = '83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d'} + + [[packages.wheels]] + name = 'numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl' + upload-time = 2025-02-13T16:50:00.079662+00:00 + url = 'https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl' + size = 16116679 + hashes = {sha256 = '3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe'} + +[tool.mousebender] +command = ['.', 'lock', '--platform', 'cpython3.12-windows-x64', '--platform', 'cpython3.12-manylinux2014-x64', 'cattrs', 'numpy'] +run-on = 2025-03-06T12:28:57.760769 diff --git a/source/specifications/pyproject-toml.rst b/source/specifications/pyproject-toml.rst new file mode 100644 index 000000000..48f35599e --- /dev/null +++ b/source/specifications/pyproject-toml.rst @@ -0,0 +1,652 @@ +.. _declaring-project-metadata: +.. _pyproject-toml-spec: + +================================ +``pyproject.toml`` specification +================================ + +.. warning:: + + This is a **technical, formal specification**. For a gentle, + user-friendly guide to ``pyproject.toml``, see + :ref:`writing-pyproject-toml`. + +The ``pyproject.toml`` file acts as a configuration file for packaging-related +tools (as well as other tools). + +.. note:: This specification was originally defined in :pep:`518` and :pep:`621`. + +The ``pyproject.toml`` file is written in `TOML `_. Three +tables are currently specified, namely +:ref:`[build-system] `, +:ref:`[project] ` and +:ref:`[tool] `. Other tables are reserved for future +use (tool-specific configuration should use the ``[tool]`` table). + +.. _pyproject-build-system-table: + +Declaring build system dependencies: the ``[build-system]`` table +================================================================= + +The ``[build-system]`` table declares any Python level dependencies that +must be installed in order to run the project's build system +successfully. + +.. TODO: merge with PEP 517 + +The ``[build-system]`` table is used to store build-related data. +Initially, only one key of the table is valid and is mandatory +for the table: ``requires``. This key must have a value of a list +of strings representing dependencies required to execute the +build system. The strings in this list follow the :ref:`version specifier +specification `. + +An example ``[build-system]`` table for a project built with +``setuptools`` is: + +.. code-block:: toml + + [build-system] + # Minimum requirements for the build system to execute. + requires = ["setuptools"] + +Build tools are expected to use the example configuration file above as +their default semantics when a ``pyproject.toml`` file is not present. + +Tools should not require the existence of the ``[build-system]`` table. +A ``pyproject.toml`` file may be used to store configuration details +other than build-related data and thus lack a ``[build-system]`` table +legitimately. If the file exists but is lacking the ``[build-system]`` +table then the default values as specified above should be used. +If the table is specified but is missing required fields then the tool +should consider it an error. + +Tools may choose to present an error to the user if the file exists, +``[build-system]`` table is missing, and there is no clear indication +that the project should be built (e.g., no setup.py/setup.cfg or other +build configuration files, and no ``[project]`` table). + +To provide a type-specific representation of the resulting data from +the TOML file for illustrative purposes only, the following +`JSON Schema `_ would match the data format: + +.. code-block:: json + + { + "$schema": "http://json-schema.org/schema#", + + "type": "object", + "additionalProperties": false, + + "properties": { + "build-system": { + "type": "object", + "additionalProperties": false, + + "properties": { + "requires": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["requires"] + }, + + "tool": { + "type": "object" + } + } + } + + +.. _pyproject-project-table: + +Declaring project metadata: the ``[project]`` table +=================================================== + +The ``[project]`` table specifies the project's :ref:`core metadata `. + +There are two kinds of metadata: *static* and *dynamic*. Static +metadata is specified in the ``pyproject.toml`` file directly and +cannot be specified or changed by a tool (this includes data +*referred* to by the metadata, e.g. the contents of files referenced +by the metadata). Dynamic metadata is listed via the ``dynamic`` key +(defined later in this specification) and represents metadata that a +tool will later provide. + +The lack of a ``[project]`` table implicitly means the :term:`build backend ` +will dynamically provide all keys. + +The only keys required to be statically defined are: + +- ``name`` + +The keys which are required but may be specified *either* statically +or listed as dynamic are: + +- ``version`` + +All other keys are considered optional and may be specified +statically, listed as dynamic, or left unspecified. + +The complete list of keys allowed in the ``[project]`` table are: + +- ``authors`` +- ``classifiers`` +- ``dependencies`` +- ``description`` +- ``dynamic`` +- ``entry-points`` +- ``gui-scripts`` +- ``import-names`` +- ``import-namespaces`` +- ``keywords`` +- ``license`` +- ``license-files`` +- ``maintainers`` +- ``name`` +- ``optional-dependencies`` +- ``readme`` +- ``requires-python`` +- ``scripts`` +- ``urls`` +- ``version`` + + +.. _pyproject-toml-name: + +``name`` +-------- + +- TOML_ type: string +- Corresponding :ref:`core metadata ` field: + :ref:`Name ` + +The name of the project. + +Tools SHOULD :ref:`normalize ` this name, as soon +as it is read for internal consistency. + + +.. _pyproject-toml-version: + +``version`` +----------- + +- TOML_ type: string +- Corresponding :ref:`core metadata ` field: + :ref:`Version ` + +The version of the project, as defined in the +:ref:`Version specifier specification `. + +Users SHOULD prefer to specify already-normalized versions. + + +.. _pyproject-toml-description: + +``description`` +--------------- + +- TOML_ type: string +- Corresponding :ref:`core metadata ` field: + :ref:`Summary ` + +The summary description of the project in one line. Tools MAY error +if this includes multiple lines. + + +.. _pyproject-toml-readme: + +``readme`` +---------- + +- TOML_ type: string or table +- Corresponding :ref:`core metadata ` field: + :ref:`Description ` and + :ref:`Description-Content-Type ` + +The full description of the project (i.e. the README). + +The key accepts either a string or a table. If it is a string then +it is a path relative to ``pyproject.toml`` to a text file containing +the full description. Tools MUST assume the file's encoding is UTF-8. +If the file path ends in a case-insensitive ``.md`` suffix, then tools +MUST assume the content-type is ``text/markdown``. If the file path +ends in a case-insensitive ``.rst``, then tools MUST assume the +content-type is ``text/x-rst``. If a tool recognizes more extensions +than this PEP, they MAY infer the content-type for the user without +specifying this key as ``dynamic``. For all unrecognized suffixes +when a content-type is not provided, tools MUST raise an error. + +The ``readme`` key may also take a table. The ``file`` key has a +string value representing a path relative to ``pyproject.toml`` to a +file containing the full description. The ``text`` key has a string +value which is the full description. These keys are +mutually-exclusive, thus tools MUST raise an error if the metadata +specifies both keys. + +A table specified in the ``readme`` key also has a ``content-type`` +key which takes a string specifying the content-type of the full +description. A tool MUST raise an error if the metadata does not +specify this key in the table. If the metadata does not specify the +``charset`` parameter, then it is assumed to be UTF-8. Tools MAY +support other encodings if they choose to. Tools MAY support +alternative content-types which they can transform to a content-type +as supported by the :ref:`core metadata `. Otherwise +tools MUST raise an error for unsupported content-types. + + +.. _pyproject-toml-requires-python: + +``requires-python`` +------------------- + +- TOML_ type: string +- Corresponding :ref:`core metadata ` field: + :ref:`Requires-Python ` + +The Python version requirements of the project. + + +.. _pyproject-toml-license: + +``license`` +----------- + +- TOML_ type: string +- Corresponding :ref:`core metadata ` field: + :ref:`License-Expression ` + +Text string that is a valid SPDX +:term:`license expression `, +as specified in :doc:`/specifications/license-expression`. +Tools SHOULD validate and perform case normalization of the expression. + +This key should **only** be specified if the license expression for any +and all distribution files created by a build backend using the +:file:`pyproject.toml` is the same as the one specified. If the license +expression will differ then it should either be specified as dynamic or +not set at all. + +Legacy specification +'''''''''''''''''''' + +- TOML_ type: table +- Corresponding :ref:`core metadata ` field: + :ref:`License ` + +The table may have one of two keys. The ``file`` key has a string +value that is a file path relative to :file:`pyproject.toml` to the file +which contains the license for the project. Tools MUST assume the +file's encoding is UTF-8. The ``text`` key has a string value which is +the license of the project. These keys are mutually exclusive, so a +tool MUST raise an error if the metadata specifies both keys. + +The table subkeys were deprecated by :pep:`639` in favor of the string value. + +.. _pyproject-toml-license-files: + +``license-files`` +----------------- + +- TOML_ type: array of strings +- Corresponding :ref:`core metadata ` field: + :ref:`License-File ` + +An array specifying paths in the project source tree relative to the project +root directory (i.e. directory containing :file:`pyproject.toml` or legacy project +configuration files, e.g. :file:`setup.py`, :file:`setup.cfg`, etc.) +to file(s) containing licenses and other legal notices to be +distributed with the package. + +The strings MUST contain valid glob patterns, as specified in +:doc:`/specifications/glob-patterns`. + +Patterns are relative to the directory containing :file:`pyproject.toml`, + +Tools MUST assume that license file content is valid UTF-8 encoded text, +and SHOULD validate this and raise an error if it is not. + +Build tools: + +- MUST include all files matched by a listed pattern in all distribution + archives. +- MUST list each matched file path under a License-File field in the + Core Metadata. + +If the ``license-files`` key is present and +is set to a value of an empty array, then tools MUST NOT include any +license files and MUST NOT raise an error. +If the ``license-files`` key is not defined, tools can decide how to handle +license files. For example they can choose not to include any files or use +their own logic to discover the appropriate files in the distribution. + + +.. _pyproject-toml-authors: +.. _pyproject-toml-maintainers: + +``authors``/``maintainers`` +--------------------------- + +- TOML_ type: Array of inline tables with string keys and values +- Corresponding :ref:`core metadata ` field: + :ref:`Author `, + :ref:`Author-email `, + :ref:`Maintainer `, and + :ref:`Maintainer-email ` + +The people or organizations considered to be the "authors" of the +project. The exact meaning is open to interpretation β€” it may list the +original or primary authors, current maintainers, or owners of the +package. + +The "maintainers" key is similar to "authors" in that its exact +meaning is open to interpretation. + +These keys accept an array of tables with 2 keys: ``name`` and +``email``. Both values must be strings. The ``name`` value MUST be a +valid email name (i.e. whatever can be put as a name, before an email, +in :rfc:`822`) and not contain commas. The ``email`` value MUST be a +valid email address. Both keys are optional, but at least one of the +keys must be specified in the table. + +Using the data to fill in :ref:`core metadata ` is as +follows: + +1. If only ``name`` is provided, the value goes in + :ref:`Author ` or + :ref:`Maintainer ` as appropriate. +2. If only ``email`` is provided, the value goes in + :ref:`Author-email ` or + :ref:`Maintainer-email ` + as appropriate. +3. If both ``email`` and ``name`` are provided, the value goes in + :ref:`Author-email ` or + :ref:`Maintainer-email ` + as appropriate, with the format ``{name} <{email}>``. +4. Multiple values should be separated by commas. + + +.. _pyproject-toml-keywords: + +``keywords`` +------------ + +- TOML_ type: array of strings +- Corresponding :ref:`core metadata ` field: + :ref:`Keywords ` + +The keywords for the project. + + +.. _pyproject-toml-classifiers: + +``classifiers`` +--------------- + +- TOML_ type: array of strings +- Corresponding :ref:`core metadata ` field: + :ref:`Classifier ` + +Trove classifiers which apply to the project. + +The use of ``License ::`` classifiers is deprecated and tools MAY issue a +warning informing users about that. +Build tools MAY raise an error if both the ``license`` string value +(translating to ``License-Expression`` metadata field) and the ``License ::`` +classifiers are used. + + +.. _pyproject-toml-urls: + +``urls`` +-------- + +- TOML_ type: table with keys and values of strings +- Corresponding :ref:`core metadata ` field: + :ref:`Project-URL ` + +A table of URLs where the key is the URL label and the value is the +URL itself. See :ref:`well-known-project-urls` for normalization rules +and well-known rules when processing metadata for presentation. + + +.. _pyproject-toml-scripts: +.. _pyproject-toml-gui-scripts: +.. _pyproject-toml-entry-points: + +Entry points +------------ + +- TOML_ type: table (``[project.scripts]``, ``[project.gui-scripts]``, + and ``[project.entry-points]``) +- :ref:`Entry points specification ` + +There are three tables related to entry points. The +``[project.scripts]`` table corresponds to the ``console_scripts`` +group in the :ref:`entry points specification `. The key +of the table is the name of the entry point and the value is the +object reference. + +The ``[project.gui-scripts]`` table corresponds to the ``gui_scripts`` +group in the :ref:`entry points specification `. Its +format is the same as ``[project.scripts]``. + +The ``[project.entry-points]`` table is a collection of tables. Each +sub-table's name is an entry point group. The key and value semantics +are the same as ``[project.scripts]``. Users MUST NOT create +nested sub-tables but instead keep the entry point groups to only one +level deep. + +Build back-ends MUST raise an error if the metadata defines a +``[project.entry-points.console_scripts]`` or +``[project.entry-points.gui_scripts]`` table, as they would +be ambiguous in the face of ``[project.scripts]`` and +``[project.gui-scripts]``, respectively. + + +.. _pyproject-toml-dependencies: +.. _pyproject-toml-optional-dependencies: + +``dependencies``/``optional-dependencies`` +------------------------------------------ + +- TOML_ type: Array of :pep:`508` strings (``dependencies``), and a + table with values of arrays of :pep:`508` strings + (``optional-dependencies``) +- Corresponding :ref:`core metadata ` field: + :ref:`Requires-Dist ` and + :ref:`Provides-Extra ` + +The (optional) dependencies of the project. + +For ``dependencies``, it is a key whose value is an array of strings. +Each string represents a dependency of the project and MUST be +formatted as a valid :pep:`508` string. Each string maps directly to +a :ref:`Requires-Dist ` entry. + +For ``optional-dependencies``, it is a table where each key specifies +an extra and whose value is an array of strings. The strings of the +arrays must be valid :pep:`508` strings. The keys MUST be valid values +for :ref:`Provides-Extra `. Each value +in the array thus becomes a corresponding +:ref:`Requires-Dist ` entry for the +matching :ref:`Provides-Extra ` +metadata. + + +.. _pyproject-toml-import-names: + +``import-names`` +---------------- + +- TOML_ type: array of strings +- Corresponding :ref:`core metadata ` field: + :ref:`Import-Name ` + +An array of strings specifying the import names that the project exclusively +provides when installed. Each string MUST be a valid Python identifier or can +be empty. An import name MAY be followed by a semicolon and the term "private" +(e.g. ``"; private"``) with any amount of whitespace surrounding the semicolon. + +Projects SHOULD list all the shortest import names that are exclusively provided +by the project. If any of the shortest names are dotted names, all intervening +names from that name to the top-level name should also be listed appropriately +in ``import-names`` and/or ``import-namespaces``. For instance, a project which +is a single package named spam with multiple submodules would only list +``project.import-names = ["spam"]``. A project that lists ``spam.bacon.eggs`` +would also need to account for ``spam`` and ``spam.bacon`` appropriately in +``import-names`` and ``import-namespaces``. Listing all names acts as a check +that the intent of the import names is as expected. As well, projects SHOULD +list all import names, public or private, using the ``; private`` modifier as +appropriate. + +If a project lists the same name in both ``import-names`` and +``import-namespaces``, then tools MUST raise an error due to ambiguity. + +Projects MAY set ``import-names`` to an empty array to represent a project with +no import names (i.e. there are no Python modules of any kind in the +distribution file). + +Build back-ends MAY support dynamically calculating the value if the user +declares the key in ``project.dynamic``. + +Examples: + +.. code-block:: toml + + [project] + name = "pillow" + import-names = ["PIL"] + +.. code-block:: toml + + [project] + name = "myproject" + import-names = ["mypackage", "_private_module ; private"] + + +.. _pyproject-toml-import-namespaces: + +``import-namespaces`` +--------------------- + +- TOML_ type: array of strings +- Corresponding :ref:`core metadata ` field: + :ref:`Import-Namespace ` + +An array of strings specifying the import names that the project provides when +installed, but not exclusively. Each string MUST be a valid Python identifier. +An import name MAY be followed by a semicolon and the term "private" (e.g. +``"; private"``) with any amount of whitespace surrounding the semicolon. Note +that unlike ``import-names``, ``import-namespaces`` CANNOT be an empty array. + +Projects SHOULD list all the shortest import names that are exclusively provided +by the project. If any of the shortest names are dotted names, all intervening +names from that name to the top-level name should also be listed appropriately +in ``import-names`` and/or ``import-namespaces``. + +This field is used for namespace packages where multiple projects can contribute +to the same import namespace. Projects all listing the same import name in +``import-namespaces`` can be installed together without shadowing each other. + +If a project lists the same name in both ``import-names`` and +``import-namespaces``, then tools MUST raise an error due to ambiguity. + +Build back-ends MAY support dynamically calculating the value if the user +declares the key in ``project.dynamic``. + +Example: + +.. code-block:: toml + + [project] + name = "zope-interface" + import-namespaces = ["zope"] + import-names = ["zope.interface"] + + +.. _pyproject-toml-dynamic: +.. _declaring-project-metadata-dynamic: + +``dynamic`` +----------- + +- TOML_ type: array of string +- Corresponding :ref:`core metadata ` field: + :ref:`Dynamic ` + +Specifies which keys listed by this PEP were intentionally +unspecified so another tool can/will provide such metadata +dynamically. This clearly delineates which metadata is purposefully +unspecified and expected to stay unspecified compared to being +provided via tooling later on. + +- A build back-end MUST honour statically-specified metadata (which + means the metadata did not list the key in ``dynamic``). +- A build back-end MUST raise an error if the metadata specifies + ``name`` in ``dynamic``. +- If the :ref:`core metadata ` specification lists a + field as "Required", then the metadata MUST specify the key + statically or list it in ``dynamic`` (build back-ends MUST raise an + error otherwise, i.e. it should not be possible for a required key + to not be listed somehow in the ``[project]`` table). +- If the :ref:`core metadata ` specification lists a + field as "Optional", the metadata MAY list it in ``dynamic`` if the + expectation is a build back-end will provide the data for the key + later. +- Build back-ends MUST raise an error if the metadata specifies a + key statically as well as being listed in ``dynamic``. +- If the metadata does not list a key in ``dynamic``, then a build + back-end CANNOT fill in the requisite metadata on behalf of the user + (i.e. ``dynamic`` is the only way to allow a tool to fill in + metadata and the user must opt into the filling in). +- Build back-ends MUST raise an error if the metadata specifies a + key in ``dynamic`` but the build back-end was unable to determine + the data for it (omitting the data, if determined to be the accurate + value, is acceptable). + + + +.. _pyproject-tool-table: + +Arbitrary tool configuration: the ``[tool]`` table +================================================== + +The ``[tool]`` table is where any tool related to your Python +project, not just build tools, can have users specify configuration +data as long as they use a sub-table within ``[tool]``, e.g. the +`flit `_ tool would store its +configuration in ``[tool.flit]``. + +A mechanism is needed to allocate names within the ``tool.*`` +namespace, to make sure that different projects do not attempt to use +the same sub-table and collide. Our rule is that a project can use +the subtable ``tool.$NAME`` if, and only if, they own the entry for +``$NAME`` in the Cheeseshop/PyPI. + + + +History +======= + +- May 2016: The initial specification of the ``pyproject.toml`` file, with just + a ``[build-system]`` containing a ``requires`` key and a ``[tool]`` table, was + approved through :pep:`518`. + +- November 2020: The specification of the ``[project]`` table was approved + through :pep:`621`. + +- December 2024: The ``license`` key was redefined, the ``license-files`` key was + added and ``License::`` classifiers were deprecated through :pep:`639`. + +- September 2025: Clarity that the ``license`` key applies to all distribution + files generated from the :file:`pyproject.toml` file. + +- October 2025: The ``import-names`` and ``import-namespaces`` keys were added + through :pep:`794`. + +.. _TOML: https://toml.io diff --git a/source/specifications/recording-installed-packages.rst b/source/specifications/recording-installed-packages.rst index 9085807f3..a689fa7fd 100644 --- a/source/specifications/recording-installed-packages.rst +++ b/source/specifications/recording-installed-packages.rst @@ -1,3 +1,5 @@ +.. highlight:: text + .. _recording-installed-packages: ============================ @@ -17,20 +19,6 @@ files in a format specific to Python tooling, it should still record the name and version of the installed project. -History and change workflow -=========================== - -The metadata described here was first specified in :pep:`376`, and later -amended in :pep:`627`. -It was formerly known as *Database of Installed Python Distributions*. -Further amendments (except trivial language or typography fixes) must be made -through the PEP process (see :pep:`1`). - -While this document is the normative specification, these PEPs that introduce -changes to it may include additional information such as rationales and -backwards compatibility considerations. - - The .dist-info directory ======================== @@ -40,12 +28,11 @@ packages (commonly, the ``site-packages`` directory). This directory is named as ``{name}-{version}.dist-info``, with ``name`` and ``version`` fields corresponding to :ref:`core-metadata`. Both fields must be -normalized (see :pep:`PEP 503 <503#normalized-names>` and -:pep:`PEP 440 <440#normalization>` for the definition of normalization for -each field respectively), and replace dash (``-``) characters with -underscore (``_``) characters, so the ``.dist-info`` directory always has -exactly one dash (``-``) character in its stem, separating the ``name`` and -``version`` fields. +normalized (see the :ref:`name normalization specification ` +and the :ref:`version normalization specification `), +and replace dash (``-``) characters with underscore (``_``) characters, +so the ``.dist-info`` directory always has exactly one dash (``-``) character in +its stem, separating the ``name`` and ``version`` fields. Historically, tools have failed to replace dot characters or normalize case in the ``name`` field, or not perform normalization in the ``version`` field. @@ -57,26 +44,34 @@ encouraged to start normalizing those fields. .. note:: - The ``.dist-info`` directory's name is formatted to unambigiously represent + The ``.dist-info`` directory's name is formatted to unambiguously represent a distribution as a filesystem path. Tools presenting a distribution name to a user should avoid using the normalized name, and instead present the specified name (when needed prior to resolution to an installed package), or read the respective fields in Core Metadata, since values listed there are unescaped and accurately reflect the distribution. Libraries should provide API for such tools to consume, so tools can have access to the - unnormalized name when displaying distrubution information. + unnormalized name when displaying distribution information. -This ``.dist-info`` directory can contain these files, described in detail -below: +This ``.dist-info`` directory may contain the following files, described in +detail below: * ``METADATA``: contains project metadata * ``RECORD``: records the list of installed files. * ``INSTALLER``: records the name of the tool used to install the project. +* ``entry_points.txt``: see :ref:`entry-points` for details +* ``direct_url.json``: see :ref:`direct-url` for details The ``METADATA`` file is mandatory. All other files may be omitted at the installing tool's discretion. Additional installer-specific files may be present. +This :file:`.dist-info/` directory may contain the following directories, described in +detail below: + +* :file:`licenses/`: contains license files. +* :file:`sboms/`: contains Software Bill-of-Materials files (SBOMs). + .. note:: The :ref:`binary-distribution-format` specification describes additional @@ -123,9 +118,9 @@ On Windows, directories may be separated either by forward- or backslashes (``/`` or ``\``). The *hash* is either an empty string or the name of a hash algorithm from -``hashlib.algorithms_guaranteed``, followed by the equals character ``=`` and +:py:data:`hashlib.algorithms_guaranteed`, followed by the equals character ``=`` and the digest of the file's contents, encoded with the urlsafe-base64-nopad -encoding (``base64.urlsafe_b64encode(digest)`` with trailing ``=`` removed). +encoding (:py:func:`base64.urlsafe_b64encode(digest) ` with trailing ``=`` removed). The *size* is either the empty string, or file's size in bytes, as a base 10 integer. @@ -155,7 +150,7 @@ Here is an example snippet of a possible ``RECORD`` file:: __pycache__/black.cpython-38.pyc,, __pycache__/blackd.cpython-38.pyc,, black-19.10b0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 - black-19.10b0.dist-info/LICENSE,sha256=nAQo8MO0d5hQz1vZbhGqqK_HLUqG1KNiI9erouWNbgA,1080 + black-19.10b0.dist-info/licenses/LICENSE,sha256=nAQo8MO0d5hQz1vZbhGqqK_HLUqG1KNiI9erouWNbgA,1080 black-19.10b0.dist-info/METADATA,sha256=UN40nGoVVTSpvLrTBwNsXgZdZIwoKFSrrDDHP6B7-A0,58841 black-19.10b0.dist-info/RECORD,, black.py,sha256=45IF72OgNfF8WpeNHnxV2QGfbCLubV5Xjl55cI65kYs,140161 @@ -169,9 +164,18 @@ Here is an example snippet of a possible ``RECORD`` file:: If the ``RECORD`` file is missing, tools that rely on ``.dist-info`` must not attempt to uninstall or upgrade the package. -(This does not apply to tools that rely on other sources of information, +(This restriction does not apply to tools that rely on other sources of information, such as system package managers in Linux distros.) +.. note:: + + It is *strongly discouraged* for an installed package to modify itself + (e.g., store cache files under its namespace in ``site-packages``). + Changes inside ``site-packages`` should be left to specialized installer + tools such as pip. If a package is nevertheless modified in this way, + then the ``RECORD`` must be updated, otherwise uninstalling the package + will leave unlisted files in place (possibly resulting in a zombie + namespace package). The INSTALLER file ================== @@ -197,6 +201,18 @@ For example, if a tool is asked to uninstall a project but finds no ``RECORD`` file, it may suggest that the tool named in ``INSTALLER`` may be able to do the uninstallation. + +The entry_points.txt file +========================= + +This file MAY be created by installers to indicate when packages contain +components intended for discovery and use by other code, including console +scripts and other applications that the installer has made available for +execution. + +Its detailed specification is at :ref:`entry-points`. + + The direct_url.json file ======================== @@ -207,3 +223,67 @@ This file MUST NOT be created when installing a distribution from an other type of requirement (i.e. name plus version specifier). Its detailed specification is at :ref:`direct-url`. + + +The :file:`licenses/` subdirectory +================================== + +If the metadata version is 2.4 or greater and one or more ``License-File`` +fields is specified, the :file:`.dist-info/` directory MUST contain a :file:`licenses/` +subdirectory which MUST contain the files listed in the ``License-File`` fields in +the :file:`METADATA` file at their respective paths relative to the +:file:`licenses/` directory. +Any files in this directory MUST be copied from wheels by the install tools. + + +The :file:`sboms/` subdirectory +================================== + +All files contained within the :file:`.dist-info/sboms/` directory MUST +be Software Bill-of-Materials (SBOM) files that describe software contained +within the installed package. +Any files in this directory MUST be copied from wheels by the install tools. + + +Intentionally preventing changes to installed packages +====================================================== + +In some cases (such as when needing to manage external dependencies in addition +to Python ecosystem dependencies), it is desirable for a tool that installs +packages into a Python environment to ensure that other tools are not used to +uninstall or otherwise modify that installed package, as doing so may cause +compatibility problems with the wider environment. + +To achieve this, affected tools should take the following steps: + +* Rename or remove the ``RECORD`` file to prevent changes via other tools (e.g. + appending a suffix to create a non-standard ``RECORD.tool`` file if the tool + itself needs the information, or omitting the file entirely if the package + contents are tracked and managed via other means) +* Write an ``INSTALLER`` file indicating the name of the tool that should be used + to manage the package (this allows ``RECORD``-aware tools to provide better + error notices when asked to modify affected packages) + +Python runtime providers may also prevent inadvertent modification of platform +provided packages by modifying the default Python package installation scheme +to use a location other than that used by platform provided packages (while also +ensuring both locations appear on the default Python import path). + +In some circumstances, it may be desirable to block even installation of +additional packages via Python-specific tools. For these cases refer to +:ref:`externally-managed-environments` + + +History +======= + +- June 2009: The original version of this specification was approved through + :pep:`376`. At the time, it was known as the *Database of Installed Python + Distributions*. +- March 2020: The specification of the ``direct_url.json`` file was approved + through :pep:`610`. It is only mentioned on this page; see :ref:`direct-url` + for the full definition. +- September 2020: Various amendments and clarifications were approved through + :pep:`627`. +- December 2024: The :file:`.dist-info/licenses/` directory was specified through + :pep:`639`. diff --git a/source/specifications/schemas/index.rst b/source/specifications/schemas/index.rst new file mode 100644 index 000000000..a80891975 --- /dev/null +++ b/source/specifications/schemas/index.rst @@ -0,0 +1,8 @@ +.. _`packaging-schemas`: + +PyPA schemas +############ + +- `direct_url.json `_ +- `build-details.json `_ +- `pylock.toml `_ diff --git a/source/specifications/section-distribution-formats.rst b/source/specifications/section-distribution-formats.rst new file mode 100644 index 000000000..b2c09f8db --- /dev/null +++ b/source/specifications/section-distribution-formats.rst @@ -0,0 +1,9 @@ +================================= +Package Distribution File Formats +================================= + +.. toctree:: + :titlesonly: + + source-distribution-format + binary-distribution-format diff --git a/source/specifications/section-distribution-metadata.rst b/source/specifications/section-distribution-metadata.rst new file mode 100644 index 000000000..551e6b730 --- /dev/null +++ b/source/specifications/section-distribution-metadata.rst @@ -0,0 +1,18 @@ +============================= +Package Distribution Metadata +============================= + +.. toctree:: + :titlesonly: + + name-normalization + core-metadata + version-specifiers + dependency-specifiers + pyproject-toml + dependency-groups + inline-script-metadata + platform-compatibility-tags + well-known-project-urls + glob-patterns + license-expression diff --git a/source/specifications/section-installation-metadata.rst b/source/specifications/section-installation-metadata.rst new file mode 100644 index 000000000..685a5aac4 --- /dev/null +++ b/source/specifications/section-installation-metadata.rst @@ -0,0 +1,13 @@ +============================= +Package Installation Metadata +============================= + +.. toctree:: + :titlesonly: + + recording-installed-packages + entry-points + direct-url + direct-url-data-structure + virtual-environments + externally-managed-environments diff --git a/source/specifications/section-package-indices.rst b/source/specifications/section-package-indices.rst new file mode 100644 index 000000000..1fcefe6ff --- /dev/null +++ b/source/specifications/section-package-indices.rst @@ -0,0 +1,12 @@ +======================== +Package Index Interfaces +======================== + +.. toctree:: + :titlesonly: + + pypirc + simple-repository-api + file-yanking + index-hosted-attestations + project-status-markers diff --git a/source/specifications/section-python-description-formats.rst b/source/specifications/section-python-description-formats.rst new file mode 100644 index 000000000..5a56c3968 --- /dev/null +++ b/source/specifications/section-python-description-formats.rst @@ -0,0 +1,8 @@ +========================== +Python Description Formats +========================== + +.. toctree:: + :titlesonly: + + build-details/index diff --git a/source/specifications/section-reproducible-environments.rst b/source/specifications/section-reproducible-environments.rst new file mode 100644 index 000000000..1f83f3673 --- /dev/null +++ b/source/specifications/section-reproducible-environments.rst @@ -0,0 +1,8 @@ +========================= +Reproducible Environments +========================= + +.. toctree:: + :titlesonly: + + pylock-toml diff --git a/source/specifications/simple-repository-api.rst b/source/specifications/simple-repository-api.rst index f54713d99..3b9a2ccac 100644 --- a/source/specifications/simple-repository-api.rst +++ b/source/specifications/simple-repository-api.rst @@ -5,9 +5,985 @@ Simple repository API ===================== -The current interface for querying available package versions and -retrieving packages from an index server is defined in :pep:`503`, -with the addition of "yank" support (allowing a kind of file deletion) -in :pep:`592`, specifying the interface version provided -by an index server in :pep:`629`, and providing package metadata -independently from a package in :pep:`658`. +The keywords "**MUST**", "**MUST NOT**", "**REQUIRED**", "**SHALL**", +"**SHALL NOT**", "**SHOULD**", "**SHOULD NOT**", "**RECOMMENDED**", "**MAY**", +and "**OPTIONAL**"" in this document are to be interpreted as described in +:rfc:`2119`. + +The interface for querying available package versions and +retrieving packages from an index server comes in two forms: +:ref:`HTML ` and +:ref:`JSON `. + +.. _simple-repository-api-base: + +Base API +======== + +A repository that implements the simple API is defined by its base URL. This is +the top level URL that all additional URLs are below. The API is named the +"simple" repository due to the fact that PyPI's base URL is +``https://pypi.org/simple/``. + +.. note:: All subsequent URLs in this document will be relative to this base + URL (so given PyPI's URL, a URL of ``/foo/`` would be + ``https://pypi.org/simple/foo/``. + +Normalized Names +---------------- + +This spec references the concept of a "normalized" project name. As per +:ref:`the name normalization specification ` +the only valid characters in a name are the ASCII alphabet, ASCII numbers, +``.``, ``-``, and ``_``. The name should be lowercased with all runs of the +characters ``.``, ``-``, or ``_`` replaced with a single ``-`` character. This +can be implemented in Python with the ``re`` module:: + + import re + + def normalize(name): + return re.sub(r"[-_.]+", "-", name).lower() + +.. _simple-repository-api-versioning: + +Versioning PyPI's Simple API +---------------------------- + +This spec proposes the inclusion of a meta tag on the responses of every +successful request to a simple API page, which contains a name attribute +of ``pypi:repository-version``, and a content that is a :ref:`version specifiers +specification ` compatible +version number, which is further constrained to ONLY be Major.Minor, and +none of the additional features supported by :ref:`the version specifiers +specification `. + +This would end up looking like: + +.. code-block:: html + + + +When interpreting the repository version: + +* Incrementing the major version is used to signal a backwards + incompatible change such that existing clients would no longer be + expected to be able to meaningfully use the API. +* Incrementing the minor version is used to signal a backwards + compatible change such that existing clients would still be + expected to be able to meaningfully use the API. + +It is left up to the discretion of any future specs as to what +specifically constitutes a backwards incompatible vs compatible change +beyond the broad suggestion that existing clients will be able to +"meaningfully" continue to use the API, and can include adding, +modifying, or removing existing features. + +It is expectation of this spec that the major version will never be +incremented, and any future major API evolutions would utilize a +different mechanism for API evolution. However the major version +is included to disambiguate with future versions (e.g. a hypothetical +simple api v2 that lived at /v2/, but which would be confusing if the +repository-version was set to a version >= 2). + +API Version History +~~~~~~~~~~~~~~~~~~~ + +This section contains only an abbreviated history of changes, +as marked by the API version number. For a full history of changes including +changes made before API versioning, see :ref:`History `. + +- API version 1.0: Initial version of the API, declared with :pep:`629`. +- API version 1.1: Added ``versions``, ``files[].size``, and ``files[].upload-time`` metadata + to the JSON serialization, declared with :pep:`700`. +- API version 1.2: Added repository "tracks" metadata, declared with :pep:`708`. +- API version 1.3: Added provenance metadata, declared with :pep:`740`. +- API version 1.4: Added status markers, declared with :pep:`792`. + +Clients +~~~~~~~ + +Clients interacting with the simple API **SHOULD** introspect each +response for the repository version, and if that data does not exist +**MUST** assume that it is version 1.0. + +When encountering a major version greater than expected, clients +**MUST** hard fail with an appropriate error message for the user. + +When encountering a minor version greater than expected, clients +**SHOULD** warn users with an appropriate message. + +Clients **MAY** still continue to use feature detection in order to +determine what features a repository uses. + +.. _simple-repository-html-serialization: + +HTML Serialization +------------------ + +.. _simple-repository-html-project-list: + +The following constraints apply to all HTML serialized responses described in +this spec: + +* All HTML responses **MUST** be a valid HTML5 document. +* HTML responses **MAY** contain one or more ``meta`` tags in the + ```` section. The semantics of these tags are defined below. + +Project List +~~~~~~~~~~~~ + +Within a repository, the root URL (``/`` for this spec which represents the base +URL) **MUST** be a valid HTML5 page with a single anchor element per project in +the repository. + +The text of each anchor tag **MUST** be the name of +the project and the ``href`` attribute **MUST** link to the URL for that particular +project. As an example: + +.. code-block:: html + + + + + frob + spamspamspam + + + +.. _simple-repository-html-project-detail: + +Project Detail +~~~~~~~~~~~~~~ + +Below the root URL is another URL for each individual project contained within +a repository. The format of this URL is ``//``, where the ```` +is replaced by the normalized name for that project. + +.. tip:: + + For example, a project named "HolyGrail" would have a URL like + ``/holygrail/``. + +The project detail URL must respond with a valid HTML5 page with a single +anchor element per file for the project. The ``href`` attribute **MUST** be a +URL that links to the location of the file for download, and the text of the +anchor tag **MUST** match the final path component (the filename) of the URL. + +Each file URL **SHOULD** include a hash in the +form of a URL fragment with the following syntax: ``#=``, +where ```` is the lowercase name of the hash function (such as +``sha256``) and ```` is the hex encoded digest. + +In addition to the above, the following constraints are placed on the API: + +* All URLs which respond with an HTML5 page **MUST** end with a ``/`` and the + repository **SHOULD** redirect the URLs without a ``/`` to add a ``/`` to the + end. + +* URLs may be either absolute or relative as long as they point to the correct + location. + +* There are no constraints on where the files must be hosted relative to the + repository. + +* There may be any other HTML elements on the API pages as long as the required + anchor elements exist. + +* Repositories **MAY** redirect unnormalized URLs to the canonical normalized + URL (e.g. ``/Foobar/`` may redirect to ``/foobar/``), however clients + **MUST NOT** rely on this redirection and **MUST** request the normalized + URL. + +* Repositories **SHOULD** choose a hash function from one of the ones + guaranteed to be available via the :py:mod:`hashlib` module in the Python standard + library (currently ``md5``, ``sha1``, ``sha224``, ``sha256``, ``sha384``, + ``sha512``). The current recommendation is to use ``sha256``. + +* If there is a GPG signature for a particular distribution file it **MUST** + live alongside that file with the same name with a ``.asc`` appended to it. + So if the file ``/packages/HolyGrail-1.0.tar.gz`` existed and had an + associated signature, the signature would be located at + ``/packages/HolyGrail-1.0.tar.gz.asc``. + +* A repository **MAY** include a ``data-core-metadata`` attribute on a file + link. + + The repository **SHOULD** provide the hash of the Core Metadata file as the + ``data-core-metadata`` attribute's value using the syntax + ``=``, where ```` is the lower cased name of + the hash function used, and ```` is the hex encoded digest. The + repository **MAY** use ``true`` as the attribute's value if a hash is unavailable. + +* A repository **MAY** include a ``data-dist-info-metadata`` attribute on a + file link. + + Index clients **MAY** consume this key if present, as a legacy fallback + for ``data-core-metadata``. + + .. important:: + + ``data-dist-info-metadata`` was standardized with :pep:`658` and renamed to + ``data-core-metadata`` with :pep:`714`. + +* A repository **MAY** include a ``data-gpg-sig`` attribute on a file link with + a value of either ``true`` or ``false`` to indicate whether or not there is a + GPG signature. Repositories that do this **SHOULD** include it on every link. + +* A repository **MAY** include a ``data-requires-python`` attribute on a file + link. This exposes the :ref:`core-metadata-requires-python` metadata field + for the corresponding release. Where this is present, installer tools + **SHOULD** ignore the download when installing to a Python version that + doesn't satisfy the requirement. For example: + + .. code-block:: html + + ... + + In the attribute value, < and > have to be HTML encoded as ``<`` and + ``>``, respectively. + +* A repository **MAY** include a ``data-yanked`` attribute on a file link. + + The ``data-yanked`` attribute may have no value, or may have an + arbitrary string as a value. The presence of a ``data-yanked`` attribute + **SHOULD** be interpreted as indicating that the file pointed to by this + particular link has been "Yanked", and should not generally be selected by + an installer, except under specific scenarios. + + The value of the ``data-yanked`` attribute, if present, is an arbitrary + string that represents the reason for why the file has been yanked. + + .. note:: + + The semantics of how tools should handle yanked files is + described in :ref:`file-yanking`. + +* A repository **MAY** include a ``data-provenance`` attribute on a file link. + The value of this attribute **MUST** be a fully qualified URL, signaling that + the file's provenance can be found at that URL. This URL **MUST** represent + a `secure origin `_. + + .. note:: + + The ``data-provenance`` attribute was added with API version 1.3. + + .. note:: + + The format of the linked provenance is defined in :ref:`index-hosted-attestations`. + +* A repository **MAY** include ``pypi:project-status`` and + ``pypi:project-status-reason`` meta tags on the response itself. + + The value of ``pypi:project-status`` **MUST** be a valid + project status marker, while the value of + ``pypi:project-status-reason`` **MUST** be an arbitrary string if present. + + .. note:: + + The set of valid project status markers and their semantics is described + in :ref:`project-status-markers`. + + .. note:: + + The ``pypi:project-status`` and ``pypi:project-status-reason`` meta tags + were added with API version 1.4. + +.. _simple-repository-api-metadata-file: + +Serve Distribution Metadata in the Simple Repository API +======================================================== + +In a simple repository's project page, each anchor tag pointing to a +distribution **MAY** have a ``data-dist-info-metadata`` attribute. The +presence of the attribute indicates the distribution represented by +the anchor tag **MUST** contain a Core Metadata file that will not be +modified when the distribution is processed and/or installed. + +If a ``data-dist-info-metadata`` attribute is present, the repository +**MUST** serve the distribution's Core Metadata file alongside the +distribution with a ``.metadata`` appended to the distribution's file +name. For example, the Core Metadata of a distribution served at +``/files/distribution-1.0-py3.none.any.whl`` would be located at +``/files/distribution-1.0-py3.none.any.whl.metadata``. This is similar +to how :ref:`the base HTML API specification ` +specifies the GPG signature file's location. + +The repository **SHOULD** provide the hash of the Core Metadata file +as the ``data-dist-info-metadata`` attribute's value using the syntax +``=``, where ```` is the lower cased +name of the hash function used, and ```` is the hex encoded +digest. The repository **MAY** use ``true`` as the attribute's value +if a hash is unavailable. + +Backwards Compatibility +----------------------- + +If an anchor tag lacks the ``data-dist-info-metadata`` attribute, +tools are expected to revert to their current behaviour of downloading +the distribution to inspect the metadata. + +Older tools not supporting the new ``data-dist-info-metadata`` +attribute are expected to ignore the attribute and maintain their +current behaviour of downloading the distribution to inspect the +metadata. This is similar to how prior ``data-`` attribute additions +expect existing tools to operate. + +.. _simple-repository-api-json: + +JSON-based Simple API for Python Package Indexes +================================================ + +To enable response parsing with only the standard library, this spec specifies that +all responses (besides the files themselves, and the HTML responses from +:ref:`the base HTML API specification `) should be +serialized using `JSON `_. + +To enable zero configuration discovery and to minimize the amount of additional HTTP +requests, this spec extends :ref:`the base HTML API specification +` such that all of the API endpoints (other than the +files themselves) will utilize HTTP content negotiation to allow client and server to +select the correct serialization format to serve, i.e. either HTML or JSON. + + +Versioning +---------- + +Versioning will adhere to :ref:`the API versioning specification +` format (``Major.Minor``), which has defined the +existing HTML responses to be ``1.0``. Since this spec does not introduce new features +into the API, rather it describes a different serialization format for the existing +features, this spec does not change the existing ``1.0`` version, and instead just +describes how to serialize that into JSON. + +Similar to :ref:`the API versioning specification +`, the major version number **MUST** be +incremented if any +changes to the new format would result in no longer being able to expect existing +clients to meaningfully understand the format. + +Likewise, the minor version **MUST** be incremented if features are +added or removed from the format, but existing clients would be expected to continue +to meaningfully understand the format. + +Changes that would not result in existing clients being unable to meaningfully +understand the format and which do not represent features being added or removed +may occur without changing the version number. + +This is intentionally vague, as this spec believes it is best left up to future specs +that make any changes to the API to investigate and decide whether or not that +change should increment the major or minor version. + +Future versions of the API may add things that can only be represented in a subset +of the available serializations of that version. All serializations version numbers, +within a major version, **SHOULD** be kept in sync, but the specifics of how a +feature serializes into each format may differ, including whether or not that feature +is present at all. + +It is the intent of this spec that the API should be thought of as URL endpoints that +return data, whose interpretation is defined by the version of that data, and then +serialized into the target serialization format. + + +.. _json-serialization: + +JSON Serialization +------------------ + +The URL structure from :ref:`the base HTML API specification +` still applies, as this spec only adds +an additional serialization format for the already existing API. + +The following constraints apply to all JSON serialized responses described in this +spec: + +* All JSON responses will *always* be a JSON object rather than an array or other + type. + +* While JSON doesn't natively support a URL type, any value that represents an + URL in this API may be either absolute or relative as long as they point to + the correct location. If relative, they are relative to the current URL as if + it were HTML. + +* Additional keys may be added to any dictionary objects in the API responses + and clients **MUST** ignore keys that they don't understand. + +* All JSON responses will have a ``meta`` key, which contains information related to + the response itself, rather than the content of the response. + +* All JSON responses will have a ``meta.api-version`` key, which will be a string that + contains the :ref:`API versioning specification + ` ``Major.Minor`` version number, with the + same fail/warn semantics as defined in :ref:`the API versioning specification + `. + +* All requirements of :ref:`the base HTML API specification + ` that are not HTML specific still apply. + +* Keys (at any level) with a leading underscore are reserved as private for + index server use. No future standard will assign a meaning to any such key. + +.. _simple-repository-json-project-list: + +Project List +~~~~~~~~~~~~ + +The root URL ``/`` for this spec (which represents the base URL) will be a JSON encoded +dictionary which has a two keys: + +- ``projects``: An array where each entry is a dictionary with a single key, ``name``, which represents string of the project name. +- ``meta``: The general response metadata as `described earlier `__. + +As an example: + +.. code-block:: json + + { + "meta": { + "api-version": "1.4" + }, + "projects": [ + {"name": "Frob"}, + {"name": "spamspamspam"} + ] + } + + +.. note:: + + The ``name`` field is the same as the one from :ref:`the base HTML API + specification `, which does not specify + whether it is the non-normalized display name or the normalized name. In practice + different implementations of these specs are choosing differently here, so relying + on it being either non-normalized or normalized is relying on an implementation + detail of the repository in question. + + +.. note:: + + While the ``projects`` key is an array, and thus is required to be in some kind + of an order, neither :ref:`the base HTML API specification + ` nor this spec requires any specific ordering nor + that the ordering is consistent from one request to the next. Mentally this is + best thought of as a set, but both JSON and HTML lack the functionality to have + sets. + +.. _simple-repository-json-project-detail: + +Project Detail +~~~~~~~~~~~~~~ + +The format of this URL is ``//`` where the ```` is replaced by the +:ref:`the base HTML API specification ` normalized +name for that project, so a project named "Silly_Walk" would +have a URL like ``/silly-walk/``. + +This URL must respond with a JSON encoded dictionary that has four keys: + +- ``name``: The normalized name of the project. +- ``files``: A list of dictionaries, each one representing an individual file. +- ``meta``: The general response metadata as `described earlier `__. + + In addition to the general response metadata, the project detail ``meta`` + dictionary **MAY** also include the following: + + - ``project-status``: If present, this **MUST** be a valid project status marker. + + .. note:: + + The set of valid project status markers and their semantics is described + in :ref:`project-status-markers`. + + .. note:: + + The ``project-status`` key was added with API version 1.4. + + - ``project-status-reason``: If present, this **MUST** be an arbitrary string + description of the project status. + + .. note:: + + The ``project-status-reason`` key was added with API version 1.4. + +- ``versions``: A list of version strings specifying all of the project versions + uploaded for this project. The value of ``versions`` is logically a set, + and as such may not contain duplicates, and the order of the versions is + not significant. + + .. note:: + + All of the files listed in the ``files`` key MUST be associated with one of the + versions in the ``versions`` key. The ``versions`` key MAY contain versions with + no associated files (to represent versions with no files uploaded, if the server + has such a concept). + + .. note:: + + Because servers may hold "legacy" data from before the adoption of + :ref:`the version specifiers specification (VSS) `, version + strings currently cannot be required to be valid VSS versions, and therefore + cannot be assumed to be orderable using the VSS rules. However, servers **SHOULD** + use normalized VSS versions where possible. + + .. note:: + + The ``versions`` key was added with API version 1.1. + +Each individual file dictionary has the following keys: + +- ``filename``: The filename that is being represented. +- ``url``: The URL that the file can be fetched from. +- ``hashes``: A dictionary mapping a hash name to a hex encoded digest of the file. + Multiple hashes can be included, and it is up to the client to decide what to do + with multiple hashes (it may validate all of them or a subset of them, or nothing + at all). These hash names **SHOULD** always be normalized to be lowercase. + + The ``hashes`` dictionary **MUST** be present, even if no hashes are available + for the file, however it is **HIGHLY** recommended that at least one secure, + guaranteed-to-be-available hash is always included. + + By default, any hash algorithm available via :py:mod:`hashlib` (specifically any that can + be passed to :py:func:`hashlib.new()` and do not require additional parameters) can + be used as a key for the hashes dictionary. At least one secure algorithm from + :py:data:`hashlib.algorithms_guaranteed` **SHOULD** always be included. At the time + of this spec, ``sha256`` specifically is recommended. +- ``requires-python``: An **optional** key that exposes the + :ref:`core-metadata-requires-python` + metadata field. Where this is present, installer tools + **SHOULD** ignore the download when installing to a Python version that + doesn't satisfy the requirement. + + Unlike ``data-requires-python`` in :ref:`the base HTML API specification + `, the ``requires-python`` key does not + require any special escaping other than anything JSON does naturally. +- ``core-metadata``: An **optional** key that indicates + that metadata for this file is available, via the same location as specified in + :ref:`the API metadata file specification + ` (``{file_url}.metadata``). Where this + is present, it **MUST** be + either a boolean to indicate if the file has an associated metadata file, or a + dictionary mapping hash names to a hex encoded digest of the metadata's hash. + + When this is a dictionary of hashes instead of a boolean, then all the same + requirements and recommendations as the ``hashes`` key hold true for this key as + well. + + If this key is missing then the metadata file may or may not exist. If the key + value is truthy, then the metadata file is present, and if it is falsey then it + is not. + + It is recommended that servers make the hashes of the metadata file available if + possible. + +- ``dist-info-metadata``: An **optional**, deprecated alias for ``core-metadata``. + + Index clients **MAY** consume this key if present, as a legacy fallback + for ``core-metadata``. + + .. important:: + + ``dist-info-metadata`` was standardized with :pep:`658` and renamed to + ``core-metadata`` with :pep:`714`. + +- ``gpg-sig``: An **optional** key that acts a boolean to indicate if the file has + an associated GPG signature or not. The URL for the signature file follows what + is specified in :ref:`the base HTML API specification + ` (``{file_url}.asc``). If this key does not exist, then + the signature may or may not exist. + +- ``yanked``: An **optional** key which may be either a boolean to indicate if the + file has been yanked, or a non empty, but otherwise arbitrary, string to indicate + that a file has been yanked with a specific reason. If the ``yanked`` key is present + and is a truthy value, then it **SHOULD** be interpreted as indicating that the + file pointed to by the ``url`` field has been "Yanked". + + .. note:: + + The semantics of how tools should handle yanked files is + described in :ref:`file-yanking`. + +- ``size``: A **mandatory** key. It **MUST** contain an integer which is the file size in bytes. + + .. note:: + + The ``size`` key was added with API version 1.1. + +- ``upload-time``: An **optional** key that, if present, **MUST** contain a valid + ISO 8601 date/time string in the format ``yyyy-mm-ddThh:mm:ss.ffffffZ`` + which represents the time the file was uploaded to the index. + + As indicated by the ``Z`` suffix, the upload time **MUST** use the UTC timezone. + The fractional seconds part of the timestamp (the ``.ffffff`` part) is optional, + and if present may contain up to 6 digits of precision. If a server does not record + upload time information for a file, it **MAY** omit the ``upload-time`` key. + + .. note:: + + The ``upload-time`` key was added with API version 1.1. + +- ``provenance``: An **optional** key which, if present **MUST** be either a JSON + string or ``null``. If not ``null``, it **MUST** be a URL to the file's + associated provenance, with the same rules as ``data-provenance`` in the + :ref:`base HTML API specification `. + + .. note:: + + The ``provenance`` field was added with API version 1.3. + +As an example: + +.. code-block:: json + + { + "meta": { + "api-version": "1.4", + "project-status": "active", + "project-status-reason": "this project is not yet haunted" + }, + "name": "holygrail", + "files": [ + { + "filename": "holygrail-1.0.tar.gz", + "url": "https://example.com/files/holygrail-1.0.tar.gz", + "hashes": {"sha256": "...", "blake2b": "..."}, + "requires-python": ">=3.7", + "yanked": "Had a vulnerability", + "size": 123456 + }, + { + "filename": "holygrail-1.0-py3-none-any.whl", + "url": "https://example.com/files/holygrail-1.0-py3-none-any.whl", + "hashes": {"sha256": "...", "blake2b": "..."}, + "requires-python": ">=3.7", + "dist-info-metadata": true, + "provenance": "https://example.com/files/holygrail-1.0-py3-none-any.whl.provenance", + "size": 1337 + } + ], + "versions": ["1.0"] + } + + +.. note:: + + While the ``files`` key is an array, and thus is required to be in some kind + of an order, neither :ref:`the base HTML API specification + ` nor this spec requires any specific ordering nor + that the ordering is consistent from one request to the next. Mentally this is + best thought of as a set, but both JSON and HTML lack the functionality to have + sets. + + +Content-Types +------------- + +This spec proposes that all responses from the Simple API will have a standard +content type that describes what the response is (a Simple API response), what +version of the API it represents, and what serialization format has been used. + +The structure of this content type will be: + +.. code-block:: text + + application/vnd.pypi.simple.$version+format + +Since only major versions should be disruptive to clients attempting to +understand one of these API responses, only the major version will be included +in the content type, and will be prefixed with a ``v`` to clarify that it is a +version number. + +Which means that for the existing 1.0 API, the content types would be: + +- **JSON:** ``application/vnd.pypi.simple.v1+json`` +- **HTML:** ``application/vnd.pypi.simple.v1+html`` + +In addition to the above, a special "meta" version is supported named ``latest``, +whose purpose is to allow clients to request the absolute latest version, without +having to know ahead of time what that version is. It is recommended however, +that clients be explicit about what versions they support. + +To support existing clients which expect the existing :ref:`the base HTML API +specification ` API responses to +use the ``text/html`` content type, this spec further defines ``text/html`` as an alias +for the ``application/vnd.pypi.simple.v1+html`` content type. + + +Version + Format Selection +-------------------------- + +Now that there is multiple possible serializations, we need a mechanism to allow +clients to indicate what serialization formats they're able to understand. In +addition, it would be beneficial if any possible new major version to the API can +be added without disrupting existing clients expecting the previous API version. + +To enable this, this spec standardizes on the use of HTTP's +`Server-Driven Content Negotiation `_. + +While this spec won't fully describe the entirety of server-driven content +negotiation, the flow is roughly: + +1. The client makes an HTTP request containing an ``Accept`` header listing all + of the version+format content types that they are able to understand. +2. The server inspects that header, selects one of the listed content types, + then returns a response using that content type (treating the absence of + an ``Accept`` header as ``Accept: */*``). +3. If the server does not support any of the content types in the ``Accept`` + header then they are able to choose between 3 different options for how to + respond: + + a. Select a default content type other than what the client has requested + and return a response with that. + b. Return a HTTP ``406 Not Acceptable`` response to indicate that none of + the requested content types were available, and the server was unable + or unwilling to select a default content type to respond with. + c. Return a HTTP ``300 Multiple Choices`` response that contains a list of + all of the possible responses that could have been chosen. +4. The client interprets the response, handling the different types of responses + that the server may have responded with. + +This spec does not specify which choices the server makes in regards to handling +a content type that it isn't able to return, and clients **SHOULD** be prepared +to handle all of the possible responses in whatever way makes the most sense for +that client. + +However, as there is no standard format for how a ``300 Multiple Choices`` +response can be interpreted, this spec highly discourages servers from utilizing +that option, as clients will have no way to understand and select a different +content-type to request. In addition, it's unlikely that the client *could* +understand a different content type anyways, so at best this response would +likely just be treated the same as a ``406 Not Acceptable`` error. + +This spec **does** require that if the meta version ``latest`` is being used, the +server **MUST** respond with the content type for the actual version that is +contained in the response +(i.e. an ``Accept: application/vnd.pypi.simple.latest+json`` request that returns +a ``v1.x`` response should have a ``Content-Type`` of +``application/vnd.pypi.simple.v1+json``). + +The ``Accept`` header is a comma separated list of content types that the client +understands and is able to process. It supports three different formats for each +content type that is being requested: + +- ``$type/$subtype`` +- ``$type/*`` +- ``*/*`` + +For the use of selecting a version+format, the most useful of these is +``$type/$subtype``, as that is the only way to actually specify the version +and format you want. + +The order of the content types listed in the ``Accept`` header does not have any +specific meaning, and the server **SHOULD** consider all of them to be equally +valid to respond with. If a client wishes to specify that they prefer a specific +content type over another, they may use the ``Accept`` header's +`quality value `_ +syntax. + +This allows a client to specify a priority for a specific entry in their +``Accept`` header, by appending a ``;q=`` followed by a value between ``0`` and +``1`` inclusive, with up to 3 decimal digits. When interpreting this value, +an entry with a higher quality has priority over an entry with a lower quality, +and any entry without a quality present will default to a quality of ``1``. + +However, clients should keep in mind that a server is free to select **any** of +the content types they've asked for, regardless of their requested priority, and +it may even return a content type that they did **not** ask for. + +To aid clients in determining the content type of the response that they have +received from an API request, this spec requires that servers always include a +``Content-Type`` header indicating the content type of the response. This is +technically a backwards incompatible change, however in practice +`pip has been enforcing this requirement `_ +so the risks for actual breakages is low. + +An example of how a client can operate would look like: + +.. code-block:: python + + import email.message + import requests + + def parse_content_type(header: str) -> str: + m = email.message.Message() + m["content-type"] = header + return m.get_content_type() + + # Construct our list of acceptable content types, we want to prefer + # that we get a v1 response serialized using JSON, however we also + # can support a v1 response serialized using HTML. For compatibility + # we also request text/html, but we prefer it least of all since we + # don't know if it's actually a Simple API response, or just some + # random HTML page that we've gotten due to a misconfiguration. + CONTENT_TYPES = [ + "application/vnd.pypi.simple.v1+json", + "application/vnd.pypi.simple.v1+html;q=0.2", + "text/html;q=0.01", # For legacy compatibility + ] + ACCEPT = ", ".join(CONTENT_TYPES) + + + # Actually make our request to the API, requesting all of the content + # types that we find acceptable, and letting the server select one of + # them out of the list. + resp = requests.get("https://pypi.org/simple/", headers={"Accept": ACCEPT}) + + # If the server does not support any of the content types you requested, + # AND it has chosen to return a HTTP 406 error instead of a default + # response then this will raise an exception for the 406 error. + resp.raise_for_status() + + + # Determine what kind of response we've gotten to ensure that it is one + # that we can support, and if it is, dispatch to a function that will + # understand how to interpret that particular version+serialization. If + # we don't understand the content type we've gotten, then we'll raise + # an exception. + content_type = parse_content_type(resp.headers.get("content-type", "")) + match content_type: + case "application/vnd.pypi.simple.v1+json": + handle_v1_json(resp) + case "application/vnd.pypi.simple.v1+html" | "text/html": + handle_v1_html(resp) + case _: + raise Exception(f"Unknown content type: {content_type}") + +If a client wishes to only support HTML or only support JSON, then they would +just remove the content types that they do not want from the ``Accept`` header, +and turn receiving them into an error. + + +Alternative Negotiation Mechanisms +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While using HTTP's Content negotiation is considered the standard way for a client +and server to coordinate to ensure that the client is getting an HTTP response that +it is able to understand, there are situations where that mechanism may not be +sufficient. For those cases this spec has alternative negotiation mechanisms that +may *optionally* be used instead. + + +URL Parameter +^^^^^^^^^^^^^ + +Servers that implement the Simple API may choose to support a URL parameter named +``format`` to allow the clients to request a specific version of the URL. + +The value of the ``format`` parameter should be **one** of the valid content types. +Passing multiple content types, wild cards, quality values, etc... is **not** +supported. + +Supporting this parameter is optional, and clients **SHOULD NOT** rely on it for +interacting with the API. This negotiation mechanism is intended to allow for easier +human based exploration of the API within a browser, or to allow documentation or +notes to link to a specific version+format. + +Servers that do not support this parameter may choose to return an error when it is +present, or they may simple ignore its presence. + +When a server does implement this parameter, it **SHOULD** take precedence over any +values in the client's ``Accept`` header, and if the server does not support the +requested format, it may choose to fall back to the ``Accept`` header, or choose any +of the error conditions that standard server-driven content negotiation typically +has (e.g. ``406 Not Available``, ``303 Multiple Choices``, or selecting a default +type to return). + + +Endpoint Configuration +^^^^^^^^^^^^^^^^^^^^^^ + +This option technically is not a special option at all, it is just a natural +consequence of using content negotiation and allowing servers to select which of the +available content types is their default. + +If a server is unwilling or unable to implement the server-driven content negotiation, +and would instead rather require users to explicitly configure their client to select +the version they want, then that is a supported configuration. + +To enable this, a server should make multiple endpoints (for instance, +``/simple/v1+html/`` and/or ``/simple/v1+json/``) for each version+format that they +wish to support. Under that endpoint, they can host a copy of their repository that +only supports one (or a subset) of the content-types. When a client makes a request +using the ``Accept`` header, the server can ignore it and return the content type +that corresponds to that endpoint. + +For clients that wish to require specific configuration, they can keep track of +which version+format a specific repository URL was configured for, and when making +a request to that server, emit an ``Accept`` header that *only* includes the correct +content type. + +Recommendations +--------------- + +This section is non-normative, and represents what the spec authors believe to be +the best default implementation decisions for something implementing this spec, but +it does **not** represent any sort of requirement to match these decisions. + +These decisions have been chosen to maximize the number of requests that can be +moved onto the newest version of an API, while maintaining the greatest amount +of compatibility. In addition, they've also tried to make using the API provide +guardrails that attempt to push clients into making the best choices it can. + +It is recommended that servers: + +- Support all 3 content types described in this spec, using server-driven + content negotiation, for as long as they reasonably can, or at least as + long as they're receiving non trivial traffic that uses the HTML responses. + +- When encountering an ``Accept`` header that does not contain any content types + that it knows how to work with, the server should not ever return a + ``300 Multiple Choice`` response, and instead return a ``406 Not Acceptable`` + response. + + - However, if choosing to use the endpoint configuration, you should prefer to + return a ``200 OK`` response in the expected content type for that endpoint. + +- When selecting an acceptable version, the server should choose the highest version + that the client supports, with the most expressive/featureful serialization format, + taking into account the specificity of the client requests as well as any + quality priority values they have expressed, and it should only use the + ``text/html`` content type as a last resort. + +It is recommended that clients: + +- Support all 3 content types described in this spec, using server-driven + content negotiation, for as long as they reasonably can. + +- When constructing an ``Accept`` header, include all of the content types + that you support. + + You should generally *not* include a quality priority value for your content + types, unless you have implementation specific reasons that you want the + server to take into account (for example, if you're using the standard library + HTML parser and you're worried that there may be some kinds of HTML responses + that you're unable to parse in some edge cases). + + The one exception to this recommendation is that it is recommended that you + *should* include a ``;q=0.01`` value on the legacy ``text/html`` content type, + unless it is the only content type that you are requesting. + +- Explicitly select what versions they are looking for, rather than using the + ``latest`` meta version during normal operation. + +- Check the ``Content-Type`` of the response and ensure it matches something + that you were expecting. + +.. _simple-repository-history: + +History +======= + +* September 2015: initial form of the HTML format, in :pep:`503` +* July 2016: Requires-Python metadata, in an update to :pep:`503` +* May 2019: "yank" support, in :pep:`592` +* July 2020: API versioning convention and metadata, and declaring the HTML + format as API v1, in :pep:`629` +* May 2021: providing package metadata independently from a package, in + :pep:`658` +* May 2022: initial form of the JSON format, with a mechanism for clients to + choose between them, and declaring both formats as API v1, in :pep:`691` +* October 2022: project versions and file size and upload-time in the JSON + format, in :pep:`700` +* June 2023: renaming the field which provides package metadata independently + from a package, in :pep:`714` +* November 2024: provenance metadata in the HTML and JSON formats, in :pep:`740` +* July 2025: project status markers in the HTML and JSON formats, in :pep:`792` +* July 2025: layout changes (dedicated page for file yanking, introduce concepts before API details) diff --git a/source/specifications/source-distribution-format.rst b/source/specifications/source-distribution-format.rst index 385828e31..9ac93be7b 100644 --- a/source/specifications/source-distribution-format.rst +++ b/source/specifications/source-distribution-format.rst @@ -5,7 +5,7 @@ Source distribution format ========================== -The current standard format of source distribution format is identified by the +The current standard source distribution format is identified by the presence of a :file:`pyproject.toml` file in the distribution archive. The layout of such a distribution was originally specified in :pep:`517` and is formally documented here. @@ -20,16 +20,20 @@ specification. Source distributions are also known as *sdists* for short. +.. _source-distribution-format-source-tree: + Source trees ============ A *source tree* is a collection of files and directories -- like a version control system checkout -- which contains a :file:`pyproject.toml` file that -can be use to build a source distribution from the contained files and +can be used to build a source distribution from the contained files and directories. :pep:`517` and :pep:`518` specify what is required to meet the definition of what :file:`pyproject.toml` must contain for something to be deemed a source tree. +.. _source-distribution-format-sdist: + Source distribution file name ============================= @@ -43,7 +47,8 @@ The name and version components of the filename MUST match the values stored in the metadata contained in the file. Code that produces a source distribution file MUST give the file a name that matches -this specification. This includes the ``build_sdist`` hook of a build backend. +this specification. This includes the ``build_sdist`` hook of a +:term:`build backend `. Code that processes source distribution files MAY recognise source distribution files by the ``.tar.gz`` suffix and the presence of precisely *one* hyphen in the filename. @@ -57,13 +62,102 @@ A ``.tar.gz`` source distribution (sdist) contains a single top-level directory called ``{name}-{version}`` (e.g. ``foo-1.0``), containing the source files of the package. The name and version MUST match the metadata stored in the file. This directory must also contain a :file:`pyproject.toml` in the format defined in -:ref:`declaring-build-dependencies`, and a ``PKG-INFO`` file containing +:ref:`pyproject-toml-spec`, and a :file:`PKG-INFO` file containing metadata in the format described in the :ref:`core-metadata` specification. The metadata MUST conform to at least version 2.2 of the metadata specification. +If the metadata version is 2.4 or greater, the source distribution MUST contain +any license files specified by the ``License-File`` field in the :file:`PKG-INFO` +at their respective paths relative to the root directory of the sdist +(containing the :file:`pyproject.toml` and the :file:`PKG-INFO` metadata). + No other content of a sdist is required or defined. Build systems can store whatever information they need in the sdist to build the project. The tarball should use the modern POSIX.1-2001 pax tar format, which specifies UTF-8 based file names. In particular, source distribution files must be readable using the standard library tarfile module with the open flag 'r:gz'. + + +.. _sdist-archive-features: + +Source distribution archive features +==================================== + +Because extracting tar files as-is is dangerous, and the results are +platform-specific, archive features of source distributions are limited. + +Unpacking with the data filter +------------------------------ + +When extracting a source distribution, tools MUST either use +:py:func:`tarfile.data_filter` (e.g. :py:meth:`TarFile.extractall(..., filter='data') `), OR +follow the *Unpacking without the data filter* section below. + +As an exception, on Python interpreters without :py:func:`hasattr(tarfile, 'data_filter') ` +(:pep:`706`), tools that normally use that filter (directly on indirectly) +MAY warn the user and ignore this specification. +The trade-off between usability (e.g. fully trusting the archive) and +security (e.g. refusing to unpack) is left up to the tool in this case. + + +Unpacking without the data filter +--------------------------------- + +Tools that do not use the ``data`` filter directly (e.g. for backwards +compatibility, allowing additional features, or not using Python) MUST follow +this section. +(At the time of this writing, the ``data`` filter also follows this section, +but it may get out of sync in the future.) + +The following files are invalid in an *sdist* archive. +Upon encountering such an entry, tools SHOULD notify the user, +MUST NOT unpack the entry, and MAY abort with a failure: + +- Files that would be placed outside the destination directory. +- Links (symbolic or hard) pointing outside the destination directory. +- Device files (including pipes). + +The following are also invalid. Tools MAY treat them as above, +but are NOT REQUIRED to do so: + +- Files with a ``..`` component in the filename or link target. +- Links pointing to a file that is not part of the archive. + +Tools MAY unpack links (symbolic or hard) as regular files, +using content from the archive. + +When extracting *sdist* archives: + +- Leading slashes in file names MUST be dropped. + (This is nowadays standard behaviour for ``tar`` unpacking.) +- For each ``mode`` (Unix permission) bit, tools MUST either: + + - use the platform's default for a new file/directory (respectively), + - set the bit according to the archive, or + - use the bit from ``rw-r--r--`` (``0o644``) for non-executable files or + ``rwxr-xr-x`` (``0o755``) for executable files and directories. + +- High ``mode`` bits (setuid, setgid, sticky) MUST be cleared. +- It is RECOMMENDED to preserve the user *executable* bit. + + +Further hints +------------- + +Tool authors are encouraged to consider how *hints for further +verification* in ``tarfile`` documentation apply to their tool. + + +History +======= + +* November 2020: The original version of this specification was approved through + :pep:`643`. +* July 2021: Defined what a source tree is. +* September 2022: The filename of a source distribution was standardized through + :pep:`625`. +* August 2023: Source distribution archive features were standardized through + :pep:`721`. +* December 2024: License files inclusion into source distribution was standardized + through :pep:`639`. diff --git a/source/specifications/version-specifiers.rst b/source/specifications/version-specifiers.rst index 66a709a1f..13015794f 100644 --- a/source/specifications/version-specifiers.rst +++ b/source/specifications/version-specifiers.rst @@ -1,3 +1,4 @@ +.. highlight:: text .. _version-specifiers: @@ -5,8 +6,1270 @@ Version specifiers ================== -Version numbering requirements and the semantics for specifying comparisons -between versions are defined in :pep:`440`. -The version specifiers section in this PEP supersedes the version specifiers -section in :pep:`345`. +This specification describes a scheme for identifying versions of Python software +distributions, and declaring dependencies on particular versions. + + +Definitions +=========== + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", +"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this +document are to be interpreted as described in :rfc:`2119`. + +"Build tools" are automated tools intended to run on development systems, +producing source and binary distribution archives. Build tools may also be +invoked by integration tools in order to build software distributed as +sdists rather than prebuilt binary archives. + +"Index servers" are active distribution registries which publish version and +dependency metadata and place constraints on the permitted metadata. + +"Publication tools" are automated tools intended to run on development +systems and upload source and binary distribution archives to index servers. + +"Installation tools" are integration tools specifically intended to run on +deployment targets, consuming source and binary distribution archives from +an index server or other designated location and deploying them to the target +system. + +"Automated tools" is a collective term covering build tools, index servers, +publication tools, integration tools and any other software that produces +or consumes distribution version and dependency metadata. + + +Version scheme +============== + +Distributions are identified by a public version identifier which +supports all defined version comparison operations + +The version scheme is used both to describe the distribution version +provided by a particular distribution archive, as well as to place +constraints on the version of dependencies needed in order to build or +run the software. + + +.. _public-version-identifiers: + +Public version identifiers +-------------------------- + +The canonical public version identifiers MUST comply with the following +scheme:: + + [N!]N(.N)*[{a|b|rc}N][.postN][.devN] + +Public version identifiers MUST NOT include leading or trailing whitespace. + +Public version identifiers MUST be unique within a given distribution. + +Installation tools SHOULD ignore any public versions which do not comply with +this scheme but MUST also include the normalizations specified below. +Installation tools MAY warn the user when non-compliant or ambiguous versions +are detected. + +See also :ref:`version-specifiers-regex` which provides a regular +expression to check strict conformance with the canonical format, as +well as a more permissive regular expression accepting inputs that may +require subsequent normalization. + +Public version identifiers are separated into up to five segments: + +* Epoch segment: ``N!`` +* Release segment: ``N(.N)*`` +* Pre-release segment: ``{a|b|rc}N`` +* Post-release segment: ``.postN`` +* Development release segment: ``.devN`` + +Any given release will be a "final release", "pre-release", "post-release" or +"developmental release" as defined in the following sections. + +All numeric components MUST be non-negative integers represented as sequences +of ASCII digits. + +All numeric components MUST be interpreted and ordered according to their +numeric value, not as text strings. + +All numeric components MAY be zero. Except as described below for the +release segment, a numeric component of zero has no special significance +aside from always being the lowest possible value in the version ordering. + +.. note:: + + Some hard to read version identifiers are permitted by this scheme in + order to better accommodate the wide range of versioning practices + across existing public and private Python projects. + + Accordingly, some of the versioning practices which are technically + permitted by the specification are strongly discouraged for new projects. Where + this is the case, the relevant details are noted in the following + sections. + + +.. _local-version-identifiers: + +Local version identifiers +------------------------- + +Local version identifiers MUST comply with the following scheme:: + + [+] + +They consist of a normal public version identifier (as defined in the +previous section), along with an arbitrary "local version label", separated +from the public version identifier by a plus. Local version labels have +no specific semantics assigned, but some syntactic restrictions are imposed. + +Local version identifiers are used to denote fully API (and, if applicable, +ABI) compatible patched versions of upstream projects. For example, these +may be created by application developers and system integrators by applying +specific backported bug fixes when upgrading to a new upstream release would +be too disruptive to the application or other integrated system (such as a +Linux distribution). + +The inclusion of the local version label makes it possible to differentiate +upstream releases from potentially altered rebuilds by downstream +integrators. The use of a local version identifier does not affect the kind +of a release but, when applied to a source distribution, does indicate that +it may not contain the exact same code as the corresponding upstream release. + +To ensure local version identifiers can be readily incorporated as part of +filenames and URLs, and to avoid formatting inconsistencies in hexadecimal +hash representations, local version labels MUST be limited to the following +set of permitted characters: + +* ASCII letters (``[a-zA-Z]``) +* ASCII digits (``[0-9]``) +* periods (``.``) + +Local version labels MUST start and end with an ASCII letter or digit. + +Comparison and ordering of local versions considers each segment of the local +version (divided by a ``.``) separately. If a segment consists entirely of +ASCII digits then that section should be considered an integer for comparison +purposes and if a segment contains any ASCII letters then that segment is +compared lexicographically with case insensitivity. When comparing a numeric +and lexicographic segment, the numeric section always compares as greater than +the lexicographic segment. Additionally a local version with a great number of +segments will always compare as greater than a local version with fewer +segments, as long as the shorter local version's segments match the beginning +of the longer local version's segments exactly. + +An "upstream project" is a project that defines its own public versions. A +"downstream project" is one which tracks and redistributes an upstream project, +potentially backporting security and bug fixes from later versions of the +upstream project. + +Local version identifiers SHOULD NOT be used when publishing upstream +projects to a public index server, but MAY be used to identify private +builds created directly from the project source. Local +version identifiers SHOULD be used by downstream projects when releasing a +version that is API compatible with the version of the upstream project +identified by the public version identifier, but contains additional changes +(such as bug fixes). As the Python Package Index is intended solely for +indexing and hosting upstream projects, it MUST NOT allow the use of local +version identifiers. + +Source distributions using a local version identifier SHOULD provide the +``python.integrator`` extension metadata (as defined in :pep:`459`). + + +Final releases +-------------- + +A version identifier that consists solely of a release segment and optionally +an epoch identifier is termed a "final release". + +The release segment consists of one or more non-negative integer +values, separated by dots:: + + N(.N)* + +Final releases within a project MUST be numbered in a consistently +increasing fashion, otherwise automated tools will not be able to upgrade +them correctly. + +Comparison and ordering of release segments considers the numeric value +of each component of the release segment in turn. When comparing release +segments with different numbers of components, the shorter segment is +padded out with additional zeros as necessary. + +While any number of additional components after the first are permitted +under this scheme, the most common variants are to use two components +("major.minor") or three components ("major.minor.micro"). + +For example:: + + 0.9 + 0.9.1 + 0.9.2 + ... + 0.9.10 + 0.9.11 + 1.0 + 1.0.1 + 1.1 + 2.0 + 2.0.1 + ... + +A release series is any set of final release numbers that start with a +common prefix. For example, ``3.3.1``, ``3.3.5`` and ``3.3.9.45`` are all +part of the ``3.3`` release series. + +.. note:: + + ``X.Y`` and ``X.Y.0`` are not considered distinct release numbers, as + the release segment comparison rules implicit expand the two component + form to ``X.Y.0`` when comparing it to any release segment that includes + three components. + +Date-based release segments are also permitted. An example of a date-based +release scheme using the year and month of the release:: + + 2012.4 + 2012.7 + 2012.10 + 2013.1 + 2013.6 + ... + + +.. _pre-release-versions: + +Pre-releases +------------ + +Some projects use an "alpha, beta, release candidate" pre-release cycle to +support testing by their users prior to a final release. + +If used as part of a project's development cycle, these pre-releases are +indicated by including a pre-release segment in the version identifier:: + + X.YaN # Alpha release + X.YbN # Beta release + X.YrcN # Release Candidate + X.Y # Final release + +A version identifier that consists solely of a release segment and a +pre-release segment is termed a "pre-release". + +The pre-release segment consists of an alphabetical identifier for the +pre-release phase, along with a non-negative integer value. Pre-releases for +a given release are ordered first by phase (alpha, beta, release candidate) +and then by the numerical component within that phase. + +Installation tools MAY accept both ``c`` and ``rc`` releases for a common +release segment in order to handle some existing legacy distributions. + +Installation tools SHOULD interpret ``c`` versions as being equivalent to +``rc`` versions (that is, ``c1`` indicates the same version as ``rc1``). + +Build tools, publication tools and index servers SHOULD disallow the creation +of both ``rc`` and ``c`` releases for a common release segment. + + +Post-releases +------------- + +Some projects use post-releases to address minor errors in a final release +that do not affect the distributed software (for example, correcting an error +in the release notes). + +If used as part of a project's development cycle, these post-releases are +indicated by including a post-release segment in the version identifier:: + + X.Y.postN # Post-release + +A version identifier that includes a post-release segment without a +developmental release segment is termed a "post-release". + +The post-release segment consists of the string ``.post``, followed by a +non-negative integer value. Post-releases are ordered by their +numerical component, immediately following the corresponding release, +and ahead of any subsequent release. + +.. note:: + + The use of post-releases to publish maintenance releases containing + actual bug fixes is strongly discouraged. In general, it is better + to use a longer release number and increment the final component + for each maintenance release. + +Post-releases are also permitted for pre-releases:: + + X.YaN.postM # Post-release of an alpha release + X.YbN.postM # Post-release of a beta release + X.YrcN.postM # Post-release of a release candidate + +.. note:: + + Creating post-releases of pre-releases is strongly discouraged, as + it makes the version identifier difficult to parse for human readers. + In general, it is substantially clearer to simply create a new + pre-release by incrementing the numeric component. + + +Developmental releases +---------------------- + +Some projects make regular developmental releases, and system packagers +(especially for Linux distributions) may wish to create early releases +directly from source control which do not conflict with later project +releases. + +If used as part of a project's development cycle, these developmental +releases are indicated by including a developmental release segment in the +version identifier:: + + X.Y.devN # Developmental release + +A version identifier that includes a developmental release segment is +termed a "developmental release". + +The developmental release segment consists of the string ``.dev``, +followed by a non-negative integer value. Developmental releases are ordered +by their numerical component, immediately before the corresponding release +(and before any pre-releases with the same release segment), and following +any previous release (including any post-releases). + +Developmental releases are also permitted for pre-releases and +post-releases:: + + X.YaN.devM # Developmental release of an alpha release + X.YbN.devM # Developmental release of a beta release + X.YrcN.devM # Developmental release of a release candidate + X.Y.postN.devM # Developmental release of a post-release + +Do note that development releases are considered a type of pre-release when +handling them. + +.. note:: + + While they may be useful for continuous integration purposes, publishing + developmental releases of pre-releases to general purpose public index + servers is strongly discouraged, as it makes the version identifier + difficult to parse for human readers. If such a release needs to be + published, it is substantially clearer to instead create a new + pre-release by incrementing the numeric component. + + Developmental releases of post-releases are also strongly discouraged, + but they may be appropriate for projects which use the post-release + notation for full maintenance releases which may include code changes. + + +Version epochs +-------------- + +If included in a version identifier, the epoch appears before all other +components, separated from the release segment by an exclamation mark:: + + E!X.Y # Version identifier with epoch + +If no explicit epoch is given, the implicit epoch is ``0``. + +Most version identifiers will not include an epoch, as an explicit epoch is +only needed if a project *changes* the way it handles version numbering in +a way that means the normal version ordering rules will give the wrong +answer. For example, if a project is using date based versions like +``2014.04`` and would like to switch to semantic versions like ``1.0``, then +the new releases would be identified as *older* than the date based releases +when using the normal sorting scheme:: + + 1.0 + 1.1 + 2.0 + 2013.10 + 2014.04 + +However, by specifying an explicit epoch, the sort order can be changed +appropriately, as all versions from a later epoch are sorted after versions +from an earlier epoch:: + + 2013.10 + 2014.04 + 1!1.0 + 1!1.1 + 1!2.0 + + +.. _version-specifiers-normalization: + +Normalization +------------- + +In order to maintain better compatibility with existing versions there are a +number of "alternative" syntaxes that MUST be taken into account when parsing +versions. These syntaxes MUST be considered when parsing a version, however +they should be "normalized" to the standard syntax defined above. + + +Case sensitivity +~~~~~~~~~~~~~~~~ + +All ascii letters should be interpreted case insensitively within a version and +the normal form is lowercase. This allows versions such as ``1.1RC1`` which +would be normalized to ``1.1rc1``. + + +Integer Normalization +~~~~~~~~~~~~~~~~~~~~~ + +All integers are interpreted via the ``int()`` built in and normalize to the +string form of the output. This means that an integer version of ``00`` would +normalize to ``0`` while ``09000`` would normalize to ``9000``. This does not +hold true for integers inside of an alphanumeric segment of a local version +such as ``1.0+foo0100`` which is already in its normalized form. + + +Pre-release separators +~~~~~~~~~~~~~~~~~~~~~~ + +Pre-releases should allow a ``.``, ``-``, or ``_`` separator between the +release segment and the pre-release segment. The normal form for this is +without a separator. This allows versions such as ``1.1.a1`` or ``1.1-a1`` +which would be normalized to ``1.1a1``. It should also allow a separator to +be used between the pre-release signifier and the numeral. This allows versions +such as ``1.0a.1`` which would be normalized to ``1.0a1``. + + +Pre-release spelling +~~~~~~~~~~~~~~~~~~~~ + +Pre-releases allow the additional spellings of ``alpha``, ``beta``, ``c``, +``pre``, and ``preview`` for ``a``, ``b``, ``rc``, ``rc``, and ``rc`` +respectively. This allows versions such as ``1.1alpha1``, ``1.1beta2``, or +``1.1c3`` which normalize to ``1.1a1``, ``1.1b2``, and ``1.1rc3``. In every +case the additional spelling should be considered equivalent to their normal +forms. + + +Implicit pre-release number +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Pre releases allow omitting the numeral in which case it is implicitly assumed +to be ``0``. The normal form for this is to include the ``0`` explicitly. This +allows versions such as ``1.2a`` which is normalized to ``1.2a0``. + + +Post release separators +~~~~~~~~~~~~~~~~~~~~~~~ + +Post releases allow a ``.``, ``-``, or ``_`` separator as well as omitting the +separator all together. The normal form of this is with the ``.`` separator. +This allows versions such as ``1.2-post2`` or ``1.2post2`` which normalize to +``1.2.post2``. Like the pre-release separator this also allows an optional +separator between the post release signifier and the numeral. This allows +versions like ``1.2.post-2`` which would normalize to ``1.2.post2``. + + +Post release spelling +~~~~~~~~~~~~~~~~~~~~~ + +Post-releases allow the additional spellings of ``rev`` and ``r``. This allows +versions such as ``1.0-r4`` which normalizes to ``1.0.post4``. As with the +pre-releases the additional spellings should be considered equivalent to their +normal forms. + + +Implicit post release number +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Post releases allow omitting the numeral in which case it is implicitly assumed +to be ``0``. The normal form for this is to include the ``0`` explicitly. This +allows versions such as ``1.2.post`` which is normalized to ``1.2.post0``. + + +Implicit post releases +~~~~~~~~~~~~~~~~~~~~~~ + +Post releases allow omitting the ``post`` signifier all together. When using +this form the separator MUST be ``-`` and no other form is allowed. This allows +versions such as ``1.0-1`` to be normalized to ``1.0.post1``. This particular +normalization MUST NOT be used in conjunction with the implicit post release +number rule. In other words, ``1.0-`` is *not* a valid version and it does *not* +normalize to ``1.0.post0``. + + +Development release separators +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Development releases allow a ``.``, ``-``, or a ``_`` separator as well as +omitting the separator all together. The normal form of this is with the ``.`` +separator. This allows versions such as ``1.2-dev2`` or ``1.2dev2`` which +normalize to ``1.2.dev2``. + + +Implicit development release number +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Development releases allow omitting the numeral in which case it is implicitly +assumed to be ``0``. The normal form for this is to include the ``0`` +explicitly. This allows versions such as ``1.2.dev`` which is normalized to +``1.2.dev0``. + + +Local version segments +~~~~~~~~~~~~~~~~~~~~~~ + +With a local version, in addition to the use of ``.`` as a separator of +segments, the use of ``-`` and ``_`` is also acceptable. The normal form is +using the ``.`` character. This allows versions such as ``1.0+ubuntu-1`` to be +normalized to ``1.0+ubuntu.1``. + + +Preceding v character +~~~~~~~~~~~~~~~~~~~~~ + +In order to support the common version notation of ``v1.0`` versions may be +preceded by a single literal ``v`` character. This character MUST be ignored +for all purposes and should be omitted from all normalized forms of the +version. The same version with and without the ``v`` is considered equivalent. + + +Leading and Trailing Whitespace +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Leading and trailing whitespace must be silently ignored and removed from all +normalized forms of a version. This includes ``" "``, ``\t``, ``\n``, ``\r``, +``\f``, and ``\v``. This allows accidental whitespace to be handled sensibly, +such as a version like ``1.0\n`` which normalizes to ``1.0``. + + +Examples of compliant version schemes +------------------------------------- + +The standard version scheme is designed to encompass a wide range of +identification practices across public and private Python projects. In +practice, a single project attempting to use the full flexibility offered +by the scheme would create a situation where human users had difficulty +figuring out the relative order of versions, even though the rules above +ensure all compliant tools will order them consistently. + +The following examples illustrate a small selection of the different +approaches projects may choose to identify their releases, while still +ensuring that the "latest release" and the "latest stable release" can +be easily determined, both by human users and automated tools. + +Simple "major.minor" versioning:: + + 0.1 + 0.2 + 0.3 + 1.0 + 1.1 + ... + +Simple "major.minor.micro" versioning:: + + 1.1.0 + 1.1.1 + 1.1.2 + 1.2.0 + ... + +"major.minor" versioning with alpha, beta and candidate +pre-releases:: + + 0.9 + 1.0a1 + 1.0a2 + 1.0b1 + 1.0rc1 + 1.0 + 1.1a1 + ... + +"major.minor" versioning with developmental releases, release candidates +and post-releases for minor corrections:: + + 0.9 + 1.0.dev1 + 1.0.dev2 + 1.0.dev3 + 1.0.dev4 + 1.0c1 + 1.0c2 + 1.0 + 1.0.post1 + 1.1.dev1 + ... + +Date based releases, using an incrementing serial within each year, skipping +zero:: + + 2012.1 + 2012.2 + 2012.3 + ... + 2012.15 + 2013.1 + 2013.2 + ... + + +Summary of permitted suffixes and relative ordering +--------------------------------------------------- + +.. note:: + + This section is intended primarily for authors of tools that + automatically process distribution metadata, rather than developers + of Python distributions deciding on a versioning scheme. + +The epoch segment of version identifiers MUST be sorted according to the +numeric value of the given epoch. If no epoch segment is present, the +implicit numeric value is ``0``. + +The release segment of version identifiers MUST be sorted in +the same order as Python's tuple sorting when the normalized release segment is +parsed as follows:: + + tuple(map(int, release_segment.split("."))) + +All release segments involved in the comparison MUST be converted to a +consistent length by padding shorter segments with zeros as needed. + +Within a numeric release (``1.0``, ``2.7.3``), the following suffixes +are permitted and MUST be ordered as shown:: + + .devN, aN, bN, rcN, , .postN + +Note that ``c`` is considered to be semantically equivalent to ``rc`` and must +be sorted as if it were ``rc``. Tools MAY reject the case of having the same +``N`` for both a ``c`` and a ``rc`` in the same release segment as ambiguous +and remain in compliance with the specification. + +Within an alpha (``1.0a1``), beta (``1.0b1``), or release candidate +(``1.0rc1``, ``1.0c1``), the following suffixes are permitted and MUST be +ordered as shown:: + + .devN, , .postN + +Within a post-release (``1.0.post1``), the following suffixes are permitted +and MUST be ordered as shown:: + + .devN, + +Note that ``devN`` and ``postN`` MUST always be preceded by a dot, even +when used immediately following a numeric version (e.g. ``1.0.dev456``, +``1.0.post1``). + +Within a pre-release, post-release or development release segment with a +shared prefix, ordering MUST be by the value of the numeric component. + +The following example covers many of the possible combinations:: + + 1.dev0 + 1.0.dev456 + 1.0a1 + 1.0a2.dev456 + 1.0a12.dev456 + 1.0a12 + 1.0b1.dev456 + 1.0b2 + 1.0b2.post345.dev456 + 1.0b2.post345 + 1.0rc1.dev456 + 1.0rc1 + 1.0 + 1.0+abc.5 + 1.0+abc.7 + 1.0+5 + 1.0.post456.dev34 + 1.0.post456 + 1.0.15 + 1.1.dev1 + + +Version ordering across different metadata versions +--------------------------------------------------- + +Metadata v1.0 (:pep:`241`) and metadata v1.1 (:pep:`314`) do not specify a standard +version identification or ordering scheme. However metadata v1.2 (:pep:`345`) +does specify a scheme which is defined in :pep:`386`. + +Due to the nature of the simple installer API it is not possible for an +installer to be aware of which metadata version a particular distribution was +using. Additionally installers required the ability to create a reasonably +prioritized list that includes all, or as many as possible, versions of +a project to determine which versions it should install. These requirements +necessitate a standardization across one parsing mechanism to be used for all +versions of a project. + +Due to the above, this specification MUST be used for all versions of metadata and +supersedes :pep:`386` even for metadata v1.2. Tools SHOULD ignore any versions +which cannot be parsed by the rules in this specification, but MAY fall back to +implementation defined version parsing and ordering schemes if no versions +complying with this specification are available. + +Distribution users may wish to explicitly remove non-compliant versions from +any private package indexes they control. + + +Compatibility with other version schemes +---------------------------------------- + +Some projects may choose to use a version scheme which requires +translation in order to comply with the public version scheme defined in +this specification. In such cases, the project specific version can be stored in the +metadata while the translated public version is published in the version field. + +This allows automated distribution tools to provide consistently correct +ordering of published releases, while still allowing developers to use +the internal versioning scheme they prefer for their projects. + + +Semantic versioning +~~~~~~~~~~~~~~~~~~~ + +`Semantic versioning`_ is a popular version identification scheme that is +more prescriptive than this specification regarding the significance of different +elements of a release number. Even if a project chooses not to abide by +the details of semantic versioning, the scheme is worth understanding as +it covers many of the issues that can arise when depending on other +distributions, and when publishing a distribution that others rely on. + +The "Major.Minor.Patch" (described in this specification as "major.minor.micro") +aspects of semantic versioning (clauses 1-8 in the 2.0.0 specification) +are fully compatible with the version scheme defined in this specification, and abiding +by these aspects is encouraged. + +Semantic versions containing a hyphen (pre-releases - clause 10) or a +plus sign (builds - clause 11) are *not* compatible with this specification +and are not permitted in the public version field. + +One possible mechanism to translate such semantic versioning based source +labels to compatible public versions is to use the ``.devN`` suffix to +specify the appropriate version order. + +Specific build information may also be included in local version labels. + +.. _Semantic versioning: https://semver.org/ + + +DVCS based version labels +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Many build tools integrate with distributed version control systems like +Git and Mercurial in order to add an identifying hash to the version +identifier. As hashes cannot be ordered reliably such versions are not +permitted in the public version field. + +As with semantic versioning, the public ``.devN`` suffix may be used to +uniquely identify such releases for publication, while the original DVCS based +label can be stored in the project metadata. + +Identifying hash information may also be included in local version labels. + + +Olson database versioning +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``pytz`` project inherits its versioning scheme from the corresponding +Olson timezone database versioning scheme: the year followed by a lowercase +character indicating the version of the database within that year. + +This can be translated to a compliant public version identifier as +``.``, where the serial starts at zero or one (for the +'a' release) and is incremented with each subsequent database +update within the year. + +As with other translated version identifiers, the corresponding Olson +database version could be recorded in the project metadata. + + +Version specifiers +================== + +A version specifier consists of a series of version clauses, separated by +commas. For example:: + + ~= 0.9, >= 1.0, != 1.3.4.*, < 2.0 + +The comparison operator determines the kind of version clause: + +* ``~=``: `Compatible release`_ clause +* ``==``: `Version matching`_ clause +* ``!=``: `Version exclusion`_ clause +* ``<=``, ``>=``: `Inclusive ordered comparison`_ clause +* ``<``, ``>``: `Exclusive ordered comparison`_ clause +* ``===``: `Arbitrary equality`_ clause. + +The comma (",") is equivalent to a logical **and** operator: a candidate +version must match all given version clauses in order to match the +specifier as a whole. + +Whitespace between a conditional operator and the following version +identifier is optional, as is the whitespace around the commas. + +When multiple candidate versions match a version specifier, the preferred +version SHOULD be the latest version as determined by the consistent +ordering defined by the standard `Version scheme`_. Whether or not +pre-releases are considered as candidate versions SHOULD be handled as +described in `Handling of pre-releases`_. + +Except where specifically noted below, local version identifiers MUST NOT be +permitted in version specifiers, and local version labels MUST be ignored +entirely when checking if candidate versions match a given version +specifier. + + +.. _version-specifiers-compatible-release: + +Compatible release +------------------ + +A compatible release clause consists of the compatible release operator ``~=`` +and a version identifier. It matches any candidate version that is expected +to be compatible with the specified version. + +The specified version identifier must be in the standard format described in +`Version scheme`_. Local version identifiers are NOT permitted in this +version specifier. + +For a given release identifier ``V.N``, the compatible release clause is +approximately equivalent to the pair of comparison clauses:: + + >= V.N, == V.* + +This operator MUST NOT be used with a single segment version number such as +``~=1``. + +For example, the following groups of version clauses are equivalent:: + + ~= 2.2 + >= 2.2, == 2.* + + ~= 1.4.5 + >= 1.4.5, == 1.4.* + +If a pre-release, post-release or developmental release is named in a +compatible release clause as ``V.N.suffix``, then the suffix is ignored +when determining the required prefix match:: + + ~= 2.2.post3 + >= 2.2.post3, == 2.* + + ~= 1.4.5a4 + >= 1.4.5a4, == 1.4.* + +The padding rules for release segment comparisons means that the assumed +degree of forward compatibility in a compatible release clause can be +controlled by appending additional zeros to the version specifier:: + + ~= 2.2.0 + >= 2.2.0, == 2.2.* + + ~= 1.4.5.0 + >= 1.4.5.0, == 1.4.5.* + + +Version matching +---------------- + +A version matching clause includes the version matching operator ``==`` +and a version identifier. + +The specified version identifier must be in the standard format described in +`Version scheme`_, but a trailing ``.*`` is permitted on public version +identifiers as described below. + +By default, the version matching operator is based on a strict equality +comparison: the specified version must be exactly the same as the requested +version. The *only* substitution performed is the zero padding of the +release segment to ensure the release segments are compared with the same +length. + +Whether or not strict version matching is appropriate depends on the specific +use case for the version specifier. Automated tools SHOULD at least issue +warnings and MAY reject them entirely when strict version matches are used +inappropriately. + +Prefix matching may be requested instead of strict comparison, by appending +a trailing ``.*`` to the version identifier in the version matching clause. +This means that additional trailing segments will be ignored when +determining whether or not a version identifier matches the clause. If the +specified version includes only a release segment, then trailing components +(or the lack thereof) in the release segment are also ignored. + +For example, given the version ``1.1.post1``, the following clauses would +match or not as shown:: + + == 1.1 # Not equal, so 1.1.post1 does not match clause + == 1.1.post1 # Equal, so 1.1.post1 matches clause + == 1.1.* # Same prefix, so 1.1.post1 matches clause + +For purposes of prefix matching, the pre-release segment is considered to +have an implied preceding ``.``, so given the version ``1.1a1``, the +following clauses would match or not as shown:: + + == 1.1 # Not equal, so 1.1a1 does not match clause + == 1.1a1 # Equal, so 1.1a1 matches clause + == 1.1.* # Same prefix, so 1.1a1 matches clause if pre-releases are requested + +An exact match is also considered a prefix match (this interpretation is +implied by the usual zero padding rules for the release segment of version +identifiers). Given the version ``1.1``, the following clauses would +match or not as shown:: + + == 1.1 # Equal, so 1.1 matches clause + == 1.1.0 # Zero padding expands 1.1 to 1.1.0, so it matches clause + == 1.1.dev1 # Not equal (dev-release), so 1.1 does not match clause + == 1.1a1 # Not equal (pre-release), so 1.1 does not match clause + == 1.1.post1 # Not equal (post-release), so 1.1 does not match clause + == 1.1.* # Same prefix, so 1.1 matches clause + +It is invalid to have a prefix match containing a development or local release +such as ``1.0.dev1.*`` or ``1.0+foo1.*``. If present, the development release +segment is always the final segment in the public version, and the local version +is ignored for comparison purposes, so using either in a prefix match wouldn't +make any sense. + +The use of ``==`` (without at least the wildcard suffix) when defining +dependencies for published distributions is strongly discouraged as it +greatly complicates the deployment of security fixes. The strict version +comparison operator is intended primarily for use when defining +dependencies for repeatable *deployments of applications* while using +a shared distribution index. + +If the specified version identifier is a public version identifier (no +local version label), then the local version label of any candidate versions +MUST be ignored when matching versions. + +If the specified version identifier is a local version identifier, then the +local version labels of candidate versions MUST be considered when matching +versions, with the public version identifier being matched as described +above, and the local version label being checked for equivalence using a +strict string equality comparison. + + +Version exclusion +----------------- + +A version exclusion clause includes the version exclusion operator ``!=`` +and a version identifier. + +The allowed version identifiers and comparison semantics are the same as +those of the `Version matching`_ operator, except that the sense of any +match is inverted. + +For example, given the version ``1.1.post1``, the following clauses would +match or not as shown:: + + != 1.1 # Not equal, so 1.1.post1 matches clause + != 1.1.post1 # Equal, so 1.1.post1 does not match clause + != 1.1.* # Same prefix, so 1.1.post1 does not match clause + + +Inclusive ordered comparison +---------------------------- + +An inclusive ordered comparison clause includes a comparison operator and a +version identifier, and will match any version where the comparison is correct +based on the relative position of the candidate version and the specified +version given the consistent ordering defined by the standard +`Version scheme`_. + +The inclusive ordered comparison operators are ``<=`` and ``>=``. + +As with version matching, the release segment is zero padded as necessary to +ensure the release segments are compared with the same length. + +Local version identifiers are NOT permitted in this version specifier. + + +Exclusive ordered comparison +---------------------------- + +The exclusive ordered comparisons ``>`` and ``<`` are similar to the inclusive +ordered comparisons in that they rely on the relative position of the candidate +version and the specified version given the consistent ordering defined by the +standard `Version scheme`_. However, they specifically exclude pre-releases, +post-releases, and local versions of the specified version. + +The exclusive ordered comparison ``>V`` **MUST NOT** allow a post-release +of the given version unless ``V`` itself is a post release. You may mandate +that releases are later than a particular post release, including additional +post releases, by using ``>V.postN``. For example, ``>1.7`` will allow +``1.7.1`` but not ``1.7.0.post1`` and ``>1.7.post2`` will allow ``1.7.1`` +and ``1.7.0.post3`` but not ``1.7.0``. + +The exclusive ordered comparison ``>V`` **MUST NOT** match a local version of +the specified version. + +The exclusive ordered comparison ``=`` entry as +part of the URL fragment. + +For version control references, the ``VCS+protocol`` scheme SHOULD be +used to identify both the version control system and the secure transport, +and a version control system with hash based commit identifiers SHOULD be +used. Automated tools MAY omit warnings about missing hashes for version +control systems that do not provide hash based commit identifiers. + +To handle version control systems that do not support including commit or +tag references directly in the URL, that information may be appended to the +end of the URL using the ``@`` or the ``@#`` +notation. + +.. note:: + + This isn't *quite* the same as the existing VCS reference notation + supported by pip. Firstly, the distribution name is moved in front rather + than embedded as part of the URL. Secondly, the commit hash is included + even when retrieving based on a tag, in order to meet the requirement + above that *every* link should include a hash to make things harder to + forge (creating a malicious repo with a particular tag is easy, creating + one with a specific *hash*, less so). + +Remote URL examples:: + + pip @ https://github.com/pypa/pip/archive/1.3.1.zip#sha1=da9234ee9982d4bbb3c72346a6de940a148ea686 + pip @ git+https://github.com/pypa/pip.git@7921be1537eac1e97bc40179a57f0349c2aee67d + pip @ git+https://github.com/pypa/pip.git@1.3.1#7921be1537eac1e97bc40179a57f0349c2aee67d + + +File URLs +--------- + +File URLs take the form of ``file:///``. If the ```` is +omitted it is assumed to be ``localhost`` and even if the ```` is omitted +the third slash MUST still exist. The ```` defines what the file path on +the filesystem that is to be accessed. + +On the various \*nix operating systems the only allowed values for ```` +is for it to be omitted, ``localhost``, or another FQDN that the current +machine believes matches its own host. In other words, on \*nix the ``file://`` +scheme can only be used to access paths on the local machine. + +On Windows the file format should include the drive letter if applicable as +part of the ```` (e.g. ``file:///c:/path/to/a/file``). Unlike \*nix on +Windows the ```` parameter may be used to specify a file residing on a +network share. In other words, in order to translate ``\\machine\volume\file`` +to a ``file://`` url, it would end up as ``file://machine/volume/file``. For +more information on ``file://`` URLs on Windows see +`MSDN `_. + + + +Summary of differences from pkg_resources.parse_version +======================================================= + +* Note: this comparison is to ``pkg_resources.parse_version`` as it existed at + the time :pep:`440` was written. After the PEP was accepted, setuptools 6.0 and + later versions adopted the behaviour described here. + +* Local versions sort differently, this specification requires that they sort as greater + than the same version without a local version, whereas + ``pkg_resources.parse_version`` considers it a pre-release marker. + +* This specification purposely restricts the syntax which constitutes a valid version + while ``pkg_resources.parse_version`` attempts to provide some meaning from + *any* arbitrary string. + +* ``pkg_resources.parse_version`` allows arbitrarily deeply nested version + signifiers like ``1.0.dev1.post1.dev5``. This specification however allows only a + single use of each type and they must exist in a certain order. + + + +.. _version-specifiers-regex: + +Appendix: Parsing version strings with regular expressions +========================================================== + +As noted earlier in the :ref:`public-version-identifiers` section, +published version identifiers SHOULD use the canonical format. This +section provides regular expressions that can be used to test whether a +version is already in that form, and if it's not, extract the various +components for subsequent normalization. + +To test whether a version identifier is in the canonical format, you can use +the following function: + +.. code-block:: python + + import re + def is_canonical(version): + return re.match(r'^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$', version) is not None + +To extract the components of a version identifier, use the following regular +expression (as defined by the `packaging `_ +project): + +.. code-block:: python + + VERSION_PATTERN = r""" + v? + (?: + (?:(?P[0-9]+)!)? # epoch + (?P[0-9]+(?:\.[0-9]+)*) # release segment + (?P
                                          # pre-release
+                [-_\.]?
+                (?P(a|b|c|rc|alpha|beta|pre|preview))
+                [-_\.]?
+                (?P[0-9]+)?
+            )?
+            (?P                                         # post release
+                (?:-(?P[0-9]+))
+                |
+                (?:
+                    [-_\.]?
+                    (?Ppost|rev|r)
+                    [-_\.]?
+                    (?P[0-9]+)?
+                )
+            )?
+            (?P                                          # dev release
+                [-_\.]?
+                (?Pdev)
+                [-_\.]?
+                (?P[0-9]+)?
+            )?
+        )
+        (?:\+(?P[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
+    """
+
+    _regex = re.compile(
+        r"^\s*" + VERSION_PATTERN + r"\s*$",
+        re.VERBOSE | re.IGNORECASE,
+    )
+
+
+
+History
+=======
+
+- August 2014: This specification was approved through :pep:`440`.
+- May 2025: Clarify that development releases are a form of pre-release when
+  they are handled.
+- Nov 2025: Make arbitrary equality case insensitivity explicit.
diff --git a/source/specifications/virtual-environments.rst b/source/specifications/virtual-environments.rst
new file mode 100644
index 000000000..c73a28c27
--- /dev/null
+++ b/source/specifications/virtual-environments.rst
@@ -0,0 +1,56 @@
+
+.. _virtual-environments:
+
+===========================
+Python Virtual Environments
+===========================
+
+For Python 3.3 and later versions, :pep:`405` introduced interpreter level support
+for the concept of "Python Virtual Environments". Each virtual environment has
+its own Python binary (allowing creation of environments with various Python
+versions) and can have its own independent set of installed Python packages in
+its site directories, but shares the standard library with the base installed
+Python. While the concept of virtual environments existed prior to this update,
+there was no previously standardised mechanism for declaring or discovering them.
+
+
+Runtime detection of virtual environments
+=========================================
+
+At runtime, virtual environments can be identified by virtue of
+:py:data:`sys.prefix` (the filesystem location of the running interpreter)
+having a different value from :py:data:`sys.base_prefix` (the default filesystem
+location of the standard library directories).
+
+:ref:`venv-explanation` in the Python standard library documentation for the
+:py:mod:`venv` module covers this along with the concept of "activating" a
+virtual environment in an interactive operating system shell (this activation
+step is optional and hence the changes it makes can't be reliably used to
+detect whether a Python program is running in a virtual environment or not).
+
+
+Declaring installation environments as Python virtual environments
+==================================================================
+
+As described in :pep:`405`, a Python virtual environment in its simplest form
+consists of nothing more than a copy or symlink of the Python binary accompanied
+by a ``site-packages`` directory and a ``pyvenv.cfg`` file with a ``home`` key
+that indicates where to find the Python standard library modules.
+
+While designed to meet the needs of the standard :py:mod:`venv` module, this
+split installation and ``pyvenv.cfg`` file approach can be used by *any*
+Python installation provider that desires Python-specific tools to be aware that
+they are already operating in a virtual environment and no further environment
+nesting is required or desired.
+
+Even in the absence of a ``pyvenv.cfg`` file, any approach (e.g.
+``sitecustomize.py``, patching the installed Python runtime) that results in
+:py:data:`sys.prefix` and :py:data:`sys.base_prefix` having different values,
+while still providing a matching default package installation scheme in
+:py:mod:`sysconfig`, will be detected and behave as a Python virtual environment.
+
+
+History
+=======
+
+- May 2012: This specification was approved through :pep:`405`.
diff --git a/source/specifications/well-known-project-urls.rst b/source/specifications/well-known-project-urls.rst
new file mode 100644
index 000000000..30fefd12b
--- /dev/null
+++ b/source/specifications/well-known-project-urls.rst
@@ -0,0 +1,176 @@
+.. _`well-known-project-urls`:
+
+===================================
+Well-known Project URLs in Metadata
+===================================
+
+.. important::
+
+    This document is primarily of interest to metadata *consumers*,
+    who should use the normalization rules and well-known list below
+    to make their presentation of project URLs consistent across the
+    Python ecosystem.
+
+    Metadata *producers* (such as build tools and individual package
+    maintainers) may continue to use any labels they please, within the
+    overall ``Project-URL`` length restrictions. However, when possible, users are
+    *encouraged* to pick meaningful labels that normalize to well-known
+    labels.
+
+.. note::
+
+    See :ref:`Writing your pyproject.toml - urls `
+    for user-oriented guidance on choosing project URL labels in your package's
+    metadata.
+
+.. note:: This specification was originally defined in :pep:`753`.
+
+:pep:`753` deprecates the :ref:`core-metadata-home-page` and
+:ref:`core-metadata-download-url` metadata fields in favor of
+:ref:`core-metadata-project-url`, and defines a normalization and
+lookup procedure for determining whether a ``Project-URL`` is
+"well-known," i.e. has the semantics assigned to ``Home-page``,
+``Download-URL``, or other common project URLs.
+
+This allows indices (such as the Python Package Index) and other downstream
+metadata consumers to present project URLs in a
+consistent manner.
+
+.. _project-url-label-normalization:
+
+Label normalization
+===================
+
+.. note::
+
+    Label normalization is performed by metadata *consumers*, not metadata
+    producers.
+
+To determine whether a ``Project-URL`` label is "well-known," metadata
+consumers should normalize the label before comparing it to the
+:ref:`list of well-known labels `.
+
+The normalization procedure for ``Project-URL`` labels is defined
+by the following Python function:
+
+.. code-block:: python
+
+    import string
+
+    def normalize_label(label: str) -> str:
+        chars_to_remove = string.punctuation + string.whitespace
+        removal_map = str.maketrans("", "", chars_to_remove)
+        return label.translate(removal_map).lower()
+
+In plain language: a label is *normalized* by deleting all ASCII punctuation
+and whitespace, and then converting the result to lowercase.
+
+The following table shows examples of labels before (raw) and after
+normalization:
+
+.. list-table::
+    :header-rows: 1
+
+    * - Raw
+      - Normalized
+    * - ``Homepage``
+      - ``homepage``
+    * - ``Home-page``
+      - ``homepage``
+    * - ``Home page``
+      - ``homepage``
+    * - ``Change_Log``
+      - ``changelog``
+    * - ``What's New?``
+      - ``whatsnew``
+    * - ``github``
+      - ``github``
+
+.. _well-known-labels:
+
+Well-known labels
+=================
+
+.. note::
+
+    The list of well-known labels is a living standard, maintained as part of
+    this document.
+
+The following table lists labels that are well-known for the purpose of
+specializing the presentation of ``Project-URL`` metadata:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Label (Human-readable equivalent)
+     - Description
+     - Aliases
+   * - ``homepage`` (Homepage)
+     - The project's home page
+     - *(none)*
+   * - ``source`` (Source Code)
+     - The project's hosted source code or repository
+     - ``repository``, ``sourcecode``, ``github``
+   * - ``download`` (Download)
+     - A download URL for the current distribution, equivalent to ``Download-URL``
+     - *(none)*
+   * - ``changelog`` (Changelog)
+     - The project's comprehensive changelog
+     - ``changes``, ``whatsnew``, ``history``
+   * - ``releasenotes`` (Release Notes)
+     - The project's curated release notes
+     - *(none)*
+   * - ``documentation`` (Documentation)
+     - The project's online documentation
+     - ``docs``
+   * - ``issues`` (Issue Tracker)
+     - The project's bug tracker
+     - ``bugs``, ``issue``, ``tracker``, ``issuetracker``, ``bugtracker``
+   * - ``funding`` (Funding)
+     - Funding Information
+     - ``sponsor``, ``donate``, ``donation``
+
+Package metadata consumers may choose to render aliased labels the same as
+their "parent" well known label, or further specialize them.
+
+Example behavior
+================
+
+The following shows the flow of project URL metadata from
+``pyproject.toml`` to core metadata to a potential index presentation:
+
+.. code-block:: toml
+    :caption: Example project URLs in standard configuration
+
+    [project.urls]
+    "Home Page" = "https://example.com"
+    DOCUMENTATION = "https://readthedocs.org"
+    Repository = "https://upstream.example.com/me/spam.git"
+    GitHub = "https://github.com/example/spam"
+
+.. code-block:: email
+    :caption: Core metadata representation
+
+    Project-URL: Home page, https://example.com
+    Project-URL: DOCUMENTATION, https://readthedocs.org
+    Project-URL: Repository, https://upstream.example.com/me/spam.git
+    Project-URL: GitHub, https://github.com/example/spam
+
+.. code-block:: text
+    :caption: Potential rendering
+
+    Homepage: https://example.com
+    Documentation: https://readthedocs.org
+    Source Code: https://upstream.example.com/me/spam.git
+    Source Code (GitHub): https://github.com/example/spam
+
+Observe that the core metadata appears in the form provided by the user
+(since metadata *producers* do not perform normalization), but the
+metadata *consumer* normalizes and identifies appropriate
+human-readable equivalents based on the normalized form:
+
+* ``Home page`` becomes ``homepage``, which is rendered as ``Homepage``
+* ``DOCUMENTATION`` becomes ``documentation``, which is rendered as ``Documentation``
+* ``Repository`` becomes ``repository``, which is rendered as ``Source Code``
+* ``GitHub`` becomes ``github``, which is rendered as ``Source Code (GitHub)``
+  (as a specialization of ``Source Code``)
diff --git a/source/tutorials/creating-documentation.rst b/source/tutorials/creating-documentation.rst
index de47c7013..5d9dae787 100644
--- a/source/tutorials/creating-documentation.rst
+++ b/source/tutorials/creating-documentation.rst
@@ -1,70 +1,7 @@
-.. _creating-documentation:
+:orphan:
 
+Creating documentation
 ======================
-Creating Documentation
-======================
-
-This section covers the basics of how to create documentation using `Sphinx`_ and
-host the documentation for free in `Read The Docs`_.
-
-.. _Sphinx: https://www.sphinx-doc.org
-.. _Read The Docs: https://readthedocs.org/
-
-Installing Sphinx
------------------
-Use ``pip`` to install Sphinx:
-
-.. tab:: Unix/macOS
-
-    .. code-block:: bash
-
-        python3 -m pip install -U sphinx
-
-.. tab:: Windows
-
-    .. code-block:: bat
-
-        py -m pip install -U sphinx
-
-For other installation methods, see this :doc:`installation guide ` by Sphinx.
-
-
-Getting Started With Sphinx
----------------------------
-
-Create a ``docs`` directory inside your project to hold your documentation:
-
-.. code-block:: bash
-
-	cd /path/to/project
-	mkdir docs
-
-Run ``sphinx-quickstart`` inside the ``docs`` directory:
-
-.. code-block:: bash
-
-	cd docs
-	sphinx-quickstart
-
-This sets up a source directory, walks you through some basic configurations, and creates an ``index.rst`` file as well as a ``conf.py`` file.
-
-You can add some information about your project in ``index.rst``, then build them:
-
-.. code-block:: bash
-
-	make html
-
-For more details on the build process, see this `guide`_ by Read The Docs.
-
-.. _guide: https://docs.readthedocs.io/en/latest/intro/import-guide.html
-
-Other Sources
--------------
-
-For a more detailed guide on how to use Sphinx and reStructuredText, please see this `documentation tutorial`_ on Hitchhiker's Guide to Python. 
-
-.. _documentation tutorial: https://docs.python-guide.org/writing/documentation/
-
-
-
 
+This tutorial has been removed since it is not related to packaging and was unmaintained.
+Please see the `Sphinx tutorial `_ instead.
diff --git a/source/tutorials/index.rst b/source/tutorials/index.rst
index 40b0620f3..33ab4f98a 100644
--- a/source/tutorials/index.rst
+++ b/source/tutorials/index.rst
@@ -11,4 +11,3 @@ topics, see :doc:`/guides/index`.
     installing-packages
     managing-dependencies
     packaging-projects
-    creating-documentation
diff --git a/source/tutorials/installing-packages.rst b/source/tutorials/installing-packages.rst
index 577d719cd..3a9aa23bb 100644
--- a/source/tutorials/installing-packages.rst
+++ b/source/tutorials/installing-packages.rst
@@ -9,7 +9,7 @@ This section covers the basics of how to install Python :term:`packages
 
 It's important to note that the term "package" in this context is being used to
 describe a bundle of software to be installed (i.e. as a synonym for a
-:term:`distribution `). It does not to refer to the kind
+:term:`distribution `). It does not refer to the kind
 of :term:`package ` that you import in your Python source code
 (i.e. a container of modules). It is common in the Python community to refer to
 a :term:`distribution ` using the term "package".  Using
@@ -18,10 +18,6 @@ confused with a Linux distribution, or another larger software distribution
 like Python itself.
 
 
-.. contents:: Contents
-   :local:
-
-
 .. _installing_requirements:
 
 Requirements for Installing Packages
@@ -56,12 +52,12 @@ please install the latest 3.x version from `python.org`_ or refer to the
 
 .. Note:: If you're a newcomer and you get an error like this:
 
-    .. code-block:: python
+    .. code-block:: pycon
 
-        >>> python --version
+        >>> python3 --version
         Traceback (most recent call last):
           File "", line 1, in 
-        NameError: name 'python' is not defined
+        NameError: name 'python3' is not defined
 
     It's because this command and other suggested commands in this tutorial
     are intended to be run in a *shell* (also called a *terminal* or
@@ -73,7 +69,7 @@ please install the latest 3.x version from `python.org`_ or refer to the
    notebook, you can run system commands like those in this tutorial by
    prefacing them with a ``!`` character:
 
-    ::
+   .. code-block:: text
 
         In [1]: import sys
                 !{sys.executable} --version
@@ -137,21 +133,21 @@ standard library:
 
 If that still doesn't allow you to run ``python -m pip``:
 
- * Securely Download `get-pip.py
-   `_ [1]_
+* Securely Download `get-pip.py
+  `_ [1]_
 
- * Run ``python get-pip.py``. [2]_  This will install or upgrade pip.
-   Additionally, it will install :ref:`setuptools` and :ref:`wheel` if they're
-   not installed already.
+* Run ``python get-pip.py``. [2]_  This will install or upgrade pip.
+  Additionally, it will install :ref:`setuptools` and :ref:`wheel` if they're
+  not installed already.
 
-   .. warning::
+  .. warning::
 
-      Be cautious if you're using a Python install that's managed by your
-      operating system or another package manager. get-pip.py does not
-      coordinate with those tools, and may leave your system in an
-      inconsistent state. You can use ``python get-pip.py --prefix=/usr/local/``
-      to install in ``/usr/local`` which is designed for locally-installed
-      software.
+     Be cautious if you're using a Python install that's managed by your
+     operating system or another package manager. get-pip.py does not
+     coordinate with those tools, and may leave your system in an
+     inconsistent state. You can use ``python get-pip.py --prefix=/usr/local/``
+     to install in ``/usr/local`` which is designed for locally-installed
+     software.
 
 
 Ensure pip, setuptools, and wheel are up to date
@@ -228,12 +224,12 @@ environments.
 Currently, there are two common tools for creating Python virtual environments:
 
 * :doc:`venv ` is available by default in Python 3.3 and later, and installs
-  :ref:`pip` and :ref:`setuptools` into created virtual environments in
-  Python 3.4 and later.
+  :ref:`pip` into created virtual environments in Python 3.4 and later
+  (Python versions prior to 3.12 also installed :ref:`setuptools`).
 * :ref:`virtualenv` needs to be installed separately, but supports Python 2.7+
   and Python 3.3+, and :ref:`pip`, :ref:`setuptools` and :ref:`wheel` are
-  always installed into created virtual environments by default (regardless of
-  Python version).
+  installed into created virtual environments by default. Note that ``setuptools`` is no longer
+  included by default starting with Python 3.12 (and ``virtualenv`` follows this behavior).
 
 The basic usage is like so:
 
@@ -277,7 +273,7 @@ that the virtual environment's variables are set within the current
 shell, and not in a subprocess (which then disappears, having no
 useful effect).
 
-In both of the above cases, Windows users should _not_ use the
+In both of the above cases, Windows users should *not* use the
 :command:`source` command, but should rather run the :command:`activate`
 script directly from the command shell like so:
 
@@ -308,9 +304,9 @@ The most common usage of :ref:`pip` is to install from the :term:`Python Package
 Index ` using a :term:`requirement specifier
 `. Generally speaking, a requirement specifier is
 composed of a project name followed by an optional :term:`version specifier
-`.  :pep:`440` contains a :pep:`full
-specification <440#version-specifiers>`
-of the currently supported specifiers. Below are some examples.
+`.  A full description of the supported specifiers can be
+found in the :ref:`Version specifier specification `.
+Below are some examples.
 
 To install the latest version of "SomeProject":
 
@@ -355,7 +351,7 @@ To install greater than or equal to one version and less than another:
         py -m pip install "SomeProject>=1,<2"
 
 
-To install a version that's :pep:`"compatible" <440#compatible-release>`
+To install a version that's :ref:`compatible `
 with a certain version: [4]_
 
 .. tab:: Unix/macOS
@@ -492,19 +488,19 @@ syntax, see pip's section on :ref:`VCS Support `.
 
     .. code-block:: bash
 
-        python3 -m pip install -e git+https://git.repo/some_pkg.git#egg=SomeProject          # from git
-        python3 -m pip install -e hg+https://hg.repo/some_pkg#egg=SomeProject                # from mercurial
-        python3 -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomeProject         # from svn
-        python3 -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomeProject  # from a branch
+        python3 -m pip install -e SomeProject @ git+https://git.repo/some_pkg.git          # from git
+        python3 -m pip install -e SomeProject @ hg+https://hg.repo/some_pkg                # from mercurial
+        python3 -m pip install -e SomeProject @ svn+svn://svn.repo/some_pkg/trunk/         # from svn
+        python3 -m pip install -e SomeProject @ git+https://git.repo/some_pkg.git@feature  # from a branch
 
 .. tab:: Windows
 
     .. code-block:: bat
 
-        py -m pip install -e git+https://git.repo/some_pkg.git#egg=SomeProject          # from git
-        py -m pip install -e hg+https://hg.repo/some_pkg#egg=SomeProject                # from mercurial
-        py -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomeProject         # from svn
-        py -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomeProject  # from a branch
+        py -m pip install -e SomeProject @ git+https://git.repo/some_pkg.git          # from git
+        py -m pip install -e SomeProject @ hg+https://hg.repo/some_pkg                # from mercurial
+        py -m pip install -e SomeProject @ svn+svn://svn.repo/some_pkg/trunk/         # from svn
+        py -m pip install -e SomeProject @ git+https://git.repo/some_pkg.git@feature  # from a branch
 
 Installing from other Indexes
 =============================
@@ -612,10 +608,10 @@ Install from a local directory containing archives (and don't check :term:`PyPI
 Installing from other sources
 =============================
 
-To install from other data sources (for example Amazon S3 storage) you can
-create a helper application that presents the data in a :pep:`503` compliant
-index format, and use the ``--extra-index-url`` flag to direct pip to use
-that index.
+To install from other data sources (for example Amazon S3 storage)
+you can create a helper application that presents the data
+in a format compliant with the :ref:`simple repository API `:,
+and use the ``--extra-index-url`` flag to direct pip to use that index.
 
 .. code-block:: bash
 
@@ -653,17 +649,17 @@ you know publishes one, you can include it in the pip installation command:
 
     .. code-block:: bash
 
-        python3 -m pip install SomePackage[PDF]
-        python3 -m pip install SomePackage[PDF]==3.0
-        python3 -m pip install -e .[PDF]  # editable project in current directory
+        python3 -m pip install 'SomePackage[PDF]'
+        python3 -m pip install 'SomePackage[PDF]==3.0'
+        python3 -m pip install -e '.[PDF]'  # editable project in current directory
 
 .. tab:: Windows
 
     .. code-block:: bat
 
-        py -m pip install SomePackage[PDF]
-        py -m pip install SomePackage[PDF]==3.0
-        py -m pip install -e .[PDF]  # editable project in current directory
+        py -m pip install "SomePackage[PDF]"
+        py -m pip install "SomePackage[PDF]==3.0"
+        py -m pip install -e ".[PDF]"  # editable project in current directory
 
 ----
 
diff --git a/source/tutorials/managing-dependencies.rst b/source/tutorials/managing-dependencies.rst
index eaa32f7ef..bb67a60e3 100644
--- a/source/tutorials/managing-dependencies.rst
+++ b/source/tutorials/managing-dependencies.rst
@@ -167,10 +167,8 @@ and techniques, listed in alphabetical order, to see if one of them is a better
   wrapper around pip that supports ``requirements.txt``, Pipenv and Poetry lock files,
   or converting them to pip-tools compatible output. Designed for containerized
   Python applications, but not limited to them.
-* `PDM `_ for a modern Python package management
-  tool supporting :pep:`582` (replacing virtual environments with ``__pypackages__``
-  directory for package installation) and relying on standards such as :pep:`517` and
-  :pep:`621`.
+* `PDM `_ for a modern Python package
+  management relying on standards such as :pep:`517` and :pep:`621`.
 * `pip-tools `_ for creating a lock file of all
   dependencies from a list of packages directly used in a project, and ensuring that
   only those dependencies are installed.
@@ -179,3 +177,5 @@ and techniques, listed in alphabetical order, to see if one of them is a better
   structured as a distributable Python package with a valid ``pyproject.toml`` file.
   By contrast, Pipenv explicitly avoids making the assumption that the application
   being worked on will support distribution as a ``pip``-installable Python package.
+* `uv `__ for a single tool that covers the entire project
+  management workflow, including dependency management, packaging, and publishing.
diff --git a/source/tutorials/packaging-projects.rst b/source/tutorials/packaging-projects.rst
index b65866c63..4f69de20b 100644
--- a/source/tutorials/packaging-projects.rst
+++ b/source/tutorials/packaging-projects.rst
@@ -54,8 +54,10 @@ Create the following file structure locally:
 The directory containing the Python files should match the project name. This
 simplifies the configuration and is more obvious to users who install the package.
 
-:file:`__init__.py` is required to import the directory as a package, and
-should be empty.
+Creating the file :file:`__init__.py` is recommended because the existence of an
+:file:`__init__.py` file allows users to import the directory as a regular package,
+even if (as is the case in this tutorial) :file:`__init__.py` is empty.
+[#namespace-packages]_
 
 :file:`example.py` is an example of a module within the package that could
 contain the logic (functions, classes, constants, etc.) of your package.
@@ -103,15 +105,20 @@ Creating a test directory
 :file:`tests/` is a placeholder for test files. Leave it empty for now.
 
 
-Creating pyproject.toml
------------------------
+.. _choosing-build-backend:
 
-.. TODO: Add an intro sentence about pyproject.toml, and a sub-heading for
-   "Configuring build tools"
+Choosing a build backend
+------------------------
+
+Tools like :ref:`pip` and :ref:`build` do not actually convert your sources
+into a :term:`distribution package ` (like a wheel);
+that job is performed by a :term:`build backend `. The build backend determines how
+your project will specify its configuration, including metadata (information
+about the project, for example, the name and tags that are displayed on PyPI)
+and input files. Build backends have different levels of functionality, such as
+whether they support building :term:`extension modules `, and
+you should choose one that suits your needs and preferences.
 
-:file:`pyproject.toml` tells "frontend" build tools like :ref:`pip` and
-:ref:`build` what "backend" tool to use to create
-:term:`distribution packages ` for your project.
 You can choose from a number of backends; this tutorial uses :ref:`Hatchling
 ` by default, but it will work identically with :ref:`setuptools`,
 :ref:`Flit `, :ref:`PDM `, and others that support the ``[project]``
@@ -124,49 +131,35 @@ table for :ref:`metadata `.
    management, as well as building, uploading, and installing packages. This
    tutorial uses single-purpose tools that work independently.
 
-Open :file:`pyproject.toml` and enter one of these ``[build-system]`` tables:
-
-.. tab:: Hatchling
-
-    .. code-block:: toml
-
-        [build-system]
-        requires = ["hatchling"]
-        build-backend = "hatchling.build"
-
-.. tab:: setuptools
-
-    .. code-block:: toml
-
-        [build-system]
-        requires = ["setuptools>=61.0"]
-        build-backend = "setuptools.build_meta"
-
-.. tab:: Flit
-
-    .. code-block:: toml
-
-        [build-system]
-        requires = ["flit_core>=3.2"]
-        build-backend = "flit_core.buildapi"
+The :file:`pyproject.toml` tells :term:`build frontend ` tools like :ref:`pip` and
+:ref:`build` which backend to use for your project. Below are some
+examples for common build backends, but check your backend's own documentation
+for more details.
 
-.. tab:: PDM
+.. include:: ../shared/build-backend-tabs.rst
 
-    .. code-block:: toml
+The ``requires`` key is a list of packages that are needed to build your package.
+The :term:`frontend ` should install them automatically when building your package.
+Frontends usually run builds in isolated environments, so omitting dependencies
+here may cause build-time errors.
+This should always include your backend's package, and might have other build-time
+dependencies.
+The minimum version specified in the above code block is the one that introduced support
+for :ref:`the new license metadata `.
 
-        [build-system]
-        requires = ["pdm-pep517"]
-        build-backend = "pdm.pep517.api"
+The ``build-backend`` key is the name of the Python object that frontends will use
+to perform the build.
 
+Both of these values will be provided by the documentation for your build
+backend, or generated by its command line interface. There should be no need for
+you to customize these settings.
 
-- ``requires`` is a list of packages that are needed to build your package. You
-  don't need to install them; build frontends like :ref:`pip` will install them
-  automatically in a temporary, isolated virtual environment for use during the
-  build process.
-- ``build-backend`` is the name of the Python object that frontends will use to
-  perform the build.
-
-.. TODO: Add note to check the tools' documentation for the current snippet?
+Additional configuration of the build tool will either be in a ``tool`` section
+of the ``pyproject.toml``, or in a special file defined by the build tool. For
+example, when using ``setuptools`` as your build backend, additional configuration
+may be added to a ``setup.py`` or ``setup.cfg`` file, and specifying
+``setuptools.build_meta`` in your build allows the tools to locate and use these
+automatically.
 
 .. _configuring metadata:
 
@@ -188,25 +181,25 @@ following this tutorial.
     ]
     description = "A small example package"
     readme = "README.md"
-    requires-python = ">=3.7"
+    requires-python = ">=3.9"
     classifiers = [
         "Programming Language :: Python :: 3",
-        "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
     ]
+    license = "MIT"
+    license-files = ["LICEN[CS]E*"]
 
     [project.urls]
-    "Homepage" = "https://github.com/pypa/sampleproject"
-    "Bug Tracker" = "https://github.com/pypa/sampleproject/issues"
+    Homepage = "https://github.com/pypa/sampleproject"
+    Issues = "https://github.com/pypa/sampleproject/issues"
 
 - ``name`` is the *distribution name* of your package. This can be any name as
   long as it only contains letters, numbers, ``.``, ``_`` , and ``-``. It also
   must not already be taken on PyPI. **Be sure to update this with your
   username** for this tutorial, as this ensures you won't try to upload a
   package with the same name as one which already exists.
-- ``version`` is the package version. See the :ref:`version specifier specification `
-  for more details on versions. Some build backends allow it to be specified
-  another way, such as from a file or a git tag.
+- ``version`` is the package version. (Some build backends allow it to be
+  specified another way, such as from a file or Git tag.)
 - ``authors`` is used to identify the author of the package; you specify a name
   and an email for each author. You can also list ``maintainers`` in the same
   format.
@@ -215,24 +208,29 @@ following this tutorial.
   package. This is shown on the package detail page on PyPI.
   In this case, the description is loaded from :file:`README.md` (which is a
   common pattern). There also is a more advanced table form described in the
-  :ref:`project metadata specification `.
+  :ref:`pyproject.toml guide `.
 - ``requires-python`` gives the versions of Python supported by your
-  project. Installers like :ref:`pip` will look back through older versions of
+  project. An installer like :ref:`pip` will look back through older versions of
   packages until it finds one that has a matching Python version.
 - ``classifiers`` gives the index and :ref:`pip` some additional metadata
   about your package. In this case, the package is only compatible with Python
-  3, is licensed under the MIT license, and is OS-independent. You should
-  always include at least which version(s) of Python your package works on,
-  which license your package is available under, and which operating systems
+  3 and is OS-independent. You should
+  always include at least which version(s) of Python your package works on
+  and which operating systems
   your package will work on. For a complete list of classifiers, see
   https://pypi.org/classifiers/.
+- ``license`` is the :term:`SPDX license expression ` of
+  your :term:`Distribution Archive` files.
+- ``license-files`` is the list of glob paths to the license files,
+  relative to the directory where :file:`pyproject.toml` is located.
 - ``urls`` lets you list any number of extra links to show on PyPI.
   Generally this could be to the source, documentation, issue trackers, etc.
 
-See the :ref:`project metadata specification ` for
-details on these and other fields that can be defined in the ``[project]``
-table. Other common fields are ``keywords`` to improve discoverability and the
-``dependencies`` that are required to install your package.
+See the :ref:`pyproject.toml guide ` for details
+on these and other fields that can be defined in the ``[project]``
+table. Other common fields are ``keywords`` to improve discoverability
+and the ``dependencies`` that are required to install your package.
+
 
 Creating README.md
 ------------------
@@ -245,19 +243,19 @@ if you'd like.
     # Example Package
 
     This is a simple example package. You can use
-    [Github-flavored Markdown](https://guides.github.com/features/mastering-markdown/)
+    [GitHub-flavored Markdown](https://guides.github.com/features/mastering-markdown/)
     to write your content.
 
 
 Creating a LICENSE
 ------------------
 
-It's important for every package uploaded to the Python Package Index to include
-a license. This tells users who install your package the terms under which they
-can use your package. For help picking a license, see
-https://choosealicense.com/. Once you have chosen a license, open
-:file:`LICENSE` and enter the license text. For example, if you had chosen the
-MIT license:
+It's important for every :term:`Distribution Archive` uploaded to the Python
+Package Index to include a license. This tells users who install your
+:term:`Distribution Archive` the terms under which they can use it. For help
+picking a license, see https://choosealicense.com/. Once you have chosen a
+license, open :file:`LICENSE` and enter the license text. For example, if you
+had chosen the MIT license:
 
 .. code-block:: text
 
@@ -283,6 +281,9 @@ MIT license:
 
 Most build backends automatically include license files in packages. See your
 backend's documentation for more details.
+If you include the path to license in the ``license-files`` key of
+:file:`pyproject.toml`, and your build backend supports :pep:`639`,
+the file will be automatically included in the package.
 
 
 Including other files
@@ -399,16 +400,15 @@ Once installed, run Twine to upload all of the archives under :file:`dist`:
 
         py -m twine upload --repository testpypi dist/*
 
-You will be prompted for a username and password. For the username,
-use ``__token__``. For the password, use the token value, including
-the ``pypi-`` prefix.
+You will be prompted for an API token. Use the token value, including the ``pypi-``
+prefix. Note that the input will be hidden, so be sure to paste correctly.
 
 After the command completes, you should see output similar to this:
 
 .. code-block::
 
     Uploading distributions to https://test.pypi.org/legacy/
-    Enter your username: __token__
+    Enter your API token:
     Uploading example_package_YOUR_USERNAME_HERE-0.0.1-py3-none-any.whl
     100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 8.2/8.2 kB β€’ 00:01 β€’ ?
     Uploading example_package_YOUR_USERNAME_HERE-0.0.1.tar.gz
@@ -473,7 +473,7 @@ Make sure you're still in your virtual environment, then run Python:
 
 and import the package:
 
-.. code-block:: python
+.. code-block:: pycon
 
     >>> from example_package_YOUR_USERNAME_HERE import example
     >>> example.add_one(2)
@@ -509,12 +509,29 @@ differences:
 At this point if you want to read more on packaging Python libraries here are
 some things you can do:
 
-.. TODO: Add links to other guides
-.. TODO: Add links to backend configuration docs
-
+* Read about advanced configuration for your chosen build backend:
+  `Hatchling `_,
+  :doc:`setuptools `,
+  :doc:`Flit `, `PDM `_.
+* Look at the :doc:`guides ` on this site for more advanced
+  practical information, or the :doc:`discussions `
+  for explanations and background on specific topics.
 * Consider packaging tools that provide a single command-line interface for
   project management and packaging, such as :ref:`hatch`, :ref:`flit`,
   :ref:`pdm`, and :ref:`poetry`.
-* Read :pep:`517` and :pep:`518` for background and details on build tool configuration.
-* Read about :doc:`/guides/packaging-binary-extensions`.
 
+
+----
+
+.. rubric:: Notes
+
+.. [#namespace-packages]
+   Technically, you can also create Python packages without an ``__init__.py`` file,
+   but those are called :doc:`namespace packages `
+   and considered an **advanced topic** (not covered in this tutorial).
+   If you are only getting started with Python packaging, it is recommended to
+   stick with *regular packages* and ``__init__.py`` (even if the file is empty).
+
+
+.. _hatchling-config: https://hatch.pypa.io/latest/config/metadata/
+.. _pdm-config: https://pdm-project.org/latest/reference/pep621/