diff --git a/.coveragerc b/.coveragerc index e9b51ab174..b5008b2b2f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,2 @@ [run] -omit = requests/packages/* \ No newline at end of file +omit = requests/packages/* diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..db51a967bf --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,5 @@ +# You can configure git to automatically use this file with the following config: +# git config --global blame.ignoreRevsFile .git-blame-ignore-revs + +# Add automatic code formatting to Requests +2a6f290bc09324406708a4d404a88a45d848ddf9 diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..ff7f10665a --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,6 @@ +# Treat each other well + +Everyone participating in the _requests_ project, and in particular in the issue tracker, +pull requests, and social media activity, is expected to treat other people with respect +and more generally to follow the guidelines articulated in the +[Python Community Code of Conduct](https://www.python.org/psf/codeofconduct/). diff --git a/CONTRIBUTING.md b/.github/CONTRIBUTING.md similarity index 92% rename from CONTRIBUTING.md rename to .github/CONTRIBUTING.md index 57aef1e0c7..3470dfee83 100644 --- a/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,10 +1,7 @@ # Contribution Guidelines -Before opening any issues or proposing any pull requests, please do the -following: - -1. Read our [Contributor's Guide](http://docs.python-requests.org/en/latest/dev/contributing/). -2. Understand our [development philosophy](http://docs.python-requests.org/en/latest/dev/philosophy/). +Before opening any issues or proposing any pull requests, please read +our [Contributor's Guide](https://requests.readthedocs.io/en/latest/dev/contributing/). To get the greatest chance of helpful responses, please also observe the following additional notes. diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000000..603c7ff9c3 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: ['https://www.python.org/psf/sponsorship/'] diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index e23f4c70ce..060d9262a5 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -25,4 +25,4 @@ import requests This command is only available on Requests v2.16.4 and greater. Otherwise, please provide some basic information about your system (Python version, -operating system, &c). \ No newline at end of file +operating system, &c). diff --git a/.github/ISSUE_TEMPLATE/Bug_report.md b/.github/ISSUE_TEMPLATE/Bug_report.md new file mode 100644 index 0000000000..cb87bd6b67 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Bug_report.md @@ -0,0 +1,36 @@ +--- +name: Bug report +about: Create a report to help us improve + +--- + + + +## Expected Result + + + +## Actual Result + + + +## Reproduction Steps + +```python +import requests + +``` + +## System Information + + $ python -m requests.help + +```json +{ + "paste": "here" +} +``` + + diff --git a/.github/ISSUE_TEMPLATE/Custom.md b/.github/ISSUE_TEMPLATE/Custom.md new file mode 100644 index 0000000000..332c3aea97 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Custom.md @@ -0,0 +1,10 @@ +--- +name: Request for Help +about: Guidance on using Requests. +labels: +- "Question/Not a bug" +- "actions/autoclose-qa" + +--- + +Please refer to our [Stack Overflow tag](https://stackoverflow.com/questions/tagged/python-requests) for guidance. diff --git a/.github/ISSUE_TEMPLATE/Feature_request.md b/.github/ISSUE_TEMPLATE/Feature_request.md new file mode 100644 index 0000000000..544113ae1c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Feature_request.md @@ -0,0 +1,10 @@ +--- +name: Feature request +about: Suggest an idea for this project +labels: +- "Feature Request" +- "actions/autoclose-feat" + +--- + +Requests is not accepting feature requests at this time. diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 0000000000..4b368617bd --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,84 @@ +# Vulnerability Disclosure + +If you think you have found a potential security vulnerability in +requests, please open a [draft Security Advisory](https://github.com/psf/requests/security/advisories/new) +via GitHub. We will coordinate verification and next steps through +that secure medium. + +If English is not your first language, please try to describe the +problem and its impact to the best of your ability. For greater detail, +please use your native language and we will try our best to translate it +using online services. + +Please also include the code you used to find the problem and the +shortest amount of code necessary to reproduce it. + +Please do not disclose this to anyone else. We will retrieve a CVE +identifier if necessary and give you full credit under whatever name or +alias you provide. We will only request an identifier when we have a fix +and can publish it in a release. + +We will respect your privacy and will only publicize your involvement if +you grant us permission. + +## Process + +This following information discusses the process the requests project +follows in response to vulnerability disclosures. If you are disclosing +a vulnerability, this section of the documentation lets you know how we +will respond to your disclosure. + +### Timeline + +When you report an issue, one of the project members will respond to you +within two days *at the outside*. In most cases responses will be +faster, usually within 12 hours. This initial response will at the very +least confirm receipt of the report. + +If we were able to rapidly reproduce the issue, the initial response +will also contain confirmation of the issue. If we are not, we will +often ask for more information about the reproduction scenario. + +Our goal is to have a fix for any vulnerability released within two +weeks of the initial disclosure. This may potentially involve shipping +an interim release that simply disables function while a more mature fix +can be prepared, but will in the vast majority of cases mean shipping a +complete release as soon as possible. + +Throughout the fix process we will keep you up to speed with how the fix +is progressing. Once the fix is prepared, we will notify you that we +believe we have a fix. Often we will ask you to confirm the fix resolves +the problem in your environment, especially if we are not confident of +our reproduction scenario. + +At this point, we will prepare for the release. We will obtain a CVE +number if one is required, providing you with full credit for the +discovery. We will also decide on a planned release date, and let you +know when it is. This release date will *always* be on a weekday. + +At this point we will reach out to our major downstream packagers to +notify them of an impending security-related patch so they can make +arrangements. In addition, these packagers will be provided with the +intended patch ahead of time, to ensure that they are able to promptly +release their downstream packages. Currently the list of people we +actively contact *ahead of a public release* is: + +- Python Maintenance Team, Red Hat (python-maint@redhat.com) +- Daniele Tricoli, Debian (@eriol) + +We will notify these individuals at least a week ahead of our planned +release date to ensure that they have sufficient time to prepare. If you +believe you should be on this list, please let one of the maintainers +know at one of the email addresses at the top of this article. + +On release day, we will push the patch to our public repository, along +with an updated changelog that describes the issue and credits you. We +will then issue a PyPI release containing the patch. + +At this point, we will publicise the release. This will involve mails to +mailing lists, Tweets, and all other communication mechanisms available +to the core team. + +We will also explicitly mention which commits contain the fix to make it +easier for other distributors and users to easily patch their own +versions of requests if upgrading is not an option. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..2be85338e3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + ignore: + # Ignore all patch releases as we can manually + # upgrade if we run into a bug and need a fix. + - dependency-name: "*" + update-types: ["version-update:semver-patch"] diff --git a/.github/workflows/close-issues.yml b/.github/workflows/close-issues.yml new file mode 100644 index 0000000000..bedc75ea5b --- /dev/null +++ b/.github/workflows/close-issues.yml @@ -0,0 +1,35 @@ +name: 'Autoclose Issues' + +on: + issues: + types: + - labeled + +permissions: + issues: write + +jobs: + close_qa: + if: github.event.label.name == 'actions/autoclose-qa' + runs-on: ubuntu-latest + steps: + - env: + ISSUE_URL: ${{ github.event.issue.html_url }} + GH_TOKEN: ${{ github.token }} + run: | + gh issue close $ISSUE_URL \ + --comment "As described in the template, we won't be able to answer questions on this issue tracker. Please use [Stack Overflow](https://stackoverflow.com/)" \ + --reason completed + gh issue lock $ISSUE_URL --reason off_topic + close_feature_request: + if: github.event.label.name == 'actions/autoclose-feat' + runs-on: ubuntu-latest + steps: + - env: + ISSUE_URL: ${{ github.event.issue.html_url }} + GH_TOKEN: ${{ github.token }} + run: | + gh issue close $ISSUE_URL \ + --comment "As described in the template, Requests is not accepting feature requests" \ + --reason "not planned" + gh issue lock $ISSUE_URL --reason off_topic diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000000..7d7a3d9f88 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,73 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + branches: [main] + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + schedule: + - cron: '0 23 * * 0' + +permissions: + contents: read + +jobs: + analyze: + permissions: + actions: read # for github/codeql-action/init to get workflow details + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/autobuild to send a status report + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@f443b600d91635bebf5b0d9ebc620189c0d6fba5 # v4.30.8 + with: + languages: "python" + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@f443b600d91635bebf5b0d9ebc620189c0d6fba5 # v4.30.8 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@f443b600d91635bebf5b0d9ebc620189c0d6fba5 # v4.30.8 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..a4d6b1c902 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,20 @@ +name: Lint code + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Set up Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: "3.x" + - name: Run pre-commit + uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # v3.0.0 diff --git a/.github/workflows/lock-issues.yml b/.github/workflows/lock-issues.yml new file mode 100644 index 0000000000..7d5a3c6525 --- /dev/null +++ b/.github/workflows/lock-issues.yml @@ -0,0 +1,19 @@ +name: 'Lock Threads' + +on: + schedule: + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write + +jobs: + action: + if: github.repository_owner == 'psf' + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@d42e5f49803f3c4e14ffee0378e31481265dda22 # v5.0.0 + with: + issue-lock-inactive-days: 90 + pr-lock-inactive-days: 90 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..28b28b2fb4 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,93 @@ +name: Publish to PyPI + +on: + push: + tags: + - "v*" + +permissions: + contents: read + +jobs: + build: + name: "Build dists" + runs-on: "ubuntu-latest" + environment: + name: "publish" + outputs: + hashes: ${{ steps.hash.outputs.hashes }} + artifact-id: ${{ steps.upload-artifact.outputs.artifact-id }} + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0 + with: + egress-policy: audit + + - name: "Checkout repository" + uses: "actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8" + with: + persist-credentials: false + + - name: "Setup Python" + uses: "actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c" + with: + python-version: "3.x" + + - name: "Install dependencies" + run: python -m pip install build==0.8.0 + + - name: "Build dists" + run: | + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) \ + python -m build + + - name: "Generate hashes" + id: hash + run: | + cd dist && echo "::set-output name=hashes::$(sha256sum * | base64 -w0)" + + - name: "Upload dists" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + id: upload-artifact + with: + name: "dist" + path: "dist/" + if-no-files-found: error + retention-days: 5 + + provenance: + needs: [build] + permissions: + actions: read + contents: write + id-token: write # Needed to access the workflow's OIDC identity. + uses: "slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.1.0" + with: + base64-subjects: "${{ needs.build.outputs.hashes }}" + upload-assets: true + compile-generator: true # Workaround for https://github.com/slsa-framework/slsa-github-generator/issues/1163 + + publish: + name: "Publish" + if: startsWith(github.ref, 'refs/tags/') + needs: ["build", "provenance"] + permissions: + contents: write + id-token: write + runs-on: "ubuntu-latest" + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0 + with: + egress-policy: audit + + - name: "Download dists" + uses: "actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0" # v5.0.0 + with: + artifact-ids: ${{ needs.build.outputs.artifact-id }} + path: "dist/" + + - name: "Publish dists to PyPI" + uses: "pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e" diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000000..b26c3d7ffa --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,75 @@ +name: Tests + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + build: + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14-dev", "pypy-3.10", "pypy-3.11"] + os: [ubuntu-22.04, macOS-latest, windows-latest] + # Pypy-3.11 can't install openssl-sys with rust + # which prevents us from testing in GHA. + exclude: + - { python-version: "pypy-3.11", os: "windows-latest" } + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + allow-prereleases: true + - name: Install dependencies + run: | + make + - name: Run tests + run: | + make ci + + no_chardet: + name: "No Character Detection" + runs-on: ubuntu-latest + strategy: + fail-fast: true + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + - name: 'Set up Python 3.9' + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c + with: + python-version: '3.9' + - name: Install dependencies + run: | + make + python -m pip uninstall -y "charset_normalizer" "chardet" + - name: Run tests + run: | + make ci + + urllib3: + name: 'urllib3 1.x' + runs-on: 'ubuntu-latest' + strategy: + fail-fast: true + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + - name: 'Set up Python 3.9' + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c + with: + python-version: '3.9' + - name: Install dependencies + run: | + make + python -m pip install "urllib3<2" + - name: Run tests + run: | + make ci diff --git a/.gitignore b/.gitignore index cd0c32e95c..de61154e3e 100644 --- a/.gitignore +++ b/.gitignore @@ -14,9 +14,21 @@ requests.egg-info/ *.swp *.egg env/ +.venv/ +.eggs/ +.tox/ +.pytest_cache/ +.vscode/ +.eggs/ .workon +# in case you work with IntelliJ/PyCharm +.idea +*.iml +.python-version + + t.py t2.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..0a0515cf87 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +exclude: 'docs/|ext/' + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort +- repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black + exclude: tests/test_lowlevel.py +- repo: https://github.com/asottile/pyupgrade + rev: v3.10.1 + hooks: + - id: pyupgrade + args: [--py37-plus] +- repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000000..0e2c719e08 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,29 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + builder: "dirhtml" + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - pdf + - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - path: . + - requirements: docs/requirements.txt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index aae4b56098..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -sudo: false -language: python -python: - # - "2.6" - - "2.7" - - "3.4" - - "3.5" - - "3.6" - - "3.7-dev" - # - "pypy" -- appears to hang - # - "pypy3" -# command to install dependencies -install: "make" -# command to run tests -script: - - | - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6" ]] ; then make test-readme; fi - - make ci -cache: pip -jobs: - include: - - stage: test - script: - - | - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6" ]] ; then make test-readme; fi - - make ci - - stage: coverage - python: 3.6 - script: codecov diff --git a/AUTHORS.rst b/AUTHORS.rst index 907687d4de..6e017c9a91 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,17 +1,16 @@ -Requests is written and maintained by Kenneth Reitz and -various contributors: +Requests was lovingly created by Kenneth Reitz. Keepers of the Crystals ``````````````````````` -- Kenneth Reitz `@kennethreitz `_, Keeper of the Master Crystal. -- Ian Cordasco `@sigmavirus24 `_. -- Nate Prewitt `@nateprewitt `_. +- Nate Prewitt `@nateprewitt `_. +- Seth M. Larson `@sethmlarson `_. Previous Keepers of Crystals ```````````````````````````` - +- Kenneth Reitz `@kennethreitz `_, reluctant Keeper of the Master Crystal. - Cory Benfield `@lukasa `_ +- Ian Cordasco `@sigmavirus24 `_. Patches and Suggestions @@ -125,7 +124,7 @@ Patches and Suggestions - Bryce Boe (`@bboe `_) - Colin Dunklau (`@cdunklau `_) - Bob Carroll (`@rcarz `_) -- Hugo Osvaldo Barrera (`@hobarrera `_) +- Hugo Osvaldo Barrera (`@hobarrera `_) - Łukasz Langa - Dave Shawley - James Clarke (`@jam `_) @@ -187,3 +186,10 @@ Patches and Suggestions - Nehal J Wani (`@nehaljwani `_) - Demetrios Bairaktaris (`@DemetriosBairaktaris `_) - Darren Dormer (`@ddormer `_) +- Rajiv Mayani (`@mayani `_) +- Antti Kaihola (`@akaihola `_) +- "Dull Bananas" (`@dullbananas `_) +- Alessio Izzo (`@aless10 `_) +- Sylvain Marié (`@smarie `_) +- Hod Bin Noon (`@hodbn `_) +- Mike Fiedler (`@miketheman `_) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 9b170d2fb1..0000000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,3 +0,0 @@ -Be cordial or be on your way. - -https://www.kennethreitz.org/essays/be-cordial-or-be-on-your-way diff --git a/HISTORY.md b/HISTORY.md new file mode 100644 index 0000000000..3828073adf --- /dev/null +++ b/HISTORY.md @@ -0,0 +1,2012 @@ +Release History +=============== + +dev +--- + +- \[Short description of non-trivial change.\] + +2.32.5 (2025-08-18) +------------------- + +**Bugfixes** + +- The SSLContext caching feature originally introduced in 2.32.0 has created + a new class of issues in Requests that have had negative impact across a number + of use cases. The Requests team has decided to revert this feature as long term + maintenance of it is proving to be unsustainable in its current iteration. + +**Deprecations** +- Added support for Python 3.14. +- Dropped support for Python 3.8 following its end of support. + +2.32.4 (2025-06-10) +------------------- + +**Security** +- CVE-2024-47081 Fixed an issue where a maliciously crafted URL and trusted + environment will retrieve credentials for the wrong hostname/machine from a + netrc file. + +**Improvements** +- Numerous documentation improvements + +**Deprecations** +- Added support for pypy 3.11 for Linux and macOS. +- Dropped support for pypy 3.9 following its end of support. + + +2.32.3 (2024-05-29) +------------------- + +**Bugfixes** +- Fixed bug breaking the ability to specify custom SSLContexts in sub-classes of + HTTPAdapter. (#6716) +- Fixed issue where Requests started failing to run on Python versions compiled + without the `ssl` module. (#6724) + +2.32.2 (2024-05-21) +------------------- + +**Deprecations** +- To provide a more stable migration for custom HTTPAdapters impacted + by the CVE changes in 2.32.0, we've renamed `_get_connection` to + a new public API, `get_connection_with_tls_context`. Existing custom + HTTPAdapters will need to migrate their code to use this new API. + `get_connection` is considered deprecated in all versions of Requests>=2.32.0. + + A minimal (2-line) example has been provided in the linked PR to ease + migration, but we strongly urge users to evaluate if their custom adapter + is subject to the same issue described in CVE-2024-35195. (#6710) + +2.32.1 (2024-05-20) +------------------- + +**Bugfixes** +- Add missing test certs to the sdist distributed on PyPI. + + +2.32.0 (2024-05-20) +------------------- + +**Security** +- Fixed an issue where setting `verify=False` on the first request from a + Session will cause subsequent requests to the _same origin_ to also ignore + cert verification, regardless of the value of `verify`. + (https://github.com/psf/requests/security/advisories/GHSA-9wx4-h78v-vm56) + +**Improvements** +- `verify=True` now reuses a global SSLContext which should improve + request time variance between first and subsequent requests. It should + also minimize certificate load time on Windows systems when using a Python + version built with OpenSSL 3.x. (#6667) +- Requests now supports optional use of character detection + (`chardet` or `charset_normalizer`) when repackaged or vendored. + This enables `pip` and other projects to minimize their vendoring + surface area. The `Response.text()` and `apparent_encoding` APIs + will default to `utf-8` if neither library is present. (#6702) + +**Bugfixes** +- Fixed bug in length detection where emoji length was incorrectly + calculated in the request content-length. (#6589) +- Fixed deserialization bug in JSONDecodeError. (#6629) +- Fixed bug where an extra leading `/` (path separator) could lead + urllib3 to unnecessarily reparse the request URI. (#6644) + +**Deprecations** + +- Requests has officially added support for CPython 3.12 (#6503) +- Requests has officially added support for PyPy 3.9 and 3.10 (#6641) +- Requests has officially dropped support for CPython 3.7 (#6642) +- Requests has officially dropped support for PyPy 3.7 and 3.8 (#6641) + +**Documentation** +- Various typo fixes and doc improvements. + +**Packaging** +- Requests has started adopting some modern packaging practices. + The source files for the projects (formerly `requests`) is now located + in `src/requests` in the Requests sdist. (#6506) +- Starting in Requests 2.33.0, Requests will migrate to a PEP 517 build system + using `hatchling`. This should not impact the average user, but extremely old + versions of packaging utilities may have issues with the new packaging format. + + +2.31.0 (2023-05-22) +------------------- + +**Security** +- Versions of Requests between v2.3.0 and v2.30.0 are vulnerable to potential + forwarding of `Proxy-Authorization` headers to destination servers when + following HTTPS redirects. + + When proxies are defined with user info (`https://user:pass@proxy:8080`), Requests + will construct a `Proxy-Authorization` header that is attached to the request to + authenticate with the proxy. + + In cases where Requests receives a redirect response, it previously reattached + the `Proxy-Authorization` header incorrectly, resulting in the value being + sent through the tunneled connection to the destination server. Users who rely on + defining their proxy credentials in the URL are *strongly* encouraged to upgrade + to Requests 2.31.0+ to prevent unintentional leakage and rotate their proxy + credentials once the change has been fully deployed. + + Users who do not use a proxy or do not supply their proxy credentials through + the user information portion of their proxy URL are not subject to this + vulnerability. + + Full details can be read in our [Github Security Advisory](https://github.com/psf/requests/security/advisories/GHSA-j8r2-6x86-q33q) + and [CVE-2023-32681](https://nvd.nist.gov/vuln/detail/CVE-2023-32681). + + +2.30.0 (2023-05-03) +------------------- + +**Dependencies** +- ⚠️ Added support for urllib3 2.0. ⚠️ + + This may contain minor breaking changes so we advise careful testing and + reviewing https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html + prior to upgrading. + + Users who wish to stay on urllib3 1.x can pin to `urllib3<2`. + +2.29.0 (2023-04-26) +------------------- + +**Improvements** + +- Requests now defers chunked requests to the urllib3 implementation to improve + standardization. (#6226) +- Requests relaxes header component requirements to support bytes/str subclasses. (#6356) + +2.28.2 (2023-01-12) +------------------- + +**Dependencies** + +- Requests now supports charset\_normalizer 3.x. (#6261) + +**Bugfixes** + +- Updated MissingSchema exception to suggest https scheme rather than http. (#6188) + +2.28.1 (2022-06-29) +------------------- + +**Improvements** + +- Speed optimization in `iter_content` with transition to `yield from`. (#6170) + +**Dependencies** + +- Added support for chardet 5.0.0 (#6179) +- Added support for charset-normalizer 2.1.0 (#6169) + +2.28.0 (2022-06-09) +------------------- + +**Deprecations** + +- ⚠️ Requests has officially dropped support for Python 2.7. ⚠️ (#6091) +- Requests has officially dropped support for Python 3.6 (including pypy3.6). (#6091) + +**Improvements** + +- Wrap JSON parsing issues in Request's JSONDecodeError for payloads without + an encoding to make `json()` API consistent. (#6097) +- Parse header components consistently, raising an InvalidHeader error in + all invalid cases. (#6154) +- Added provisional 3.11 support with current beta build. (#6155) +- Requests got a makeover and we decided to paint it black. (#6095) + +**Bugfixes** + +- Fixed bug where setting `CURL_CA_BUNDLE` to an empty string would disable + cert verification. All Requests 2.x versions before 2.28.0 are affected. (#6074) +- Fixed urllib3 exception leak, wrapping `urllib3.exceptions.SSLError` with + `requests.exceptions.SSLError` for `content` and `iter_content`. (#6057) +- Fixed issue where invalid Windows registry entries caused proxy resolution + to raise an exception rather than ignoring the entry. (#6149) +- Fixed issue where entire payload could be included in the error message for + JSONDecodeError. (#6036) + +2.27.1 (2022-01-05) +------------------- + +**Bugfixes** + +- Fixed parsing issue that resulted in the `auth` component being + dropped from proxy URLs. (#6028) + +2.27.0 (2022-01-03) +------------------- + +**Improvements** + +- Officially added support for Python 3.10. (#5928) + +- Added a `requests.exceptions.JSONDecodeError` to unify JSON exceptions between + Python 2 and 3. This gets raised in the `response.json()` method, and is + backwards compatible as it inherits from previously thrown exceptions. + Can be caught from `requests.exceptions.RequestException` as well. (#5856) + +- Improved error text for misnamed `InvalidSchema` and `MissingSchema` + exceptions. This is a temporary fix until exceptions can be renamed + (Schema->Scheme). (#6017) + +- Improved proxy parsing for proxy URLs missing a scheme. This will address + recent changes to `urlparse` in Python 3.9+. (#5917) + +**Bugfixes** + +- Fixed defect in `extract_zipped_paths` which could result in an infinite loop + for some paths. (#5851) + +- Fixed handling for `AttributeError` when calculating length of files obtained + by `Tarfile.extractfile()`. (#5239) + +- Fixed urllib3 exception leak, wrapping `urllib3.exceptions.InvalidHeader` with + `requests.exceptions.InvalidHeader`. (#5914) + +- Fixed bug where two Host headers were sent for chunked requests. (#5391) + +- Fixed regression in Requests 2.26.0 where `Proxy-Authorization` was + incorrectly stripped from all requests sent with `Session.send`. (#5924) + +- Fixed performance regression in 2.26.0 for hosts with a large number of + proxies available in the environment. (#5924) + +- Fixed idna exception leak, wrapping `UnicodeError` with + `requests.exceptions.InvalidURL` for URLs with a leading dot (.) in the + domain. (#5414) + +**Deprecations** + +- Requests support for Python 2.7 and 3.6 will be ending in 2022. While we + don't have exact dates, Requests 2.27.x is likely to be the last release + series providing support. + +2.26.0 (2021-07-13) +------------------- + +**Improvements** + +- Requests now supports Brotli compression, if either the `brotli` or + `brotlicffi` package is installed. (#5783) + +- `Session.send` now correctly resolves proxy configurations from both + the Session and Request. Behavior now matches `Session.request`. (#5681) + +**Bugfixes** + +- Fixed a race condition in zip extraction when using Requests in parallel + from zip archive. (#5707) + +**Dependencies** + +- Instead of `chardet`, use the MIT-licensed `charset_normalizer` for Python3 + to remove license ambiguity for projects bundling requests. If `chardet` + is already installed on your machine it will be used instead of `charset_normalizer` + to keep backwards compatibility. (#5797) + + You can also install `chardet` while installing requests by + specifying `[use_chardet_on_py3]` extra as follows: + + ```shell + pip install "requests[use_chardet_on_py3]" + ``` + + Python2 still depends upon the `chardet` module. + +- Requests now supports `idna` 3.x on Python 3. `idna` 2.x will continue to + be used on Python 2 installations. (#5711) + +**Deprecations** + +- The `requests[security]` extra has been converted to a no-op install. + PyOpenSSL is no longer the recommended secure option for Requests. (#5867) + +- Requests has officially dropped support for Python 3.5. (#5867) + +2.25.1 (2020-12-16) +------------------- + +**Bugfixes** + +- Requests now treats `application/json` as `utf8` by default. Resolving + inconsistencies between `r.text` and `r.json` output. (#5673) + +**Dependencies** + +- Requests now supports chardet v4.x. + +2.25.0 (2020-11-11) +------------------- + +**Improvements** + +- Added support for NETRC environment variable. (#5643) + +**Dependencies** + +- Requests now supports urllib3 v1.26. + +**Deprecations** + +- Requests v2.25.x will be the last release series with support for Python 3.5. +- The `requests[security]` extra is officially deprecated and will be removed + in Requests v2.26.0. + +2.24.0 (2020-06-17) +------------------- + +**Improvements** + +- pyOpenSSL TLS implementation is now only used if Python + either doesn't have an `ssl` module or doesn't support + SNI. Previously pyOpenSSL was unconditionally used if available. + This applies even if pyOpenSSL is installed via the + `requests[security]` extra (#5443) + +- Redirect resolution should now only occur when + `allow_redirects` is True. (#5492) + +- No longer perform unnecessary Content-Length calculation for + requests that won't use it. (#5496) + +2.23.0 (2020-02-19) +------------------- + +**Improvements** + +- Remove defunct reference to `prefetch` in Session `__attrs__` (#5110) + +**Bugfixes** + +- Requests no longer outputs password in basic auth usage warning. (#5099) + +**Dependencies** + +- Pinning for `chardet` and `idna` now uses major version instead of minor. + This hopefully reduces the need for releases every time a dependency is updated. + +2.22.0 (2019-05-15) +------------------- + +**Dependencies** + +- Requests now supports urllib3 v1.25.2. + (note: 1.25.0 and 1.25.1 are incompatible) + +**Deprecations** + +- Requests has officially stopped support for Python 3.4. + +2.21.0 (2018-12-10) +------------------- + +**Dependencies** + +- Requests now supports idna v2.8. + +2.20.1 (2018-11-08) +------------------- + +**Bugfixes** + +- Fixed bug with unintended Authorization header stripping for + redirects using default ports (http/80, https/443). + +2.20.0 (2018-10-18) +------------------- + +**Bugfixes** + +- Content-Type header parsing is now case-insensitive (e.g. + charset=utf8 v Charset=utf8). +- Fixed exception leak where certain redirect urls would raise + uncaught urllib3 exceptions. +- Requests removes Authorization header from requests redirected + from https to http on the same hostname. (CVE-2018-18074) +- `should_bypass_proxies` now handles URIs without hostnames (e.g. + files). + +**Dependencies** + +- Requests now supports urllib3 v1.24. + +**Deprecations** + +- Requests has officially stopped support for Python 2.6. + +2.19.1 (2018-06-14) +------------------- + +**Bugfixes** + +- Fixed issue where status\_codes.py's `init` function failed trying + to append to a `__doc__` value of `None`. + +2.19.0 (2018-06-12) +------------------- + +**Improvements** + +- Warn user about possible slowdown when using cryptography version + < 1.3.4 +- Check for invalid host in proxy URL, before forwarding request to + adapter. +- Fragments are now properly maintained across redirects. (RFC7231 + 7.1.2) +- Removed use of cgi module to expedite library load time. +- Added support for SHA-256 and SHA-512 digest auth algorithms. +- Minor performance improvement to `Request.content`. +- Migrate to using collections.abc for 3.7 compatibility. + +**Bugfixes** + +- Parsing empty `Link` headers with `parse_header_links()` no longer + return one bogus entry. +- Fixed issue where loading the default certificate bundle from a zip + archive would raise an `IOError`. +- Fixed issue with unexpected `ImportError` on windows system which do + not support `winreg` module. +- DNS resolution in proxy bypass no longer includes the username and + password in the request. This also fixes the issue of DNS queries + failing on macOS. +- Properly normalize adapter prefixes for url comparison. +- Passing `None` as a file pointer to the `files` param no longer + raises an exception. +- Calling `copy` on a `RequestsCookieJar` will now preserve the cookie + policy correctly. + +**Dependencies** + +- We now support idna v2.7. +- We now support urllib3 v1.23. + +2.18.4 (2017-08-15) +------------------- + +**Improvements** + +- Error messages for invalid headers now include the header name for + easier debugging + +**Dependencies** + +- We now support idna v2.6. + +2.18.3 (2017-08-02) +------------------- + +**Improvements** + +- Running `$ python -m requests.help` now includes the installed + version of idna. + +**Bugfixes** + +- Fixed issue where Requests would raise `ConnectionError` instead of + `SSLError` when encountering SSL problems when using urllib3 v1.22. + +2.18.2 (2017-07-25) +------------------- + +**Bugfixes** + +- `requests.help` no longer fails on Python 2.6 due to the absence of + `ssl.OPENSSL_VERSION_NUMBER`. + +**Dependencies** + +- We now support urllib3 v1.22. + +2.18.1 (2017-06-14) +------------------- + +**Bugfixes** + +- Fix an error in the packaging whereby the `*.whl` contained + incorrect data that regressed the fix in v2.17.3. + +2.18.0 (2017-06-14) +------------------- + +**Improvements** + +- `Response` is now a context manager, so can be used directly in a + `with` statement without first having to be wrapped by + `contextlib.closing()`. + +**Bugfixes** + +- Resolve installation failure if multiprocessing is not available +- Resolve tests crash if multiprocessing is not able to determine the + number of CPU cores +- Resolve error swallowing in utils set\_environ generator + +2.17.3 (2017-05-29) +------------------- + +**Improvements** + +- Improved `packages` namespace identity support, for monkeypatching + libraries. + +2.17.2 (2017-05-29) +------------------- + +**Improvements** + +- Improved `packages` namespace identity support, for monkeypatching + libraries. + +2.17.1 (2017-05-29) +------------------- + +**Improvements** + +- Improved `packages` namespace identity support, for monkeypatching + libraries. + +2.17.0 (2017-05-29) +------------------- + +**Improvements** + +- Removal of the 301 redirect cache. This improves thread-safety. + +2.16.5 (2017-05-28) +------------------- + +- Improvements to `$ python -m requests.help`. + +2.16.4 (2017-05-27) +------------------- + +- Introduction of the `$ python -m requests.help` command, for + debugging with maintainers! + +2.16.3 (2017-05-27) +------------------- + +- Further restored the `requests.packages` namespace for compatibility + reasons. + +2.16.2 (2017-05-27) +------------------- + +- Further restored the `requests.packages` namespace for compatibility + reasons. + +No code modification (noted below) should be necessary any longer. + +2.16.1 (2017-05-27) +------------------- + +- Restored the `requests.packages` namespace for compatibility + reasons. +- Bugfix for `urllib3` version parsing. + +**Note**: code that was written to import against the +`requests.packages` namespace previously will have to import code that +rests at this module-level now. + +For example: + + from requests.packages.urllib3.poolmanager import PoolManager + +Will need to be re-written to be: + + from requests.packages import urllib3 + urllib3.poolmanager.PoolManager + +Or, even better: + + from urllib3.poolmanager import PoolManager + +2.16.0 (2017-05-26) +------------------- + +- Unvendor ALL the things! + +2.15.1 (2017-05-26) +------------------- + +- Everyone makes mistakes. + +2.15.0 (2017-05-26) +------------------- + +**Improvements** + +- Introduction of the `Response.next` property, for getting the next + `PreparedResponse` from a redirect chain (when + `allow_redirects=False`). +- Internal refactoring of `__version__` module. + +**Bugfixes** + +- Restored once-optional parameter for + `requests.utils.get_environ_proxies()`. + +2.14.2 (2017-05-10) +------------------- + +**Bugfixes** + +- Changed a less-than to an equal-to and an or in the dependency + markers to widen compatibility with older setuptools releases. + +2.14.1 (2017-05-09) +------------------- + +**Bugfixes** + +- Changed the dependency markers to widen compatibility with older pip + releases. + +2.14.0 (2017-05-09) +------------------- + +**Improvements** + +- It is now possible to pass `no_proxy` as a key to the `proxies` + dictionary to provide handling similar to the `NO_PROXY` environment + variable. +- When users provide invalid paths to certificate bundle files or + directories Requests now raises `IOError`, rather than failing at + the time of the HTTPS request with a fairly inscrutable certificate + validation error. +- The behavior of `SessionRedirectMixin` was slightly altered. + `resolve_redirects` will now detect a redirect by calling + `get_redirect_target(response)` instead of directly querying + `Response.is_redirect` and `Response.headers['location']`. Advanced + users will be able to process malformed redirects more easily. +- Changed the internal calculation of elapsed request time to have + higher resolution on Windows. +- Added `win_inet_pton` as conditional dependency for the `[socks]` + extra on Windows with Python 2.7. +- Changed the proxy bypass implementation on Windows: the proxy bypass + check doesn't use forward and reverse DNS requests anymore +- URLs with schemes that begin with `http` but are not `http` or + `https` no longer have their host parts forced to lowercase. + +**Bugfixes** + +- Much improved handling of non-ASCII `Location` header values in + redirects. Fewer `UnicodeDecodeErrors` are encountered on Python 2, + and Python 3 now correctly understands that Latin-1 is unlikely to + be the correct encoding. +- If an attempt to `seek` file to find out its length fails, we now + appropriately handle that by aborting our content-length + calculations. +- Restricted `HTTPDigestAuth` to only respond to auth challenges made + on 4XX responses, rather than to all auth challenges. +- Fixed some code that was firing `DeprecationWarning` on Python 3.6. +- The dismayed person emoticon (`/o\\`) no longer has a big head. I'm + sure this is what you were all worrying about most. + +**Miscellaneous** + +- Updated bundled urllib3 to v1.21.1. +- Updated bundled chardet to v3.0.2. +- Updated bundled idna to v2.5. +- Updated bundled certifi to 2017.4.17. + +2.13.0 (2017-01-24) +------------------- + +**Features** + +- Only load the `idna` library when we've determined we need it. This + will save some memory for users. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.20. +- Updated bundled idna to 2.2. + +2.12.5 (2017-01-18) +------------------- + +**Bugfixes** + +- Fixed an issue with JSON encoding detection, specifically detecting + big-endian UTF-32 with BOM. + +2.12.4 (2016-12-14) +------------------- + +**Bugfixes** + +- Fixed regression from 2.12.2 where non-string types were rejected in + the basic auth parameters. While support for this behaviour has been + re-added, the behaviour is deprecated and will be removed in the + future. + +2.12.3 (2016-12-01) +------------------- + +**Bugfixes** + +- Fixed regression from v2.12.1 for URLs with schemes that begin with + "http". These URLs have historically been processed as though they + were HTTP-schemed URLs, and so have had parameters added. This was + removed in v2.12.2 in an overzealous attempt to resolve problems + with IDNA-encoding those URLs. This change was reverted: the other + fixes for IDNA-encoding have been judged to be sufficient to return + to the behaviour Requests had before v2.12.0. + +2.12.2 (2016-11-30) +------------------- + +**Bugfixes** + +- Fixed several issues with IDNA-encoding URLs that are technically + invalid but which are widely accepted. Requests will now attempt to + IDNA-encode a URL if it can but, if it fails, and the host contains + only ASCII characters, it will be passed through optimistically. + This will allow users to opt-in to using IDNA2003 themselves if they + want to, and will also allow technically invalid but still common + hostnames. +- Fixed an issue where URLs with leading whitespace would raise + `InvalidSchema` errors. +- Fixed an issue where some URLs without the HTTP or HTTPS schemes + would still have HTTP URL preparation applied to them. +- Fixed an issue where Unicode strings could not be used in basic + auth. +- Fixed an issue encountered by some Requests plugins where + constructing a Response object would cause `Response.content` to + raise an `AttributeError`. + +2.12.1 (2016-11-16) +------------------- + +**Bugfixes** + +- Updated setuptools 'security' extra for the new PyOpenSSL backend in + urllib3. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.19.1. + +2.12.0 (2016-11-15) +------------------- + +**Improvements** + +- Updated support for internationalized domain names from IDNA2003 to + IDNA2008. This updated support is required for several forms of IDNs + and is mandatory for .de domains. +- Much improved heuristics for guessing content lengths: Requests will + no longer read an entire `StringIO` into memory. +- Much improved logic for recalculating `Content-Length` headers for + `PreparedRequest` objects. +- Improved tolerance for file-like objects that have no `tell` method + but do have a `seek` method. +- Anything that is a subclass of `Mapping` is now treated like a + dictionary by the `data=` keyword argument. +- Requests now tolerates empty passwords in proxy credentials, rather + than stripping the credentials. +- If a request is made with a file-like object as the body and that + request is redirected with a 307 or 308 status code, Requests will + now attempt to rewind the body object so it can be replayed. + +**Bugfixes** + +- When calling `response.close`, the call to `close` will be + propagated through to non-urllib3 backends. +- Fixed issue where the `ALL_PROXY` environment variable would be + preferred over scheme-specific variables like `HTTP_PROXY`. +- Fixed issue where non-UTF8 reason phrases got severely mangled by + falling back to decoding using ISO 8859-1 instead. +- Fixed a bug where Requests would not correctly correlate cookies set + when using custom Host headers if those Host headers did not use the + native string type for the platform. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.19. +- Updated bundled certifi certs to 2016.09.26. + +2.11.1 (2016-08-17) +------------------- + +**Bugfixes** + +- Fixed a bug when using `iter_content` with `decode_unicode=True` for + streamed bodies would raise `AttributeError`. This bug was + introduced in 2.11. +- Strip Content-Type and Transfer-Encoding headers from the header + block when following a redirect that transforms the verb from + POST/PUT to GET. + +2.11.0 (2016-08-08) +------------------- + +**Improvements** + +- Added support for the `ALL_PROXY` environment variable. +- Reject header values that contain leading whitespace or newline + characters to reduce risk of header smuggling. + +**Bugfixes** + +- Fixed occasional `TypeError` when attempting to decode a JSON + response that occurred in an error case. Now correctly returns a + `ValueError`. +- Requests would incorrectly ignore a non-CIDR IP address in the + `NO_PROXY` environment variables: Requests now treats it as a + specific IP. +- Fixed a bug when sending JSON data that could cause us to encounter + obscure OpenSSL errors in certain network conditions (yes, really). +- Added type checks to ensure that `iter_content` only accepts + integers and `None` for chunk sizes. +- Fixed issue where responses whose body had not been fully consumed + would have the underlying connection closed but not returned to the + connection pool, which could cause Requests to hang in situations + where the `HTTPAdapter` had been configured to use a blocking + connection pool. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.16. +- Some previous releases accidentally accepted non-strings as + acceptable header values. This release does not. + +2.10.0 (2016-04-29) +------------------- + +**New Features** + +- SOCKS Proxy Support! (requires PySocks; + `$ pip install requests[socks]`) + +**Miscellaneous** + +- Updated bundled urllib3 to 1.15.1. + +2.9.2 (2016-04-29) +------------------ + +**Improvements** + +- Change built-in CaseInsensitiveDict (used for headers) to use + OrderedDict as its underlying datastore. + +**Bugfixes** + +- Don't use redirect\_cache if allow\_redirects=False +- When passed objects that throw exceptions from `tell()`, send them + via chunked transfer encoding instead of failing. +- Raise a ProxyError for proxy related connection issues. + +2.9.1 (2015-12-21) +------------------ + +**Bugfixes** + +- Resolve regression introduced in 2.9.0 that made it impossible to + send binary strings as bodies in Python 3. +- Fixed errors when calculating cookie expiration dates in certain + locales. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.13.1. + +2.9.0 (2015-12-15) +------------------ + +**Minor Improvements** (Backwards compatible) + +- The `verify` keyword argument now supports being passed a path to a + directory of CA certificates, not just a single-file bundle. +- Warnings are now emitted when sending files opened in text mode. +- Added the 511 Network Authentication Required status code to the + status code registry. + +**Bugfixes** + +- For file-like objects that are not sought to the very beginning, we + now send the content length for the number of bytes we will actually + read, rather than the total size of the file, allowing partial file + uploads. +- When uploading file-like objects, if they are empty or have no + obvious content length we set `Transfer-Encoding: chunked` rather + than `Content-Length: 0`. +- We correctly receive the response in buffered mode when uploading + chunked bodies. +- We now handle being passed a query string as a bytestring on Python + 3, by decoding it as UTF-8. +- Sessions are now closed in all cases (exceptional and not) when + using the functional API rather than leaking and waiting for the + garbage collector to clean them up. +- Correctly handle digest auth headers with a malformed `qop` + directive that contains no token, by treating it the same as if no + `qop` directive was provided at all. +- Minor performance improvements when removing specific cookies by + name. + +**Miscellaneous** + +- Updated urllib3 to 1.13. + +2.8.1 (2015-10-13) +------------------ + +**Bugfixes** + +- Update certificate bundle to match `certifi` 2015.9.6.2's weak + certificate bundle. +- Fix a bug in 2.8.0 where requests would raise `ConnectTimeout` + instead of `ConnectionError` +- When using the PreparedRequest flow, requests will now correctly + respect the `json` parameter. Broken in 2.8.0. +- When using the PreparedRequest flow, requests will now correctly + handle a Unicode-string method name on Python 2. Broken in 2.8.0. + +2.8.0 (2015-10-05) +------------------ + +**Minor Improvements** (Backwards Compatible) + +- Requests now supports per-host proxies. This allows the `proxies` + dictionary to have entries of the form + `{'://': ''}`. Host-specific proxies will + be used in preference to the previously-supported scheme-specific + ones, but the previous syntax will continue to work. +- `Response.raise_for_status` now prints the URL that failed as part + of the exception message. +- `requests.utils.get_netrc_auth` now takes an `raise_errors` kwarg, + defaulting to `False`. When `True`, errors parsing `.netrc` files + cause exceptions to be thrown. +- Change to bundled projects import logic to make it easier to + unbundle requests downstream. +- Changed the default User-Agent string to avoid leaking data on + Linux: now contains only the requests version. + +**Bugfixes** + +- The `json` parameter to `post()` and friends will now only be used + if neither `data` nor `files` are present, consistent with the + documentation. +- We now ignore empty fields in the `NO_PROXY` environment variable. +- Fixed problem where `httplib.BadStatusLine` would get raised if + combining `stream=True` with `contextlib.closing`. +- Prevented bugs where we would attempt to return the same connection + back to the connection pool twice when sending a Chunked body. +- Miscellaneous minor internal changes. +- Digest Auth support is now thread safe. + +**Updates** + +- Updated urllib3 to 1.12. + +2.7.0 (2015-05-03) +------------------ + +This is the first release that follows our new release process. For +more, see [our +documentation](https://requests.readthedocs.io/en/latest/community/release-process/). + +**Bugfixes** + +- Updated urllib3 to 1.10.4, resolving several bugs involving chunked + transfer encoding and response framing. + +2.6.2 (2015-04-23) +------------------ + +**Bugfixes** + +- Fix regression where compressed data that was sent as chunked data + was not properly decompressed. (\#2561) + +2.6.1 (2015-04-22) +------------------ + +**Bugfixes** + +- Remove VendorAlias import machinery introduced in v2.5.2. +- Simplify the PreparedRequest.prepare API: We no longer require the + user to pass an empty list to the hooks keyword argument. (c.f. + \#2552) +- Resolve redirects now receives and forwards all of the original + arguments to the adapter. (\#2503) +- Handle UnicodeDecodeErrors when trying to deal with a unicode URL + that cannot be encoded in ASCII. (\#2540) +- Populate the parsed path of the URI field when performing Digest + Authentication. (\#2426) +- Copy a PreparedRequest's CookieJar more reliably when it is not an + instance of RequestsCookieJar. (\#2527) + +2.6.0 (2015-03-14) +------------------ + +**Bugfixes** + +- CVE-2015-2296: Fix handling of cookies on redirect. Previously a + cookie without a host value set would use the hostname for the + redirected URL exposing requests users to session fixation attacks + and potentially cookie stealing. This was disclosed privately by + Matthew Daley of [BugFuzz](https://bugfuzz.com). This affects all + versions of requests from v2.1.0 to v2.5.3 (inclusive on both ends). +- Fix error when requests is an `install_requires` dependency and + `python setup.py test` is run. (\#2462) +- Fix error when urllib3 is unbundled and requests continues to use + the vendored import location. +- Include fixes to `urllib3`'s header handling. +- Requests' handling of unvendored dependencies is now more + restrictive. + +**Features and Improvements** + +- Support bytearrays when passed as parameters in the `files` + argument. (\#2468) +- Avoid data duplication when creating a request with `str`, `bytes`, + or `bytearray` input to the `files` argument. + +2.5.3 (2015-02-24) +------------------ + +**Bugfixes** + +- Revert changes to our vendored certificate bundle. For more context + see (\#2455, \#2456, and ) + +2.5.2 (2015-02-23) +------------------ + +**Features and Improvements** + +- Add sha256 fingerprint support. + ([shazow/urllib3\#540](https://github.com/shazow/urllib3/pull/540)) +- Improve the performance of headers. + ([shazow/urllib3\#544](https://github.com/shazow/urllib3/pull/544)) + +**Bugfixes** + +- Copy pip's import machinery. When downstream redistributors remove + requests.packages.urllib3 the import machinery will continue to let + those same symbols work. Example usage in requests' documentation + and 3rd-party libraries relying on the vendored copies of urllib3 + will work without having to fallback to the system urllib3. +- Attempt to quote parts of the URL on redirect if unquoting and then + quoting fails. (\#2356) +- Fix filename type check for multipart form-data uploads. (\#2411) +- Properly handle the case where a server issuing digest + authentication challenges provides both auth and auth-int + qop-values. (\#2408) +- Fix a socket leak. + ([shazow/urllib3\#549](https://github.com/shazow/urllib3/pull/549)) +- Fix multiple `Set-Cookie` headers properly. + ([shazow/urllib3\#534](https://github.com/shazow/urllib3/pull/534)) +- Disable the built-in hostname verification. + ([shazow/urllib3\#526](https://github.com/shazow/urllib3/pull/526)) +- Fix the behaviour of decoding an exhausted stream. + ([shazow/urllib3\#535](https://github.com/shazow/urllib3/pull/535)) + +**Security** + +- Pulled in an updated `cacert.pem`. +- Drop RC4 from the default cipher list. + ([shazow/urllib3\#551](https://github.com/shazow/urllib3/pull/551)) + +2.5.1 (2014-12-23) +------------------ + +**Behavioural Changes** + +- Only catch HTTPErrors in raise\_for\_status (\#2382) + +**Bugfixes** + +- Handle LocationParseError from urllib3 (\#2344) +- Handle file-like object filenames that are not strings (\#2379) +- Unbreak HTTPDigestAuth handler. Allow new nonces to be negotiated + (\#2389) + +2.5.0 (2014-12-01) +------------------ + +**Improvements** + +- Allow usage of urllib3's Retry object with HTTPAdapters (\#2216) +- The `iter_lines` method on a response now accepts a delimiter with + which to split the content (\#2295) + +**Behavioural Changes** + +- Add deprecation warnings to functions in requests.utils that will be + removed in 3.0 (\#2309) +- Sessions used by the functional API are always closed (\#2326) +- Restrict requests to HTTP/1.1 and HTTP/1.0 (stop accepting HTTP/0.9) + (\#2323) + +**Bugfixes** + +- Only parse the URL once (\#2353) +- Allow Content-Length header to always be overridden (\#2332) +- Properly handle files in HTTPDigestAuth (\#2333) +- Cap redirect\_cache size to prevent memory abuse (\#2299) +- Fix HTTPDigestAuth handling of redirects after authenticating + successfully (\#2253) +- Fix crash with custom method parameter to Session.request (\#2317) +- Fix how Link headers are parsed using the regular expression library + (\#2271) + +**Documentation** + +- Add more references for interlinking (\#2348) +- Update CSS for theme (\#2290) +- Update width of buttons and sidebar (\#2289) +- Replace references of Gittip with Gratipay (\#2282) +- Add link to changelog in sidebar (\#2273) + +2.4.3 (2014-10-06) +------------------ + +**Bugfixes** + +- Unicode URL improvements for Python 2. +- Re-order JSON param for backwards compat. +- Automatically defrag authentication schemes from host/pass URIs. + ([\#2249](https://github.com/psf/requests/issues/2249)) + +2.4.2 (2014-10-05) +------------------ + +**Improvements** + +- FINALLY! Add json parameter for uploads! + ([\#2258](https://github.com/psf/requests/pull/2258)) +- Support for bytestring URLs on Python 3.x + ([\#2238](https://github.com/psf/requests/pull/2238)) + +**Bugfixes** + +- Avoid getting stuck in a loop + ([\#2244](https://github.com/psf/requests/pull/2244)) +- Multiple calls to iter\* fail with unhelpful error. + ([\#2240](https://github.com/psf/requests/issues/2240), + [\#2241](https://github.com/psf/requests/issues/2241)) + +**Documentation** + +- Correct redirection introduction + ([\#2245](https://github.com/psf/requests/pull/2245/)) +- Added example of how to send multiple files in one request. + ([\#2227](https://github.com/psf/requests/pull/2227/)) +- Clarify how to pass a custom set of CAs + ([\#2248](https://github.com/psf/requests/pull/2248/)) + +2.4.1 (2014-09-09) +------------------ + +- Now has a "security" package extras set, + `$ pip install requests[security]` +- Requests will now use Certifi if it is available. +- Capture and re-raise urllib3 ProtocolError +- Bugfix for responses that attempt to redirect to themselves forever + (wtf?). + +2.4.0 (2014-08-29) +------------------ + +**Behavioral Changes** + +- `Connection: keep-alive` header is now sent automatically. + +**Improvements** + +- Support for connect timeouts! Timeout now accepts a tuple (connect, + read) which is used to set individual connect and read timeouts. +- Allow copying of PreparedRequests without headers/cookies. +- Updated bundled urllib3 version. +- Refactored settings loading from environment -- new + Session.merge\_environment\_settings. +- Handle socket errors in iter\_content. + +2.3.0 (2014-05-16) +------------------ + +**API Changes** + +- New `Response` property `is_redirect`, which is true when the + library could have processed this response as a redirection (whether + or not it actually did). +- The `timeout` parameter now affects requests with both `stream=True` + and `stream=False` equally. +- The change in v2.0.0 to mandate explicit proxy schemes has been + reverted. Proxy schemes now default to `http://`. +- The `CaseInsensitiveDict` used for HTTP headers now behaves like a + normal dictionary when references as string or viewed in the + interpreter. + +**Bugfixes** + +- No longer expose Authorization or Proxy-Authorization headers on + redirect. Fix CVE-2014-1829 and CVE-2014-1830 respectively. +- Authorization is re-evaluated each redirect. +- On redirect, pass url as native strings. +- Fall-back to autodetected encoding for JSON when Unicode detection + fails. +- Headers set to `None` on the `Session` are now correctly not sent. +- Correctly honor `decode_unicode` even if it wasn't used earlier in + the same response. +- Stop advertising `compress` as a supported Content-Encoding. +- The `Response.history` parameter is now always a list. +- Many, many `urllib3` bugfixes. + +2.2.1 (2014-01-23) +------------------ + +**Bugfixes** + +- Fixes incorrect parsing of proxy credentials that contain a literal + or encoded '\#' character. +- Assorted urllib3 fixes. + +2.2.0 (2014-01-09) +------------------ + +**API Changes** + +- New exception: `ContentDecodingError`. Raised instead of `urllib3` + `DecodeError` exceptions. + +**Bugfixes** + +- Avoid many many exceptions from the buggy implementation of + `proxy_bypass` on OS X in Python 2.6. +- Avoid crashing when attempting to get authentication credentials + from \~/.netrc when running as a user without a home directory. +- Use the correct pool size for pools of connections to proxies. +- Fix iteration of `CookieJar` objects. +- Ensure that cookies are persisted over redirect. +- Switch back to using chardet, since it has merged with charade. + +2.1.0 (2013-12-05) +------------------ + +- Updated CA Bundle, of course. +- Cookies set on individual Requests through a `Session` (e.g. via + `Session.get()`) are no longer persisted to the `Session`. +- Clean up connections when we hit problems during chunked upload, + rather than leaking them. +- Return connections to the pool when a chunked upload is successful, + rather than leaking it. +- Match the HTTPbis recommendation for HTTP 301 redirects. +- Prevent hanging when using streaming uploads and Digest Auth when a + 401 is received. +- Values of headers set by Requests are now always the native string + type. +- Fix previously broken SNI support. +- Fix accessing HTTP proxies using proxy authentication. +- Unencode HTTP Basic usernames and passwords extracted from URLs. +- Support for IP address ranges for no\_proxy environment variable +- Parse headers correctly when users override the default `Host:` + header. +- Avoid munging the URL in case of case-sensitive servers. +- Looser URL handling for non-HTTP/HTTPS urls. +- Accept unicode methods in Python 2.6 and 2.7. +- More resilient cookie handling. +- Make `Response` objects pickleable. +- Actually added MD5-sess to Digest Auth instead of pretending to like + last time. +- Updated internal urllib3. +- Fixed @Lukasa's lack of taste. + +2.0.1 (2013-10-24) +------------------ + +- Updated included CA Bundle with new mistrusts and automated process + for the future +- Added MD5-sess to Digest Auth +- Accept per-file headers in multipart file POST messages. +- Fixed: Don't send the full URL on CONNECT messages. +- Fixed: Correctly lowercase a redirect scheme. +- Fixed: Cookies not persisted when set via functional API. +- Fixed: Translate urllib3 ProxyError into a requests ProxyError + derived from ConnectionError. +- Updated internal urllib3 and chardet. + +2.0.0 (2013-09-24) +------------------ + +**API Changes:** + +- Keys in the Headers dictionary are now native strings on all Python + versions, i.e. bytestrings on Python 2, unicode on Python 3. +- Proxy URLs now *must* have an explicit scheme. A `MissingSchema` + exception will be raised if they don't. +- Timeouts now apply to read time if `Stream=False`. +- `RequestException` is now a subclass of `IOError`, not + `RuntimeError`. +- Added new method to `PreparedRequest` objects: + `PreparedRequest.copy()`. +- Added new method to `Session` objects: `Session.update_request()`. + This method updates a `Request` object with the data (e.g. cookies) + stored on the `Session`. +- Added new method to `Session` objects: `Session.prepare_request()`. + This method updates and prepares a `Request` object, and returns the + corresponding `PreparedRequest` object. +- Added new method to `HTTPAdapter` objects: + `HTTPAdapter.proxy_headers()`. This should not be called directly, + but improves the subclass interface. +- `httplib.IncompleteRead` exceptions caused by incorrect chunked + encoding will now raise a Requests `ChunkedEncodingError` instead. +- Invalid percent-escape sequences now cause a Requests `InvalidURL` + exception to be raised. +- HTTP 208 no longer uses reason phrase `"im_used"`. Correctly uses + `"already_reported"`. +- HTTP 226 reason added (`"im_used"`). + +**Bugfixes:** + +- Vastly improved proxy support, including the CONNECT verb. Special + thanks to the many contributors who worked towards this improvement. +- Cookies are now properly managed when 401 authentication responses + are received. +- Chunked encoding fixes. +- Support for mixed case schemes. +- Better handling of streaming downloads. +- Retrieve environment proxies from more locations. +- Minor cookies fixes. +- Improved redirect behaviour. +- Improved streaming behaviour, particularly for compressed data. +- Miscellaneous small Python 3 text encoding bugs. +- `.netrc` no longer overrides explicit auth. +- Cookies set by hooks are now correctly persisted on Sessions. +- Fix problem with cookies that specify port numbers in their host + field. +- `BytesIO` can be used to perform streaming uploads. +- More generous parsing of the `no_proxy` environment variable. +- Non-string objects can be passed in data values alongside files. + +1.2.3 (2013-05-25) +------------------ + +- Simple packaging fix + +1.2.2 (2013-05-23) +------------------ + +- Simple packaging fix + +1.2.1 (2013-05-20) +------------------ + +- 301 and 302 redirects now change the verb to GET for all verbs, not + just POST, improving browser compatibility. +- Python 3.3.2 compatibility +- Always percent-encode location headers +- Fix connection adapter matching to be most-specific first +- new argument to the default connection adapter for passing a block + argument +- prevent a KeyError when there's no link headers + +1.2.0 (2013-03-31) +------------------ + +- Fixed cookies on sessions and on requests +- Significantly change how hooks are dispatched - hooks now receive + all the arguments specified by the user when making a request so + hooks can make a secondary request with the same parameters. This is + especially necessary for authentication handler authors +- certifi support was removed +- Fixed bug where using OAuth 1 with body `signature_type` sent no + data +- Major proxy work thanks to @Lukasa including parsing of proxy + authentication from the proxy url +- Fix DigestAuth handling too many 401s +- Update vendored urllib3 to include SSL bug fixes +- Allow keyword arguments to be passed to `json.loads()` via the + `Response.json()` method +- Don't send `Content-Length` header by default on `GET` or `HEAD` + requests +- Add `elapsed` attribute to `Response` objects to time how long a + request took. +- Fix `RequestsCookieJar` +- Sessions and Adapters are now picklable, i.e., can be used with the + multiprocessing library +- Update charade to version 1.0.3 + +The change in how hooks are dispatched will likely cause a great deal of +issues. + +1.1.0 (2013-01-10) +------------------ + +- CHUNKED REQUESTS +- Support for iterable response bodies +- Assume servers persist redirect params +- Allow explicit content types to be specified for file data +- Make merge\_kwargs case-insensitive when looking up keys + +1.0.3 (2012-12-18) +------------------ + +- Fix file upload encoding bug +- Fix cookie behavior + +1.0.2 (2012-12-17) +------------------ + +- Proxy fix for HTTPAdapter. + +1.0.1 (2012-12-17) +------------------ + +- Cert verification exception bug. +- Proxy fix for HTTPAdapter. + +1.0.0 (2012-12-17) +------------------ + +- Massive Refactor and Simplification +- Switch to Apache 2.0 license +- Swappable Connection Adapters +- Mountable Connection Adapters +- Mutable ProcessedRequest chain +- /s/prefetch/stream +- Removal of all configuration +- Standard library logging +- Make Response.json() callable, not property. +- Usage of new charade project, which provides python 2 and 3 + simultaneous chardet. +- Removal of all hooks except 'response' +- Removal of all authentication helpers (OAuth, Kerberos) + +This is not a backwards compatible change. + +0.14.2 (2012-10-27) +------------------- + +- Improved mime-compatible JSON handling +- Proxy fixes +- Path hack fixes +- Case-Insensitive Content-Encoding headers +- Support for CJK parameters in form posts + +0.14.1 (2012-10-01) +------------------- + +- Python 3.3 Compatibility +- Simply default accept-encoding +- Bugfixes + +0.14.0 (2012-09-02) +------------------- + +- No more iter\_content errors if already downloaded. + +0.13.9 (2012-08-25) +------------------- + +- Fix for OAuth + POSTs +- Remove exception eating from dispatch\_hook +- General bugfixes + +0.13.8 (2012-08-21) +------------------- + +- Incredible Link header support :) + +0.13.7 (2012-08-19) +------------------- + +- Support for (key, value) lists everywhere. +- Digest Authentication improvements. +- Ensure proxy exclusions work properly. +- Clearer UnicodeError exceptions. +- Automatic casting of URLs to strings (fURL and such) +- Bugfixes. + +0.13.6 (2012-08-06) +------------------- + +- Long awaited fix for hanging connections! + +0.13.5 (2012-07-27) +------------------- + +- Packaging fix + +0.13.4 (2012-07-27) +------------------- + +- GSSAPI/Kerberos authentication! +- App Engine 2.7 Fixes! +- Fix leaking connections (from urllib3 update) +- OAuthlib path hack fix +- OAuthlib URL parameters fix. + +0.13.3 (2012-07-12) +------------------- + +- Use simplejson if available. +- Do not hide SSLErrors behind Timeouts. +- Fixed param handling with urls containing fragments. +- Significantly improved information in User Agent. +- client certificates are ignored when verify=False + +0.13.2 (2012-06-28) +------------------- + +- Zero dependencies (once again)! +- New: Response.reason +- Sign querystring parameters in OAuth 1.0 +- Client certificates no longer ignored when verify=False +- Add openSUSE certificate support + +0.13.1 (2012-06-07) +------------------- + +- Allow passing a file or file-like object as data. +- Allow hooks to return responses that indicate errors. +- Fix Response.text and Response.json for body-less responses. + +0.13.0 (2012-05-29) +------------------- + +- Removal of Requests.async in favor of + [grequests](https://github.com/kennethreitz/grequests) +- Allow disabling of cookie persistence. +- New implementation of safe\_mode +- cookies.get now supports default argument +- Session cookies not saved when Session.request is called with + return\_response=False +- Env: no\_proxy support. +- RequestsCookieJar improvements. +- Various bug fixes. + +0.12.1 (2012-05-08) +------------------- + +- New `Response.json` property. +- Ability to add string file uploads. +- Fix out-of-range issue with iter\_lines. +- Fix iter\_content default size. +- Fix POST redirects containing files. + +0.12.0 (2012-05-02) +------------------- + +- EXPERIMENTAL OAUTH SUPPORT! +- Proper CookieJar-backed cookies interface with awesome dict-like + interface. +- Speed fix for non-iterated content chunks. +- Move `pre_request` to a more usable place. +- New `pre_send` hook. +- Lazily encode data, params, files. +- Load system Certificate Bundle if `certify` isn't available. +- Cleanups, fixes. + +0.11.2 (2012-04-22) +------------------- + +- Attempt to use the OS's certificate bundle if `certifi` isn't + available. +- Infinite digest auth redirect fix. +- Multi-part file upload improvements. +- Fix decoding of invalid %encodings in URLs. +- If there is no content in a response don't throw an error the second + time that content is attempted to be read. +- Upload data on redirects. + +0.11.1 (2012-03-30) +------------------- + +- POST redirects now break RFC to do what browsers do: Follow up with + a GET. +- New `strict_mode` configuration to disable new redirect behavior. + +0.11.0 (2012-03-14) +------------------- + +- Private SSL Certificate support +- Remove select.poll from Gevent monkeypatching +- Remove redundant generator for chunked transfer encoding +- Fix: Response.ok raises Timeout Exception in safe\_mode + +0.10.8 (2012-03-09) +------------------- + +- Generate chunked ValueError fix +- Proxy configuration by environment variables +- Simplification of iter\_lines. +- New trust\_env configuration for disabling system/environment hints. +- Suppress cookie errors. + +0.10.7 (2012-03-07) +------------------- + +- encode\_uri = False + +0.10.6 (2012-02-25) +------------------- + +- Allow '=' in cookies. + +0.10.5 (2012-02-25) +------------------- + +- Response body with 0 content-length fix. +- New async.imap. +- Don't fail on netrc. + +0.10.4 (2012-02-20) +------------------- + +- Honor netrc. + +0.10.3 (2012-02-20) +------------------- + +- HEAD requests don't follow redirects anymore. +- raise\_for\_status() doesn't raise for 3xx anymore. +- Make Session objects picklable. +- ValueError for invalid schema URLs. + +0.10.2 (2012-01-15) +------------------- + +- Vastly improved URL quoting. +- Additional allowed cookie key values. +- Attempted fix for "Too many open files" Error +- Replace unicode errors on first pass, no need for second pass. +- Append '/' to bare-domain urls before query insertion. +- Exceptions now inherit from RuntimeError. +- Binary uploads + auth fix. +- Bugfixes. + +0.10.1 (2012-01-23) +------------------- + +- PYTHON 3 SUPPORT! +- Dropped 2.5 Support. (*Backwards Incompatible*) + +0.10.0 (2012-01-21) +------------------- + +- `Response.content` is now bytes-only. (*Backwards Incompatible*) +- New `Response.text` is unicode-only. +- If no `Response.encoding` is specified and `chardet` is available, + `Response.text` will guess an encoding. +- Default to ISO-8859-1 (Western) encoding for "text" subtypes. +- Removal of decode\_unicode. (*Backwards Incompatible*) +- New multiple-hooks system. +- New `Response.register_hook` for registering hooks within the + pipeline. +- `Response.url` is now Unicode. + +0.9.3 (2012-01-18) +------------------ + +- SSL verify=False bugfix (apparent on windows machines). + +0.9.2 (2012-01-18) +------------------ + +- Asynchronous async.send method. +- Support for proper chunk streams with boundaries. +- session argument for Session classes. +- Print entire hook tracebacks, not just exception instance. +- Fix response.iter\_lines from pending next line. +- Fix but in HTTP-digest auth w/ URI having query strings. +- Fix in Event Hooks section. +- Urllib3 update. + +0.9.1 (2012-01-06) +------------------ + +- danger\_mode for automatic Response.raise\_for\_status() +- Response.iter\_lines refactor + +0.9.0 (2011-12-28) +------------------ + +- verify ssl is default. + +0.8.9 (2011-12-28) +------------------ + +- Packaging fix. + +0.8.8 (2011-12-28) +------------------ + +- SSL CERT VERIFICATION! +- Release of Cerifi: Mozilla's cert list. +- New 'verify' argument for SSL requests. +- Urllib3 update. + +0.8.7 (2011-12-24) +------------------ + +- iter\_lines last-line truncation fix +- Force safe\_mode for async requests +- Handle safe\_mode exceptions more consistently +- Fix iteration on null responses in safe\_mode + +0.8.6 (2011-12-18) +------------------ + +- Socket timeout fixes. +- Proxy Authorization support. + +0.8.5 (2011-12-14) +------------------ + +- Response.iter\_lines! + +0.8.4 (2011-12-11) +------------------ + +- Prefetch bugfix. +- Added license to installed version. + +0.8.3 (2011-11-27) +------------------ + +- Converted auth system to use simpler callable objects. +- New session parameter to API methods. +- Display full URL while logging. + +0.8.2 (2011-11-19) +------------------ + +- New Unicode decoding system, based on over-ridable + Response.encoding. +- Proper URL slash-quote handling. +- Cookies with `[`, `]`, and `_` allowed. + +0.8.1 (2011-11-15) +------------------ + +- URL Request path fix +- Proxy fix. +- Timeouts fix. + +0.8.0 (2011-11-13) +------------------ + +- Keep-alive support! +- Complete removal of Urllib2 +- Complete removal of Poster +- Complete removal of CookieJars +- New ConnectionError raising +- Safe\_mode for error catching +- prefetch parameter for request methods +- OPTION method +- Async pool size throttling +- File uploads send real names +- Vendored in urllib3 + +0.7.6 (2011-11-07) +------------------ + +- Digest authentication bugfix (attach query data to path) + +0.7.5 (2011-11-04) +------------------ + +- Response.content = None if there was an invalid response. +- Redirection auth handling. + +0.7.4 (2011-10-26) +------------------ + +- Session Hooks fix. + +0.7.3 (2011-10-23) +------------------ + +- Digest Auth fix. + +0.7.2 (2011-10-23) +------------------ + +- PATCH Fix. + +0.7.1 (2011-10-23) +------------------ + +- Move away from urllib2 authentication handling. +- Fully Remove AuthManager, AuthObject, &c. +- New tuple-based auth system with handler callbacks. + +0.7.0 (2011-10-22) +------------------ + +- Sessions are now the primary interface. +- Deprecated InvalidMethodException. +- PATCH fix. +- New config system (no more global settings). + +0.6.6 (2011-10-19) +------------------ + +- Session parameter bugfix (params merging). + +0.6.5 (2011-10-18) +------------------ + +- Offline (fast) test suite. +- Session dictionary argument merging. + +0.6.4 (2011-10-13) +------------------ + +- Automatic decoding of unicode, based on HTTP Headers. +- New `decode_unicode` setting. +- Removal of `r.read/close` methods. +- New `r.faw` interface for advanced response usage.\* +- Automatic expansion of parameterized headers. + +0.6.3 (2011-10-13) +------------------ + +- Beautiful `requests.async` module, for making async requests w/ + gevent. + +0.6.2 (2011-10-09) +------------------ + +- GET/HEAD obeys allow\_redirects=False. + +0.6.1 (2011-08-20) +------------------ + +- Enhanced status codes experience `\o/` +- Set a maximum number of redirects (`settings.max_redirects`) +- Full Unicode URL support +- Support for protocol-less redirects. +- Allow for arbitrary request types. +- Bugfixes + +0.6.0 (2011-08-17) +------------------ + +- New callback hook system +- New persistent sessions object and context manager +- Transparent Dict-cookie handling +- Status code reference object +- Removed Response.cached +- Added Response.request +- All args are kwargs +- Relative redirect support +- HTTPError handling improvements +- Improved https testing +- Bugfixes + +0.5.1 (2011-07-23) +------------------ + +- International Domain Name Support! +- Access headers without fetching entire body (`read()`) +- Use lists as dicts for parameters +- Add Forced Basic Authentication +- Forced Basic is default authentication type +- `python-requests.org` default User-Agent header +- CaseInsensitiveDict lower-case caching +- Response.history bugfix + +0.5.0 (2011-06-21) +------------------ + +- PATCH Support +- Support for Proxies +- HTTPBin Test Suite +- Redirect Fixes +- settings.verbose stream writing +- Querystrings for all methods +- URLErrors (Connection Refused, Timeout, Invalid URLs) are treated as + explicitly raised + `r.requests.get('hwe://blah'); r.raise_for_status()` + +0.4.1 (2011-05-22) +------------------ + +- Improved Redirection Handling +- New 'allow\_redirects' param for following non-GET/HEAD Redirects +- Settings module refactoring + +0.4.0 (2011-05-15) +------------------ + +- Response.history: list of redirected responses +- Case-Insensitive Header Dictionaries! +- Unicode URLs + +0.3.4 (2011-05-14) +------------------ + +- Urllib2 HTTPAuthentication Recursion fix (Basic/Digest) +- Internal Refactor +- Bytes data upload Bugfix + +0.3.3 (2011-05-12) +------------------ + +- Request timeouts +- Unicode url-encoded data +- Settings context manager and module + +0.3.2 (2011-04-15) +------------------ + +- Automatic Decompression of GZip Encoded Content +- AutoAuth Support for Tupled HTTP Auth + +0.3.1 (2011-04-01) +------------------ + +- Cookie Changes +- Response.read() +- Poster fix + +0.3.0 (2011-02-25) +------------------ + +- Automatic Authentication API Change +- Smarter Query URL Parameterization +- Allow file uploads and POST data together +- + + New Authentication Manager System + + : - Simpler Basic HTTP System + - Supports all built-in urllib2 Auths + - Allows for custom Auth Handlers + +0.2.4 (2011-02-19) +------------------ + +- Python 2.5 Support +- PyPy-c v1.4 Support +- Auto-Authentication tests +- Improved Request object constructor + +0.2.3 (2011-02-15) +------------------ + +- + + New HTTPHandling Methods + + : - Response.\_\_nonzero\_\_ (false if bad HTTP Status) + - Response.ok (True if expected HTTP Status) + - Response.error (Logged HTTPError if bad HTTP Status) + - Response.raise\_for\_status() (Raises stored HTTPError) + +0.2.2 (2011-02-14) +------------------ + +- Still handles request in the event of an HTTPError. (Issue \#2) +- Eventlet and Gevent Monkeypatch support. +- Cookie Support (Issue \#1) + +0.2.1 (2011-02-14) +------------------ + +- Added file attribute to POST and PUT requests for multipart-encode + file uploads. +- Added Request.url attribute for context and redirects + +0.2.0 (2011-02-14) +------------------ + +- Birth! + +0.0.1 (2011-02-13) +------------------ + +- Frustration +- Conception diff --git a/HISTORY.rst b/HISTORY.rst deleted file mode 100644 index 61f7e232b4..0000000000 --- a/HISTORY.rst +++ /dev/null @@ -1,1545 +0,0 @@ -.. :changelog: - -Release History ---------------- - -dev -+++ - -**Improvements** - -- Warn user about possible slowdown when using cryptography version < 1.3.4 -- Check for invalid host in proxy URL, before forwarding request to adapter. -- Fragments are now properly maintained across redirects. (RFC7231 7.1.2) - -**Bugfixes** - -- Parsing empty ``Link`` headers with ``parse_header_links()`` no longer return one bogus entry -- Fixed issue where loading the default certificate bundle from a zip archive - would raise an ``IOError`` -- Fixed issue with unexpected ``ImportError`` on windows system which do not support ``winreg`` module -- DNS resolution in proxy bypass no longer includes the username and password in - the request. This also fixes the issue of DNS queries failing on macOS. - -2.18.4 (2017-08-15) -+++++++++++++++++++ - -**Improvements** - -- Error messages for invalid headers now include the header name for easier debugging - -**Dependencies** - -- We now support idna v2.6. - -2.18.3 (2017-08-02) -+++++++++++++++++++ - -**Improvements** - -- Running ``$ python -m requests.help`` now includes the installed version of idna. - -**Bugfixes** - -- Fixed issue where Requests would raise ``ConnectionError`` instead of - ``SSLError`` when encountering SSL problems when using urllib3 v1.22. - -2.18.2 (2017-07-25) -+++++++++++++++++++ - -**Bugfixes** - -- ``requests.help`` no longer fails on Python 2.6 due to the absence of - ``ssl.OPENSSL_VERSION_NUMBER``. - -**Dependencies** - -- We now support urllib3 v1.22. - -2.18.1 (2017-06-14) -+++++++++++++++++++ - -**Bugfixes** - -- Fix an error in the packaging whereby the ``*.whl`` contained incorrect data - that regressed the fix in v2.17.3. - -2.18.0 (2017-06-14) -+++++++++++++++++++ - -**Improvements** - -- ``Response`` is now a context manager, so can be used directly in a ``with`` statement - without first having to be wrapped by ``contextlib.closing()``. - -**Bugfixes** - -- Resolve installation failure if multiprocessing is not available -- Resolve tests crash if multiprocessing is not able to determine the number of CPU cores -- Resolve error swallowing in utils set_environ generator - - -2.17.3 (2017-05-29) -+++++++++++++++++++ - -**Improvements** - -- Improved ``packages`` namespace identity support, for monkeypatching libraries. - - -2.17.2 (2017-05-29) -+++++++++++++++++++ - -**Improvements** - -- Improved ``packages`` namespace identity support, for monkeypatching libraries. - - -2.17.1 (2017-05-29) -+++++++++++++++++++ - -**Improvements** - -- Improved ``packages`` namespace identity support, for monkeypatching libraries. - - -2.17.0 (2017-05-29) -+++++++++++++++++++ - -**Improvements** - -- Removal of the 301 redirect cache. This improves thread-safety. - - -2.16.5 (2017-05-28) -+++++++++++++++++++ - -- Improvements to ``$ python -m requests.help``. - -2.16.4 (2017-05-27) -+++++++++++++++++++ - -- Introduction of the ``$ python -m requests.help`` command, for debugging with maintainers! - -2.16.3 (2017-05-27) -+++++++++++++++++++ - -- Further restored the ``requests.packages`` namespace for compatibility reasons. - -2.16.2 (2017-05-27) -+++++++++++++++++++ - -- Further restored the ``requests.packages`` namespace for compatibility reasons. - -No code modification (noted below) should be necessary any longer. - -2.16.1 (2017-05-27) -+++++++++++++++++++ - -- Restored the ``requests.packages`` namespace for compatibility reasons. -- Bugfix for ``urllib3`` version parsing. - -**Note**: code that was written to import against the ``requests.packages`` -namespace previously will have to import code that rests at this module-level -now. - -For example:: - - from requests.packages.urllib3.poolmanager import PoolManager - -Will need to be re-written to be:: - - from requests.packages import urllib3 - urllib3.poolmanager.PoolManager - -Or, even better:: - - from urllib3.poolmanager import PoolManager - -2.16.0 (2017-05-26) -+++++++++++++++++++ - -- Unvendor ALL the things! - -2.15.1 (2017-05-26) -+++++++++++++++++++ - -- Everyone makes mistakes. - -2.15.0 (2017-05-26) -+++++++++++++++++++ - -**Improvements** - -- Introduction of the ``Response.next`` property, for getting the next - ``PreparedResponse`` from a redirect chain (when ``allow_redirects=False``). -- Internal refactoring of ``__version__`` module. - -**Bugfixes** - -- Restored once-optional parameter for ``requests.utils.get_environ_proxies()``. - -2.14.2 (2017-05-10) -+++++++++++++++++++ - -**Bugfixes** - -- Changed a less-than to an equal-to and an or in the dependency markers to - widen compatibility with older setuptools releases. - -2.14.1 (2017-05-09) -+++++++++++++++++++ - -**Bugfixes** - -- Changed the dependency markers to widen compatibility with older pip - releases. - -2.14.0 (2017-05-09) -+++++++++++++++++++ - -**Improvements** - -- It is now possible to pass ``no_proxy`` as a key to the ``proxies`` - dictionary to provide handling similar to the ``NO_PROXY`` environment - variable. -- When users provide invalid paths to certificate bundle files or directories - Requests now raises ``IOError``, rather than failing at the time of the HTTPS - request with a fairly inscrutable certificate validation error. -- The behavior of ``SessionRedirectMixin`` was slightly altered. - ``resolve_redirects`` will now detect a redirect by calling - ``get_redirect_target(response)`` instead of directly - querying ``Response.is_redirect`` and ``Response.headers['location']``. - Advanced users will be able to process malformed redirects more easily. -- Changed the internal calculation of elapsed request time to have higher - resolution on Windows. -- Added ``win_inet_pton`` as conditional dependency for the ``[socks]`` extra - on Windows with Python 2.7. -- Changed the proxy bypass implementation on Windows: the proxy bypass - check doesn't use forward and reverse DNS requests anymore -- URLs with schemes that begin with ``http`` but are not ``http`` or ``https`` - no longer have their host parts forced to lowercase. - -**Bugfixes** - -- Much improved handling of non-ASCII ``Location`` header values in redirects. - Fewer ``UnicodeDecodeErrors`` are encountered on Python 2, and Python 3 now - correctly understands that Latin-1 is unlikely to be the correct encoding. -- If an attempt to ``seek`` file to find out its length fails, we now - appropriately handle that by aborting our content-length calculations. -- Restricted ``HTTPDigestAuth`` to only respond to auth challenges made on 4XX - responses, rather than to all auth challenges. -- Fixed some code that was firing ``DeprecationWarning`` on Python 3.6. -- The dismayed person emoticon (``/o\\``) no longer has a big head. I'm sure - this is what you were all worrying about most. - - -**Miscellaneous** - -- Updated bundled urllib3 to v1.21.1. -- Updated bundled chardet to v3.0.2. -- Updated bundled idna to v2.5. -- Updated bundled certifi to 2017.4.17. - -2.13.0 (2017-01-24) -+++++++++++++++++++ - -**Features** - -- Only load the ``idna`` library when we've determined we need it. This will - save some memory for users. - -**Miscellaneous** - -- Updated bundled urllib3 to 1.20. -- Updated bundled idna to 2.2. - -2.12.5 (2017-01-18) -+++++++++++++++++++ - -**Bugfixes** - -- Fixed an issue with JSON encoding detection, specifically detecting - big-endian UTF-32 with BOM. - -2.12.4 (2016-12-14) -+++++++++++++++++++ - -**Bugfixes** - -- Fixed regression from 2.12.2 where non-string types were rejected in the - basic auth parameters. While support for this behaviour has been readded, - the behaviour is deprecated and will be removed in the future. - -2.12.3 (2016-12-01) -+++++++++++++++++++ - -**Bugfixes** - -- Fixed regression from v2.12.1 for URLs with schemes that begin with "http". - These URLs have historically been processed as though they were HTTP-schemed - URLs, and so have had parameters added. This was removed in v2.12.2 in an - overzealous attempt to resolve problems with IDNA-encoding those URLs. This - change was reverted: the other fixes for IDNA-encoding have been judged to - be sufficient to return to the behaviour Requests had before v2.12.0. - -2.12.2 (2016-11-30) -+++++++++++++++++++ - -**Bugfixes** - -- Fixed several issues with IDNA-encoding URLs that are technically invalid but - which are widely accepted. Requests will now attempt to IDNA-encode a URL if - it can but, if it fails, and the host contains only ASCII characters, it will - be passed through optimistically. This will allow users to opt-in to using - IDNA2003 themselves if they want to, and will also allow technically invalid - but still common hostnames. -- Fixed an issue where URLs with leading whitespace would raise - ``InvalidSchema`` errors. -- Fixed an issue where some URLs without the HTTP or HTTPS schemes would still - have HTTP URL preparation applied to them. -- Fixed an issue where Unicode strings could not be used in basic auth. -- Fixed an issue encountered by some Requests plugins where constructing a - Response object would cause ``Response.content`` to raise an - ``AttributeError``. - -2.12.1 (2016-11-16) -+++++++++++++++++++ - -**Bugfixes** - -- Updated setuptools 'security' extra for the new PyOpenSSL backend in urllib3. - -**Miscellaneous** - -- Updated bundled urllib3 to 1.19.1. - -2.12.0 (2016-11-15) -+++++++++++++++++++ - -**Improvements** - -- Updated support for internationalized domain names from IDNA2003 to IDNA2008. - This updated support is required for several forms of IDNs and is mandatory - for .de domains. -- Much improved heuristics for guessing content lengths: Requests will no - longer read an entire ``StringIO`` into memory. -- Much improved logic for recalculating ``Content-Length`` headers for - ``PreparedRequest`` objects. -- Improved tolerance for file-like objects that have no ``tell`` method but - do have a ``seek`` method. -- Anything that is a subclass of ``Mapping`` is now treated like a dictionary - by the ``data=`` keyword argument. -- Requests now tolerates empty passwords in proxy credentials, rather than - stripping the credentials. -- If a request is made with a file-like object as the body and that request is - redirected with a 307 or 308 status code, Requests will now attempt to - rewind the body object so it can be replayed. - -**Bugfixes** - -- When calling ``response.close``, the call to ``close`` will be propagated - through to non-urllib3 backends. -- Fixed issue where the ``ALL_PROXY`` environment variable would be preferred - over scheme-specific variables like ``HTTP_PROXY``. -- Fixed issue where non-UTF8 reason phrases got severely mangled by falling - back to decoding using ISO 8859-1 instead. -- Fixed a bug where Requests would not correctly correlate cookies set when - using custom Host headers if those Host headers did not use the native - string type for the platform. - -**Miscellaneous** - -- Updated bundled urllib3 to 1.19. -- Updated bundled certifi certs to 2016.09.26. - -2.11.1 (2016-08-17) -+++++++++++++++++++ - -**Bugfixes** - -- Fixed a bug when using ``iter_content`` with ``decode_unicode=True`` for - streamed bodies would raise ``AttributeError``. This bug was introduced in - 2.11. -- Strip Content-Type and Transfer-Encoding headers from the header block when - following a redirect that transforms the verb from POST/PUT to GET. - -2.11.0 (2016-08-08) -+++++++++++++++++++ - -**Improvements** - -- Added support for the ``ALL_PROXY`` environment variable. -- Reject header values that contain leading whitespace or newline characters to - reduce risk of header smuggling. - -**Bugfixes** - -- Fixed occasional ``TypeError`` when attempting to decode a JSON response that - occurred in an error case. Now correctly returns a ``ValueError``. -- Requests would incorrectly ignore a non-CIDR IP address in the ``NO_PROXY`` - environment variables: Requests now treats it as a specific IP. -- Fixed a bug when sending JSON data that could cause us to encounter obscure - OpenSSL errors in certain network conditions (yes, really). -- Added type checks to ensure that ``iter_content`` only accepts integers and - ``None`` for chunk sizes. -- Fixed issue where responses whose body had not been fully consumed would have - the underlying connection closed but not returned to the connection pool, - which could cause Requests to hang in situations where the ``HTTPAdapter`` - had been configured to use a blocking connection pool. - -**Miscellaneous** - -- Updated bundled urllib3 to 1.16. -- Some previous releases accidentally accepted non-strings as acceptable header values. This release does not. - -2.10.0 (2016-04-29) -+++++++++++++++++++ - -**New Features** - -- SOCKS Proxy Support! (requires PySocks; ``$ pip install requests[socks]``) - -**Miscellaneous** - -- Updated bundled urllib3 to 1.15.1. - -2.9.2 (2016-04-29) -++++++++++++++++++ - -**Improvements** - -- Change built-in CaseInsensitiveDict (used for headers) to use OrderedDict - as its underlying datastore. - -**Bugfixes** - -- Don't use redirect_cache if allow_redirects=False -- When passed objects that throw exceptions from ``tell()``, send them via - chunked transfer encoding instead of failing. -- Raise a ProxyError for proxy related connection issues. - -2.9.1 (2015-12-21) -++++++++++++++++++ - -**Bugfixes** - -- Resolve regression introduced in 2.9.0 that made it impossible to send binary - strings as bodies in Python 3. -- Fixed errors when calculating cookie expiration dates in certain locales. - -**Miscellaneous** - -- Updated bundled urllib3 to 1.13.1. - -2.9.0 (2015-12-15) -++++++++++++++++++ - -**Minor Improvements** (Backwards compatible) - -- The ``verify`` keyword argument now supports being passed a path to a - directory of CA certificates, not just a single-file bundle. -- Warnings are now emitted when sending files opened in text mode. -- Added the 511 Network Authentication Required status code to the status code - registry. - -**Bugfixes** - -- For file-like objects that are not seeked to the very beginning, we now - send the content length for the number of bytes we will actually read, rather - than the total size of the file, allowing partial file uploads. -- When uploading file-like objects, if they are empty or have no obvious - content length we set ``Transfer-Encoding: chunked`` rather than - ``Content-Length: 0``. -- We correctly receive the response in buffered mode when uploading chunked - bodies. -- We now handle being passed a query string as a bytestring on Python 3, by - decoding it as UTF-8. -- Sessions are now closed in all cases (exceptional and not) when using the - functional API rather than leaking and waiting for the garbage collector to - clean them up. -- Correctly handle digest auth headers with a malformed ``qop`` directive that - contains no token, by treating it the same as if no ``qop`` directive was - provided at all. -- Minor performance improvements when removing specific cookies by name. - -**Miscellaneous** - -- Updated urllib3 to 1.13. - -2.8.1 (2015-10-13) -++++++++++++++++++ - -**Bugfixes** - -- Update certificate bundle to match ``certifi`` 2015.9.6.2's weak certificate - bundle. -- Fix a bug in 2.8.0 where requests would raise ``ConnectTimeout`` instead of - ``ConnectionError`` -- When using the PreparedRequest flow, requests will now correctly respect the - ``json`` parameter. Broken in 2.8.0. -- When using the PreparedRequest flow, requests will now correctly handle a - Unicode-string method name on Python 2. Broken in 2.8.0. - -2.8.0 (2015-10-05) -++++++++++++++++++ - -**Minor Improvements** (Backwards Compatible) - -- Requests now supports per-host proxies. This allows the ``proxies`` - dictionary to have entries of the form - ``{'://': ''}``. Host-specific proxies will be used - in preference to the previously-supported scheme-specific ones, but the - previous syntax will continue to work. -- ``Response.raise_for_status`` now prints the URL that failed as part of the - exception message. -- ``requests.utils.get_netrc_auth`` now takes an ``raise_errors`` kwarg, - defaulting to ``False``. When ``True``, errors parsing ``.netrc`` files cause - exceptions to be thrown. -- Change to bundled projects import logic to make it easier to unbundle - requests downstream. -- Changed the default User-Agent string to avoid leaking data on Linux: now - contains only the requests version. - -**Bugfixes** - -- The ``json`` parameter to ``post()`` and friends will now only be used if - neither ``data`` nor ``files`` are present, consistent with the - documentation. -- We now ignore empty fields in the ``NO_PROXY`` environment variable. -- Fixed problem where ``httplib.BadStatusLine`` would get raised if combining - ``stream=True`` with ``contextlib.closing``. -- Prevented bugs where we would attempt to return the same connection back to - the connection pool twice when sending a Chunked body. -- Miscellaneous minor internal changes. -- Digest Auth support is now thread safe. - -**Updates** - -- Updated urllib3 to 1.12. - -2.7.0 (2015-05-03) -++++++++++++++++++ - -This is the first release that follows our new release process. For more, see -`our documentation -`_. - -**Bugfixes** - -- Updated urllib3 to 1.10.4, resolving several bugs involving chunked transfer - encoding and response framing. - -2.6.2 (2015-04-23) -++++++++++++++++++ - -**Bugfixes** - -- Fix regression where compressed data that was sent as chunked data was not - properly decompressed. (#2561) - -2.6.1 (2015-04-22) -++++++++++++++++++ - -**Bugfixes** - -- Remove VendorAlias import machinery introduced in v2.5.2. - -- Simplify the PreparedRequest.prepare API: We no longer require the user to - pass an empty list to the hooks keyword argument. (c.f. #2552) - -- Resolve redirects now receives and forwards all of the original arguments to - the adapter. (#2503) - -- Handle UnicodeDecodeErrors when trying to deal with a unicode URL that - cannot be encoded in ASCII. (#2540) - -- Populate the parsed path of the URI field when performing Digest - Authentication. (#2426) - -- Copy a PreparedRequest's CookieJar more reliably when it is not an instance - of RequestsCookieJar. (#2527) - -2.6.0 (2015-03-14) -++++++++++++++++++ - -**Bugfixes** - -- CVE-2015-2296: Fix handling of cookies on redirect. Previously a cookie - without a host value set would use the hostname for the redirected URL - exposing requests users to session fixation attacks and potentially cookie - stealing. This was disclosed privately by Matthew Daley of - `BugFuzz `_. This affects all versions of requests from - v2.1.0 to v2.5.3 (inclusive on both ends). - -- Fix error when requests is an ``install_requires`` dependency and ``python - setup.py test`` is run. (#2462) - -- Fix error when urllib3 is unbundled and requests continues to use the - vendored import location. - -- Include fixes to ``urllib3``'s header handling. - -- Requests' handling of unvendored dependencies is now more restrictive. - -**Features and Improvements** - -- Support bytearrays when passed as parameters in the ``files`` argument. - (#2468) - -- Avoid data duplication when creating a request with ``str``, ``bytes``, or - ``bytearray`` input to the ``files`` argument. - -2.5.3 (2015-02-24) -++++++++++++++++++ - -**Bugfixes** - -- Revert changes to our vendored certificate bundle. For more context see - (#2455, #2456, and http://bugs.python.org/issue23476) - -2.5.2 (2015-02-23) -++++++++++++++++++ - -**Features and Improvements** - -- Add sha256 fingerprint support. (`shazow/urllib3#540`_) - -- Improve the performance of headers. (`shazow/urllib3#544`_) - -**Bugfixes** - -- Copy pip's import machinery. When downstream redistributors remove - requests.packages.urllib3 the import machinery will continue to let those - same symbols work. Example usage in requests' documentation and 3rd-party - libraries relying on the vendored copies of urllib3 will work without having - to fallback to the system urllib3. - -- Attempt to quote parts of the URL on redirect if unquoting and then quoting - fails. (#2356) - -- Fix filename type check for multipart form-data uploads. (#2411) - -- Properly handle the case where a server issuing digest authentication - challenges provides both auth and auth-int qop-values. (#2408) - -- Fix a socket leak. (`shazow/urllib3#549`_) - -- Fix multiple ``Set-Cookie`` headers properly. (`shazow/urllib3#534`_) - -- Disable the built-in hostname verification. (`shazow/urllib3#526`_) - -- Fix the behaviour of decoding an exhausted stream. (`shazow/urllib3#535`_) - -**Security** - -- Pulled in an updated ``cacert.pem``. - -- Drop RC4 from the default cipher list. (`shazow/urllib3#551`_) - -.. _shazow/urllib3#551: https://github.com/shazow/urllib3/pull/551 -.. _shazow/urllib3#549: https://github.com/shazow/urllib3/pull/549 -.. _shazow/urllib3#544: https://github.com/shazow/urllib3/pull/544 -.. _shazow/urllib3#540: https://github.com/shazow/urllib3/pull/540 -.. _shazow/urllib3#535: https://github.com/shazow/urllib3/pull/535 -.. _shazow/urllib3#534: https://github.com/shazow/urllib3/pull/534 -.. _shazow/urllib3#526: https://github.com/shazow/urllib3/pull/526 - -2.5.1 (2014-12-23) -++++++++++++++++++ - -**Behavioural Changes** - -- Only catch HTTPErrors in raise_for_status (#2382) - -**Bugfixes** - -- Handle LocationParseError from urllib3 (#2344) -- Handle file-like object filenames that are not strings (#2379) -- Unbreak HTTPDigestAuth handler. Allow new nonces to be negotiated (#2389) - -2.5.0 (2014-12-01) -++++++++++++++++++ - -**Improvements** - -- Allow usage of urllib3's Retry object with HTTPAdapters (#2216) -- The ``iter_lines`` method on a response now accepts a delimiter with which - to split the content (#2295) - -**Behavioural Changes** - -- Add deprecation warnings to functions in requests.utils that will be removed - in 3.0 (#2309) -- Sessions used by the functional API are always closed (#2326) -- Restrict requests to HTTP/1.1 and HTTP/1.0 (stop accepting HTTP/0.9) (#2323) - -**Bugfixes** - -- Only parse the URL once (#2353) -- Allow Content-Length header to always be overridden (#2332) -- Properly handle files in HTTPDigestAuth (#2333) -- Cap redirect_cache size to prevent memory abuse (#2299) -- Fix HTTPDigestAuth handling of redirects after authenticating successfully - (#2253) -- Fix crash with custom method parameter to Session.request (#2317) -- Fix how Link headers are parsed using the regular expression library (#2271) - -**Documentation** - -- Add more references for interlinking (#2348) -- Update CSS for theme (#2290) -- Update width of buttons and sidebar (#2289) -- Replace references of Gittip with Gratipay (#2282) -- Add link to changelog in sidebar (#2273) - -2.4.3 (2014-10-06) -++++++++++++++++++ - -**Bugfixes** - -- Unicode URL improvements for Python 2. -- Re-order JSON param for backwards compat. -- Automatically defrag authentication schemes from host/pass URIs. (`#2249 `_) - - -2.4.2 (2014-10-05) -++++++++++++++++++ - -**Improvements** - -- FINALLY! Add json parameter for uploads! (`#2258 `_) -- Support for bytestring URLs on Python 3.x (`#2238 `_) - -**Bugfixes** - -- Avoid getting stuck in a loop (`#2244 `_) -- Multiple calls to iter* fail with unhelpful error. (`#2240 `_, `#2241 `_) - -**Documentation** - -- Correct redirection introduction (`#2245 `_) -- Added example of how to send multiple files in one request. (`#2227 `_) -- Clarify how to pass a custom set of CAs (`#2248 `_) - - - -2.4.1 (2014-09-09) -++++++++++++++++++ - -- Now has a "security" package extras set, ``$ pip install requests[security]`` -- Requests will now use Certifi if it is available. -- Capture and re-raise urllib3 ProtocolError -- Bugfix for responses that attempt to redirect to themselves forever (wtf?). - - -2.4.0 (2014-08-29) -++++++++++++++++++ - -**Behavioral Changes** - -- ``Connection: keep-alive`` header is now sent automatically. - -**Improvements** - -- Support for connect timeouts! Timeout now accepts a tuple (connect, read) which is used to set individual connect and read timeouts. -- Allow copying of PreparedRequests without headers/cookies. -- Updated bundled urllib3 version. -- Refactored settings loading from environment -- new `Session.merge_environment_settings`. -- Handle socket errors in iter_content. - - -2.3.0 (2014-05-16) -++++++++++++++++++ - -**API Changes** - -- New ``Response`` property ``is_redirect``, which is true when the - library could have processed this response as a redirection (whether - or not it actually did). -- The ``timeout`` parameter now affects requests with both ``stream=True`` and - ``stream=False`` equally. -- The change in v2.0.0 to mandate explicit proxy schemes has been reverted. - Proxy schemes now default to ``http://``. -- The ``CaseInsensitiveDict`` used for HTTP headers now behaves like a normal - dictionary when references as string or viewed in the interpreter. - -**Bugfixes** - -- No longer expose Authorization or Proxy-Authorization headers on redirect. - Fix CVE-2014-1829 and CVE-2014-1830 respectively. -- Authorization is re-evaluated each redirect. -- On redirect, pass url as native strings. -- Fall-back to autodetected encoding for JSON when Unicode detection fails. -- Headers set to ``None`` on the ``Session`` are now correctly not sent. -- Correctly honor ``decode_unicode`` even if it wasn't used earlier in the same - response. -- Stop advertising ``compress`` as a supported Content-Encoding. -- The ``Response.history`` parameter is now always a list. -- Many, many ``urllib3`` bugfixes. - -2.2.1 (2014-01-23) -++++++++++++++++++ - -**Bugfixes** - -- Fixes incorrect parsing of proxy credentials that contain a literal or encoded '#' character. -- Assorted urllib3 fixes. - -2.2.0 (2014-01-09) -++++++++++++++++++ - -**API Changes** - -- New exception: ``ContentDecodingError``. Raised instead of ``urllib3`` - ``DecodeError`` exceptions. - -**Bugfixes** - -- Avoid many many exceptions from the buggy implementation of ``proxy_bypass`` on OS X in Python 2.6. -- Avoid crashing when attempting to get authentication credentials from ~/.netrc when running as a user without a home directory. -- Use the correct pool size for pools of connections to proxies. -- Fix iteration of ``CookieJar`` objects. -- Ensure that cookies are persisted over redirect. -- Switch back to using chardet, since it has merged with charade. - -2.1.0 (2013-12-05) -++++++++++++++++++ - -- Updated CA Bundle, of course. -- Cookies set on individual Requests through a ``Session`` (e.g. via ``Session.get()``) are no longer persisted to the ``Session``. -- Clean up connections when we hit problems during chunked upload, rather than leaking them. -- Return connections to the pool when a chunked upload is successful, rather than leaking it. -- Match the HTTPbis recommendation for HTTP 301 redirects. -- Prevent hanging when using streaming uploads and Digest Auth when a 401 is received. -- Values of headers set by Requests are now always the native string type. -- Fix previously broken SNI support. -- Fix accessing HTTP proxies using proxy authentication. -- Unencode HTTP Basic usernames and passwords extracted from URLs. -- Support for IP address ranges for no_proxy environment variable -- Parse headers correctly when users override the default ``Host:`` header. -- Avoid munging the URL in case of case-sensitive servers. -- Looser URL handling for non-HTTP/HTTPS urls. -- Accept unicode methods in Python 2.6 and 2.7. -- More resilient cookie handling. -- Make ``Response`` objects pickleable. -- Actually added MD5-sess to Digest Auth instead of pretending to like last time. -- Updated internal urllib3. -- Fixed @Lukasa's lack of taste. - -2.0.1 (2013-10-24) -++++++++++++++++++ - -- Updated included CA Bundle with new mistrusts and automated process for the future -- Added MD5-sess to Digest Auth -- Accept per-file headers in multipart file POST messages. -- Fixed: Don't send the full URL on CONNECT messages. -- Fixed: Correctly lowercase a redirect scheme. -- Fixed: Cookies not persisted when set via functional API. -- Fixed: Translate urllib3 ProxyError into a requests ProxyError derived from ConnectionError. -- Updated internal urllib3 and chardet. - -2.0.0 (2013-09-24) -++++++++++++++++++ - -**API Changes:** - -- Keys in the Headers dictionary are now native strings on all Python versions, - i.e. bytestrings on Python 2, unicode on Python 3. -- Proxy URLs now *must* have an explicit scheme. A ``MissingSchema`` exception - will be raised if they don't. -- Timeouts now apply to read time if ``Stream=False``. -- ``RequestException`` is now a subclass of ``IOError``, not ``RuntimeError``. -- Added new method to ``PreparedRequest`` objects: ``PreparedRequest.copy()``. -- Added new method to ``Session`` objects: ``Session.update_request()``. This - method updates a ``Request`` object with the data (e.g. cookies) stored on - the ``Session``. -- Added new method to ``Session`` objects: ``Session.prepare_request()``. This - method updates and prepares a ``Request`` object, and returns the - corresponding ``PreparedRequest`` object. -- Added new method to ``HTTPAdapter`` objects: ``HTTPAdapter.proxy_headers()``. - This should not be called directly, but improves the subclass interface. -- ``httplib.IncompleteRead`` exceptions caused by incorrect chunked encoding - will now raise a Requests ``ChunkedEncodingError`` instead. -- Invalid percent-escape sequences now cause a Requests ``InvalidURL`` - exception to be raised. -- HTTP 208 no longer uses reason phrase ``"im_used"``. Correctly uses - ``"already_reported"``. -- HTTP 226 reason added (``"im_used"``). - -**Bugfixes:** - -- Vastly improved proxy support, including the CONNECT verb. Special thanks to - the many contributors who worked towards this improvement. -- Cookies are now properly managed when 401 authentication responses are - received. -- Chunked encoding fixes. -- Support for mixed case schemes. -- Better handling of streaming downloads. -- Retrieve environment proxies from more locations. -- Minor cookies fixes. -- Improved redirect behaviour. -- Improved streaming behaviour, particularly for compressed data. -- Miscellaneous small Python 3 text encoding bugs. -- ``.netrc`` no longer overrides explicit auth. -- Cookies set by hooks are now correctly persisted on Sessions. -- Fix problem with cookies that specify port numbers in their host field. -- ``BytesIO`` can be used to perform streaming uploads. -- More generous parsing of the ``no_proxy`` environment variable. -- Non-string objects can be passed in data values alongside files. - -1.2.3 (2013-05-25) -++++++++++++++++++ - -- Simple packaging fix - - -1.2.2 (2013-05-23) -++++++++++++++++++ - -- Simple packaging fix - - -1.2.1 (2013-05-20) -++++++++++++++++++ - -- 301 and 302 redirects now change the verb to GET for all verbs, not just - POST, improving browser compatibility. -- Python 3.3.2 compatibility -- Always percent-encode location headers -- Fix connection adapter matching to be most-specific first -- new argument to the default connection adapter for passing a block argument -- prevent a KeyError when there's no link headers - -1.2.0 (2013-03-31) -++++++++++++++++++ - -- Fixed cookies on sessions and on requests -- Significantly change how hooks are dispatched - hooks now receive all the - arguments specified by the user when making a request so hooks can make a - secondary request with the same parameters. This is especially necessary for - authentication handler authors -- certifi support was removed -- Fixed bug where using OAuth 1 with body ``signature_type`` sent no data -- Major proxy work thanks to @Lukasa including parsing of proxy authentication - from the proxy url -- Fix DigestAuth handling too many 401s -- Update vendored urllib3 to include SSL bug fixes -- Allow keyword arguments to be passed to ``json.loads()`` via the - ``Response.json()`` method -- Don't send ``Content-Length`` header by default on ``GET`` or ``HEAD`` - requests -- Add ``elapsed`` attribute to ``Response`` objects to time how long a request - took. -- Fix ``RequestsCookieJar`` -- Sessions and Adapters are now picklable, i.e., can be used with the - multiprocessing library -- Update charade to version 1.0.3 - -The change in how hooks are dispatched will likely cause a great deal of -issues. - -1.1.0 (2013-01-10) -++++++++++++++++++ - -- CHUNKED REQUESTS -- Support for iterable response bodies -- Assume servers persist redirect params -- Allow explicit content types to be specified for file data -- Make merge_kwargs case-insensitive when looking up keys - -1.0.3 (2012-12-18) -++++++++++++++++++ - -- Fix file upload encoding bug -- Fix cookie behavior - -1.0.2 (2012-12-17) -++++++++++++++++++ - -- Proxy fix for HTTPAdapter. - -1.0.1 (2012-12-17) -++++++++++++++++++ - -- Cert verification exception bug. -- Proxy fix for HTTPAdapter. - -1.0.0 (2012-12-17) -++++++++++++++++++ - -- Massive Refactor and Simplification -- Switch to Apache 2.0 license -- Swappable Connection Adapters -- Mountable Connection Adapters -- Mutable ProcessedRequest chain -- /s/prefetch/stream -- Removal of all configuration -- Standard library logging -- Make Response.json() callable, not property. -- Usage of new charade project, which provides python 2 and 3 simultaneous chardet. -- Removal of all hooks except 'response' -- Removal of all authentication helpers (OAuth, Kerberos) - -This is not a backwards compatible change. - -0.14.2 (2012-10-27) -+++++++++++++++++++ - -- Improved mime-compatible JSON handling -- Proxy fixes -- Path hack fixes -- Case-Insensitive Content-Encoding headers -- Support for CJK parameters in form posts - - -0.14.1 (2012-10-01) -+++++++++++++++++++ - -- Python 3.3 Compatibility -- Simply default accept-encoding -- Bugfixes - - -0.14.0 (2012-09-02) -++++++++++++++++++++ - -- No more iter_content errors if already downloaded. - -0.13.9 (2012-08-25) -+++++++++++++++++++ - -- Fix for OAuth + POSTs -- Remove exception eating from dispatch_hook -- General bugfixes - -0.13.8 (2012-08-21) -+++++++++++++++++++ - -- Incredible Link header support :) - -0.13.7 (2012-08-19) -+++++++++++++++++++ - -- Support for (key, value) lists everywhere. -- Digest Authentication improvements. -- Ensure proxy exclusions work properly. -- Clearer UnicodeError exceptions. -- Automatic casting of URLs to strings (fURL and such) -- Bugfixes. - -0.13.6 (2012-08-06) -+++++++++++++++++++ - -- Long awaited fix for hanging connections! - -0.13.5 (2012-07-27) -+++++++++++++++++++ - -- Packaging fix - -0.13.4 (2012-07-27) -+++++++++++++++++++ - -- GSSAPI/Kerberos authentication! -- App Engine 2.7 Fixes! -- Fix leaking connections (from urllib3 update) -- OAuthlib path hack fix -- OAuthlib URL parameters fix. - -0.13.3 (2012-07-12) -+++++++++++++++++++ - -- Use simplejson if available. -- Do not hide SSLErrors behind Timeouts. -- Fixed param handling with urls containing fragments. -- Significantly improved information in User Agent. -- client certificates are ignored when verify=False - -0.13.2 (2012-06-28) -+++++++++++++++++++ - -- Zero dependencies (once again)! -- New: Response.reason -- Sign querystring parameters in OAuth 1.0 -- Client certificates no longer ignored when verify=False -- Add openSUSE certificate support - -0.13.1 (2012-06-07) -+++++++++++++++++++ - -- Allow passing a file or file-like object as data. -- Allow hooks to return responses that indicate errors. -- Fix Response.text and Response.json for body-less responses. - -0.13.0 (2012-05-29) -+++++++++++++++++++ - -- Removal of Requests.async in favor of `grequests `_ -- Allow disabling of cookie persistence. -- New implementation of safe_mode -- cookies.get now supports default argument -- Session cookies not saved when Session.request is called with return_response=False -- Env: no_proxy support. -- RequestsCookieJar improvements. -- Various bug fixes. - -0.12.1 (2012-05-08) -+++++++++++++++++++ - -- New ``Response.json`` property. -- Ability to add string file uploads. -- Fix out-of-range issue with iter_lines. -- Fix iter_content default size. -- Fix POST redirects containing files. - -0.12.0 (2012-05-02) -+++++++++++++++++++ - -- EXPERIMENTAL OAUTH SUPPORT! -- Proper CookieJar-backed cookies interface with awesome dict-like interface. -- Speed fix for non-iterated content chunks. -- Move ``pre_request`` to a more usable place. -- New ``pre_send`` hook. -- Lazily encode data, params, files. -- Load system Certificate Bundle if ``certify`` isn't available. -- Cleanups, fixes. - -0.11.2 (2012-04-22) -+++++++++++++++++++ - -- Attempt to use the OS's certificate bundle if ``certifi`` isn't available. -- Infinite digest auth redirect fix. -- Multi-part file upload improvements. -- Fix decoding of invalid %encodings in URLs. -- If there is no content in a response don't throw an error the second time that content is attempted to be read. -- Upload data on redirects. - -0.11.1 (2012-03-30) -+++++++++++++++++++ - -* POST redirects now break RFC to do what browsers do: Follow up with a GET. -* New ``strict_mode`` configuration to disable new redirect behavior. - - -0.11.0 (2012-03-14) -+++++++++++++++++++ - -* Private SSL Certificate support -* Remove select.poll from Gevent monkeypatching -* Remove redundant generator for chunked transfer encoding -* Fix: Response.ok raises Timeout Exception in safe_mode - -0.10.8 (2012-03-09) -+++++++++++++++++++ - -* Generate chunked ValueError fix -* Proxy configuration by environment variables -* Simplification of iter_lines. -* New `trust_env` configuration for disabling system/environment hints. -* Suppress cookie errors. - -0.10.7 (2012-03-07) -+++++++++++++++++++ - -* `encode_uri` = False - -0.10.6 (2012-02-25) -+++++++++++++++++++ - -* Allow '=' in cookies. - -0.10.5 (2012-02-25) -+++++++++++++++++++ - -* Response body with 0 content-length fix. -* New async.imap. -* Don't fail on netrc. - - -0.10.4 (2012-02-20) -+++++++++++++++++++ - -* Honor netrc. - -0.10.3 (2012-02-20) -+++++++++++++++++++ - -* HEAD requests don't follow redirects anymore. -* raise_for_status() doesn't raise for 3xx anymore. -* Make Session objects picklable. -* ValueError for invalid schema URLs. - -0.10.2 (2012-01-15) -+++++++++++++++++++ - -* Vastly improved URL quoting. -* Additional allowed cookie key values. -* Attempted fix for "Too many open files" Error -* Replace unicode errors on first pass, no need for second pass. -* Append '/' to bare-domain urls before query insertion. -* Exceptions now inherit from RuntimeError. -* Binary uploads + auth fix. -* Bugfixes. - - -0.10.1 (2012-01-23) -+++++++++++++++++++ - -* PYTHON 3 SUPPORT! -* Dropped 2.5 Support. (*Backwards Incompatible*) - -0.10.0 (2012-01-21) -+++++++++++++++++++ - -* ``Response.content`` is now bytes-only. (*Backwards Incompatible*) -* New ``Response.text`` is unicode-only. -* If no ``Response.encoding`` is specified and ``chardet`` is available, ``Response.text`` will guess an encoding. -* Default to ISO-8859-1 (Western) encoding for "text" subtypes. -* Removal of `decode_unicode`. (*Backwards Incompatible*) -* New multiple-hooks system. -* New ``Response.register_hook`` for registering hooks within the pipeline. -* ``Response.url`` is now Unicode. - -0.9.3 (2012-01-18) -++++++++++++++++++ - -* SSL verify=False bugfix (apparent on windows machines). - -0.9.2 (2012-01-18) -++++++++++++++++++ - -* Asynchronous async.send method. -* Support for proper chunk streams with boundaries. -* session argument for Session classes. -* Print entire hook tracebacks, not just exception instance. -* Fix response.iter_lines from pending next line. -* Fix but in HTTP-digest auth w/ URI having query strings. -* Fix in Event Hooks section. -* Urllib3 update. - - -0.9.1 (2012-01-06) -++++++++++++++++++ - -* danger_mode for automatic Response.raise_for_status() -* Response.iter_lines refactor - -0.9.0 (2011-12-28) -++++++++++++++++++ - -* verify ssl is default. - - -0.8.9 (2011-12-28) -++++++++++++++++++ - -* Packaging fix. - - -0.8.8 (2011-12-28) -++++++++++++++++++ - -* SSL CERT VERIFICATION! -* Release of Cerifi: Mozilla's cert list. -* New 'verify' argument for SSL requests. -* Urllib3 update. - -0.8.7 (2011-12-24) -++++++++++++++++++ - -* iter_lines last-line truncation fix -* Force safe_mode for async requests -* Handle safe_mode exceptions more consistently -* Fix iteration on null responses in safe_mode - -0.8.6 (2011-12-18) -++++++++++++++++++ - -* Socket timeout fixes. -* Proxy Authorization support. - -0.8.5 (2011-12-14) -++++++++++++++++++ - -* Response.iter_lines! - -0.8.4 (2011-12-11) -++++++++++++++++++ - -* Prefetch bugfix. -* Added license to installed version. - -0.8.3 (2011-11-27) -++++++++++++++++++ - -* Converted auth system to use simpler callable objects. -* New session parameter to API methods. -* Display full URL while logging. - -0.8.2 (2011-11-19) -++++++++++++++++++ - -* New Unicode decoding system, based on over-ridable `Response.encoding`. -* Proper URL slash-quote handling. -* Cookies with ``[``, ``]``, and ``_`` allowed. - -0.8.1 (2011-11-15) -++++++++++++++++++ - -* URL Request path fix -* Proxy fix. -* Timeouts fix. - -0.8.0 (2011-11-13) -++++++++++++++++++ - -* Keep-alive support! -* Complete removal of Urllib2 -* Complete removal of Poster -* Complete removal of CookieJars -* New ConnectionError raising -* Safe_mode for error catching -* prefetch parameter for request methods -* OPTION method -* Async pool size throttling -* File uploads send real names -* Vendored in urllib3 - -0.7.6 (2011-11-07) -++++++++++++++++++ - -* Digest authentication bugfix (attach query data to path) - -0.7.5 (2011-11-04) -++++++++++++++++++ - -* Response.content = None if there was an invalid response. -* Redirection auth handling. - -0.7.4 (2011-10-26) -++++++++++++++++++ - -* Session Hooks fix. - -0.7.3 (2011-10-23) -++++++++++++++++++ - -* Digest Auth fix. - - -0.7.2 (2011-10-23) -++++++++++++++++++ - -* PATCH Fix. - - -0.7.1 (2011-10-23) -++++++++++++++++++ - -* Move away from urllib2 authentication handling. -* Fully Remove AuthManager, AuthObject, &c. -* New tuple-based auth system with handler callbacks. - - -0.7.0 (2011-10-22) -++++++++++++++++++ - -* Sessions are now the primary interface. -* Deprecated InvalidMethodException. -* PATCH fix. -* New config system (no more global settings). - - -0.6.6 (2011-10-19) -++++++++++++++++++ - -* Session parameter bugfix (params merging). - - -0.6.5 (2011-10-18) -++++++++++++++++++ - -* Offline (fast) test suite. -* Session dictionary argument merging. - - -0.6.4 (2011-10-13) -++++++++++++++++++ - -* Automatic decoding of unicode, based on HTTP Headers. -* New ``decode_unicode`` setting. -* Removal of ``r.read/close`` methods. -* New ``r.faw`` interface for advanced response usage.* -* Automatic expansion of parameterized headers. - - -0.6.3 (2011-10-13) -++++++++++++++++++ - -* Beautiful ``requests.async`` module, for making async requests w/ gevent. - - -0.6.2 (2011-10-09) -++++++++++++++++++ - -* GET/HEAD obeys allow_redirects=False. - - -0.6.1 (2011-08-20) -++++++++++++++++++ - -* Enhanced status codes experience ``\o/`` -* Set a maximum number of redirects (``settings.max_redirects``) -* Full Unicode URL support -* Support for protocol-less redirects. -* Allow for arbitrary request types. -* Bugfixes - - -0.6.0 (2011-08-17) -++++++++++++++++++ - -* New callback hook system -* New persistent sessions object and context manager -* Transparent Dict-cookie handling -* Status code reference object -* Removed Response.cached -* Added Response.request -* All args are kwargs -* Relative redirect support -* HTTPError handling improvements -* Improved https testing -* Bugfixes - - -0.5.1 (2011-07-23) -++++++++++++++++++ - -* International Domain Name Support! -* Access headers without fetching entire body (``read()``) -* Use lists as dicts for parameters -* Add Forced Basic Authentication -* Forced Basic is default authentication type -* ``python-requests.org`` default User-Agent header -* CaseInsensitiveDict lower-case caching -* Response.history bugfix - - -0.5.0 (2011-06-21) -++++++++++++++++++ - -* PATCH Support -* Support for Proxies -* HTTPBin Test Suite -* Redirect Fixes -* settings.verbose stream writing -* Querystrings for all methods -* URLErrors (Connection Refused, Timeout, Invalid URLs) are treated as explicitly raised - ``r.requests.get('hwe://blah'); r.raise_for_status()`` - - -0.4.1 (2011-05-22) -++++++++++++++++++ - -* Improved Redirection Handling -* New 'allow_redirects' param for following non-GET/HEAD Redirects -* Settings module refactoring - - -0.4.0 (2011-05-15) -++++++++++++++++++ - -* Response.history: list of redirected responses -* Case-Insensitive Header Dictionaries! -* Unicode URLs - - -0.3.4 (2011-05-14) -++++++++++++++++++ - -* Urllib2 HTTPAuthentication Recursion fix (Basic/Digest) -* Internal Refactor -* Bytes data upload Bugfix - - - -0.3.3 (2011-05-12) -++++++++++++++++++ - -* Request timeouts -* Unicode url-encoded data -* Settings context manager and module - - -0.3.2 (2011-04-15) -++++++++++++++++++ - -* Automatic Decompression of GZip Encoded Content -* AutoAuth Support for Tupled HTTP Auth - - -0.3.1 (2011-04-01) -++++++++++++++++++ - -* Cookie Changes -* Response.read() -* Poster fix - - -0.3.0 (2011-02-25) -++++++++++++++++++ - -* Automatic Authentication API Change -* Smarter Query URL Parameterization -* Allow file uploads and POST data together -* New Authentication Manager System - - Simpler Basic HTTP System - - Supports all build-in urllib2 Auths - - Allows for custom Auth Handlers - - -0.2.4 (2011-02-19) -++++++++++++++++++ - -* Python 2.5 Support -* PyPy-c v1.4 Support -* Auto-Authentication tests -* Improved Request object constructor - -0.2.3 (2011-02-15) -++++++++++++++++++ - -* New HTTPHandling Methods - - Response.__nonzero__ (false if bad HTTP Status) - - Response.ok (True if expected HTTP Status) - - Response.error (Logged HTTPError if bad HTTP Status) - - Response.raise_for_status() (Raises stored HTTPError) - - -0.2.2 (2011-02-14) -++++++++++++++++++ - -* Still handles request in the event of an HTTPError. (Issue #2) -* Eventlet and Gevent Monkeypatch support. -* Cookie Support (Issue #1) - - -0.2.1 (2011-02-14) -++++++++++++++++++ - -* Added file attribute to POST and PUT requests for multipart-encode file uploads. -* Added Request.url attribute for context and redirects - - -0.2.0 (2011-02-14) -++++++++++++++++++ - -* Birth! - - -0.0.1 (2011-02-13) -++++++++++++++++++ - -* Frustration -* Conception diff --git a/LICENSE b/LICENSE index db78ea69f4..67db858821 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,175 @@ -Copyright 2017 Kenneth Reitz - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/MANIFEST.in b/MANIFEST.in index 2c0fb95ce9..9dd81e6f0f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ -include README.rst LICENSE NOTICE HISTORY.rst pytest.ini requirements.txt +include README.md LICENSE NOTICE HISTORY.md requirements-dev.txt recursive-include tests *.py +recursive-include tests/certs * diff --git a/Makefile b/Makefile index 317a7c76fb..5401f406d5 100644 --- a/Makefile +++ b/Makefile @@ -1,28 +1,27 @@ .PHONY: docs init: - pip install pipenv --upgrade - pipenv install --dev --skip-lock + python -m pip install -r requirements-dev.txt test: - # This runs all of the tests, on both Python 2 and Python 3. - detox + python -m pytest tests + ci: - pipenv run py.test -n 8 --boxed --junitxml=report.xml + python -m pytest tests --junitxml=report.xml test-readme: - @pipenv run python setup.py check --restructuredtext --strict && ([ $$? -eq 0 ] && echo "README.rst and HISTORY.rst ok") || echo "Invalid markup in README.rst or HISTORY.rst!" - -flake8: - pipenv run flake8 --ignore=E501,F401,E128,E402,E731,F821 requests + python setup.py check --restructuredtext --strict && ([ $$? -eq 0 ] && echo "README.rst and HISTORY.rst ok") || echo "Invalid markup in README.rst or HISTORY.rst!" coverage: - pipenv run py.test --cov-config .coveragerc --verbose --cov-report term --cov-report xml --cov=requests tests + python -m pytest --cov-config .coveragerc --verbose --cov-report term --cov-report xml --cov=src/requests tests + +.publishenv: + python -m venv .publishenv + .publishenv/bin/pip install 'twine>=1.5.0' build -publish: - pip install 'twine>=1.5.0' - python setup.py sdist bdist_wheel - twine upload dist/* +publish: .publishenv + .publishenv/bin/python -m build + .publishenv/bin/python -m twine upload --skip-existing dist/* rm -fr build dist .egg requests.egg-info docs: cd docs && make html - @echo "\033[95m\n\nBuild successful! View the docs homepage at docs/_build/html/index.html.\n\033[0m" \ No newline at end of file + @echo "\033[95m\n\nBuild successful! View the docs homepage at docs/_build/html/index.html.\n\033[0m" diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000..1ff62db688 --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +Requests +Copyright 2019 Kenneth Reitz diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 51e8e8469f..0000000000 --- a/Pipfile +++ /dev/null @@ -1,24 +0,0 @@ -[[source]] -url = "https://pypi.org/simple/" -verify_ssl = true - -[dev-packages] - -pytest = ">=2.8.0" -codecov = "*" -"pytest-httpbin" = "==0.0.7" -"pytest-mock" = "*" -"pytest-cov" = "*" -"pytest-xdist" = "*" -alabaster = "*" -"readme-renderer" = "*" -sphinx = "<=1.5.5" -pysocks = "*" -docutils = "*" -"flake8" = "*" -tox = "*" -detox = "*" -httpbin = "==0.5.0" - -[packages] -"e1839a8" = {path = ".", editable = true, extras=["socks"]} \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index fd9a1a10b3..0000000000 --- a/Pipfile.lock +++ /dev/null @@ -1,517 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "72b5a08e9c266b930d308024036e928e6b99ed4b7a50f22af377a463b7867a14" - }, - "host-environment-markers": { - "implementation_name": "cpython", - "implementation_version": "3.6.2", - "os_name": "posix", - "platform_machine": "x86_64", - "platform_python_implementation": "CPython", - "platform_release": "16.7.0", - "platform_system": "Darwin", - "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64", - "python_full_version": "3.6.2", - "python_version": "3.6", - "sys_platform": "darwin" - }, - "pipfile-spec": 6, - "requires": {}, - "sources": [ - { - "url": "https://pypi.python.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "certifi": { - "hashes": [ - "sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704", - "sha256:40523d2efb60523e113b44602298f0960e900388cf3bb6043f645cf57ea9e3f5" - ], - "version": "==2017.7.27.1" - }, - "chardet": { - "hashes": [ - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691", - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae" - ], - "version": "==3.0.4" - }, - "e1839a8": { - "editable": true, - "extras": [ - "socks" - ], - "path": "." - }, - "idna": { - "hashes": [ - "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4", - "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f" - ], - "version": "==2.6" - }, - "pysocks": { - "hashes": [ - "sha256:18842328a4e6061f084cfba70f6950d9140ecf7418b3df7cef558ebb217bac8d", - "sha256:d00329f27efa157db7efe3ca26fcd69033cd61f83822461ee3f8a353b48e33cf" - ], - "version": "==1.6.7" - }, - "urllib3": { - "hashes": [ - "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", - "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" - ], - "version": "==1.22" - } - }, - "develop": { - "alabaster": { - "hashes": [ - "sha256:2eef172f44e8d301d25aff8068fddd65f767a3f04b5f15b0f4922f113aa1c732", - "sha256:37cdcb9e9954ed60912ebc1ca12a9d12178c26637abdf124e3cde2341c257fe0" - ], - "version": "==0.7.10" - }, - "apipkg": { - "hashes": [ - "sha256:65d2aa68b28e7d31233bb2ba8eb31cda40e4671f8ac2d6b241e358c9652a74b9", - "sha256:2e38399dbe842891fe85392601aab8f40a8f4cc5a9053c326de35a1cc0297ac6" - ], - "version": "==1.4" - }, - "babel": { - "hashes": [ - "sha256:f20b2acd44f587988ff185d8949c3e208b4b3d5d20fcab7d91fe481ffa435528", - "sha256:6007daf714d0cd5524bbe436e2d42b3c20e68da66289559341e48d2cd6d25811" - ], - "version": "==2.5.1" - }, - "bleach": { - "hashes": [ - "sha256:a6d9d5f5b7368c1689ad7f128af8e792beea23393688872b576c0271e6564a16", - "sha256:b9522130003e4caedf4f00a39c120a906dcd4242329c1c8f621f5370203cbc30" - ], - "version": "==2.0.0" - }, - "certifi": { - "hashes": [ - "sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704", - "sha256:40523d2efb60523e113b44602298f0960e900388cf3bb6043f645cf57ea9e3f5" - ], - "version": "==2017.7.27.1" - }, - "chardet": { - "hashes": [ - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691", - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", - "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" - ], - "version": "==6.7" - }, - "codecov": { - "hashes": [ - "sha256:ad82f054837b02081f86ed1eb6c04cddc029fbc734eaf92ff73da1db3a79188b", - "sha256:db1c182ca896244d8644d8410a33f6f6dd1cc24d80209907a65077445923f00c" - ], - "version": "==2.0.9" - }, - "configparser": { - "hashes": [ - "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a" - ], - "version": "==3.5.0" - }, - "coverage": { - "hashes": [ - "sha256:c1456f66c536010cf9e4633a8853a9153e8fd588393695295afd4d0fc16c1d74", - "sha256:97a7ec51cdde3a386e390b159b20f247ccb478084d925c75f1faa3d26c01335e", - "sha256:83e955b975666b5a07d217135e7797857ce844eb340a99e46cc25525120417c4", - "sha256:483ed14080c5301048128bb027b77978c632dd9e92e3ecb09b7e28f5b92abfcf", - "sha256:ef574ab9640bcfa2f3c671831faf03f65788945fdf8efa4d4a1fffc034838e2a", - "sha256:c5a205b4da3c624f5119dc4d84240789b5906bb8468902ec22dcc4aad8aa4638", - "sha256:5dea90ed140e7fa9bc00463313f9bc4a6e6aff297b4969615e7a688615c4c4d2", - "sha256:f9e83b39d29c2815a38e4118d776b482d4082b5bf9c9147fbc99a3f83abe480a", - "sha256:700040c354f0230287906b1276635552a3def4b646e0145555bc9e2e5da9e365", - "sha256:7f1eacae700c66c3d7362a433b228599c9d94a5a3a52613dddd9474e04deb6bc", - "sha256:13ef9f799c8fb45c446a239df68034de3a6f3de274881b088bebd7f5661f79f8", - "sha256:dfb011587e2b7299112f08a2a60d2601706aac9abde37aa1177ea825adaed923", - "sha256:381be5d31d3f0d912334cf2c159bc7bea6bfe6b0e3df6061a3bf2bf88359b1f6", - "sha256:83a477ac4f55a6ef59552683a0544d47b68a85ce6a80fd0ca6b3dc767f6495fb", - "sha256:dfd35f1979da31bcabbe27bcf78d4284d69870731874af629082590023a77336", - "sha256:9681efc2d310cfc53863cc6f63e88ebe7a48124550fa822147996cb09390b6ab", - "sha256:53770b20ac5b4a12e99229d4bae57af0945be87cc257fce6c6c7571a39f0c5d4", - "sha256:8801880d32f11b6df11c32a961e186774b4634ae39d7c43235f5a24368a85f07", - "sha256:16db2c69a1acbcb3c13211e9f954e22b22a729909d81f983b6b9badacc466eda", - "sha256:ef43a06a960b46c73c018704051e023ee6082030f145841ffafc8728039d5a88", - "sha256:c3e2736664a6074fc9bd54fb643f5af0fc60bfedb2963b3d3f98c7450335e34c", - "sha256:17709e22e4c9f5412ba90f446fb13b245cc20bf4a60377021bbff6c0f1f63e7c", - "sha256:a2f7106d1167825c4115794c2ba57cc3b15feb6183db5328fa66f94c12902d8b", - "sha256:2a08e978f402696c6956eee9d1b7e95d3ad042959b71bafe1f3e4557cbd6e0ac", - "sha256:57f510bb16efaec0b6f371b64a8000c62e7e3b3e48e8b0a5745ade078d849814", - "sha256:0f1883eab9c19aa243f51308751b8a2a547b9b817b721cc0ecf3efb99fafbea7", - "sha256:e00fe141e22ce6e9395aa24d862039eb180c6b7e89df0bbaf9765e9aebe560a9", - "sha256:ec596e4401553caa6dd2e3349ce47f9ef82c1f1bcba5d8ac3342724f0df8d6ff", - "sha256:c820a533a943ebc860acc0ce6a00dd36e0fdf2c6f619ff8225755169428c5fa2", - "sha256:b7f7283eb7badd2b8a9c6a9d6eeca200a0a24db6be79baee2c11398f978edcaa", - "sha256:a5ed27ad3e8420b2d6b625dcbd3e59488c14ccc06030167bcf14ffb0f4189b77", - "sha256:d7b70b7b4eb14d0753d33253fe4f121ca99102612e2719f0993607deb30c6f33", - "sha256:4047dc83773869701bde934fb3c4792648eda7c0e008a77a0aec64157d246801", - "sha256:7a9c44400ee0f3b4546066e0710e1250fd75831adc02ab99dda176ad8726f424", - "sha256:0f649e68db74b1b5b8ca4161d08eb2b8fa8ae11af1ebfb80e80e112eb0ef5300", - "sha256:52964fae0fafef8bd283ad8e9a9665205a9fdf912535434defc0ec3def1da26b", - "sha256:36aa6c8db83bc27346ddcd8c2a60846a7178ecd702672689d3ea1828eb1a4d11", - "sha256:9824e15b387d331c0fc0fef905a539ab69784368a1d6ac3db864b4182e520948", - "sha256:4a678e1b9619a29c51301af61ab84122e2f8cc7a0a6b40854b808ac6be604300", - "sha256:8bb7c8dca54109b61013bc4114d96effbf10dea136722c586bce3a5d9fc4e730", - "sha256:1a41d621aa9b6ab6457b557a754d50aaff0813fad3453434de075496fca8a183", - "sha256:0fa423599fc3d9e18177f913552cdb34a8d9ad33efcf52a98c9d4b644edb42c5", - "sha256:e61a4ba0b2686040cb4828297c7e37bcaf3a1a1c0bc0dbe46cc789dde51a80fa", - "sha256:ce9ef0fc99d11d418662e36fd8de6d71b19ec87c2eab961a117cc9d087576e72" - ], - "version": "==4.4.1" - }, - "decorator": { - "hashes": [ - "sha256:95a26b17806e284452bfd97fa20aa1e8cb4ee23542bda4dbac5e4562aa1642cd", - "sha256:7cb64d38cb8002971710c8899fbdfb859a23a364b7c99dab19d1f719c2ba16b5" - ], - "version": "==4.1.2" - }, - "detox": { - "hashes": [ - "sha256:af0097ea01263f68f546826df69b9301458d6cec0ed278c53c01f9529fbd349e", - "sha256:4719ca48c4ea5ffd908b1bc3d5d1b593b41e71dee17180d58d8a3e7e8f588d45" - ], - "version": "==0.11" - }, - "docutils": { - "hashes": [ - "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6", - "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", - "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274" - ], - "version": "==0.14" - }, - "enum-compat": { - "hashes": [ - "sha256:939ceff18186a5762ae4db9fa7bfe017edbd03b66526b798dd8245394c8a4192" - ], - "version": "==0.0.2" - }, - "enum34": { - "hashes": [ - "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", - "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", - "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1", - "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850" - ], - "version": "==1.1.6" - }, - "eventlet": { - "hashes": [ - "sha256:0a7d1e1d2f4dd2e0b2cb627dadf7a0f23de0eca88ba2d6af4229abe32a24dec9", - "sha256:08faffab88c1b08bd53ea28bf084a572c89f7e7648bd9d71e6116ac17a51a15d" - ], - "version": "==0.21.0" - }, - "execnet": { - "hashes": [ - "sha256:d2b909c7945832e1c19cfacd96e78da68bdadc656440cfc7dfe59b766744eb8c", - "sha256:f66dd4a7519725a1b7e14ad9ae7d3df8e09b2da88062386e08e941cafc0ef3e6" - ], - "version": "==1.4.1" - }, - "flake8": { - "hashes": [ - "sha256:f1a9d8886a9cbefb52485f4f4c770832c7fb569c084a9a314fb1eaa37c0c2c86", - "sha256:c20044779ff848f67f89c56a0e4624c04298cd476e25253ac0c36f910a1a11d8" - ], - "version": "==3.4.1" - }, - "flask": { - "hashes": [ - "sha256:0749df235e3ff61ac108f69ac178c9770caeaccad2509cb762ce1f65570a8856", - "sha256:49f44461237b69ecd901cc7ce66feea0319b9158743dd27a2899962ab214dac1" - ], - "version": "==0.12.2" - }, - "funcsigs": { - "hashes": [ - "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", - "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" - ], - "version": "==1.0.2" - }, - "greenlet": { - "hashes": [ - "sha256:96888e47898a471073b394ea641b7d675c1d054c580dd4a04a382bd34e67d89e", - "sha256:d2d5103f6cba131e1be660230018e21f276911d2b68b629ead1c5cb5e5472ac7", - "sha256:bc339de0e0969de5118d0b62a080a7611e2ba729a90f4a3ad78559c51bc5576d", - "sha256:b8ab98f8ae25938326dc4c21e3689a933531500ae4f3bfcefe36e3e25fda4dbf", - "sha256:416a3328d7e0a19aa1df3ec09524a109061fd7b80e010ef0dff9f695b4ac5e20", - "sha256:21232907c8c26838b16915bd8fbbf82fc70c996073464cc70981dd4a96bc841c", - "sha256:6803d8c6b235c861c50afddf00c7467ffbcd5ab960d137ff0f9c36f2cb11ee4b", - "sha256:76dab055476dd4dabb00a967b4df1990b25542d17eaa40a18f66971d10193e0b", - "sha256:70b9ff28921f5a3c03df4896ec8c55f5f94c593d7a79abd98b4c5c4a692ba873", - "sha256:7114b757b4146f4c87a0f00f1e58abd4c4729836679af0fc37266910a4a72eb0", - "sha256:0d90c709355ed13f16676f84e5a9cd67826a9f5c5143381c21e8fc3100ade1f1", - "sha256:ebae83b6247f83b1e8d887733dfa8046ce6e29d8b3e2a7380256e9de5c6ae55d", - "sha256:e841e3ece633acae5e2bf6102140a605ffee7d5d4921dca1625c5fdc0f0b3248", - "sha256:3e5e9be157ece49e4f97f3225460caf758ccb00f934fcbc5db34367cc1ff0aee", - "sha256:e77b708c37b652c7501b9f8f6056b23633c567aaa0d29edfef1c11673c64b949", - "sha256:0da1fc809c3bdb93fbacd0f921f461aacd53e554a7b7d4e9953ba09131c4206e", - "sha256:66fa5b101fcf4521138c1a29668074268d938bbb7de739c8faa9f92ea1f05e1f", - "sha256:e5451e1ce06b74a4861576c2db74405a4398c4809a105774550a9e52cfc8c4da", - "sha256:9c407aa6adfd4eea1232e81aa9f3cb3d9b955a9891c4819bf9b498c77efba14b", - "sha256:b56ac981f07b77e72ad5154278b93396d706572ea52c2fce79fee2abfcc8bfa6", - "sha256:e4c99c6010a5d153d481fdaf63b8a0782825c0721506d880403a3b9b82ae347e" - ], - "version": "==0.4.12" - }, - "html5lib": { - "hashes": [ - "sha256:b8934484cf22f1db684c0fae27569a0db404d0208d20163fbf51cc537245d008", - "sha256:ee747c0ffd3028d2722061936b5c65ee4fe13c8e4613519b4447123fc4546298" - ], - "version": "==0.999999999" - }, - "httpbin": { - "hashes": [ - "sha256:710069973216d4bbf9ab6757f1e9a1f3be05832ce77da023adce0a98dfeecfee", - "sha256:79fbc5d27e4194ea908b0fa18e09a59d95d287c91667aa69bcd010342d1589b5" - ], - "version": "==0.5.0" - }, - "idna": { - "hashes": [ - "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4", - "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f" - ], - "version": "==2.6" - }, - "imagesize": { - "hashes": [ - "sha256:6ebdc9e0ad188f9d1b2cdd9bc59cbe42bf931875e829e7a595e6b3abdc05cdfb", - "sha256:0ab2c62b87987e3252f89d30b7cedbec12a01af9274af9ffa48108f2c13c6062" - ], - "version": "==0.7.1" - }, - "itsdangerous": { - "hashes": [ - "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" - ], - "version": "==0.24" - }, - "jinja2": { - "hashes": [ - "sha256:2231bace0dfd8d2bf1e5d7e41239c06c9e0ded46e70cc1094a0aa64b0afeb054", - "sha256:ddaa01a212cd6d641401cb01b605f4a4d9f37bfc93043d7f760ec70fb99ff9ff" - ], - "version": "==2.9.6" - }, - "markupsafe": { - "hashes": [ - "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" - ], - "version": "==1.0" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, - "mock": { - "hashes": [ - "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1", - "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba" - ], - "version": "==2.0.0" - }, - "pbr": { - "hashes": [ - "sha256:60c25b7dfd054ef9bb0ae327af949dd4676aa09ac3a9471cdc871d8a9213f9ac", - "sha256:05f61c71aaefc02d8e37c0a3eeb9815ff526ea28b3b76324769e6158d7f95be1" - ], - "version": "==3.1.1" - }, - "pluggy": { - "hashes": [ - "sha256:bd60171dbb250fdebafad46ed16d97065369da40568ae948ef7117eee8536e94" - ], - "version": "==0.5.2" - }, - "py": { - "hashes": [ - "sha256:2ccb79b01769d99115aa600d7eed99f524bf752bba8f041dc1c184853514655a", - "sha256:0f2d585d22050e90c7d293b6451c83db097df77871974d90efd5a30dc12fcde3" - ], - "version": "==1.4.34" - }, - "pycodestyle": { - "hashes": [ - "sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9", - "sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766" - ], - "version": "==2.3.1" - }, - "pyflakes": { - "hashes": [ - "sha256:cc5eadfb38041f8366128786b4ca12700ed05bbf1403d808e89d57d67a3875a7", - "sha256:aa0d4dff45c0cc2214ba158d29280f8fa1129f3e87858ef825930845146337f4" - ], - "version": "==1.5.0" - }, - "pygments": { - "hashes": [ - "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d", - "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc" - ], - "version": "==2.2.0" - }, - "pysocks": { - "hashes": [ - "sha256:18842328a4e6061f084cfba70f6950d9140ecf7418b3df7cef558ebb217bac8d", - "sha256:d00329f27efa157db7efe3ca26fcd69033cd61f83822461ee3f8a353b48e33cf" - ], - "version": "==1.6.7" - }, - "pytest": { - "hashes": [ - "sha256:b84f554f8ddc23add65c411bf112b2d88e2489fd45f753b1cae5936358bdf314", - "sha256:f46e49e0340a532764991c498244a60e3a37d7424a532b3ff1a6a7653f1a403a" - ], - "version": "==3.2.2" - }, - "pytest-cov": { - "hashes": [ - "sha256:890fe5565400902b0c78b5357004aab1c814115894f4f21370e2433256a3eeec", - "sha256:03aa752cf11db41d281ea1d807d954c4eda35cfa1b21d6971966cc041bbf6e2d" - ], - "version": "==2.5.1" - }, - "pytest-forked": { - "hashes": [ - "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08", - "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805" - ], - "version": "==0.2" - }, - "pytest-httpbin": { - "hashes": [ - "sha256:f430f0b5742a9d325148a3428f890f538f331cb7b244a49873cc322f838c85ea", - "sha256:03af8a7055c8bbcb68b14d9a14c103c82c97aeb86a8f1b29cd63d83644c2f021" - ], - "version": "==0.0.7" - }, - "pytest-mock": { - "hashes": [ - "sha256:7ed6e7e8c636fd320927c5d73aedb77ac2eeb37196c3410e6176b7c92fdf2f69", - "sha256:920d1167af5c2c2ad3fa0717d0c6c52e97e97810160c15721ac895cac53abb1c" - ], - "version": "==1.6.3" - }, - "pytest-xdist": { - "hashes": [ - "sha256:7924d45c2430191fe3679a58116c74ceea13307d7822c169d65fd59a24b3a4fe" - ], - "version": "==1.20.0" - }, - "pytz": { - "hashes": [ - "sha256:c883c2d6670042c7bc1688645cac73dd2b03193d1f7a6847b6154e96890be06d", - "sha256:03c9962afe00e503e2d96abab4e8998a0f84d4230fa57afe1e0528473698cdd9", - "sha256:487e7d50710661116325747a9cd1744d3323f8e49748e287bc9e659060ec6bf9", - "sha256:43f52d4c6a0be301d53ebd867de05e2926c35728b3260157d274635a0a947f1c", - "sha256:d1d6729c85acea5423671382868627129432fba9a89ecbb248d8d1c7a9f01c67", - "sha256:54a935085f7bf101f86b2aff75bd9672b435f51c3339db2ff616e66845f2b8f9", - "sha256:39504670abb5dae77f56f8eb63823937ce727d7cdd0088e6909e6dcac0f89043", - "sha256:ddc93b6d41cfb81266a27d23a79e13805d4a5521032b512643af8729041a81b4", - "sha256:f5c056e8f62d45ba8215e5cb8f50dfccb198b4b9fbea8500674f3443e4689589" - ], - "version": "==2017.2" - }, - "readme-renderer": { - "hashes": [ - "sha256:c9637bfcf1ff40f7683b3439f4b97eb0f9a1cffc2a1fad5fa01debd667ddb111", - "sha256:9deab442963a63a71ab494bf581b1c844473995a2357f4b3228a1df1c8cba8da" - ], - "version": "==17.2" - }, - "requests": { - "hashes": [ - "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", - "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" - ], - "version": "==2.18.4" - }, - "six": { - "hashes": [ - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" - ], - "version": "==1.11.0" - }, - "snowballstemmer": { - "hashes": [ - "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89", - "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128" - ], - "version": "==1.2.1" - }, - "sphinx": { - "hashes": [ - "sha256:11f271e7a9398385ed730e90f0bb41dc3815294bdcd395b46ed2d033bc2e7d87", - "sha256:4064ea6c56feeb268838cb8fbbee507d0c3d5d92fa63a7df935a916b52c9e2f5" - ], - "version": "==1.5.5" - }, - "tox": { - "hashes": [ - "sha256:49d88f2c217352c499450e9f61ca82fd9c8873d01a45555bb342a32f2b6753a2", - "sha256:d9c279e707d2cfef8d77d10f13b38b3e68b7e470018b45747560f6c3c66d6b83" - ], - "version": "==2.8.2" - }, - "urllib3": { - "hashes": [ - "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", - "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" - ], - "version": "==1.22" - }, - "virtualenv": { - "hashes": [ - "sha256:39d88b533b422825d644087a21e78c45cf5af0ef7a99a1fc9fbb7b481e5c85b0", - "sha256:02f8102c2436bb03b3ee6dede1919d1dac8a427541652e5ec95171ec8adbc93a" - ], - "version": "==15.1.0" - }, - "webencodings": { - "hashes": [ - "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", - "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" - ], - "version": "==0.5.1" - }, - "werkzeug": { - "hashes": [ - "sha256:e8549c143af3ce6559699a01e26fa4174f4c591dbee0a499f3cd4c3781cdec3d", - "sha256:903a7b87b74635244548b30d30db4c8947fe64c5198f58899ddcd3a13c23bb26" - ], - "version": "==0.12.2" - } - } -} diff --git a/README.md b/README.md new file mode 100644 index 0000000000..74adab80dd --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +# Requests + +**Requests** is a simple, yet elegant, HTTP library. + +```python +>>> import requests +>>> r = requests.get('https://httpbin.org/basic-auth/user/pass', auth=('user', 'pass')) +>>> r.status_code +200 +>>> r.headers['content-type'] +'application/json; charset=utf8' +>>> r.encoding +'utf-8' +>>> r.text +'{"authenticated": true, ...' +>>> r.json() +{'authenticated': True, ...} +``` + +Requests allows you to send HTTP/1.1 requests extremely easily. There’s no need to manually add query strings to your URLs, or to form-encode your `PUT` & `POST` data — but nowadays, just use the `json` method! + +Requests is one of the most downloaded Python packages today, pulling in around `30M downloads / week`— according to GitHub, Requests is currently [depended upon](https://github.com/psf/requests/network/dependents?package_id=UGFja2FnZS01NzA4OTExNg%3D%3D) by `1,000,000+` repositories. You may certainly put your trust in this code. + +[![Downloads](https://static.pepy.tech/badge/requests/month)](https://pepy.tech/project/requests) +[![Supported Versions](https://img.shields.io/pypi/pyversions/requests.svg)](https://pypi.org/project/requests) +[![Contributors](https://img.shields.io/github/contributors/psf/requests.svg)](https://github.com/psf/requests/graphs/contributors) + +## Installing Requests and Supported Versions + +Requests is available on PyPI: + +```console +$ python -m pip install requests +``` + +Requests officially supports Python 3.9+. + +## Supported Features & Best–Practices + +Requests is ready for the demands of building robust and reliable HTTP–speaking applications, for the needs of today. + +- Keep-Alive & Connection Pooling +- International Domains and URLs +- Sessions with Cookie Persistence +- Browser-style TLS/SSL Verification +- Basic & Digest Authentication +- Familiar `dict`–like Cookies +- Automatic Content Decompression and Decoding +- Multi-part File Uploads +- SOCKS Proxy Support +- Connection Timeouts +- Streaming Downloads +- Automatic honoring of `.netrc` +- Chunked HTTP Requests + +## API Reference and User Guide available on [Read the Docs](https://requests.readthedocs.io) + +[![Read the Docs](https://raw.githubusercontent.com/psf/requests/main/ext/ss.png)](https://requests.readthedocs.io) + +## Cloning the repository + +When cloning the Requests repository, you may need to add the `-c +fetch.fsck.badTimezone=ignore` flag to avoid an error about a bad commit timestamp (see +[this issue](https://github.com/psf/requests/issues/2690) for more background): + +```shell +git clone -c fetch.fsck.badTimezone=ignore https://github.com/psf/requests.git +``` + +You can also apply this setting to your global Git config: + +```shell +git config --global fetch.fsck.badTimezone ignore +``` + +--- + +[![Kenneth Reitz](https://raw.githubusercontent.com/psf/requests/main/ext/kr.png)](https://kennethreitz.org) [![Python Software Foundation](https://raw.githubusercontent.com/psf/requests/main/ext/psf.png)](https://www.python.org/psf) diff --git a/README.rst b/README.rst deleted file mode 100644 index d65fd9479b..0000000000 --- a/README.rst +++ /dev/null @@ -1,110 +0,0 @@ -Requests: HTTP for Humans -========================= - -.. image:: https://img.shields.io/pypi/v/requests.svg - :target: https://pypi.org/project/requests/ - -.. image:: https://img.shields.io/pypi/l/requests.svg - :target: https://pypi.org/project/requests/ - -.. image:: https://img.shields.io/pypi/pyversions/requests.svg - :target: https://pypi.org/project/requests/ - -.. image:: https://codecov.io/github/requests/requests/coverage.svg?branch=master - :target: https://codecov.io/github/requests/requests - :alt: codecov.io - -.. image:: https://img.shields.io/github/contributors/requests/requests.svg - :target: https://github.com/requests/requests/graphs/contributors - -.. image:: https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg - :target: https://saythanks.io/to/kennethreitz - -Requests is the only *Non-GMO* HTTP library for Python, safe for human -consumption. - -.. image:: https://farm5.staticflickr.com/4317/35198386374_1939af3de6_k_d.jpg - -Behold, the power of Requests: - -.. code-block:: python - - >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) - >>> r.status_code - 200 - >>> r.headers['content-type'] - 'application/json; charset=utf8' - >>> r.encoding - 'utf-8' - >>> r.text - u'{"type":"User"...' - >>> r.json() - {u'disk_usage': 368627, u'private_gists': 484, ...} - -See `the similar code, sans Requests `_. - -.. image:: https://raw.githubusercontent.com/requests/requests/master/docs/_static/requests-logo-small.png - :target: http://docs.python-requests.org/ - - -Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, without the -need for manual labor. There's no need to manually add query strings to your -URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling -are 100% automatic, thanks to `urllib3 `_. - -Besides, all the cool kids are doing it. Requests is one of the most -downloaded Python packages of all time, pulling in over 11,000,000 downloads -every month. You don't want to be left out! - -Feature Support ---------------- - -Requests is ready for today's web. - -- International Domains and URLs -- Keep-Alive & Connection Pooling -- Sessions with Cookie Persistence -- Browser-style SSL Verification -- Basic/Digest Authentication -- Elegant Key/Value Cookies -- Automatic Decompression -- Automatic Content Decoding -- Unicode Response Bodies -- Multipart File Uploads -- HTTP(S) Proxy Support -- Connection Timeouts -- Streaming Downloads -- ``.netrc`` Support -- Chunked Requests - -Requests officially supports Python 2.6–2.7 & 3.4–3.6, and runs great on PyPy. - -Installation ------------- - -To install Requests, simply use `pipenv `_ (or pip, of course): - -.. code-block:: bash - - $ pipenv install requests - ✨🍰✨ - -Satisfaction guaranteed. - -Documentation -------------- - -Fantastic documentation is available at http://docs.python-requests.org/, for a limited time only. - - -How to Contribute ------------------ - -#. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug. There is a `Contributor Friendly`_ tag for issues that should be ideal for people who are not very familiar with the codebase yet. -#. Fork `the repository`_ on GitHub to start making your changes to the **master** branch (or branch off of it). -#. Write a test which shows that the bug was fixed or that the feature works as expected. -#. Send a pull request and bug the maintainer until it gets merged and published. :) Make sure to add yourself to AUTHORS_. - -.. _`the repository`: https://github.com/requests/requests -.. _AUTHORS: https://github.com/requests/requests/blob/master/AUTHORS.rst -.. _Contributor Friendly: https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open diff --git a/_appveyor/install.ps1 b/_appveyor/install.ps1 deleted file mode 100644 index 94d6f01813..0000000000 --- a/_appveyor/install.ps1 +++ /dev/null @@ -1,229 +0,0 @@ -# Sample script to install Python and pip under Windows -# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ - -$MINICONDA_URL = "http://repo.continuum.io/miniconda/" -$BASE_URL = "https://www.python.org/ftp/python/" -$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" -$GET_PIP_PATH = "C:\get-pip.py" - -$PYTHON_PRERELEASE_REGEX = @" -(?x) -(?\d+) -\. -(?\d+) -\. -(?\d+) -(?[a-z]{1,2}\d+) -"@ - - -function Download ($filename, $url) { - $webclient = New-Object System.Net.WebClient - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 3 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 2 - for ($i = 0; $i -lt $retry_attempts; $i++) { - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - Start-Sleep 1 - } - } - if (Test-Path $filepath) { - Write-Host "File saved at" $filepath - } else { - # Retry once to get the error message if any at the last try - $webclient.DownloadFile($url, $filepath) - } - return $filepath -} - - -function ParsePythonVersion ($python_version) { - if ($python_version -match $PYTHON_PRERELEASE_REGEX) { - return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro, - $matches.prerelease) - } - $version_obj = [version]$python_version - return ($version_obj.major, $version_obj.minor, $version_obj.build, "") -} - - -function DownloadPython ($python_version, $platform_suffix) { - $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version - - if (($major -le 2 -and $micro -eq 0) ` - -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) ` - ) { - $dir = "$major.$minor" - $python_version = "$major.$minor$prerelease" - } else { - $dir = "$major.$minor.$micro" - } - - if ($prerelease) { - if (($major -le 2) ` - -or ($major -eq 3 -and $minor -eq 1) ` - -or ($major -eq 3 -and $minor -eq 2) ` - -or ($major -eq 3 -and $minor -eq 3) ` - ) { - $dir = "$dir/prev" - } - } - - if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) { - $ext = "msi" - if ($platform_suffix) { - $platform_suffix = ".$platform_suffix" - } - } else { - $ext = "exe" - if ($platform_suffix) { - $platform_suffix = "-$platform_suffix" - } - } - - $filename = "python-$python_version$platform_suffix.$ext" - $url = "$BASE_URL$dir/$filename" - $filepath = Download $filename $url - return $filepath -} - - -function InstallPython ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -eq "32") { - $platform_suffix = "" - } else { - $platform_suffix = "amd64" - } - $installer_path = DownloadPython $python_version $platform_suffix - $installer_ext = [System.IO.Path]::GetExtension($installer_path) - Write-Host "Installing $installer_path to $python_home" - $install_log = $python_home + ".log" - if ($installer_ext -eq '.msi') { - InstallPythonMSI $installer_path $python_home $install_log - } else { - InstallPythonEXE $installer_path $python_home $install_log - } - if (Test-Path $python_home) { - Write-Host "Python $python_version ($architecture) installation complete" - } else { - Write-Host "Failed to install Python in $python_home" - Get-Content -Path $install_log - Exit 1 - } -} - - -function InstallPythonEXE ($exepath, $python_home, $install_log) { - $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home" - RunCommand $exepath $install_args -} - - -function InstallPythonMSI ($msipath, $python_home, $install_log) { - $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home" - $uninstall_args = "/qn /x $msipath" - RunCommand "msiexec.exe" $install_args - if (-not(Test-Path $python_home)) { - Write-Host "Python seems to be installed else-where, reinstalling." - RunCommand "msiexec.exe" $uninstall_args - RunCommand "msiexec.exe" $install_args - } -} - -function RunCommand ($command, $command_args) { - Write-Host $command $command_args - Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru -} - - -function InstallPip ($python_home) { - $pip_path = $python_home + "\Scripts\pip.exe" - $python_path = $python_home + "\python.exe" - if (-not(Test-Path $pip_path)) { - Write-Host "Installing pip..." - $webclient = New-Object System.Net.WebClient - $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) - Write-Host "Executing:" $python_path $GET_PIP_PATH - & $python_path $GET_PIP_PATH - } else { - Write-Host "pip already installed." - } -} - - -function DownloadMiniconda ($python_version, $platform_suffix) { - if ($python_version -eq "3.4") { - $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe" - } else { - $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe" - } - $url = $MINICONDA_URL + $filename - $filepath = Download $filename $url - return $filepath -} - - -function InstallMiniconda ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -eq "32") { - $platform_suffix = "x86" - } else { - $platform_suffix = "x86_64" - } - $filepath = DownloadMiniconda $python_version $platform_suffix - Write-Host "Installing" $filepath "to" $python_home - $install_log = $python_home + ".log" - $args = "/S /D=$python_home" - Write-Host $filepath $args - Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru - if (Test-Path $python_home) { - Write-Host "Python $python_version ($architecture) installation complete" - } else { - Write-Host "Failed to install Python in $python_home" - Get-Content -Path $install_log - Exit 1 - } -} - - -function InstallMinicondaPip ($python_home) { - $pip_path = $python_home + "\Scripts\pip.exe" - $conda_path = $python_home + "\Scripts\conda.exe" - if (-not(Test-Path $pip_path)) { - Write-Host "Installing pip..." - $args = "install --yes pip" - Write-Host $conda_path $args - Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru - } else { - Write-Host "pip already installed." - } -} - -function main () { - InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON - InstallPip $env:PYTHON -} - -main \ No newline at end of file diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 8c84a9f3d5..0000000000 --- a/appveyor.yml +++ /dev/null @@ -1,51 +0,0 @@ -# AppVeyor.yml from https://github.com/ogrisel/python-appveyor-demo -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ - -build: off - -environment: - matrix: - - PYTHON: "C:\\Python27-x64" - PYTHON_VERSION: "2.7.x" - PYTHON_ARCH: "64" - TOXENV: "py27" - - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.x" - PYTHON_ARCH: "64" - TOXENV: "py34" - - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5.x" - PYTHON_ARCH: "64" - TOXENV: "py35" - - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "64" - TOXENV: "py36" - -install: - # Install Python (from the official .msi of http://python.org) and pip when - # not already installed. - - ps: if (-not(Test-Path($env:PYTHON))) { & _appveyor\install.ps1 } - - # Prepend newly installed Python to the PATH of this build (this cannot be - # done from inside the powershell script as it would require to restart - # the parent CMD process). - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - # Check that we have the expected version and architecture for Python - - "python --version" - - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" - - # Upgrade to the latest version of pip to avoid it displaying warnings - # about it being out of date. - - "python -m pip install --upgrade pip wheel" - - "C:\\MinGW\\bin\\mingw32-make" - -test_script: - - "C:\\MinGW\\bin\\mingw32-make coverage" - -on_success: - - "pipenv run codecov -f coverage.xml" diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/docs/.nojekyll @@ -0,0 +1 @@ + diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 54def686eb..465e8a9a60 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -6,15 +6,16 @@ body > div.document > div.sphinxsidebar > div > form > table > tbody > tr:nth-ch color: white; } +/* Carbon by BuySellAds */ #carbonads { display: block; overflow: hidden; + margin: 1.5em 0 2em; padding: 1em; - background-color: #eeeeee; - text-align: center; border: solid 1px #cccccc; - margin: 1.5em 0 2em; border-radius: 2px; + background-color: #eeeeee; + text-align: center; line-height: 1.5; } @@ -42,6 +43,135 @@ body > div.document > div.sphinxsidebar > div > form > table > tbody > tr:nth-ch display: block; text-transform: uppercase; letter-spacing: 1px; + font-size: 10px; line-height: 1; +} + + +/* Native CPC by BuySellAds */ + +#native-ribbon #_custom_ { + position: fixed; + right: 0; + bottom: 0; + left: 0; + box-shadow: 0 -1px 4px 1px hsla(0, 0%, 0%, .15); + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, + Cantarell, "Helvetica Neue", Helvetica, Arial, sans-serif; + transition: all .25s ease-in-out; + transform: translateY(calc(100% - 35px)); + + flex-flow: column nowrap; +} + +#native-ribbon #_custom_:hover { + transform: translateY(0); +} + +.native-img { + margin-right: 20px; + max-height: 50px; + border-radius: 3px; +} + +.native-sponsor { + margin: 10px 20px; + text-align: center; + text-transform: uppercase; + letter-spacing: .5px; + font-size: 12px; + transition: all .3s ease-in-out; + transform-origin: left; +} + +#native-ribbon #_custom_:hover .native-sponsor { + margin: 0 20px; + opacity: 0; + transform: scaleY(0); +} + +.native-flex { + display: flex; + padding: 10px 20px 25px; + text-decoration: none; + + flex-flow: row nowrap; + justify-content: center; + align-items: center; +} + +.native-main { + display: flex; + + flex-flow: row nowrap; + align-items: center; +} + +.native-details { + display: flex; + margin-right: 30px; + + flex-flow: column nowrap; +} + +.native-company { + margin-bottom: 4px; + text-transform: uppercase; + letter-spacing: 2px; font-size: 10px; } + +.native-desc { + letter-spacing: 1px; + font-weight: 300; + font-size: 14px; + line-height: 1.4; +} + +.native-cta { + padding: 10px 14px; + border-radius: 3px; + box-shadow: 0 6px 13px 0 hsla(0, 0%, 0%, .15); + text-transform: uppercase; + white-space: nowrap; + letter-spacing: 1px; + font-weight: 400; + font-size: 12px; + transition: all .3s ease-in-out; + transform: translateY(-1px); +} + +.native-cta:hover { + box-shadow: none; + transform: translateY(1px); +} + +@media only screen and (min-width: 320px) and (max-width: 759px) { + .native-flex { + padding: 5px 5px 15px; + flex-direction: column; + + flex-wrap: wrap; + } + + .native-img { + margin: 0; + display: none; + } + + .native-details { + margin: 0; + } + + .native-main { + flex-direction: column; + text-align: left; + + flex-wrap: wrap; + align-content: center; + } + + .native-cta { + display: none; + } +} diff --git a/docs/_static/konami.js b/docs/_static/konami.js deleted file mode 100644 index d72cf9df89..0000000000 --- a/docs/_static/konami.js +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Konami-JS ~ - * :: Now with support for touch events and multiple instances for - * :: those situations that call for multiple easter eggs! - * Code: http://konami-js.googlecode.com/ - * Examples: http://www.snaptortoise.com/konami-js - * Copyright (c) 2009 George Mandis (georgemandis.com, snaptortoise.com) - * Version: 1.4.2 (9/2/2013) - * Licensed under the MIT License (http://opensource.org/licenses/MIT) - * Tested in: Safari 4+, Google Chrome 4+, Firefox 3+, IE7+, Mobile Safari 2.2.1 and Dolphin Browser - */ - -var Konami = function (callback) { - var konami = { - addEvent: function (obj, type, fn, ref_obj) { - if (obj.addEventListener) - obj.addEventListener(type, fn, false); - else if (obj.attachEvent) { - // IE - obj["e" + type + fn] = fn; - obj[type + fn] = function () { - obj["e" + type + fn](window.event, ref_obj); - }; - obj.attachEvent("on" + type, obj[type + fn]); - } - }, - input: "", - pattern: "38384040373937396665", - load: function (link) { - this.addEvent(document, "keydown", function (e, ref_obj) { - if (ref_obj) konami = ref_obj; // IE - konami.input += e ? e.keyCode : event.keyCode; - if (konami.input.length > konami.pattern.length) - konami.input = konami.input.substr((konami.input.length - konami.pattern.length)); - if (konami.input == konami.pattern) { - konami.code(link); - konami.input = ""; - e.preventDefault(); - return false; - } - }, this); - this.iphone.load(link); - }, - code: function (link) { - window.location = link - }, - iphone: { - start_x: 0, - start_y: 0, - stop_x: 0, - stop_y: 0, - tapTolerance: 8, - capture: false, - orig_keys: "", - keys: ["UP", "UP", "DOWN", "DOWN", "LEFT", "RIGHT", "LEFT", "RIGHT", "TAP", "TAP"], - code: function (link) { - konami.code(link); - }, - touchCapture: function(evt) { - konami.iphone.start_x = evt.changedTouches[0].pageX; - konami.iphone.start_y = evt.changedTouches[0].pageY; - konami.iphone.capture = true; - }, - load: function (link) { - this.orig_keys = this.keys; - konami.addEvent(document, "touchmove", function (e) { - if (e.touches.length == 1 && konami.iphone.capture == true) { - var touch = e.touches[0]; - konami.iphone.stop_x = touch.pageX; - konami.iphone.stop_y = touch.pageY; - konami.iphone.check_direction(); - } - }); - konami.addEvent(document, "touchend", function (evt) { - konami.touchCapture(evt); - konami.iphone.check_direction(link); - }, false); - konami.addEvent(document, "touchstart", function (evt) { - konami.touchCapture(evt); - }); - }, - check_direction: function (link) { - var x_magnitude = Math.abs(this.start_x - this.stop_x); - var y_magnitude = Math.abs(this.start_y - this.stop_y); - var hasMoved = (x_magnitude > this.tapTolerance || y_magnitude > this.tapTolerance); - var result; - if (this.capture === true && hasMoved) { - this.capture = false; - var x = ((this.start_x - this.stop_x) < 0) ? "RIGHT" : "LEFT"; - var y = ((this.start_y - this.stop_y) < 0) ? "DOWN" : "UP"; - var result = (x_magnitude > y_magnitude) ? x : y; - } - else if (this.capture === false && !hasMoved) { - result = (this.tap == true) ? "TAP" : result; - result = "TAP"; - } - if (result) { - if (result == this.keys[0]) this.keys = this.keys.slice(1, this.keys.length); - else this.keys = this.orig_keys; - } - if (this.keys.length == 0) { - this.keys = this.orig_keys; - this.code(link); - } - } - } - } - - typeof callback === "string" && konami.load(callback); - if (typeof callback === "function") { - konami.code = callback; - konami.load(); - } - - return konami; -}; diff --git a/docs/_static/requests-logo-small.png b/docs/_static/requests-logo-small.png deleted file mode 100644 index afadeaa4f6..0000000000 Binary files a/docs/_static/requests-logo-small.png and /dev/null differ diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html index c3fe2d1ea6..eca5dffcb5 100644 --- a/docs/_templates/hacks.html +++ b/docs/_templates/hacks.html @@ -1,33 +1,4 @@ - - - - - - + - - - + - - - +
+
- - - \ No newline at end of file diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index bdb1c02430..607bf92c4e 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -1,11 +1,11 @@

-

@@ -13,59 +13,25 @@ Requests is an elegant and simple HTTP library for Python, built for human beings.

-

Sponsored by Linode and other wonderful organizations.

- - - -

Requests Stickers!

-

Stay Informed

-

Receive updates on new releases and upcoming projects.

- -

- -

-

Join Mailing List.

-

Other Projects

- -

More Kenneth Reitz projects:

+

Useful Links

+
  • Quickstart
    • +
  • Advanced Usage
    • +
  • API Reference
    • +
  • Release History
    • +
  • Contributors Guide
    • +

    -

    Useful Links

    - - -

    Translations

    - - +
    +
    diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 1b7afbd805..71eb82e726 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -1,65 +1,30 @@ -

    - +

    -

    Requests is an elegant and simple HTTP library for Python, built for human beings. You are currently looking at the documentation of the development release.

    -

    Sponsored by Linode and other wonderful organizations.

    - -

    Stay Informed

    -

    Receive updates on new releases and upcoming projects.

    - -

    Requests Stickers!

    - -

    Join Mailing List.

    - -
    - - - -

    +

    Useful Links

    +
      +
    • Quickstart
      • +
    • Advanced Usage
      • +
    • API Reference
      • +
    • Release History
      • +
    • Contributors Guide
      • -

      +

      +
    • Recommended Packages and Extensions
      • -

      Other Projects

      +

      -

      More Kenneth Reitz projects:

      - - -

      Translations

      - - diff --git a/docs/api.rst b/docs/api.rst index ef84bf6077..3ae5b9d856 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -31,11 +31,11 @@ Exceptions .. autoexception:: requests.RequestException .. autoexception:: requests.ConnectionError .. autoexception:: requests.HTTPError -.. autoexception:: requests.URLRequired .. autoexception:: requests.TooManyRedirects .. autoexception:: requests.ConnectTimeout .. autoexception:: requests.ReadTimeout .. autoexception:: requests.Timeout +.. autoexception:: requests.JSONDecodeError Request Sessions @@ -127,7 +127,7 @@ API Changes :: import requests - r = requests.get('https://github.com/timeline.json') + r = requests.get('https://api.github.com/events') r.json() # This *call* raises an exception if JSON decoding fails * The ``Session`` API has changed. Sessions objects no longer take parameters. @@ -139,7 +139,7 @@ API Changes s = requests.Session() # formerly, session took parameters s.auth = auth s.headers.update(headers) - r = s.get('http://httpbin.org/headers') + r = s.get('https://httpbin.org/headers') * All request hooks have been removed except 'response'. @@ -156,7 +156,7 @@ API Changes :: # in 0.x, passing prefetch=False would accomplish the same thing - r = requests.get('https://github.com/timeline.json', stream=True) + r = requests.get('https://api.github.com/events', stream=True) for chunk in r.iter_content(8192): ... @@ -185,7 +185,7 @@ API Changes requests_log.setLevel(logging.DEBUG) requests_log.propagate = True - requests.get('http://httpbin.org/headers') + requests.get('https://httpbin.org/headers') @@ -197,8 +197,8 @@ license from the ISC_ license to the `Apache 2.0`_ license. The Apache 2.0 license ensures that contributions to Requests are also covered by the Apache 2.0 license. -.. _ISC: http://opensource.org/licenses/ISC -.. _Apache 2.0: http://opensource.org/licenses/Apache-2.0 +.. _ISC: https://opensource.org/licenses/ISC +.. _Apache 2.0: https://opensource.org/licenses/Apache-2.0 Migrating to 2.x @@ -213,7 +213,7 @@ For more details on the changes in this release including new APIs, links to the relevant GitHub issues and some of the bug fixes, read Cory's blog_ on the subject. -.. _blog: http://lukasa.co.uk/2013/09/Requests_20/ +.. _blog: https://lukasa.co.uk/2013/09/Requests_20/ API Changes diff --git a/docs/community/faq.rst b/docs/community/faq.rst index 7f5a0e1e76..2804b1d3de 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -3,8 +3,6 @@ Frequently Asked Questions ========================== -.. image:: https://farm5.staticflickr.com/4290/35294660055_42c02b2316_k_d.jpg - This part of the documentation answers common questions about Requests. Encoded Data? @@ -13,6 +11,9 @@ Encoded Data? Requests automatically decompresses gzip-encoded responses, and does its best to decode response content to unicode when possible. +When either the `brotli `_ or `brotlicffi `_ +package is installed, requests also decodes Brotli-encoded responses. + You can get direct access to the raw response (and even the socket), if needed as well. @@ -21,7 +22,8 @@ Custom User-Agents? ------------------- Requests allows you to easily override User-Agent strings, along with -any other HTTP Header. +any other HTTP Header. See :ref:`documentation about headers `. + Why not Httplib2? @@ -53,15 +55,18 @@ Chris Adams gave an excellent summary on Python 3 Support? ----------------- -Yes! Here's a list of Python platforms that are officially -supported: +Yes! Requests supports all `officially supported versions of Python `_ +and recent releases of PyPy. + +Python 2 Support? +----------------- + +No! As of Requests 2.28.0, Requests no longer supports Python 2.7. Users who +have been unable to migrate should pin to `requests<2.28`. Full information +can be found in `psf/requests#6023 `_. -* Python 2.6 -* Python 2.7 -* Python 3.4 -* Python 3.5 -* Python 3.6 -* PyPy +It is *highly* recommended users migrate to a supported Python 3.x version now since +Python 2.7 is no longer receiving bug fixes or security updates as of January 1, 2020. What are "hostname doesn't match" errors? ----------------------------------------- @@ -70,7 +75,7 @@ These errors occur when :ref:`SSL certificate verification ` fails to match the certificate the server responds with to the hostname Requests thinks it's contacting. If you're certain the server's SSL setup is correct (for example, because you can visit the site with your browser) and -you're using Python 2.6 or 2.7, a possible explanation is that you need +you're using Python 2.7, a possible explanation is that you need Server-Name-Indication. `Server-Name-Indication`_, or SNI, is an official extension to SSL where the @@ -79,10 +84,7 @@ when servers are using `Virtual Hosting`_. When such servers are hosting more than one SSL site they need to be able to return the appropriate certificate based on the hostname the client is connecting to. -Python3 and Python 2.7.9+ include native support for SNI in their SSL modules. -For information on using SNI with Requests on Python < 2.7.9 refer to this -`Stack Overflow answer`_. +Python 3 already includes native support for SNI in their SSL modules. .. _`Server-Name-Indication`: https://en.wikipedia.org/wiki/Server_Name_Indication .. _`virtual hosting`: https://en.wikipedia.org/wiki/Virtual_hosting -.. _`Stack Overflow answer`: https://stackoverflow.com/questions/18578439/using-requests-with-tls-doesnt-give-sni-support/18579484#18579484 diff --git a/docs/community/out-there.rst b/docs/community/out-there.rst index 63e70169f9..c75c71f6a2 100644 --- a/docs/community/out-there.rst +++ b/docs/community/out-there.rst @@ -1,24 +1,10 @@ Integrations ============ -.. image:: https://farm5.staticflickr.com/4239/34450900674_15863ddea0_k_d.jpg - -Python for iOS --------------- - -Requests is built into the wonderful `Python for iOS `_ runtime! - -To give it a try, simply:: - - import requests - - Articles & Talks ================ -- `Python for the Web `_ teaches how to use Python to interact with the web, using Requests. -- `Daniel Greenfeld's Review of Requests `_ -- `My 'Python for Humans' talk `_ ( `audio `_ ) -- `Issac Kelly's 'Consuming Web APIs' talk `_ -- `Blog post about Requests via Yum `_ -- `Russian blog post introducing Requests `_ +- `Daniel Greenfeld's Review of Requests `_ +- `Issac Kelly's 'Consuming Web APIs' talk `_ +- `Blog post about Requests via Yum `_ +- `Russian blog post introducing Requests `_ - `Sending JSON in Requests `_ diff --git a/docs/community/recommended.rst b/docs/community/recommended.rst index 8fcd47a436..517f4b1895 100644 --- a/docs/community/recommended.rst +++ b/docs/community/recommended.rst @@ -3,8 +3,6 @@ Recommended Packages and Extensions =================================== -.. image:: https://farm5.staticflickr.com/4218/35224319272_cfc0e621fb_k_d.jpg - Requests has a great variety of powerful and useful third-party extensions. This page provides an overview of some of the best of them. @@ -40,7 +38,7 @@ requested by users within the community. Requests-Threads ---------------- -`Requests-Threads` is a Requests session that returns the amazing Twisted's awaitable Deferreds instead of Response objects. This allows the use of ``async``/``await`` keyword usage on Python 3, or Twisted's style of programming, if desired. +`Requests-Threads`_ is a Requests session that returns the amazing Twisted's awaitable Deferreds instead of Response objects. This allows the use of ``async``/``await`` keyword usage on Python 3, or Twisted's style of programming, if desired. .. _Requests-Threads: https://github.com/requests/requests-threads @@ -61,4 +59,4 @@ Betamax `Betamax`_ records your HTTP interactions so the NSA does not have to. A VCR imitation designed only for Python-Requests. -.. _betamax: https://github.com/sigmavirus24/betamax +.. _betamax: https://github.com/betamaxpy/betamax diff --git a/docs/community/release-process.rst b/docs/community/release-process.rst index 18f71168a5..4aa98f755e 100644 --- a/docs/community/release-process.rst +++ b/docs/community/release-process.rst @@ -1,8 +1,6 @@ Release Process and Rules ========================= -.. image:: https://farm5.staticflickr.com/4215/34450901614_b74ae720db_k_d.jpg - .. versionadded:: v2.6.2 Starting with the version to be released after ``v2.6.2``, the following rules diff --git a/docs/community/sponsors.rst b/docs/community/sponsors.rst deleted file mode 100644 index f1e11efdd0..0000000000 --- a/docs/community/sponsors.rst +++ /dev/null @@ -1,96 +0,0 @@ -Community Sponsors -================== - -**tl;dr**: Requests development is currently `funded by the Python community `_, and -some wonderful organizations that utilize the software in their businesses. - - -------------------- - - -Requests is one of the most heavily–utilized Python packages in the world. - -It is used by major corporations worldwide for all tasks, both small and large — from writing one–off scripts to orchestrating millions of dollars of critical infrastructure. - -It's even embedded within pip, that tool that you use to install packages and deploy with every day! - -After losing our primary open source maintainer (who was sponsored by a company to work on Requests, and other projects, full–time), we are seeking community financial contributions towards the development of Requests 3.0. - -Patron Sponsors ----------------- - - -`Linode — SSD Cloud Hosting & Linux Servers `_ -////////////////////////////////////////////////////////////////////// - -Whether you’re just getting started or deploying a complex system, launching a Linode cloud server has never been easier. They offer the fastest hardware and network in the industry with scalable environments, and their 24x7 customer support team is always standing by to help with any questions. - -✨🍰✨ -////// - ----------------------------------- - -This slot is reserved for ethical organizations willing to invest $10,000 or more in Requests per year. - -By becoming a patron–level sponsor, your organization will receive the following benefits: - -- Prominent placement on the Requests documentation sidebar (~11,000 uniques / day). -- Honorable mention here, with logo. -- Peace of mind knowing that the infrastructure you rely on is being actively maintained. - -Organizations that sign up will be listed in order — first come first serve! - -Major Sponsors --------------- - -The following organizations have significantly contributed towards Requests' sustainability: - -`Slack — Bring your team together `_ -/////////////////////////////////////////////////////// - -Slack was extremely kind to be the first organization to generously donate a large sum towards the `2018 Requests 3.0 fundraiser `_, surpassing our entire fundraising goal immediately! They are helping the world become a better place through connectiveness, and reducing the amount of email we all have -to deal with on a daily basis. - -P.S. They're `hiring `_! - - -`Twilio — Voice, SMS, and Video for Humans `_ -///////////////////////////////////////////////////////////////////// - -Twilio was the second organization to generously donate a large sum towards the `2018 Requests 3.0 fundraiser `_, matching the donation of Slack! They are helping the world become a better place through interconnectivity, -providing easy–to–use APIs, and empowering developers world-over to help humans communicate in meaningful and effictive ways. - - -`Azure Cloud Developer Advocates `_ -///////////////////////////////////////////////////////////////////////////////////// - -Azure was the third organization to generously donate a large sum towards the `2018 Requests 3.0 fundraiser `_, matching the donation of Twilio! Awesome group of generous folks :) - - -`Niteo — Web Systems Development `_ -///////////////////////////////////////////////////////////// - -Niteo was the fourth company to generously donate towards the `2018 Requests 3.0 fundraiser `_. Niteo is a company employing tech enthusiasts from all over the world -who love to build great stuff. - - -`Heroku `_ -///////////////////////////////////// - -Heroku has allowed Kenneth Reitz to work on some open source projects during work hours, -including Requests (but mostly Pipenv), from time–to–time, so they are listed -here as an honorable mention. - ----------------- - -If your organization is interested in becoming either a sponsor or a patron, please `send us an email `_. - - -Individual Sponsors -------------------- - -Countless individuals, too many to list here, have individually contributed towards the sustainability of the Requests -project over the years. Some, financially, others, with code. Contributions (from humans) of all kinds are greatly -appreciated. - -✨🍰✨ \ No newline at end of file diff --git a/docs/community/support.rst b/docs/community/support.rst index 02e8da7515..ee905f543d 100644 --- a/docs/community/support.rst +++ b/docs/community/support.rst @@ -3,51 +3,29 @@ Support ======= -.. image:: https://farm5.staticflickr.com/4198/34080352913_5c13ffb336_k_d.jpg - If you have questions or issues about Requests, there are several options: -StackOverflow -------------- +Stack Overflow +-------------- If your question does not contain sensitive (possibly proprietary) information or can be properly anonymized, please ask a question on -`StackOverflow `_ +`Stack Overflow `_ and use the tag ``python-requests``. -Send a Tweet ------------- - -If your question is less than 140 characters, feel free to send a tweet to -`@kennethreitz `_, -`@sigmavirus24 `_, or -`@lukasaoz `_. File an Issue ------------- If you notice some unexpected behaviour in Requests, or want to see support for a new feature, -`file an issue on GitHub `_. - +`file an issue on GitHub `_. -E-mail ------- -I'm more than happy to answer any personal or in-depth questions about -Requests. Feel free to email -`requests@kennethreitz.com `_. - - -IRC ---- - -The official Freenode channel for Requests is -`#python-requests `_ - -The core developers of requests are on IRC throughout the day. -You can find them in ``#python-requests`` as: +Send a Tweet +------------ -- kennethreitz -- lukasa -- sigmavirus24 +If your question is less than 280 characters, feel free to send a tweet to +`@nateprewitt `_, +`@sethmlarson `_, or +`@sigmavirus24 `_. diff --git a/docs/community/updates.rst b/docs/community/updates.rst index 3b9a30970e..c787c45bcf 100644 --- a/docs/community/updates.rst +++ b/docs/community/updates.rst @@ -4,8 +4,6 @@ Community Updates ================= -.. image:: https://farm5.staticflickr.com/4244/34080354873_516c283ad0_k_d.jpg - If you'd like to stay up to date on the community and development of Requests, there are several options: @@ -14,18 +12,7 @@ GitHub ------ The best way to track the development of Requests is through -`the GitHub repo `_. - -Twitter -------- - -The author, Kenneth Reitz, often tweets about new features and releases of Requests. - -Follow `@kennethreitz `_ for updates. - - +`the GitHub repo `_. -Release and Version History -=========================== -.. include:: ../../HISTORY.rst +.. include:: ../../HISTORY.md diff --git a/docs/community/vulnerabilities.rst b/docs/community/vulnerabilities.rst index a6ab887ee7..764d34223d 100644 --- a/docs/community/vulnerabilities.rst +++ b/docs/community/vulnerabilities.rst @@ -1,107 +1,5 @@ Vulnerability Disclosure ======================== -.. image:: https://farm5.staticflickr.com/4211/34709353644_b041e9e1c2_k_d.jpg - -If you think you have found a potential security vulnerability in requests, -please email `sigmavirus24 `_ and -`Lukasa `_ directly. **Do not file a public issue.** - -Our PGP Key fingerprints are: - -- 0161 BB7E B208 B5E0 4FDC 9F81 D9DA 0A04 9113 F853 (@sigmavirus24) - -- 90DC AE40 FEA7 4B14 9B70 662D F25F 2144 EEC1 373D (@lukasa) - -If English is not your first language, please try to describe the problem and -its impact to the best of your ability. For greater detail, please use your -native language and we will try our best to translate it using online services. - -Please also include the code you used to find the problem and the shortest -amount of code necessary to reproduce it. - -Please do not disclose this to anyone else. We will retrieve a CVE identifier -if necessary and give you full credit under whatever name or alias you provide. -We will only request an identifier when we have a fix and can publish it in a -release. - -We will respect your privacy and will only publicize your involvement if you -grant us permission. - -Process -------- - -This following information discusses the process the requests project follows -in response to vulnerability disclosures. If you are disclosing a -vulnerability, this section of the documentation lets you know how we will -respond to your disclosure. - -Timeline -~~~~~~~~ - -When you report an issue, one of the project members will respond to you within -two days *at the outside*. In most cases responses will be faster, usually -within 12 hours. This initial response will at the very least confirm receipt -of the report. - -If we were able to rapidly reproduce the issue, the initial response will also -contain confirmation of the issue. If we are not, we will often ask for more -information about the reproduction scenario. - -Our goal is to have a fix for any vulnerability released within two weeks of -the initial disclosure. This may potentially involve shipping an interim -release that simply disables function while a more mature fix can be prepared, -but will in the vast majority of cases mean shipping a complete release as soon -as possible. - -Throughout the fix process we will keep you up to speed with how the fix is -progressing. Once the fix is prepared, we will notify you that we believe we -have a fix. Often we will ask you to confirm the fix resolves the problem in -your environment, especially if we are not confident of our reproduction -scenario. - -At this point, we will prepare for the release. We will obtain a CVE number -if one is required, providing you with full credit for the discovery. We will -also decide on a planned release date, and let you know when it is. This -release date will *always* be on a weekday. - -At this point we will reach out to our major downstream packagers to notify -them of an impending security-related patch so they can make arrangements. In -addition, these packagers will be provided with the intended patch ahead of -time, to ensure that they are able to promptly release their downstream -packages. Currently the list of people we actively contact *ahead of a public -release* is: - -- Jeremy Cline, Red Hat (@jeremycline) -- Daniele Tricoli, Debian (@eriol) - -We will notify these individuals at least a week ahead of our planned release -date to ensure that they have sufficient time to prepare. If you believe you -should be on this list, please let one of the maintainers know at one of the -email addresses at the top of this article. - -On release day, we will push the patch to our public repository, along with an -updated changelog that describes the issue and credits you. We will then issue -a PyPI release containing the patch. - -At this point, we will publicise the release. This will involve mails to -mailing lists, Tweets, and all other communication mechanisms available to the -core team. - -We will also explicitly mention which commits contain the fix to make it easier -for other distributors and users to easily patch their own versions of requests -if upgrading is not an option. - -Previous CVEs -------------- - -- Fixed in 2.6.0 - - - `CVE 2015-2296 `_, - reported by Matthew Daley of `BugFuzz `_. - -- Fixed in 2.3.0 - - - `CVE 2014-1829 `_ - - - `CVE 2014-1830 `_ +The latest vulnerability disclosure information can be found on GitHub in our +`Security Policy `_. diff --git a/docs/conf.py b/docs/conf.py index c952fe7941..9b81db0810 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,11 +18,11 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # Insert Requests' path into the system. -sys.path.insert(0, os.path.abspath('..')) -sys.path.insert(0, os.path.abspath('_themes')) +sys.path.insert(0, os.path.abspath("..")) +sys.path.insert(0, os.path.abspath("_themes")) import requests @@ -30,36 +30,36 @@ # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.viewcode', + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.viewcode", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'Requests' -copyright = u'MMXVIII. A Kenneth Reitz Project' -author = u'Kenneth Reitz' +project = u"Requests" +copyright = u'MMXVIX. A Kenneth Reitz Project' +author = u"Kenneth Reitz" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -79,17 +79,17 @@ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = False @@ -100,16 +100,16 @@ # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'flask_theme_support.FlaskyStyle' +pygments_style = "flask_theme_support.FlaskyStyle" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True @@ -119,52 +119,52 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - 'show_powered_by': False, - 'github_user': 'requests', - 'github_repo': 'requests', - 'github_banner': True, - 'show_related': False, - 'note_bg': '#FFF59C' + "show_powered_by": False, + "github_user": "requests", + "github_repo": "requests", + "github_banner": True, + "show_related": False, + "note_bg": "#FFF59C", } # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. @@ -172,24 +172,29 @@ # Custom sidebar templates, maps document names to template names. html_sidebars = { - 'index': ['sidebarintro.html', 'sourcelink.html', 'searchbox.html', - 'hacks.html'], - '**': ['sidebarlogo.html', 'localtoc.html', 'relations.html', - 'sourcelink.html', 'searchbox.html', 'hacks.html'] + "index": ["sidebarintro.html", "sourcelink.html", "searchbox.html", "hacks.html"], + "**": [ + "sidebarlogo.html", + "localtoc.html", + "relations.html", + "sourcelink.html", + "searchbox.html", + "hacks.html", + ], } # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = False @@ -203,84 +208,77 @@ # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'Requestsdoc' +htmlhelp_basename = "Requestsdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', + # Latex figure (float) alignment + #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'Requests.tex', u'Requests Documentation', - u'Kenneth Reitz', 'manual'), + (master_doc, "Requests.tex", u"Requests Documentation", u"Kenneth Reitz", "manual") ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'requests', u'Requests Documentation', - [author], 1) -] +man_pages = [(master_doc, "requests", u"Requests Documentation", [author], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -289,22 +287,28 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'Requests', u'Requests Documentation', - author, 'Requests', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "Requests", + u"Requests Documentation", + author, + "Requests", + "One line description of project.", + "Miscellaneous", + ) ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False # -- Options for Epub output ---------------------------------------------- @@ -316,64 +320,67 @@ epub_copyright = copyright # The basename for the epub file. It defaults to the project name. -#epub_basename = project +# epub_basename = project # The HTML theme for the epub output. Since the default themes are not # optimized for small screen space, using the same theme for HTML and epub # output is usually not wise. This defaults to 'epub', a theme designed to save # visual space. -#epub_theme = 'epub' +# epub_theme = 'epub' # The language of the text. It defaults to the language option # or 'en' if the language is not set. -#epub_language = '' +# epub_language = '' # The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' +# epub_scheme = '' # The unique identifier of the text. This can be a ISBN number # or the project homepage. -#epub_identifier = '' +# epub_identifier = '' # A unique identification for the text. -#epub_uid = '' +# epub_uid = '' # A tuple containing the cover image and cover page html template filenames. -#epub_cover = () +# epub_cover = () # A sequence of (type, uri, title) tuples for the guide element of content.opf. -#epub_guide = () +# epub_guide = () # HTML files that should be inserted before the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_pre_files = [] +# epub_pre_files = [] # HTML files that should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_post_files = [] +# epub_post_files = [] # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 +# epub_tocdepth = 3 # Allow duplicate toc entries. -#epub_tocdup = True +# epub_tocdup = True # Choose between 'default' and 'includehidden'. -#epub_tocscope = 'default' +# epub_tocscope = 'default' # Fix unsupported image types using the Pillow. -#epub_fix_images = False +# epub_fix_images = False # Scale large images. -#epub_max_image_width = 0 +# epub_max_image_width = 0 # How to display URL addresses: 'footnote', 'no', or 'inline'. -#epub_show_urls = 'inline' +# epub_show_urls = 'inline' # If false, no index is generated. -#epub_use_index = True +# epub_use_index = True -intersphinx_mapping = {'urllib3': ('https://urllib3.readthedocs.io/en/latest', None)} +intersphinx_mapping = { + "python": ("https://docs.python.org/3/", None), + "urllib3": ("https://urllib3.readthedocs.io/en/latest", None), +} diff --git a/docs/dev/authors.rst b/docs/dev/authors.rst index 4cdd14cd43..e9799a9142 100644 --- a/docs/dev/authors.rst +++ b/docs/dev/authors.rst @@ -1,6 +1,4 @@ Authors ======= -.. image:: https://static1.squarespace.com/static/533ad9bde4b098d084a846b1/t/534f6e1ce4b09b70f38ee6c1/1432265542589/DSCF3147.jpg?format=2500w - .. include:: ../../AUTHORS.rst diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index efc79bbfff..214e2b48ce 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -3,8 +3,6 @@ Contributor's Guide =================== -.. image:: https://farm5.staticflickr.com/4237/35550408335_7671fde302_k_d.jpg - If you're reading this, you're probably interested in contributing to Requests. Thank you very much! Open source projects live-and-die based on the support they receive from others, and the fact that you're even considering @@ -13,35 +11,28 @@ contributing to the Requests project is *very* generous of you. This document lays out guidelines and advice for contributing to this project. If you're thinking of contributing, please start by reading this document and getting a feel for how contributing to this project works. If you have any -questions, feel free to reach out to either `Ian Cordasco`_ or `Cory Benfield`_, -the primary maintainers. +questions, feel free to reach out to either `Nate Prewitt`_, `Ian Cordasco`_, +or `Seth Michael Larson`_, the primary maintainers. .. _Ian Cordasco: http://www.coglib.com/~icordasc/ -.. _Cory Benfield: https://lukasa.co.uk/about - -If you have non-technical feedback, philosophical ponderings, crazy ideas, or -other general thoughts about Requests or its position within the Python -ecosystem, the BDFL, `Kenneth Reitz`_, would love to hear from you. +.. _Nate Prewitt: https://www.nateprewitt.com/ +.. _Seth Michael Larson: https://sethmlarson.dev/ The guide is split into sections based on the type of contribution you're thinking of making, with a section that covers general guidelines for all contributors. -.. _Kenneth Reitz: mailto:me@kennethreitz.org - -Be Cordial ----------- +Code of Conduct +--------------- - **Be cordial or be on your way**. *—Kenneth Reitz* +The Python community is made up of members from around the globe with a diverse +set of skills, personalities, and experiences. It is through these differences +that our community experiences great successes and continued growth. When you're +working with members of the community, follow the +`Python Software Foundation Code of Conduct`_ to help steer your interactions +and keep Python a positive, successful, and growing community. -Requests has one very important rule governing all forms of contribution, -including reporting bugs or requesting features. This golden rule is -"`be cordial or be on your way`_". - -**All contributions are welcome**, as long as -everyone involved is treated with respect. - -.. _be cordial or be on your way: http://kennethreitz.org/be-cordial-or-be-on-your-way/ +.. _Python Software Foundation Code of Conduct: https://policies.python.org/python.org/code-of-conduct/ .. _early-feedback: @@ -85,7 +76,7 @@ When contributing code, you'll want to follow this checklist: 4. Make your change. 5. Run the entire test suite again, confirming that all tests pass *including the ones you just added*. -6. Send a GitHub Pull Request to the main repository's ``master`` branch. +6. Send a GitHub Pull Request to the main repository's ``main`` branch. GitHub Pull Requests are the expected method of code collaboration on this project. @@ -100,6 +91,21 @@ event that you object to the code review feedback, you should make your case clearly and calmly. If, after doing so, the feedback is judged to still apply, you must either apply the feedback or withdraw your contribution. +Code Style +~~~~~~~~~~ + +Requests uses a collection of tools to ensure the code base has a consistent +style as it grows. We have these orchestrated using a tool called +`pre-commit`_. This can be installed locally and run over your changes prior +to opening a PR, and will also be run as part of the CI approval process +before a change is merged. + +You can find the full list of formatting requirements specified in the +`.pre-commit-config.yaml`_ at the top level directory of Requests. + +.. _pre-commit: https://pre-commit.com/ +.. _.pre-commit-config.yaml: https://github.com/psf/requests/blob/main/.pre-commit-config.yaml + New Contributors ~~~~~~~~~~~~~~~~ @@ -110,55 +116,6 @@ asking for help. Please also check the :ref:`early-feedback` section. -Kenneth Reitz's Code Style™ -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The Requests codebase uses the `PEP 8`_ code style. - -In addition to the standards outlined in PEP 8, we have a few guidelines: - -- Line-length can exceed 79 characters, to 100, when convenient. -- Line-length can exceed 100 characters, when doing otherwise would be *terribly* inconvenient. -- Always use single-quoted strings (e.g. ``'#flatearth'``), unless a single-quote occurs within the string. - -Additionally, one of the styles that PEP8 recommends for `line continuations`_ -completely lacks all sense of taste, and is not to be permitted within -the Requests codebase:: - - # Aligned with opening delimiter. - foo = long_function_name(var_one, var_two, - var_three, var_four) - -No. Just don't. Please. - -Docstrings are to follow the following syntaxes:: - - def the_earth_is_flat(): - """NASA divided up the seas into thirty-three degrees.""" - pass - -:: - - def fibonacci_spiral_tool(): - """With my feet upon the ground I lose myself / between the sounds - and open wide to suck it in. / I feel it move across my skin. / I'm - reaching up and reaching out. / I'm reaching for the random or - whatever will bewilder me. / Whatever will bewilder me. / And - following our will and wind we may just go where no one's been. / - We'll ride the spiral to the end and may just go where no one's - been. - - Spiral out. Keep going... - """ - pass - -All functions, methods, and classes are to contain docstrings. Object data -model methods (e.g. ``__repr__``) are typically the exception to this rule. - -Thanks for helping to make the world a better place! - -.. _PEP 8: http://pep8.org -.. _line continuations: https://www.python.org/dev/peps/pep-0008/#indentation Documentation Contributions --------------------------- @@ -189,7 +146,7 @@ through the `GitHub issues`_, **both open and closed**, to confirm that the bug hasn't been reported before. Duplicate bug reports are a huge drain on the time of other contributors, and should be avoided as much as possible. -.. _GitHub issues: https://github.com/requests/requests/issues +.. _GitHub issues: https://github.com/psf/requests/issues Feature Requests diff --git a/docs/dev/philosophy.rst b/docs/dev/philosophy.rst deleted file mode 100644 index 563c551c82..0000000000 --- a/docs/dev/philosophy.rst +++ /dev/null @@ -1,47 +0,0 @@ -Development Philosophy -====================== - -.. image:: https://farm5.staticflickr.com/4231/34484831073_636008a23d_k_d.jpg - -Requests is an open but opinionated library, created by an open but opinionated developer. - - -Management Style -~~~~~~~~~~~~~~~~ - -`Kenneth Reitz `_ is the BDFL. He has final say in any decision related to the Requests project. Kenneth is responsible for the direction and form of the library, as well as its presentation. In addition to making decisions based on technical merit, he is responsible for making decisions based on the development philosophy of Requests. - -`Ian Cordasco `_ and `Cory Benfield `_ are the core contributors. They are responsible for triaging bug reports, reviewing pull requests and ensuring that Kenneth is kept up to speed with developments around the library. The day-to-day managing of the project is done by the core contributors. They are responsible for making judgements about whether or not a feature request is likely to be accepted by Kenneth. Their word is, in some ways, more final than Kenneth's. - -Values -~~~~~~ - -- Simplicity is always better than functionality. -- Listen to everyone, then disregard it. -- The API is all that matters. Everything else is secondary. -- Fit the 90% use-case. Ignore the nay-sayers. - -Semantic Versioning -~~~~~~~~~~~~~~~~~~~ - -For many years, the open source community has been plagued with version number dystonia. Numbers vary so greatly from project to project, they are practically meaningless. - -Requests uses `Semantic Versioning `_. This specification seeks to put an end to this madness with a small set of practical guidelines for you and your colleagues to use in your next project. - -Standard Library? -~~~~~~~~~~~~~~~~~ - -Requests has no *active* plans to be included in the standard library. This decision has been discussed at length with Guido as well as numerous core developers. - -.. raw:: html - - - -Essentially, the standard library is where a library goes to die. It is appropriate for a module to be included when active development is no longer necessary. - -Linux Distro Packages -~~~~~~~~~~~~~~~~~~~~~ - -Distributions have been made for many Linux repositories, including: Ubuntu, Debian, RHEL, and Arch. - -These distributions are sometimes divergent forks, or are otherwise not kept up-to-date with the latest code and bugfixes. PyPI (and its mirrors) and GitHub are the official distribution sources; alternatives are not supported by the Requests project. diff --git a/docs/dev/todo.rst b/docs/dev/todo.rst deleted file mode 100644 index 1766a28a8f..0000000000 --- a/docs/dev/todo.rst +++ /dev/null @@ -1,64 +0,0 @@ -How to Help -=========== - -.. image:: https://farm5.staticflickr.com/4290/34450900104_bc1d424213_k_d.jpg - -Requests is under active development, and contributions are more than welcome! - -#. Check for open issues or open a fresh issue to start a discussion around a bug. - There is a Contributor Friendly tag for issues that should be ideal for people who are not very - familiar with the codebase yet. -#. Fork `the repository `_ on GitHub and start making your - changes to a new branch. -#. Write a test which shows that the bug was fixed. -#. Send a pull request and bug the maintainer until it gets merged and published. :) - Make sure to add yourself to `AUTHORS `_. - -Feature Freeze --------------- - -As of v1.0.0, Requests has now entered a feature freeze. Requests for new -features and Pull Requests implementing those features will not be accepted. - -Development Dependencies ------------------------- - -You'll need to install py.test in order to run the Requests' test suite:: - - $ venv .venv - $ source .venv/bin/activate - - $ make - $ python setup.py test - ============================= test session starts ============================== - platform darwin -- Python 3.4.4, pytest-3.0.6, py-1.4.32, pluggy-0.4.0 - ... - collected 445 items - - tests/test_hooks.py ... - tests/test_lowlevel.py ............ - tests/test_requests.py ........................................................... - tests/test_structures.py .................... - tests/test_testserver.py ........... - tests/test_utils.py ..s........................................................... - - ============== 442 passed, 1 skipped, 2 xpassed in 46.48 seconds =============== - -You can also run ``$ make tests`` to run against all supported Python versions, using tox/detox. - -Runtime Environments --------------------- - -Requests currently supports the following versions of Python: - -- Python 2.6 -- Python 2.7 -- Python 3.4 -- Python 3.5 -- Python 3.6 -- PyPy - -Google AppEngine is not officially supported although support is available -with the `Requests-Toolbelt`_. - -.. _Requests-Toolbelt: https://toolbelt.readthedocs.io/ diff --git a/docs/index.rst b/docs/index.rst index 918e2dcf80..aef47a8906 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,34 +3,29 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Requests: HTTP for Humans -========================= +Requests: HTTP for Humans™ +========================== Release v\ |version|. (:ref:`Installation `) + +.. image:: https://static.pepy.tech/badge/requests/month + :target: https://pepy.tech/project/requests + :alt: Requests Downloads Per Month Badge + .. image:: https://img.shields.io/pypi/l/requests.svg :target: https://pypi.org/project/requests/ + :alt: License Badge .. image:: https://img.shields.io/pypi/wheel/requests.svg :target: https://pypi.org/project/requests/ + :alt: Wheel Support Badge .. image:: https://img.shields.io/pypi/pyversions/requests.svg :target: https://pypi.org/project/requests/ + :alt: Python Version Support Badge -.. image:: https://codecov.io/github/requests/requests/coverage.svg?branch=master - :target: https://codecov.io/github/requests/requests - :alt: codecov.io - -.. image:: https://img.shields.io/badge/Say%20Thanks!-🦉-1EAEDB.svg - :target: https://saythanks.io/to/kennethreitz - - -**Requests** is the only *Non-GMO* HTTP library for Python, safe for human -consumption. - -.. note:: The use of **Python 3** is *highly* preferred over Python 2. Consider upgrading your applications and infrastructure if you find yourself *still* using Python 2 in production today. If you are using Python 3, congratulations — you are indeed a person of excellent taste. - —*Kenneth Reitz* - +**Requests** is an elegant and simple HTTP library for Python, built for human beings. ------------------- @@ -44,48 +39,17 @@ consumption. >>> r.encoding 'utf-8' >>> r.text - u'{"type":"User"...' + '{"type":"User"...' >>> r.json() - {u'private_gists': 419, u'total_private_repos': 77, ...} + {'private_gists': 419, 'total_private_repos': 77, ...} See `similar code, sans Requests `_. -**Requests** allows you to send *organic, grass-fed* HTTP/1.1 requests, without the -need for manual labor. There's no need to manually add query strings to your +**Requests** allows you to send HTTP/1.1 requests extremely easily. +There's no need to manually add query strings to your URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling -are 100% automatic, thanks to `urllib3 `_. - -User Testimonials ------------------ - -Nike, Twitter, Spotify, Microsoft, Amazon, Lyft, BuzzFeed, Reddit, The NSA, Her Majesty's Government, Google, Twilio, Runscope, Mozilla, Heroku, -PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington -Post, SoundCloud, Kippt, Sony, and Federal U.S. -Institutions that prefer to be unnamed claim to use Requests internally. - -**Armin Ronacher**, creator of Flask— - *Requests is the perfect example how beautiful an API can be with the - right level of abstraction.* - -**Matt DeBoard**— - *I'm going to get Kenneth Reitz's Python requests module tattooed - on my body, somehow. The whole thing.* - -**Daniel Greenfeld**— - *Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to - Kenneth Reitz's Requests library. Today has been AWESOME.* - -**Kenny Meyers**— - *Python HTTP: When in doubt, or when not in doubt, use Requests. Beautiful, - simple, Pythonic.* - -Requests is one of the most downloaded Python packages of all time, pulling in -over 400,000 downloads **each day**. Join the party! - -If your organization uses Requests internally, consider `supporting the development of 3.0 `_. Your -generosity will be greatly appreciated, and help drive the project forward -into the future. +are 100% automatic, thanks to `urllib3 `_. Beloved Features ---------------- @@ -108,7 +72,7 @@ Requests is ready for today's web. - Chunked Requests - ``.netrc`` Support -Requests officially supports Python 2.6–2.7 & 3.4–3.7, and runs great on PyPy. +Requests officially supports Python 3.9+, and runs great on PyPy. The User Guide @@ -121,7 +85,6 @@ instructions for getting the most out of Requests. .. toctree:: :maxdepth: 2 - user/intro user/install user/quickstart user/advanced @@ -137,15 +100,18 @@ Requests ecosystem and community. .. toctree:: :maxdepth: 2 - community/sponsors community/recommended community/faq community/out-there community/support community/vulnerabilities - community/updates community/release-process +.. toctree:: + :maxdepth: 1 + + community/updates + The API Documentation / Guide ----------------------------- @@ -168,8 +134,6 @@ you. :maxdepth: 3 dev/contributing - dev/philosophy - dev/todo dev/authors There are no more guides. You are now guideless. diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000000..2af334d57b --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +# Pinning to avoid unexpected breakages. +# Used by RTD to generate docs. +Sphinx==7.2.6 diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 3e78813de6..2ff0c7dfbf 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -3,8 +3,6 @@ Advanced Usage ============== -.. image:: https://farm5.staticflickr.com/4263/35163665790_d182d84f5e_k_d.jpg - This document covers some of Requests more advanced features. .. _session-objects: @@ -25,8 +23,8 @@ Let's persist some cookies across requests:: s = requests.Session() - s.get('http://httpbin.org/cookies/set/sessioncookie/123456789') - r = s.get('http://httpbin.org/cookies') + s.get('https://httpbin.org/cookies/set/sessioncookie/123456789') + r = s.get('https://httpbin.org/cookies') print(r.text) # '{"cookies": {"sessioncookie": "123456789"}}' @@ -40,7 +38,7 @@ is done by providing data to the properties on a Session object:: s.headers.update({'x-test': 'true'}) # both 'x-test' and 'x-test2' are sent - s.get('http://httpbin.org/headers', headers={'x-test2': 'true'}) + s.get('https://httpbin.org/headers', headers={'x-test2': 'true'}) Any dictionaries that you pass to a request method will be merged with the @@ -53,11 +51,11 @@ with the first request, but not the second:: s = requests.Session() - r = s.get('http://httpbin.org/cookies', cookies={'from-my': 'browser'}) + r = s.get('https://httpbin.org/cookies', cookies={'from-my': 'browser'}) print(r.text) # '{"cookies": {"from-my": "browser"}}' - r = s.get('http://httpbin.org/cookies') + r = s.get('https://httpbin.org/cookies') print(r.text) # '{"cookies": {}}' @@ -69,7 +67,7 @@ If you want to manually add cookies to your session, use the Sessions can also be used as context managers:: with requests.Session() as s: - s.get('http://httpbin.org/cookies/set/sessioncookie/123456789') + s.get('https://httpbin.org/cookies/set/sessioncookie/123456789') This will make sure the session is closed as soon as the ``with`` block is exited, even if unhandled exceptions occurred. @@ -97,7 +95,7 @@ The ``Response`` object contains all of the information returned by the server a also contains the ``Request`` object you created originally. Here is a simple request to get some very important information from Wikipedia's servers:: - >>> r = requests.get('http://en.wikipedia.org/wiki/Monty_Python') + >>> r = requests.get('https://en.wikipedia.org/wiki/Monty_Python') If we want to access the headers the server sent back to us, we do this:: @@ -193,7 +191,7 @@ When you are using the prepared request flow, keep in mind that it does not take This can cause problems if you are using environment variables to change the behaviour of requests. For example: Self-signed SSL certificates specified in ``REQUESTS_CA_BUNDLE`` will not be taken into account. As a result an ``SSL: CERTIFICATE_VERIFY_FAILED`` is thrown. -You can get around this behaviour by explicity merging the environment settings into your session:: +You can get around this behaviour by explicitly merging the environment settings into your session:: from requests import Request, Session @@ -203,7 +201,7 @@ You can get around this behaviour by explicity merging the environment settings prepped = s.prepare_request(req) # Merge environment settings into session - settings = s.merge_environment_settings(prepped.url, None, None, None, None) + settings = s.merge_environment_settings(prepped.url, {}, None, None, None) resp = s.send(prepped, **settings) print(resp.status_code) @@ -235,15 +233,22 @@ or persistent:: s.verify = '/path/to/certfile' .. note:: If ``verify`` is set to a path to a directory, the directory must have been processed using - the c_rehash utility supplied with OpenSSL. + the ``c_rehash`` utility supplied with OpenSSL. This list of trusted CAs can also be specified through the ``REQUESTS_CA_BUNDLE`` environment variable. +If ``REQUESTS_CA_BUNDLE`` is not set, ``CURL_CA_BUNDLE`` will be used as fallback. Requests can also ignore verifying the SSL certificate if you set ``verify`` to False:: >>> requests.get('https://kennethreitz.org', verify=False) +Note that when ``verify`` is set to ``False``, requests will accept any TLS +certificate presented by the server, and will ignore hostname mismatches +and/or expired certificates, which will make your application vulnerable to +man-in-the-middle (MitM) attacks. Setting verify to ``False`` may be useful +during local development or testing. + By default, ``verify`` is set to True. Option ``verify`` only applies to host certs. Client Side Certificates @@ -286,8 +291,8 @@ versions of Requests. For the sake of security we recommend upgrading certifi frequently! .. _HTTP persistent connection: https://en.wikipedia.org/wiki/HTTP_persistent_connection -.. _connection pooling: https://urllib3.readthedocs.io/en/latest/reference/index.html#module-urllib3.connectionpool -.. _certifi: http://certifi.io/ +.. _connection pooling: https://urllib3.readthedocs.io/en/latest/reference/urllib3.connectionpool.html +.. _certifi: https://certifiio.readthedocs.io/ .. _Mozilla trust store: https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt .. _body-content-workflow: @@ -300,7 +305,7 @@ immediately. You can override this behaviour and defer downloading the response body until you access the :attr:`Response.content ` attribute with the ``stream`` parameter:: - tarball_url = 'https://github.com/requests/requests/tarball/master' + tarball_url = 'https://github.com/psf/requests/tarball/main' r = requests.get(tarball_url, stream=True) At this point only the response headers have been downloaded and the connection @@ -323,7 +328,7 @@ inefficiency with connections. If you find yourself partially reading request bodies (or not reading them at all) while using ``stream=True``, you should make the request within a ``with`` statement to ensure it's always closed:: - with requests.get('http://httpbin.org/get', stream=True) as r: + with requests.get('https://httpbin.org/get', stream=True) as r: # Do things with the response here. .. _keep-alive: @@ -351,13 +356,11 @@ file-like object for your body:: with open('massive-body', 'rb') as f: requests.post('http://some.url/streamed', data=f) -.. warning:: It is strongly recommended that you open files in `binary mode`_. - This is because Requests may attempt to provide the - ``Content-Length`` header for you, and if it does this value will - be set to the number of *bytes* in the file. Errors may occur if - you open the file in *text mode*. - -.. _binary mode: https://docs.python.org/2/tutorial/inputoutput.html#reading-and-writing-files +.. warning:: It is strongly recommended that you open files in :ref:`binary + mode `. This is because Requests may attempt to provide + the ``Content-Length`` header for you, and if it does this value + will be set to the number of *bytes* in the file. Errors may occur + if you open the file in *text mode*. .. _chunk-encoding: @@ -395,10 +398,10 @@ upload image files to an HTML form with a multiple file field 'images':: To do that, just set files to a list of tuples of ``(form_field_name, file_info)``:: - >>> url = 'http://httpbin.org/post' + >>> url = 'https://httpbin.org/post' >>> multiple_files = [ - ('images', ('foo.png', open('foo.png', 'rb'), 'image/png')), - ('images', ('bar.png', open('bar.png', 'rb'), 'image/png'))] + ... ('images', ('foo.png', open('foo.png', 'rb'), 'image/png')), + ... ('images', ('bar.png', open('bar.png', 'rb'), 'image/png'))] >>> r = requests.post(url, files=multiple_files) >>> r.text { @@ -408,13 +411,11 @@ To do that, just set files to a list of tuples of ``(form_field_name, file_info) ... } -.. warning:: It is strongly recommended that you open files in `binary mode`_. - This is because Requests may attempt to provide the - ``Content-Length`` header for you, and if it does this value will - be set to the number of *bytes* in the file. Errors may occur if - you open the file in *text mode*. - -.. _binary mode: https://docs.python.org/2/tutorial/inputoutput.html#reading-and-writing-files +.. warning:: It is strongly recommended that you open files in :ref:`binary + mode `. This is because Requests may attempt to provide + the ``Content-Length`` header for you, and if it does this value + will be set to the number of *bytes* in the file. Errors may occur + if you open the file in *text mode*. .. _event-hooks: @@ -445,7 +446,7 @@ argument. def print_url(r, *args, **kwargs): print(r.url) -If an error occurs while executing your callback, a warning is given. +Your callback function must handle its own exceptions. Any unhandled exception won't be passed silently and thus should be handled by the code calling Requests. If the callback function returns a value, it is assumed that it is to replace the data that was passed in. If the function doesn't return @@ -459,13 +460,13 @@ anything, nothing else is affected. Let's print some request method arguments at runtime:: - >>> requests.get('http://httpbin.org', hooks={'response': print_url}) - http://httpbin.org + >>> requests.get('https://httpbin.org/', hooks={'response': print_url}) + https://httpbin.org/ You can add multiple hooks to a single request. Let's call two hooks at once:: - >>> r = requests.get('http://httpbin.org', hooks={'response': [print_url, record_hook]}) + >>> r = requests.get('https://httpbin.org/', hooks={'response': [print_url, record_hook]}) >>> r.hook_called True @@ -474,8 +475,8 @@ be called on every request made to the session. For example:: >>> s = requests.Session() >>> s.hooks['response'].append(print_url) - >>> s.get('http://httpbin.org') - http://httpbin.org + >>> s.get('https://httpbin.org/') + https://httpbin.org/ A ``Session`` can have multiple hooks, which will be called in the order @@ -486,7 +487,7 @@ they are added. Custom Authentication --------------------- -Requests allows you to use specify your own authentication mechanism. +Requests allows you to specify your own authentication mechanism. Any callable which is passed as the ``auth`` argument to a request method will have the opportunity to modify the request before it is dispatched. @@ -533,7 +534,7 @@ set ``stream`` to ``True`` and iterate over the response with import json import requests - r = requests.get('http://httpbin.org/stream/20', stream=True) + r = requests.get('https://httpbin.org/stream/20', stream=True) for line in r.iter_lines(): @@ -547,7 +548,7 @@ When using `decode_unicode=True` with :meth:`Response.iter_content() `, you'll want to provide a fallback encoding in the event the server doesn't provide one:: - r = requests.get('http://httpbin.org/stream/20', stream=True) + r = requests.get('https://httpbin.org/stream/20', stream=True) if r.encoding is None: r.encoding = 'utf-8' @@ -588,21 +589,55 @@ If you need to use a proxy, you can configure individual requests with the requests.get('http://example.org', proxies=proxies) -You can also configure proxies by setting the environment variables -``HTTP_PROXY`` and ``HTTPS_PROXY``. +Alternatively you can configure it once for an entire +:class:`Session `:: -:: + import requests + + proxies = { + 'http': 'http://10.10.1.10:3128', + 'https': 'http://10.10.1.10:1080', + } + session = requests.Session() + session.proxies.update(proxies) + + session.get('http://example.org') + +.. warning:: Setting ``session.proxies`` may behave differently than expected. + Values provided will be overwritten by environmental proxies + (those returned by `urllib.request.getproxies `_). + To ensure the use of proxies in the presence of environmental proxies, + explicitly specify the ``proxies`` argument on all individual requests as + initially explained above. + + See `#2018 `_ for details. + +When the proxies configuration is not overridden per request as shown above, +Requests relies on the proxy configuration defined by standard +environment variables ``http_proxy``, ``https_proxy``, ``no_proxy``, +and ``all_proxy``. Uppercase variants of these variables are also supported. +You can therefore set them to configure Requests (only set the ones relevant +to your needs):: $ export HTTP_PROXY="http://10.10.1.10:3128" $ export HTTPS_PROXY="http://10.10.1.10:1080" + $ export ALL_PROXY="socks5://10.10.1.10:3434" $ python >>> import requests >>> requests.get('http://example.org') -To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` syntax:: +To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` +syntax in any of the above configuration entries:: + + $ export HTTPS_PROXY="http://user:pass@10.10.1.10:1080" + + $ python + >>> proxies = {'http': 'http://user:pass@10.10.1.10:3128/'} - proxies = {'http': 'http://user:pass@10.10.1.10:3128/'} +.. warning:: Storing sensitive username and password information in an + environment variable or a version-controlled file is a security risk and is + highly discouraged. To give a proxy for a specific scheme and host, use the `scheme://hostname` form for the key. This will match for @@ -614,6 +649,25 @@ any request to the given scheme and exact hostname. Note that proxy URLs must include the scheme. +Finally, note that using a proxy for https connections typically requires your +local machine to trust the proxy's root certificate. By default the list of +certificates trusted by Requests can be found with:: + + from requests.utils import DEFAULT_CA_BUNDLE_PATH + print(DEFAULT_CA_BUNDLE_PATH) + +You override this default certificate bundle by setting the ``REQUESTS_CA_BUNDLE`` +(or ``CURL_CA_BUNDLE``) environment variable to another file path:: + + $ export REQUESTS_CA_BUNDLE="/usr/local/myproxy_info/cacert.pem" + $ export https_proxy="http://10.10.1.10:1080" + + $ python + >>> import requests + >>> requests.get('https://example.org') + +.. _socks: + SOCKS ^^^^^ @@ -627,7 +681,7 @@ You can get the dependencies for this feature from ``pip``: .. code-block:: bash - $ pip install requests[socks] + $ python -m pip install 'requests[socks]' Once you've installed those dependencies, using a SOCKS proxy is just as easy as using a HTTP one:: @@ -655,13 +709,24 @@ Encodings When you receive a response, Requests makes a guess at the encoding to use for decoding the response when you access the :attr:`Response.text ` attribute. Requests will first check for an -encoding in the HTTP header, and if none is present, will use `chardet -`_ to attempt to guess the encoding. +encoding in the HTTP header, and if none is present, will use +`charset_normalizer `_ +or `chardet `_ to attempt to +guess the encoding. + +If ``chardet`` is installed, ``requests`` uses it, however for python3 +``chardet`` is no longer a mandatory dependency. The ``chardet`` +library is an LGPL-licenced dependency and some users of requests +cannot depend on mandatory LGPL-licensed dependencies. -The only time Requests will not do this is if no explicit charset +When you install ``requests`` without specifying ``[use_chardet_on_py3]`` extra, +and ``chardet`` is not already installed, ``requests`` uses ``charset-normalizer`` +(MIT-licensed) to guess the encoding. + +The only time Requests will not guess the encoding is if no explicit charset is present in the HTTP headers **and** the ``Content-Type`` header contains ``text``. In this situation, `RFC 2616 -`_ specifies +`_ specifies that the default charset must be ``ISO-8859-1``. Requests follows the specification in this case. If you require a different encoding, you can manually set the :attr:`Response.encoding ` @@ -684,7 +749,7 @@ from GitHub. Suppose we wanted commit ``a050faf`` on Requests. We would get it like so:: >>> import requests - >>> r = requests.get('https://api.github.com/repos/requests/requests/git/commits/a050faf084662f3a352dd1a941f2c7c9f886d4ad') + >>> r = requests.get('https://api.github.com/repos/psf/requests/git/commits/a050faf084662f3a352dd1a941f2c7c9f886d4ad') We should confirm that GitHub responded correctly. If it has, we want to work out what type of content it is. Do this like so:: @@ -702,12 +767,12 @@ So, GitHub returns JSON. That's great, we can use the :meth:`r.json >>> commit_data = r.json() >>> print(commit_data.keys()) - [u'committer', u'author', u'url', u'tree', u'sha', u'parents', u'message'] + ['committer', 'author', 'url', 'tree', 'sha', 'parents', 'message'] - >>> print(commit_data[u'committer']) - {u'date': u'2012-05-10T11:10:50-07:00', u'email': u'me@kennethreitz.com', u'name': u'Kenneth Reitz'} + >>> print(commit_data['committer']) + {'date': '2012-05-10T11:10:50-07:00', 'email': 'me@kennethreitz.com', 'name': 'Kenneth Reitz'} - >>> print(commit_data[u'message']) + >>> print(commit_data['message']) makin' history So far, so simple. Well, let's investigate the GitHub API a little bit. Now, @@ -739,37 +804,37 @@ we should probably avoid making ham-handed POSTS to it. Instead, let's play with the Issues feature of GitHub. This documentation was added in response to -`Issue #482 `_. Given that +`Issue #482 `_. Given that this issue already exists, we will use it as an example. Let's start by getting it. :: - >>> r = requests.get('https://api.github.com/repos/requests/requests/issues/482') + >>> r = requests.get('https://api.github.com/repos/psf/requests/issues/482') >>> r.status_code 200 >>> issue = json.loads(r.text) - >>> print(issue[u'title']) + >>> print(issue['title']) Feature any http verb in docs - >>> print(issue[u'comments']) + >>> print(issue['comments']) 3 Cool, we have three comments. Let's take a look at the last of them. :: - >>> r = requests.get(r.url + u'/comments') + >>> r = requests.get(r.url + '/comments') >>> r.status_code 200 >>> comments = r.json() >>> print(comments[0].keys()) - [u'body', u'url', u'created_at', u'updated_at', u'user', u'id'] + ['body', 'url', 'created_at', 'updated_at', 'user', 'id'] - >>> print(comments[2][u'body']) + >>> print(comments[2]['body']) Probably in the "advanced" section Well, that seems like a silly place. Let's post a comment telling the poster @@ -777,7 +842,7 @@ that he's silly. Who is the poster, anyway? :: - >>> print(comments[2][u'user'][u'login']) + >>> print(comments[2]['user']['login']) kennethreitz OK, so let's tell this Kenneth guy that we think this example should go in the @@ -787,7 +852,7 @@ is to POST to the thread. Let's do it. :: >>> body = json.dumps({u"body": u"Sounds great! I'll get right on it!"}) - >>> url = u"https://api.github.com/repos/requests/requests/issues/482/comments" + >>> url = u"https://api.github.com/repos/psf/requests/issues/482/comments" >>> r = requests.post(url=url, data=body) >>> r.status_code @@ -807,7 +872,7 @@ the very common Basic Auth. 201 >>> content = r.json() - >>> print(content[u'body']) + >>> print(content['body']) Sounds great! I'll get right on it. Brilliant. Oh, wait, no! I meant to add that it would take me a while, because @@ -821,7 +886,7 @@ that. 5804413 >>> body = json.dumps({u"body": u"Sounds great! I'll get right on it once I feed my cat."}) - >>> url = u"https://api.github.com/repos/requests/requests/issues/comments/5804413" + >>> url = u"https://api.github.com/repos/psf/requests/issues/comments/5804413" >>> r = requests.patch(url=url, data=body, auth=auth) >>> r.status_code @@ -883,7 +948,7 @@ Link Headers Many HTTP APIs feature Link headers. They make APIs more self describing and discoverable. -GitHub uses these for `pagination `_ +GitHub uses these for `pagination `_ in their API, for example:: >>> url = 'https://api.github.com/users/kennethreitz/repos?page=1&per_page=10' @@ -904,11 +969,9 @@ Requests will automatically parse these link headers and make them easily consum Transport Adapters ------------------ -As of v1.0.0, Requests has moved to a modular internal design. Part of the -reason this was done was to implement Transport Adapters, originally -`described here`_. Transport Adapters provide a mechanism to define interaction -methods for an HTTP service. In particular, they allow you to apply per-service -configuration. +As of v1.0.0, Requests has moved to a modular internal design using Transport +Adapters. These objects provide a mechanism to define interaction methods for an +HTTP service. In particular, they allow you to apply per-service configuration. Requests ships with a single Transport Adapter, the :class:`HTTPAdapter `. This adapter provides the default Requests @@ -925,12 +988,16 @@ it should apply to. :: >>> s = requests.Session() - >>> s.mount('http://www.github.com', MyAdapter()) + >>> s.mount('https://github.com/', MyAdapter()) The mount call registers a specific instance of a Transport Adapter to a prefix. Once mounted, any HTTP request made using that session whose URL starts with the given prefix will use the given Transport Adapter. +.. note:: The adapter will be chosen based on a longest prefix match. Be mindful + prefixes such as ``http://localhost`` will also match ``http://localhost.other.com`` + or ``http://localhost@other.com``. It's recommended to terminate full hostnames with a ``/``. + Many of the details of implementing a Transport Adapter are beyond the scope of this documentation, but take a look at the next example for a simple SSL use- case. For more than that, you might look at subclassing the @@ -963,8 +1030,29 @@ library to use SSLv3:: num_pools=connections, maxsize=maxsize, block=block, ssl_version=ssl.PROTOCOL_SSLv3) -.. _`described here`: http://www.kennethreitz.org/essays/the-future-of-python-http -.. _`urllib3`: https://github.com/shazow/urllib3 +Example: Automatic Retries +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, Requests does not retry failed connections. However, it is possible +to implement automatic retries with a powerful array of features, including +backoff, within a Requests :class:`Session ` using the +`urllib3.util.Retry`_ class:: + + from urllib3.util import Retry + from requests import Session + from requests.adapters import HTTPAdapter + + s = Session() + retries = Retry( + total=3, + backoff_factor=0.1, + status_forcelist=[502, 503, 504], + allowed_methods={'POST'}, + ) + s.mount('https://', HTTPAdapter(max_retries=retries)) + +.. _`urllib3`: https://github.com/urllib3/urllib3 +.. _`urllib3.util.Retry`: https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html#urllib3.util.Retry .. _blocking-or-nonblocking: @@ -980,18 +1068,19 @@ response at a time. However, these calls will still block. If you are concerned about the use of blocking IO, there are lots of projects out there that combine Requests with one of Python's asynchronicity frameworks. -Some excellent examples are `requests-threads`_, `grequests`_, and `requests-futures`_. +Some excellent examples are `requests-threads`_, `grequests`_, `requests-futures`_, and `httpx`_. .. _`requests-threads`: https://github.com/requests/requests-threads -.. _`grequests`: https://github.com/kennethreitz/grequests +.. _`grequests`: https://github.com/spyoungtech/grequests .. _`requests-futures`: https://github.com/ross/requests-futures +.. _`httpx`: https://github.com/encode/httpx Header Ordering --------------- In unusual circumstances you may want to provide headers in an ordered manner. If you pass an ``OrderedDict`` to the ``headers`` keyword argument, that will provide the headers with an ordering. *However*, the ordering of the default headers used by Requests will be preferred, which means that if you override default headers in the ``headers`` keyword argument, they may appear out of order compared to other headers in that keyword argument. -If this is problematic, users should consider setting the default headers on a :class:`Session ` object, by setting :attr:`Session ` to a custom ``OrderedDict``. That ordering will always be preferred. +If this is problematic, users should consider setting the default headers on a :class:`Session ` object, by setting :attr:`Session.headers ` to a custom ``OrderedDict``. That ordering will always be preferred. .. _timeouts: @@ -1007,7 +1096,7 @@ The **connect** timeout is the number of seconds Requests will wait for your client to establish a connection to a remote machine (corresponding to the `connect()`_) call on the socket. It's a good practice to set connect timeouts to slightly larger than a multiple of 3, which is the default `TCP packet -retransmission window `_. +retransmission window `_. Once your client has connected to the server and sent the HTTP request, the **read** timeout is the number of seconds the client will wait for the server @@ -1032,4 +1121,18 @@ coffee. r = requests.get('https://github.com', timeout=None) -.. _`connect()`: http://linux.die.net/man/2/connect +.. note:: The connect timeout applies to each connection attempt to an IP address. + If multiple addresses exist for a domain name, the underlying ``urllib3`` will + try each address sequentially until one successfully connects. + This may lead to an effective total connection timeout *multiple* times longer + than the specified time, e.g. an unresponsive server having both IPv4 and IPv6 + addresses will have its perceived timeout *doubled*, so take that into account + when setting the connection timeout. +.. note:: Neither the connect nor read timeouts are `wall clock`_. This means + that if you start a request, and look at the time, and then look at + the time when the request finishes or times out, the real-world time + may be greater than what you specified. + + +.. _`wall clock`: https://wiki.php.net/rfc/max_execution_wall_time +.. _`connect()`: https://linux.die.net/man/2/connect diff --git a/docs/user/authentication.rst b/docs/user/authentication.rst index 411f79fd8a..76be9cccaf 100644 --- a/docs/user/authentication.rst +++ b/docs/user/authentication.rst @@ -3,8 +3,6 @@ Authentication ============== -.. image:: https://farm5.staticflickr.com/4258/35550409215_3b08d49d22_k_d.jpg - This document discusses using various kinds of authentication with Requests. Many web services require authentication, and there are many different types. @@ -21,13 +19,14 @@ the simplest kind, and Requests supports it straight out of the box. Making requests with HTTP Basic Auth is very simple:: >>> from requests.auth import HTTPBasicAuth - >>> requests.get('https://api.github.com/user', auth=HTTPBasicAuth('user', 'pass')) + >>> basic = HTTPBasicAuth('user', 'pass') + >>> requests.get('https://httpbin.org/basic-auth/user/pass', auth=basic) In fact, HTTP Basic Auth is so common that Requests provides a handy shorthand for using it:: - >>> requests.get('https://api.github.com/user', auth=('user', 'pass')) + >>> requests.get('https://httpbin.org/basic-auth/user/pass', auth=('user', 'pass')) Providing the credentials in a tuple like this is exactly the same as the @@ -45,6 +44,16 @@ set with `headers=`. If credentials for the hostname are found, the request is sent with HTTP Basic Auth. +Requests will search for the netrc file at `~/.netrc`, `~/_netrc`, or at the path +specified by the `NETRC` environment variable. `~` denotes the user's home +directory, which is `$HOME` on Unix based systems and `%USERPROFILE%` on Windows. + +Usage of netrc file can be disabled by setting `trust_env` to `False` in the +Requests session:: + + >>> s = requests.Session() + >>> s.trust_env = False + >>> s.get('https://httpbin.org/basic-auth/user/pass') Digest Authentication --------------------- @@ -53,7 +62,7 @@ Another very popular form of HTTP Authentication is Digest Authentication, and Requests supports this out of the box as well:: >>> from requests.auth import HTTPDigestAuth - >>> url = 'http://httpbin.org/digest-auth/auth/user/pass' + >>> url = 'https://httpbin.org/digest-auth/auth/user/pass' >>> requests.get(url, auth=HTTPDigestAuth('user', 'pass')) @@ -122,7 +131,7 @@ To do so, subclass :class:`AuthBase ` and implement the ... # Implement my authentication ... return r ... - >>> url = 'http://httpbin.org/get' + >>> url = 'https://httpbin.org/get' >>> requests.get(url, auth=MyAuth()) @@ -134,7 +143,7 @@ authentication will additionally add hooks to provide further functionality. Further examples can be found under the `Requests organization`_ and in the ``auth.py`` file. -.. _OAuth: http://oauth.net/ +.. _OAuth: https://oauth.net/ .. _requests_oauthlib: https://github.com/requests/requests-oauthlib .. _requests-oauthlib OAuth2 documentation: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html .. _Web Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#web-application-flow diff --git a/docs/user/install.rst b/docs/user/install.rst index 1dd9de8e5c..7fa9a606d2 100644 --- a/docs/user/install.rst +++ b/docs/user/install.rst @@ -3,41 +3,34 @@ Installation of Requests ======================== -.. image:: https://farm5.staticflickr.com/4230/35550376215_da1bf77a8c_k_d.jpg - - This part of the documentation covers the installation of Requests. The first step to using any software package is getting it properly installed. -$ pipenv install requests -------------------------- +$ python -m pip install requests +-------------------------------- To install Requests, simply run this simple command in your terminal of choice:: - $ pipenv install requests - -If you don't have `pipenv `_ installed (tisk tisk!), head over to the Pipenv website for installation instructions. Or, if you prefer to just use pip and don't have it installed, -`this Python installation guide `_ -can guide you through the process. + $ python -m pip install requests Get the Source Code ------------------- Requests is actively developed on GitHub, where the code is -`always available `_. +`always available `_. You can either clone the public repository:: - $ git clone git://github.com/requests/requests.git + $ git clone https://github.com/psf/requests.git -Or, download the `tarball `_:: +Or, download the `tarball `_:: - $ curl -OL https://github.com/requests/requests/tarball/master + $ curl -OL https://github.com/psf/requests/tarball/main # optionally, zipball is also available (for Windows users). Once you have a copy of the source, you can embed it in your own Python package, or install it into your site-packages easily:: $ cd requests - $ pip install . + $ python -m pip install . diff --git a/docs/user/intro.rst b/docs/user/intro.rst deleted file mode 100644 index e8395d9d0c..0000000000 --- a/docs/user/intro.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. _introduction: - -Introduction -============ - -.. image:: https://farm5.staticflickr.com/4317/35198386374_1939af3de6_k_d.jpg - -Philosophy ----------- - -Requests was developed with a few :pep:`20` idioms in mind. - - -#. Beautiful is better than ugly. -#. Explicit is better than implicit. -#. Simple is better than complex. -#. Complex is better than complicated. -#. Readability counts. - -All contributions to Requests should keep these important rules in mind. - -.. _`apache2`: - -Apache2 License ---------------- - -A large number of open source projects you find today are `GPL Licensed`_. -While the GPL has its time and place, it should most certainly not be your -go-to license for your next open source project. - -A project that is released as GPL cannot be used in any commercial product -without the product itself also being offered as open source. - -The MIT, BSD, ISC, and Apache2 licenses are great alternatives to the GPL -that allow your open-source software to be used freely in proprietary, -closed-source software. - -Requests is released under terms of `Apache2 License`_. - -.. _`GPL Licensed`: http://www.opensource.org/licenses/gpl-license.php -.. _`Apache2 License`: http://opensource.org/licenses/Apache-2.0 - - -Requests License ----------------- - - .. include:: ../../LICENSE diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 032e70f837..3755d26239 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -3,8 +3,6 @@ Quickstart ========== -.. image:: https://farm5.staticflickr.com/4259/35163667010_8bfcaef274_k_d.jpg - .. module:: requests.models Eager to get started? This page gives a good introduction in how to get started @@ -39,15 +37,15 @@ get all the information we need from this object. Requests' simple API means that all forms of HTTP request are as obvious. For example, this is how you make an HTTP POST request:: - >>> r = requests.post('http://httpbin.org/post', data = {'key':'value'}) + >>> r = requests.post('https://httpbin.org/post', data={'key': 'value'}) Nice, right? What about the other HTTP request types: PUT, DELETE, HEAD and OPTIONS? These are all just as simple:: - >>> r = requests.put('http://httpbin.org/put', data = {'key':'value'}) - >>> r = requests.delete('http://httpbin.org/delete') - >>> r = requests.head('http://httpbin.org/get') - >>> r = requests.options('http://httpbin.org/get') + >>> r = requests.put('https://httpbin.org/put', data={'key': 'value'}) + >>> r = requests.delete('https://httpbin.org/delete') + >>> r = requests.head('https://httpbin.org/get') + >>> r = requests.options('https://httpbin.org/get') That's all well and good, but it's also only the start of what Requests can do. @@ -65,12 +63,12 @@ using the ``params`` keyword argument. As an example, if you wanted to pass following code:: >>> payload = {'key1': 'value1', 'key2': 'value2'} - >>> r = requests.get('http://httpbin.org/get', params=payload) + >>> r = requests.get('https://httpbin.org/get', params=payload) You can see that the URL has been correctly encoded by printing the URL:: >>> print(r.url) - http://httpbin.org/get?key2=value2&key1=value1 + https://httpbin.org/get?key2=value2&key1=value1 Note that any dictionary key whose value is ``None`` will not be added to the URL's query string. @@ -79,9 +77,9 @@ You can also pass a list of items as a value:: >>> payload = {'key1': 'value1', 'key2': ['value2', 'value3']} - >>> r = requests.get('http://httpbin.org/get', params=payload) + >>> r = requests.get('https://httpbin.org/get', params=payload) >>> print(r.url) - http://httpbin.org/get?key1=value1&key2=value2&key2=value3 + https://httpbin.org/get?key1=value1&key2=value2&key2=value3 Response Content ---------------- @@ -93,7 +91,7 @@ again:: >>> r = requests.get('https://api.github.com/events') >>> r.text - u'[{"repository":{"open_issues":0,"url":"https://github.com/... + '[{"repository":{"open_issues":0,"url":"https://github.com/... Requests will automatically decode content from the server. Most unicode charsets are seamlessly decoded. @@ -130,6 +128,9 @@ You can also access the response body as bytes, for non-text requests:: The ``gzip`` and ``deflate`` transfer-encodings are automatically decoded for you. +The ``br`` transfer-encoding is automatically decoded for you if a Brotli library +like `brotli `_ or `brotlicffi `_ is installed. + For example, to create an image from binary data returned by a request, you can use the following code:: @@ -148,11 +149,13 @@ There's also a builtin JSON decoder, in case you're dealing with JSON data:: >>> r = requests.get('https://api.github.com/events') >>> r.json() - [{u'repository': {u'open_issues': 0, u'url': 'https://github.com/... + [{'repository': {'open_issues': 0, 'url': 'https://github.com/... In case the JSON decoding fails, ``r.json()`` raises an exception. For example, if the response gets a 204 (No Content), or if the response contains invalid JSON, -attempting ``r.json()`` raises ``ValueError: No JSON object could be decoded``. +attempting ``r.json()`` raises ``requests.exceptions.JSONDecodeError``. This wrapper exception +provides interoperability for multiple exceptions that may be thrown by different +python versions and json serialization libraries. It should be noted that the success of the call to ``r.json()`` does **not** indicate the success of the response. Some servers may return a JSON object in a @@ -174,7 +177,7 @@ server, you can access ``r.raw``. If you want to do this, make sure you set >>> r.raw.read(10) - '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03' + b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03' In general, however, you should use a pattern like this to save what is being streamed to a file:: @@ -198,6 +201,8 @@ may better fit your use cases. were returned, use ``Response.raw``. +.. _custom-headers: + Custom Headers -------------- @@ -215,7 +220,9 @@ Note: Custom headers are given less precedence than more specific sources of inf * Authorization headers set with `headers=` will be overridden if credentials are specified in ``.netrc``, which in turn will be overridden by the ``auth=`` - parameter. + parameter. Requests will search for the netrc file at `~/.netrc`, `~/_netrc`, + or at the path specified by the `NETRC` environment variable. + Check details in :ref:`netrc authentication `. * Authorization headers will be removed if you get redirected off-host. * Proxy-Authorization headers will be overridden by proxy credentials provided in the URL. * Content-Length headers will be overridden when we can determine the length of the content. @@ -233,7 +240,7 @@ dictionary of data will automatically be form-encoded when the request is made:: >>> payload = {'key1': 'value1', 'key2': 'value2'} - >>> r = requests.post("http://httpbin.org/post", data=payload) + >>> r = requests.post('https://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -244,12 +251,16 @@ dictionary of data will automatically be form-encoded when the request is made:: ... } -You can also pass a list of tuples to the ``data`` argument. This is particularly -useful when the form has multiple elements that use the same key:: +The ``data`` argument can also have multiple values for each key. This can be +done by making ``data`` either a list of tuples or a dictionary with lists +as values. This is particularly useful when the form has multiple elements that +use the same key:: - >>> payload = (('key1', 'value1'), ('key1', 'value2')) - >>> r = requests.post('http://httpbin.org/post', data=payload) - >>> print(r.text) + >>> payload_tuples = [('key1', 'value1'), ('key1', 'value2')] + >>> r1 = requests.post('https://httpbin.org/post', data=payload_tuples) + >>> payload_dict = {'key1': ['value1', 'value2']} + >>> r2 = requests.post('https://httpbin.org/post', data=payload_dict) + >>> print(r1.text) { ... "form": { @@ -260,6 +271,8 @@ useful when the form has multiple elements that use the same key:: }, ... } + >>> r1.text == r2.text + True There are times that you may want to send data that is not form-encoded. If you pass in a ``string`` instead of a ``dict``, that data will be posted directly. @@ -273,8 +286,12 @@ For example, the GitHub API v3 accepts JSON-Encoded POST/PATCH data:: >>> r = requests.post(url, data=json.dumps(payload)) -Instead of encoding the ``dict`` yourself, you can also pass it directly using -the ``json`` parameter (added in version 2.4.2) and it will be encoded automatically:: +Please note that the above code will NOT add the ``Content-Type`` header +(so in particular it will NOT set it to ``application/json``). + +If you need that header set and you don't want to encode the ``dict`` yourself, +you can also pass it directly using the ``json`` parameter (added in version 2.4.2) +and it will be encoded automatically: >>> url = 'https://api.github.com/some/endpoint' >>> payload = {'some': 'data'} @@ -283,14 +300,12 @@ the ``json`` parameter (added in version 2.4.2) and it will be encoded automatic Note, the ``json`` parameter is ignored if either ``data`` or ``files`` is passed. -Using the ``json`` parameter in the request will change the ``Content-Type`` in the header to ``application/json``. - POST a Multipart-Encoded File ----------------------------- Requests makes it simple to upload Multipart-encoded files:: - >>> url = 'http://httpbin.org/post' + >>> url = 'https://httpbin.org/post' >>> files = {'file': open('report.xls', 'rb')} >>> r = requests.post(url, files=files) @@ -305,7 +320,7 @@ Requests makes it simple to upload Multipart-encoded files:: You can set the filename, content_type and headers explicitly:: - >>> url = 'http://httpbin.org/post' + >>> url = 'https://httpbin.org/post' >>> files = {'file': ('report.xls', open('report.xls', 'rb'), 'application/vnd.ms-excel', {'Expires': '0'})} >>> r = requests.post(url, files=files) @@ -320,7 +335,7 @@ You can set the filename, content_type and headers explicitly:: If you want, you can send strings to be received as files:: - >>> url = 'http://httpbin.org/post' + >>> url = 'https://httpbin.org/post' >>> files = {'file': ('report.csv', 'some,data,to,send\nanother,row,to,send\n')} >>> r = requests.post(url, files=files) @@ -342,13 +357,11 @@ support this, but there is a separate package which does - For sending multiple files in one request refer to the :ref:`advanced ` section. -.. warning:: It is strongly recommended that you open files in `binary mode`_. - This is because Requests may attempt to provide the - ``Content-Length`` header for you, and if it does this value will - be set to the number of *bytes* in the file. Errors may occur if - you open the file in *text mode*. - -.. _binary mode: https://docs.python.org/2/tutorial/inputoutput.html#reading-and-writing-files +.. warning:: It is strongly recommended that you open files in :ref:`binary + mode `. This is because Requests may attempt to provide + the ``Content-Length`` header for you, and if it does this value + will be set to the number of *bytes* in the file. Errors may occur + if you open the file in *text mode*. Response Status Codes @@ -356,7 +369,7 @@ Response Status Codes We can check the response status code:: - >>> r = requests.get('http://httpbin.org/get') + >>> r = requests.get('https://httpbin.org/get') >>> r.status_code 200 @@ -370,7 +383,7 @@ If we made a bad request (a 4XX client error or 5XX server error response), we can raise it with :meth:`Response.raise_for_status() `:: - >>> bad_r = requests.get('http://httpbin.org/status/404') + >>> bad_r = requests.get('https://httpbin.org/status/404') >>> bad_r.status_code 404 @@ -406,7 +419,7 @@ We can view the server's response headers using a Python dictionary:: } The dictionary is special, though: it's made just for HTTP headers. According to -`RFC 7230 `_, HTTP Header names +`RFC 7230 `_, HTTP Header names are case-insensitive. So, we can access the headers using any capitalization we want:: @@ -420,7 +433,7 @@ So, we can access the headers using any capitalization we want:: It is also special in that the server could have sent the same header multiple times with different values, but requests combines them so they can be represented in the dictionary within a single mapping, as per -`RFC 7230 `_: +`RFC 7230 `_: A recipient MAY combine multiple header fields with the same field name into one "field-name: field-value" pair, without changing the semantics @@ -441,7 +454,7 @@ If a response contains some Cookies, you can quickly access them:: To send your own cookies to the server, you can use the ``cookies`` parameter:: - >>> url = 'http://httpbin.org/cookies' + >>> url = 'https://httpbin.org/cookies' >>> cookies = dict(cookies_are='working') >>> r = requests.get(url, cookies=cookies) @@ -456,7 +469,7 @@ also be passed in to requests:: >>> jar = requests.cookies.RequestsCookieJar() >>> jar.set('tasty_cookie', 'yum', domain='httpbin.org', path='/cookies') >>> jar.set('gross_cookie', 'blech', domain='httpbin.org', path='/elsewhere') - >>> url = 'http://httpbin.org/cookies' + >>> url = 'https://httpbin.org/cookies' >>> r = requests.get(url, cookies=jar) >>> r.text '{"cookies": {"tasty_cookie": "yum"}}' @@ -477,7 +490,7 @@ response. For example, GitHub redirects all HTTP requests to HTTPS:: - >>> r = requests.get('http://github.com') + >>> r = requests.get('http://github.com/') >>> r.url 'https://github.com/' @@ -492,7 +505,7 @@ For example, GitHub redirects all HTTP requests to HTTPS:: If you're using GET, OPTIONS, POST, PUT, PATCH or DELETE, you can disable redirection handling with the ``allow_redirects`` parameter:: - >>> r = requests.get('http://github.com', allow_redirects=False) + >>> r = requests.get('http://github.com/', allow_redirects=False) >>> r.status_code 301 @@ -502,7 +515,7 @@ redirection handling with the ``allow_redirects`` parameter:: If you're using HEAD, you can enable redirection as well:: - >>> r = requests.head('http://github.com', allow_redirects=True) + >>> r = requests.head('http://github.com/', allow_redirects=True) >>> r.url 'https://github.com/' @@ -519,7 +532,7 @@ seconds with the ``timeout`` parameter. Nearly all production code should use this parameter in nearly all requests. Failure to do so can cause your program to hang indefinitely:: - >>> requests.get('http://github.com', timeout=0.001) + >>> requests.get('https://github.com/', timeout=0.001) Traceback (most recent call last): File "", line 1, in requests.exceptions.Timeout: HTTPConnectionPool(host='github.com', port=80): Request timed out. (timeout=0.001) diff --git a/ext/LICENSE b/ext/LICENSE new file mode 100644 index 0000000000..c38aa5c3f5 --- /dev/null +++ b/ext/LICENSE @@ -0,0 +1 @@ +Copyright 2019 Kenneth Reitz. All rights reserved. diff --git a/ext/flower-of-life.jpg b/ext/flower-of-life.jpg new file mode 100644 index 0000000000..f92cc3b17b Binary files /dev/null and b/ext/flower-of-life.jpg differ diff --git a/ext/kr-compressed.png b/ext/kr-compressed.png new file mode 100644 index 0000000000..53210649cc Binary files /dev/null and b/ext/kr-compressed.png differ diff --git a/ext/kr.png b/ext/kr.png new file mode 100644 index 0000000000..b18d76bfe0 Binary files /dev/null and b/ext/kr.png differ diff --git a/ext/psf-compressed.png b/ext/psf-compressed.png new file mode 100644 index 0000000000..3bc0d5c130 Binary files /dev/null and b/ext/psf-compressed.png differ diff --git a/ext/psf.png b/ext/psf.png new file mode 100644 index 0000000000..c5815e260a Binary files /dev/null and b/ext/psf.png differ diff --git a/ext/requests-logo-compressed.png b/ext/requests-logo-compressed.png new file mode 100644 index 0000000000..cb4bc64241 Binary files /dev/null and b/ext/requests-logo-compressed.png differ diff --git a/ext/requests-logo.png b/ext/requests-logo.png new file mode 100644 index 0000000000..cb4bc64241 Binary files /dev/null and b/ext/requests-logo.png differ diff --git a/ext/ss-compressed.png b/ext/ss-compressed.png new file mode 100644 index 0000000000..149016d3b1 Binary files /dev/null and b/ext/ss-compressed.png differ diff --git a/ext/ss.png b/ext/ss.png new file mode 100644 index 0000000000..149016d3b1 Binary files /dev/null and b/ext/ss.png differ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..1b7901e155 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +[tool.isort] +profile = "black" +src_paths = ["src/requests", "test"] +honor_noqa = true + +[tool.pytest.ini_options] +addopts = "--doctest-modules" +doctest_optionflags = "NORMALIZE_WHITESPACE ELLIPSIS" +minversion = "6.2" +testpaths = ["tests"] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index c1fa878547..0000000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts = -p no:warnings \ No newline at end of file diff --git a/requests/__init__.py b/requests/__init__.py deleted file mode 100644 index 6fa855dfc2..0000000000 --- a/requests/__init__.py +++ /dev/null @@ -1,136 +0,0 @@ -# -*- coding: utf-8 -*- - -# __ -# /__) _ _ _ _ _/ _ -# / ( (- (/ (/ (- _) / _) -# / - -""" -Requests HTTP Library -~~~~~~~~~~~~~~~~~~~~~ - -Requests is an HTTP library, written in Python, for human beings. Basic GET -usage: - - >>> import requests - >>> r = requests.get('https://www.python.org') - >>> r.status_code - 200 - >>> 'Python is a programming language' in r.content - True - -... or POST: - - >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post('http://httpbin.org/post', data=payload) - >>> print(r.text) - { - ... - "form": { - "key2": "value2", - "key1": "value1" - }, - ... - } - -The other HTTP methods are supported - see `requests.api`. Full documentation -is at . - -:copyright: (c) 2017 by Kenneth Reitz. -:license: Apache 2.0, see LICENSE for more details. -""" - -import urllib3 -import chardet -import warnings -from .exceptions import RequestsDependencyWarning - - -def check_compatibility(urllib3_version, chardet_version): - urllib3_version = urllib3_version.split('.') - assert urllib3_version != ['dev'] # Verify urllib3 isn't installed from git. - - # Sometimes, urllib3 only reports its version as 16.1. - if len(urllib3_version) == 2: - urllib3_version.append('0') - - # Check urllib3 for compatibility. - major, minor, patch = urllib3_version # noqa: F811 - major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.22 - assert major == 1 - assert minor >= 21 - assert minor <= 22 - - # Check chardet for compatibility. - major, minor, patch = chardet_version.split('.')[:3] - major, minor, patch = int(major), int(minor), int(patch) - # chardet >= 3.0.2, < 3.1.0 - assert major == 3 - assert minor < 1 - assert patch >= 2 - - -def _check_cryptography(cryptography_version): - # cryptography < 1.3.4 - try: - cryptography_version = list(map(int, cryptography_version.split('.'))) - except ValueError: - return - - if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) - warnings.warn(warning, RequestsDependencyWarning) - -# Check imported dependencies for compatibility. -try: - check_compatibility(urllib3.__version__, chardet.__version__) -except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " - "version!".format(urllib3.__version__, chardet.__version__), - RequestsDependencyWarning) - -# Attempt to enable urllib3's SNI support, if possible -try: - from urllib3.contrib import pyopenssl - pyopenssl.inject_into_urllib3() - - # Check cryptography version - from cryptography import __version__ as cryptography_version - _check_cryptography(cryptography_version) -except ImportError: - pass - -# urllib3's DependencyWarnings should be silenced. -from urllib3.exceptions import DependencyWarning -warnings.simplefilter('ignore', DependencyWarning) - -from .__version__ import __title__, __description__, __url__, __version__ -from .__version__ import __build__, __author__, __author_email__, __license__ -from .__version__ import __copyright__, __cake__ - -from . import utils -from . import packages -from .models import Request, Response, PreparedRequest -from .api import request, get, head, post, patch, put, delete, options -from .sessions import session, Session -from .status_codes import codes -from .exceptions import ( - RequestException, Timeout, URLRequired, - TooManyRedirects, HTTPError, ConnectionError, - FileModeWarning, ConnectTimeout, ReadTimeout -) - -# Set default logging handler to avoid "No handler found" warnings. -import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass - -logging.getLogger(__name__).addHandler(NullHandler()) - -# FileModeWarnings go off per the default. -warnings.simplefilter('default', FileModeWarning, append=True) diff --git a/requests/__version__.py b/requests/__version__.py deleted file mode 100644 index dc33eef651..0000000000 --- a/requests/__version__.py +++ /dev/null @@ -1,14 +0,0 @@ -# .-. .-. .-. . . .-. .-. .-. .-. -# |( |- |.| | | |- `-. | `-. -# ' ' `-' `-`.`-' `-' `-' ' `-' - -__title__ = 'requests' -__description__ = 'Python HTTP for Humans.' -__url__ = 'http://python-requests.org' -__version__ = '2.18.4' -__build__ = 0x021804 -__author__ = 'Kenneth Reitz' -__author_email__ = 'me@kennethreitz.org' -__license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2017 Kenneth Reitz' -__cake__ = u'\u2728 \U0001f370 \u2728' diff --git a/requests/compat.py b/requests/compat.py deleted file mode 100644 index f417cfd8d0..0000000000 --- a/requests/compat.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -requests.compat -~~~~~~~~~~~~~~~ - -This module handles import compatibility issues between Python 2 and -Python 3. -""" - -import chardet - -import sys - -# ------- -# Pythons -# ------- - -# Syntax sugar. -_ver = sys.version_info - -#: Python 2.x? -is_py2 = (_ver[0] == 2) - -#: Python 3.x? -is_py3 = (_ver[0] == 3) - -try: - import simplejson as json -except ImportError: - import json - -# --------- -# Specifics -# --------- - -if is_py2: - from urllib import ( - quote, unquote, quote_plus, unquote_plus, urlencode, getproxies, - proxy_bypass, proxy_bypass_environment, getproxies_environment) - from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag - from urllib2 import parse_http_list - import cookielib - from Cookie import Morsel - from StringIO import StringIO - - from urllib3.packages.ordered_dict import OrderedDict - - builtin_str = str - bytes = str - str = unicode - basestring = basestring - numeric_types = (int, long, float) - integer_types = (int, long) - -elif is_py3: - from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag - from urllib.request import parse_http_list, getproxies, proxy_bypass, proxy_bypass_environment, getproxies_environment - from http import cookiejar as cookielib - from http.cookies import Morsel - from io import StringIO - from collections import OrderedDict - - builtin_str = str - str = str - bytes = bytes - basestring = (str, bytes) - numeric_types = (int, float) - integer_types = (int,) diff --git a/requests/help.py b/requests/help.py deleted file mode 100644 index 06e06b2a75..0000000000 --- a/requests/help.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Module containing bug report helper(s).""" -from __future__ import print_function - -import json -import platform -import sys -import ssl - -import idna -import urllib3 -import chardet - -from . import __version__ as requests_version - -try: - from urllib3.contrib import pyopenssl -except ImportError: - pyopenssl = None - OpenSSL = None - cryptography = None -else: - import OpenSSL - import cryptography - - -def _implementation(): - """Return a dict with the Python implementation and version. - - Provide both the name and the version of the Python implementation - currently running. For example, on CPython 2.7.5 it will return - {'name': 'CPython', 'version': '2.7.5'}. - - This function works best on CPython and PyPy: in particular, it probably - doesn't work for Jython or IronPython. Future investigation should be done - to work out the correct shape of the code for those platforms. - """ - implementation = platform.python_implementation() - - if implementation == 'CPython': - implementation_version = platform.python_version() - elif implementation == 'PyPy': - implementation_version = '%s.%s.%s' % (sys.pypy_version_info.major, - sys.pypy_version_info.minor, - sys.pypy_version_info.micro) - if sys.pypy_version_info.releaselevel != 'final': - implementation_version = ''.join([ - implementation_version, sys.pypy_version_info.releaselevel - ]) - elif implementation == 'Jython': - implementation_version = platform.python_version() # Complete Guess - elif implementation == 'IronPython': - implementation_version = platform.python_version() # Complete Guess - else: - implementation_version = 'Unknown' - - return {'name': implementation, 'version': implementation_version} - - -def info(): - """Generate information for a bug report.""" - try: - platform_info = { - 'system': platform.system(), - 'release': platform.release(), - } - except IOError: - platform_info = { - 'system': 'Unknown', - 'release': 'Unknown', - } - - implementation_info = _implementation() - urllib3_info = {'version': urllib3.__version__} - chardet_info = {'version': chardet.__version__} - - pyopenssl_info = { - 'version': None, - 'openssl_version': '', - } - if OpenSSL: - pyopenssl_info = { - 'version': OpenSSL.__version__, - 'openssl_version': '%x' % OpenSSL.SSL.OPENSSL_VERSION_NUMBER, - } - cryptography_info = { - 'version': getattr(cryptography, '__version__', ''), - } - idna_info = { - 'version': getattr(idna, '__version__', ''), - } - - # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. - system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) - system_ssl_info = { - 'version': '%x' % system_ssl if system_ssl is not None else '' - } - - return { - 'platform': platform_info, - 'implementation': implementation_info, - 'system_ssl': system_ssl_info, - 'using_pyopenssl': pyopenssl is not None, - 'pyOpenSSL': pyopenssl_info, - 'urllib3': urllib3_info, - 'chardet': chardet_info, - 'cryptography': cryptography_info, - 'idna': idna_info, - 'requests': { - 'version': requests_version, - }, - } - - -def main(): - """Pretty-print the bug information as JSON.""" - print(json.dumps(info(), sort_keys=True, indent=2)) - - -if __name__ == '__main__': - main() diff --git a/requests/packages.py b/requests/packages.py deleted file mode 100644 index 7232fe0ff7..0000000000 --- a/requests/packages.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -# This code exists for backwards compatibility reasons. -# I don't like it either. Just look the other way. :) - -for package in ('urllib3', 'idna', 'chardet'): - locals()[package] = __import__(package) - # This traversal is apparently necessary such that the identities are - # preserved (requests.packages.urllib3.* is urllib3.*) - for mod in list(sys.modules): - if mod == package or mod.startswith(package + '.'): - sys.modules['requests.packages.' + mod] = sys.modules[mod] - -# Kinda cool, though, right? diff --git a/requests/status_codes.py b/requests/status_codes.py deleted file mode 100644 index 96b86ddb97..0000000000 --- a/requests/status_codes.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -The ``codes`` object defines a mapping from common names for HTTP statuses -to their numerical codes, accessible either as attributes or as dictionary -items. - ->>> requests.codes['temporary_redirect'] -307 ->>> requests.codes.teapot -418 ->>> requests.codes['\o/'] -200 - -Some codes have multiple names, and both upper- and lower-case versions of -the names are allowed. For example, ``codes.ok``, ``codes.OK``, and -``codes.okay`` all correspond to the HTTP status code 200. -""" - -from .structures import LookupDict - -_codes = { - - # Informational. - 100: ('continue',), - 101: ('switching_protocols',), - 102: ('processing',), - 103: ('checkpoint',), - 122: ('uri_too_long', 'request_uri_too_long'), - 200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/', '✓'), - 201: ('created',), - 202: ('accepted',), - 203: ('non_authoritative_info', 'non_authoritative_information'), - 204: ('no_content',), - 205: ('reset_content', 'reset'), - 206: ('partial_content', 'partial'), - 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'), - 208: ('already_reported',), - 226: ('im_used',), - - # Redirection. - 300: ('multiple_choices',), - 301: ('moved_permanently', 'moved', '\\o-'), - 302: ('found',), - 303: ('see_other', 'other'), - 304: ('not_modified',), - 305: ('use_proxy',), - 306: ('switch_proxy',), - 307: ('temporary_redirect', 'temporary_moved', 'temporary'), - 308: ('permanent_redirect', - 'resume_incomplete', 'resume',), # These 2 to be removed in 3.0 - - # Client Error. - 400: ('bad_request', 'bad'), - 401: ('unauthorized',), - 402: ('payment_required', 'payment'), - 403: ('forbidden',), - 404: ('not_found', '-o-'), - 405: ('method_not_allowed', 'not_allowed'), - 406: ('not_acceptable',), - 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), - 408: ('request_timeout', 'timeout'), - 409: ('conflict',), - 410: ('gone',), - 411: ('length_required',), - 412: ('precondition_failed', 'precondition'), - 413: ('request_entity_too_large',), - 414: ('request_uri_too_large',), - 415: ('unsupported_media_type', 'unsupported_media', 'media_type'), - 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), - 417: ('expectation_failed',), - 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), - 421: ('misdirected_request',), - 422: ('unprocessable_entity', 'unprocessable'), - 423: ('locked',), - 424: ('failed_dependency', 'dependency'), - 425: ('unordered_collection', 'unordered'), - 426: ('upgrade_required', 'upgrade'), - 428: ('precondition_required', 'precondition'), - 429: ('too_many_requests', 'too_many'), - 431: ('header_fields_too_large', 'fields_too_large'), - 444: ('no_response', 'none'), - 449: ('retry_with', 'retry'), - 450: ('blocked_by_windows_parental_controls', 'parental_controls'), - 451: ('unavailable_for_legal_reasons', 'legal_reasons'), - 499: ('client_closed_request',), - - # Server Error. - 500: ('internal_server_error', 'server_error', '/o\\', '✗'), - 501: ('not_implemented',), - 502: ('bad_gateway',), - 503: ('service_unavailable', 'unavailable'), - 504: ('gateway_timeout',), - 505: ('http_version_not_supported', 'http_version'), - 506: ('variant_also_negotiates',), - 507: ('insufficient_storage',), - 509: ('bandwidth_limit_exceeded', 'bandwidth'), - 510: ('not_extended',), - 511: ('network_authentication_required', 'network_auth', 'network_authentication'), -} - -codes = LookupDict(name='status_codes') - -def _init(): - for code, titles in _codes.items(): - for title in titles: - setattr(codes, title, code) - if not title.startswith(('\\', '/')): - setattr(codes, title.upper(), code) - - def doc(code): - names = ', '.join('``%s``' % n for n in _codes[code]) - return '* %d: %s' % (code, names) - - global __doc__ - __doc__ = (__doc__ + '\n' + - '\n'.join(doc(code) for code in sorted(_codes))) - -_init() diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000000..77fedb9716 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +-e .[socks] +pytest>=2.8.0,<9 +pytest-cov +pytest-httpbin==2.1.0 +httpbin~=0.10.0 +trustme +wheel diff --git a/setup.cfg b/setup.cfg index ed8a958e0a..8d44e0e14b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,17 @@ -[bdist_wheel] -universal = 1 - [metadata] license_file = LICENSE +provides-extra = + socks + use_chardet_on_py3 +requires-dist = + certifi>=2017.4.17 + charset_normalizer>=2,<4 + idna>=2.5,<4 + urllib3>=1.21.1,<3 + +[flake8] +ignore = E203, E501, W503 +per-file-ignores = + src/requests/__init__.py:E402, F401 + src/requests/compat.py:E402, F401 + tests/compat.py:F401 diff --git a/setup.py b/setup.py index f32cca7540..ff65d39102 100755 --- a/setup.py +++ b/setup.py @@ -1,102 +1,107 @@ #!/usr/bin/env python -# Learn more: https://github.com/kennethreitz/setup.py - import os -import re import sys - from codecs import open from setuptools import setup -from setuptools.command.test import test as TestCommand - -here = os.path.abspath(os.path.dirname(__file__)) - -class PyTest(TestCommand): - user_options = [('pytest-args=', 'a', "Arguments to pass into py.test")] - def initialize_options(self): - TestCommand.initialize_options(self) - try: - from multiprocessing import cpu_count - self.pytest_args = ['-n', str(cpu_count()), '--boxed'] - except (ImportError, NotImplementedError): - self.pytest_args = ['-n', '1', '--boxed'] +CURRENT_PYTHON = sys.version_info[:2] +REQUIRED_PYTHON = (3, 9) - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True +if CURRENT_PYTHON < REQUIRED_PYTHON: + sys.stderr.write( + """ +========================== +Unsupported Python version +========================== +This version of Requests requires at least Python {}.{}, but +you're trying to install it on Python {}.{}. To resolve this, +consider upgrading to a supported Python version. - def run_tests(self): - import pytest +If you can't upgrade your Python version, you'll need to +pin to an older version of Requests (<2.32.0). +""".format( + *(REQUIRED_PYTHON + CURRENT_PYTHON) + ) + ) + sys.exit(1) - errno = pytest.main(self.pytest_args) - sys.exit(errno) # 'setup.py publish' shortcut. -if sys.argv[-1] == 'publish': - os.system('python setup.py sdist bdist_wheel') - os.system('twine upload dist/*') +if sys.argv[-1] == "publish": + os.system("python setup.py sdist bdist_wheel") + os.system("twine upload dist/*") sys.exit() -packages = ['requests'] - requires = [ - 'chardet>=3.0.2,<3.1.0', - 'idna>=2.5,<2.7', - 'urllib3>=1.21.1,<1.23', - 'certifi>=2017.4.17' - + "charset_normalizer>=2,<4", + "idna>=2.5,<4", + "urllib3>=1.21.1,<3", + "certifi>=2017.4.17", +] +test_requirements = [ + "pytest-httpbin==2.1.0", + "pytest-cov", + "pytest-mock", + "pytest-xdist", + "PySocks>=1.5.6, !=1.5.7", + "pytest>=3", ] -test_requirements = ['pytest-httpbin==0.0.7', 'pytest-cov', 'pytest-mock', 'pytest-xdist', 'PySocks>=1.5.6, !=1.5.7', 'pytest>=2.8.0'] about = {} -with open(os.path.join(here, 'requests', '__version__.py'), 'r', 'utf-8') as f: +here = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(here, "src", "requests", "__version__.py"), "r", "utf-8") as f: exec(f.read(), about) -with open('README.rst', 'r', 'utf-8') as f: +with open("README.md", "r", "utf-8") as f: readme = f.read() -with open('HISTORY.rst', 'r', 'utf-8') as f: - history = f.read() setup( - name=about['__title__'], - version=about['__version__'], - description=about['__description__'], - long_description=readme + '\n\n' + history, - author=about['__author__'], - author_email=about['__author_email__'], - url=about['__url__'], - packages=packages, - package_data={'': ['LICENSE', 'NOTICE'], 'requests': ['*.pem']}, - package_dir={'requests': 'requests'}, + name=about["__title__"], + version=about["__version__"], + description=about["__description__"], + long_description=readme, + long_description_content_type="text/markdown", + author=about["__author__"], + author_email=about["__author_email__"], + url=about["__url__"], + packages=["requests"], + package_data={"": ["LICENSE", "NOTICE"]}, + package_dir={"": "src"}, include_package_data=True, - python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + python_requires=">=3.9", install_requires=requires, - license=about['__license__'], + license=about["__license__"], zip_safe=False, - classifiers=( - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Natural Language :: English', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy' - ), - cmdclass={'test': PyTest}, + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries", + ], tests_require=test_requirements, extras_require={ - 'security': ['pyOpenSSL>=0.14', 'cryptography>=1.3.4', 'idna>=2.0.0'], - 'socks': ['PySocks>=1.5.6, !=1.5.7'], - 'socks:sys_platform == "win32" and (python_version == "2.7" or python_version == "2.6")': ['win_inet_pton'], + "security": [], + "socks": ["PySocks>=1.5.6, !=1.5.7"], + "use_chardet_on_py3": ["chardet>=3.0.2,<6"], + }, + project_urls={ + "Documentation": "https://requests.readthedocs.io", + "Source": "https://github.com/psf/requests", }, ) diff --git a/src/requests/__init__.py b/src/requests/__init__.py new file mode 100644 index 0000000000..051cda1340 --- /dev/null +++ b/src/requests/__init__.py @@ -0,0 +1,184 @@ +# __ +# /__) _ _ _ _ _/ _ +# / ( (- (/ (/ (- _) / _) +# / + +""" +Requests HTTP Library +~~~~~~~~~~~~~~~~~~~~~ + +Requests is an HTTP library, written in Python, for human beings. +Basic GET usage: + + >>> import requests + >>> r = requests.get('https://www.python.org') + >>> r.status_code + 200 + >>> b'Python is a programming language' in r.content + True + +... or POST: + + >>> payload = dict(key1='value1', key2='value2') + >>> r = requests.post('https://httpbin.org/post', data=payload) + >>> print(r.text) + { + ... + "form": { + "key1": "value1", + "key2": "value2" + }, + ... + } + +The other HTTP methods are supported - see `requests.api`. Full documentation +is at . + +:copyright: (c) 2017 by Kenneth Reitz. +:license: Apache 2.0, see LICENSE for more details. +""" + +import warnings + +import urllib3 + +from .exceptions import RequestsDependencyWarning + +try: + from charset_normalizer import __version__ as charset_normalizer_version +except ImportError: + charset_normalizer_version = None + +try: + from chardet import __version__ as chardet_version +except ImportError: + chardet_version = None + + +def check_compatibility(urllib3_version, chardet_version, charset_normalizer_version): + urllib3_version = urllib3_version.split(".") + assert urllib3_version != ["dev"] # Verify urllib3 isn't installed from git. + + # Sometimes, urllib3 only reports its version as 16.1. + if len(urllib3_version) == 2: + urllib3_version.append("0") + + # Check urllib3 for compatibility. + major, minor, patch = urllib3_version # noqa: F811 + major, minor, patch = int(major), int(minor), int(patch) + # urllib3 >= 1.21.1 + assert major >= 1 + if major == 1: + assert minor >= 21 + + # Check charset_normalizer for compatibility. + if chardet_version: + major, minor, patch = chardet_version.split(".")[:3] + major, minor, patch = int(major), int(minor), int(patch) + # chardet_version >= 3.0.2, < 6.0.0 + assert (3, 0, 2) <= (major, minor, patch) < (6, 0, 0) + elif charset_normalizer_version: + major, minor, patch = charset_normalizer_version.split(".")[:3] + major, minor, patch = int(major), int(minor), int(patch) + # charset_normalizer >= 2.0.0 < 4.0.0 + assert (2, 0, 0) <= (major, minor, patch) < (4, 0, 0) + else: + warnings.warn( + "Unable to find acceptable character detection dependency " + "(chardet or charset_normalizer).", + RequestsDependencyWarning, + ) + + +def _check_cryptography(cryptography_version): + # cryptography < 1.3.4 + try: + cryptography_version = list(map(int, cryptography_version.split("."))) + except ValueError: + return + + if cryptography_version < [1, 3, 4]: + warning = "Old version of cryptography ({}) may cause slowdown.".format( + cryptography_version + ) + warnings.warn(warning, RequestsDependencyWarning) + + +# Check imported dependencies for compatibility. +try: + check_compatibility( + urllib3.__version__, chardet_version, charset_normalizer_version + ) +except (AssertionError, ValueError): + warnings.warn( + "urllib3 ({}) or chardet ({})/charset_normalizer ({}) doesn't match a supported " + "version!".format( + urllib3.__version__, chardet_version, charset_normalizer_version + ), + RequestsDependencyWarning, + ) + +# Attempt to enable urllib3's fallback for SNI support +# if the standard library doesn't support SNI or the +# 'ssl' library isn't available. +try: + try: + import ssl + except ImportError: + ssl = None + + if not getattr(ssl, "HAS_SNI", False): + from urllib3.contrib import pyopenssl + + pyopenssl.inject_into_urllib3() + + # Check cryptography version + from cryptography import __version__ as cryptography_version + + _check_cryptography(cryptography_version) +except ImportError: + pass + +# urllib3's DependencyWarnings should be silenced. +from urllib3.exceptions import DependencyWarning + +warnings.simplefilter("ignore", DependencyWarning) + +# Set default logging handler to avoid "No handler found" warnings. +import logging +from logging import NullHandler + +from . import packages, utils +from .__version__ import ( + __author__, + __author_email__, + __build__, + __cake__, + __copyright__, + __description__, + __license__, + __title__, + __url__, + __version__, +) +from .api import delete, get, head, options, patch, post, put, request +from .exceptions import ( + ConnectionError, + ConnectTimeout, + FileModeWarning, + HTTPError, + JSONDecodeError, + ReadTimeout, + RequestException, + Timeout, + TooManyRedirects, + URLRequired, +) +from .models import PreparedRequest, Request, Response +from .sessions import Session, session +from .status_codes import codes + +logging.getLogger(__name__).addHandler(NullHandler()) + +# FileModeWarnings go off per the default. +warnings.simplefilter("default", FileModeWarning, append=True) diff --git a/src/requests/__version__.py b/src/requests/__version__.py new file mode 100644 index 0000000000..effdd98cf1 --- /dev/null +++ b/src/requests/__version__.py @@ -0,0 +1,14 @@ +# .-. .-. .-. . . .-. .-. .-. .-. +# |( |- |.| | | |- `-. | `-. +# ' ' `-' `-`.`-' `-' `-' ' `-' + +__title__ = "requests" +__description__ = "Python HTTP for Humans." +__url__ = "https://requests.readthedocs.io" +__version__ = "2.32.5" +__build__ = 0x023205 +__author__ = "Kenneth Reitz" +__author_email__ = "me@kennethreitz.org" +__license__ = "Apache-2.0" +__copyright__ = "Copyright Kenneth Reitz" +__cake__ = "\u2728 \U0001f370 \u2728" diff --git a/requests/_internal_utils.py b/src/requests/_internal_utils.py similarity index 55% rename from requests/_internal_utils.py rename to src/requests/_internal_utils.py index 759d9a56ba..f2cf635e29 100644 --- a/requests/_internal_utils.py +++ b/src/requests/_internal_utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ requests._internal_utils ~~~~~~~~~~~~~~ @@ -7,11 +5,24 @@ Provides utility functions that are consumed internally by Requests which depend on extremely few external helpers (such as compat) """ +import re + +from .compat import builtin_str + +_VALID_HEADER_NAME_RE_BYTE = re.compile(rb"^[^:\s][^:\r\n]*$") +_VALID_HEADER_NAME_RE_STR = re.compile(r"^[^:\s][^:\r\n]*$") +_VALID_HEADER_VALUE_RE_BYTE = re.compile(rb"^\S[^\r\n]*$|^$") +_VALID_HEADER_VALUE_RE_STR = re.compile(r"^\S[^\r\n]*$|^$") -from .compat import is_py2, builtin_str, str +_HEADER_VALIDATORS_STR = (_VALID_HEADER_NAME_RE_STR, _VALID_HEADER_VALUE_RE_STR) +_HEADER_VALIDATORS_BYTE = (_VALID_HEADER_NAME_RE_BYTE, _VALID_HEADER_VALUE_RE_BYTE) +HEADER_VALIDATORS = { + bytes: _HEADER_VALIDATORS_BYTE, + str: _HEADER_VALIDATORS_STR, +} -def to_native_string(string, encoding='ascii'): +def to_native_string(string, encoding="ascii"): """Given a string object, regardless of type, returns a representation of that string in the native string type, encoding and decoding where necessary. This assumes ASCII unless told otherwise. @@ -19,10 +30,7 @@ def to_native_string(string, encoding='ascii'): if isinstance(string, builtin_str): out = string else: - if is_py2: - out = string.encode(encoding) - else: - out = string.decode(encoding) + out = string.decode(encoding) return out @@ -36,7 +44,7 @@ def unicode_is_ascii(u_string): """ assert isinstance(u_string, str) try: - u_string.encode('ascii') + u_string.encode("ascii") return True except UnicodeEncodeError: return False diff --git a/requests/adapters.py b/src/requests/adapters.py similarity index 56% rename from requests/adapters.py rename to src/requests/adapters.py index a4b0284208..670c92767c 100644 --- a/requests/adapters.py +++ b/src/requests/adapters.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ requests.adapters ~~~~~~~~~~~~~~~~~ @@ -9,55 +7,118 @@ """ import os.path -import socket +import socket # noqa: F401 +import typing +import warnings -from urllib3.poolmanager import PoolManager, proxy_from_url -from urllib3.response import HTTPResponse -from urllib3.util import parse_url -from urllib3.util import Timeout as TimeoutSauce -from urllib3.util.retry import Retry -from urllib3.exceptions import ClosedPoolError -from urllib3.exceptions import ConnectTimeoutError +from urllib3.exceptions import ClosedPoolError, ConnectTimeoutError from urllib3.exceptions import HTTPError as _HTTPError -from urllib3.exceptions import MaxRetryError -from urllib3.exceptions import NewConnectionError +from urllib3.exceptions import InvalidHeader as _InvalidHeader +from urllib3.exceptions import ( + LocationValueError, + MaxRetryError, + NewConnectionError, + ProtocolError, +) from urllib3.exceptions import ProxyError as _ProxyError -from urllib3.exceptions import ProtocolError -from urllib3.exceptions import ReadTimeoutError +from urllib3.exceptions import ReadTimeoutError, ResponseError from urllib3.exceptions import SSLError as _SSLError -from urllib3.exceptions import ResponseError +from urllib3.poolmanager import PoolManager, proxy_from_url +from urllib3.util import Timeout as TimeoutSauce +from urllib3.util import parse_url +from urllib3.util.retry import Retry +from .auth import _basic_auth_str +from .compat import basestring, urlparse +from .cookies import extract_cookies_to_jar +from .exceptions import ( + ConnectionError, + ConnectTimeout, + InvalidHeader, + InvalidProxyURL, + InvalidSchema, + InvalidURL, + ProxyError, + ReadTimeout, + RetryError, + SSLError, +) from .models import Response -from .compat import urlparse, basestring -from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, - get_encoding_from_headers, prepend_scheme_if_needed, - get_auth_from_url, urldefragauth, select_proxy) from .structures import CaseInsensitiveDict -from .cookies import extract_cookies_to_jar -from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) -from .auth import _basic_auth_str +from .utils import ( + DEFAULT_CA_BUNDLE_PATH, + extract_zipped_paths, + get_auth_from_url, + get_encoding_from_headers, + prepend_scheme_if_needed, + select_proxy, + urldefragauth, +) try: from urllib3.contrib.socks import SOCKSProxyManager except ImportError: + def SOCKSProxyManager(*args, **kwargs): raise InvalidSchema("Missing dependencies for SOCKS support.") + +if typing.TYPE_CHECKING: + from .models import PreparedRequest + + DEFAULT_POOLBLOCK = False DEFAULT_POOLSIZE = 10 DEFAULT_RETRIES = 0 DEFAULT_POOL_TIMEOUT = None -class BaseAdapter(object): +def _urllib3_request_context( + request: "PreparedRequest", + verify: "bool | str | None", + client_cert: "typing.Tuple[str, str] | str | None", + poolmanager: "PoolManager", +) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])": + host_params = {} + pool_kwargs = {} + parsed_request_url = urlparse(request.url) + scheme = parsed_request_url.scheme.lower() + port = parsed_request_url.port + + cert_reqs = "CERT_REQUIRED" + if verify is False: + cert_reqs = "CERT_NONE" + elif isinstance(verify, str): + if not os.path.isdir(verify): + pool_kwargs["ca_certs"] = verify + else: + pool_kwargs["ca_cert_dir"] = verify + pool_kwargs["cert_reqs"] = cert_reqs + if client_cert is not None: + if isinstance(client_cert, tuple) and len(client_cert) == 2: + pool_kwargs["cert_file"] = client_cert[0] + pool_kwargs["key_file"] = client_cert[1] + else: + # According to our docs, we allow users to specify just the client + # cert path + pool_kwargs["cert_file"] = client_cert + host_params = { + "scheme": scheme, + "host": parsed_request_url.hostname, + "port": port, + } + return host_params, pool_kwargs + + +class BaseAdapter: """The Base Transport Adapter""" def __init__(self): - super(BaseAdapter, self).__init__() + super().__init__() - def send(self, request, stream=False, timeout=None, verify=True, - cert=None, proxies=None): + def send( + self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None + ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest ` being sent. @@ -105,12 +166,22 @@ class HTTPAdapter(BaseAdapter): >>> a = requests.adapters.HTTPAdapter(max_retries=3) >>> s.mount('http://', a) """ - __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize', - '_pool_block'] - def __init__(self, pool_connections=DEFAULT_POOLSIZE, - pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES, - pool_block=DEFAULT_POOLBLOCK): + __attrs__ = [ + "max_retries", + "config", + "_pool_connections", + "_pool_maxsize", + "_pool_block", + ] + + def __init__( + self, + pool_connections=DEFAULT_POOLSIZE, + pool_maxsize=DEFAULT_POOLSIZE, + max_retries=DEFAULT_RETRIES, + pool_block=DEFAULT_POOLBLOCK, + ): if max_retries == DEFAULT_RETRIES: self.max_retries = Retry(0, read=False) else: @@ -118,7 +189,7 @@ def __init__(self, pool_connections=DEFAULT_POOLSIZE, self.config = {} self.proxy_manager = {} - super(HTTPAdapter, self).__init__() + super().__init__() self._pool_connections = pool_connections self._pool_maxsize = pool_maxsize @@ -127,8 +198,7 @@ def __init__(self, pool_connections=DEFAULT_POOLSIZE, self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -139,10 +209,13 @@ def __setstate__(self, state): for attr, value in state.items(): setattr(self, attr, value) - self.init_poolmanager(self._pool_connections, self._pool_maxsize, - block=self._pool_block) + self.init_poolmanager( + self._pool_connections, self._pool_maxsize, block=self._pool_block + ) - def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs): + def init_poolmanager( + self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs + ): """Initializes a urllib3 PoolManager. This method should not be called from user code, and is only @@ -159,8 +232,12 @@ def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool self._pool_maxsize = maxsize self._pool_block = block - self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, - block=block, strict=True, **pool_kwargs) + self.poolmanager = PoolManager( + num_pools=connections, + maxsize=maxsize, + block=block, + **pool_kwargs, + ) def proxy_manager_for(self, proxy, **proxy_kwargs): """Return urllib3 ProxyManager for the given proxy. @@ -176,7 +253,7 @@ def proxy_manager_for(self, proxy, **proxy_kwargs): """ if proxy in self.proxy_manager: manager = self.proxy_manager[proxy] - elif proxy.lower().startswith('socks'): + elif proxy.lower().startswith("socks"): username, password = get_auth_from_url(proxy) manager = self.proxy_manager[proxy] = SOCKSProxyManager( proxy, @@ -185,7 +262,7 @@ def proxy_manager_for(self, proxy, **proxy_kwargs): num_pools=self._pool_connections, maxsize=self._pool_maxsize, block=self._pool_block, - **proxy_kwargs + **proxy_kwargs, ) else: proxy_headers = self.proxy_headers(proxy) @@ -195,7 +272,8 @@ def proxy_manager_for(self, proxy, **proxy_kwargs): num_pools=self._pool_connections, maxsize=self._pool_maxsize, block=self._pool_block, - **proxy_kwargs) + **proxy_kwargs, + ) return manager @@ -211,8 +289,7 @@ def cert_verify(self, conn, url, verify, cert): to a CA bundle to use :param cert: The SSL certificate to verify. """ - if url.lower().startswith('https') and verify: - + if url.lower().startswith("https") and verify: cert_loc = None # Allow self-specified cert location. @@ -223,17 +300,19 @@ def cert_verify(self, conn, url, verify, cert): cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) if not cert_loc or not os.path.exists(cert_loc): - raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + raise OSError( + f"Could not find a suitable TLS CA certificate bundle, " + f"invalid path: {cert_loc}" + ) - conn.cert_reqs = 'CERT_REQUIRED' + conn.cert_reqs = "CERT_REQUIRED" if not os.path.isdir(cert_loc): conn.ca_certs = cert_loc else: conn.ca_cert_dir = cert_loc else: - conn.cert_reqs = 'CERT_NONE' + conn.cert_reqs = "CERT_NONE" conn.ca_certs = None conn.ca_cert_dir = None @@ -245,11 +324,14 @@ def cert_verify(self, conn, url, verify, cert): conn.cert_file = cert conn.key_file = None if conn.cert_file and not os.path.exists(conn.cert_file): - raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) + raise OSError( + f"Could not find the TLS certificate file, " + f"invalid path: {conn.cert_file}" + ) if conn.key_file and not os.path.exists(conn.key_file): - raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) + raise OSError( + f"Could not find the TLS key file, invalid path: {conn.key_file}" + ) def build_response(self, req, resp): """Builds a :class:`Response ` object from a urllib3 @@ -264,10 +346,10 @@ def build_response(self, req, resp): response = Response() # Fallback to None if there's no status_code, for whatever reason. - response.status_code = getattr(resp, 'status', None) + response.status_code = getattr(resp, "status", None) # Make headers case-insensitive. - response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {})) + response.headers = CaseInsensitiveDict(getattr(resp, "headers", {})) # Set encoding. response.encoding = get_encoding_from_headers(response.headers) @@ -275,7 +357,7 @@ def build_response(self, req, resp): response.reason = response.raw.reason if isinstance(req.url, bytes): - response.url = req.url.decode('utf-8') + response.url = req.url.decode("utf-8") else: response.url = req.url @@ -288,8 +370,110 @@ def build_response(self, req, resp): return response + def build_connection_pool_key_attributes(self, request, verify, cert=None): + """Build the PoolKey attributes used by urllib3 to return a connection. + + This looks at the PreparedRequest, the user-specified verify value, + and the value of the cert parameter to determine what PoolKey values + to use to select a connection from a given urllib3 Connection Pool. + + The SSL related pool key arguments are not consistently set. As of + this writing, use the following to determine what keys may be in that + dictionary: + + * If ``verify`` is ``True``, ``"ssl_context"`` will be set and will be the + default Requests SSL Context + * If ``verify`` is ``False``, ``"ssl_context"`` will not be set but + ``"cert_reqs"`` will be set + * If ``verify`` is a string, (i.e., it is a user-specified trust bundle) + ``"ca_certs"`` will be set if the string is not a directory recognized + by :py:func:`os.path.isdir`, otherwise ``"ca_cert_dir"`` will be + set. + * If ``"cert"`` is specified, ``"cert_file"`` will always be set. If + ``"cert"`` is a tuple with a second item, ``"key_file"`` will also + be present + + To override these settings, one may subclass this class, call this + method and use the above logic to change parameters as desired. For + example, if one wishes to use a custom :py:class:`ssl.SSLContext` one + must both set ``"ssl_context"`` and based on what else they require, + alter the other keys to ensure the desired behaviour. + + :param request: + The PreparedReqest being sent over the connection. + :type request: + :class:`~requests.models.PreparedRequest` + :param verify: + Either a boolean, in which case it controls whether + we verify the server's TLS certificate, or a string, in which case it + must be a path to a CA bundle to use. + :param cert: + (optional) Any user-provided SSL certificate for client + authentication (a.k.a., mTLS). This may be a string (i.e., just + the path to a file which holds both certificate and key) or a + tuple of length 2 with the certificate file path and key file + path. + :returns: + A tuple of two dictionaries. The first is the "host parameters" + portion of the Pool Key including scheme, hostname, and port. The + second is a dictionary of SSLContext related parameters. + """ + return _urllib3_request_context(request, verify, cert, self.poolmanager) + + def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): + """Returns a urllib3 connection for the given request and TLS settings. + This should not be called from user code, and is only exposed for use + when subclassing the :class:`HTTPAdapter `. + + :param request: + The :class:`PreparedRequest ` object to be sent + over the connection. + :param verify: + Either a boolean, in which case it controls whether we verify the + server's TLS certificate, or a string, in which case it must be a + path to a CA bundle to use. + :param proxies: + (optional) The proxies dictionary to apply to the request. + :param cert: + (optional) Any user-provided SSL certificate to be used for client + authentication (a.k.a., mTLS). + :rtype: + urllib3.ConnectionPool + """ + proxy = select_proxy(request.url, proxies) + try: + host_params, pool_kwargs = self.build_connection_pool_key_attributes( + request, + verify, + cert, + ) + except ValueError as e: + raise InvalidURL(e, request=request) + if proxy: + proxy = prepend_scheme_if_needed(proxy, "http") + proxy_url = parse_url(proxy) + if not proxy_url.host: + raise InvalidProxyURL( + "Please check proxy URL. It is malformed " + "and could be missing the host." + ) + proxy_manager = self.proxy_manager_for(proxy) + conn = proxy_manager.connection_from_host( + **host_params, pool_kwargs=pool_kwargs + ) + else: + # Only scheme should be lower case + conn = self.poolmanager.connection_from_host( + **host_params, pool_kwargs=pool_kwargs + ) + + return conn + def get_connection(self, url, proxies=None): - """Returns a urllib3 connection for the given URL. This should not be + """DEPRECATED: Users should move to `get_connection_with_tls_context` + for all subclasses of HTTPAdapter using Requests>=2.32.2. + + Returns a urllib3 connection for the given URL. This should not be called from user code, and is only exposed for use when subclassing the :class:`HTTPAdapter `. @@ -297,14 +481,25 @@ def get_connection(self, url, proxies=None): :param proxies: (optional) A Requests-style dictionary of proxies used on this request. :rtype: urllib3.ConnectionPool """ + warnings.warn( + ( + "`get_connection` has been deprecated in favor of " + "`get_connection_with_tls_context`. Custom HTTPAdapter subclasses " + "will need to migrate for Requests>=2.32.2. Please see " + "https://github.com/psf/requests/pull/6710 for more details." + ), + DeprecationWarning, + ) proxy = select_proxy(url, proxies) if proxy: - proxy = prepend_scheme_if_needed(proxy, 'http') + proxy = prepend_scheme_if_needed(proxy, "http") proxy_url = parse_url(proxy) if not proxy_url.host: - raise InvalidProxyURL("Please check proxy URL. It is malformed" - " and could be missing the host.") + raise InvalidProxyURL( + "Please check proxy URL. It is malformed " + "and could be missing the host." + ) proxy_manager = self.proxy_manager_for(proxy) conn = proxy_manager.connection_from_url(url) else: @@ -342,13 +537,16 @@ def request_url(self, request, proxies): proxy = select_proxy(request.url, proxies) scheme = urlparse(request.url).scheme - is_proxied_http_request = (proxy and scheme != 'https') + is_proxied_http_request = proxy and scheme != "https" using_socks_proxy = False if proxy: proxy_scheme = urlparse(proxy).scheme.lower() - using_socks_proxy = proxy_scheme.startswith('socks') + using_socks_proxy = proxy_scheme.startswith("socks") url = request.path_url + if url.startswith("//"): # Don't confuse urllib3 + url = f"/{url.lstrip('/')}" + if is_proxied_http_request and not using_socks_proxy: url = urldefragauth(request.url) @@ -378,19 +576,20 @@ def proxy_headers(self, proxy): when subclassing the :class:`HTTPAdapter `. - :param proxies: The url of the proxy being used for this request. + :param proxy: The url of the proxy being used for this request. :rtype: dict """ headers = {} username, password = get_auth_from_url(proxy) if username: - headers['Proxy-Authorization'] = _basic_auth_str(username, - password) + headers["Proxy-Authorization"] = _basic_auth_str(username, password) return headers - def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): + def send( + self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None + ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest ` being sent. @@ -407,91 +606,56 @@ def send(self, request, stream=False, timeout=None, verify=True, cert=None, prox :rtype: requests.Response """ - conn = self.get_connection(request.url, proxies) + try: + conn = self.get_connection_with_tls_context( + request, verify, proxies=proxies, cert=cert + ) + except LocationValueError as e: + raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) - self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) + self.add_headers( + request, + stream=stream, + timeout=timeout, + verify=verify, + cert=cert, + proxies=proxies, + ) - chunked = not (request.body is None or 'Content-Length' in request.headers) + chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) - except ValueError as e: - # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " - "timeout tuple, or a single float to set " - "both timeouts to the same value".format(timeout)) - raise ValueError(err) + except ValueError: + raise ValueError( + f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " + f"or a single float to set both timeouts to the same value." + ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: - if not chunked: - resp = conn.urlopen( - method=request.method, - url=url, - body=request.body, - headers=request.headers, - redirect=False, - assert_same_host=False, - preload_content=False, - decode_content=False, - retries=self.max_retries, - timeout=timeout - ) + resp = conn.urlopen( + method=request.method, + url=url, + body=request.body, + headers=request.headers, + redirect=False, + assert_same_host=False, + preload_content=False, + decode_content=False, + retries=self.max_retries, + timeout=timeout, + chunked=chunked, + ) - # Send the request. - else: - if hasattr(conn, 'proxy_pool'): - conn = conn.proxy_pool - - low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT) - - try: - low_conn.putrequest(request.method, - url, - skip_accept_encoding=True) - - for header, value in request.headers.items(): - low_conn.putheader(header, value) - - low_conn.endheaders() - - for i in request.body: - low_conn.send(hex(len(i))[2:].encode('utf-8')) - low_conn.send(b'\r\n') - low_conn.send(i) - low_conn.send(b'\r\n') - low_conn.send(b'0\r\n\r\n') - - # Receive the response from the server - try: - # For Python 2.7+ versions, use buffering of HTTP - # responses - r = low_conn.getresponse(buffering=True) - except TypeError: - # For compatibility with Python 2.6 versions and back - r = low_conn.getresponse() - - resp = HTTPResponse.from_httplib( - r, - pool=conn, - connection=low_conn, - preload_content=False, - decode_content=False - ) - except: - # If we hit any problems here, clean up the connection. - # Then, reraise so that we can handle the actual exception. - low_conn.close() - raise - - except (ProtocolError, socket.error) as err: + except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: @@ -524,6 +688,8 @@ def send(self, request, stream=False, timeout=None, verify=True, cert=None, prox raise SSLError(e, request=request) elif isinstance(e, ReadTimeoutError): raise ReadTimeout(e, request=request) + elif isinstance(e, _InvalidHeader): + raise InvalidHeader(e, request=request) else: raise diff --git a/requests/api.py b/src/requests/api.py similarity index 69% rename from requests/api.py rename to src/requests/api.py index a2cc84d769..5960744552 100644 --- a/requests/api.py +++ b/src/requests/api.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ requests.api ~~~~~~~~~~~~ @@ -16,16 +14,18 @@ def request(method, url, **kwargs): """Constructs and sends a :class:`Request `. - :param method: method for the new :class:`Request` object. + :param method: method for the new :class:`Request` object: ``GET``, ``OPTIONS``, ``HEAD``, ``POST``, ``PUT``, ``PATCH``, or ``DELETE``. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the query string for the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. ``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')`` - or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string + or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content_type'`` is a string defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers to add for the file. :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. @@ -47,7 +47,8 @@ def request(method, url, **kwargs): Usage:: >>> import requests - >>> req = requests.request('GET', 'http://httpbin.org/get') + >>> req = requests.request('GET', 'https://httpbin.org/get') + >>> req """ @@ -62,14 +63,14 @@ def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the query string for the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) - return request('get', url, params=params, **kwargs) + return request("get", url, params=params, **kwargs) def options(url, **kwargs): @@ -81,63 +82,67 @@ def options(url, **kwargs): :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) - return request('options', url, **kwargs) + return request("options", url, **kwargs) def head(url, **kwargs): r"""Sends a HEAD request. :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. If + `allow_redirects` is not provided, it will be set to `False` (as + opposed to the default :meth:`request` behavior). :return: :class:`Response ` object :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', False) - return request('head', url, **kwargs) + kwargs.setdefault("allow_redirects", False) + return request("head", url, **kwargs) def post(url, data=None, json=None, **kwargs): r"""Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. + :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response """ - return request('post', url, data=data, json=json, **kwargs) + return request("post", url, data=data, json=json, **kwargs) def put(url, data=None, **kwargs): r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. + :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response """ - return request('put', url, data=data, **kwargs) + return request("put", url, data=data, **kwargs) def patch(url, data=None, **kwargs): r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. + :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response """ - return request('patch', url, data=data, **kwargs) + return request("patch", url, data=data, **kwargs) def delete(url, **kwargs): @@ -149,4 +154,4 @@ def delete(url, **kwargs): :rtype: requests.Response """ - return request('delete', url, **kwargs) + return request("delete", url, **kwargs) diff --git a/requests/auth.py b/src/requests/auth.py similarity index 63% rename from requests/auth.py rename to src/requests/auth.py index 1a182dffdd..4a7ce6dc14 100644 --- a/requests/auth.py +++ b/src/requests/auth.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ requests.auth ~~~~~~~~~~~~~ @@ -7,22 +5,21 @@ This module contains the authentication handlers for Requests. """ +import hashlib import os import re -import time -import hashlib import threading +import time import warnings - from base64 import b64encode -from .compat import urlparse, str, basestring -from .cookies import extract_cookies_to_jar from ._internal_utils import to_native_string +from .compat import basestring, str, urlparse +from .cookies import extract_cookies_to_jar from .utils import parse_dict_header -CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' -CONTENT_TYPE_MULTI_PART = 'multipart/form-data' +CONTENT_TYPE_FORM_URLENCODED = "application/x-www-form-urlencoded" +CONTENT_TYPE_MULTI_PART = "multipart/form-data" def _basic_auth_str(username, password): @@ -38,7 +35,7 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(username), category=DeprecationWarning, @@ -48,32 +45,32 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " - "problems.".format(password), + "problems.".format(type(password)), category=DeprecationWarning, ) password = str(password) # -- End Removal -- if isinstance(username, str): - username = username.encode('latin1') + username = username.encode("latin1") if isinstance(password, str): - password = password.encode('latin1') + password = password.encode("latin1") - authstr = 'Basic ' + to_native_string( - b64encode(b':'.join((username, password))).strip() + authstr = "Basic " + to_native_string( + b64encode(b":".join((username, password))).strip() ) return authstr -class AuthBase(object): +class AuthBase: """Base class that all auth implementations derive from""" def __call__(self, r): - raise NotImplementedError('Auth hooks must be callable.') + raise NotImplementedError("Auth hooks must be callable.") class HTTPBasicAuth(AuthBase): @@ -84,16 +81,18 @@ def __init__(self, username, password): self.password = password def __eq__(self, other): - return all([ - self.username == getattr(other, 'username', None), - self.password == getattr(other, 'password', None) - ]) + return all( + [ + self.username == getattr(other, "username", None), + self.password == getattr(other, "password", None), + ] + ) def __ne__(self, other): return not self == other def __call__(self, r): - r.headers['Authorization'] = _basic_auth_str(self.username, self.password) + r.headers["Authorization"] = _basic_auth_str(self.username, self.password) return r @@ -101,7 +100,7 @@ class HTTPProxyAuth(HTTPBasicAuth): """Attaches HTTP Proxy Authentication to a given Request object.""" def __call__(self, r): - r.headers['Proxy-Authorization'] = _basic_auth_str(self.username, self.password) + r.headers["Proxy-Authorization"] = _basic_auth_str(self.username, self.password) return r @@ -116,9 +115,9 @@ def __init__(self, username, password): def init_per_thread_state(self): # Ensure state is initialized just once per-thread - if not hasattr(self._thread_local, 'init'): + if not hasattr(self._thread_local, "init"): self._thread_local.init = True - self._thread_local.last_nonce = '' + self._thread_local.last_nonce = "" self._thread_local.nonce_count = 0 self._thread_local.chal = {} self._thread_local.pos = None @@ -129,32 +128,52 @@ def build_digest_header(self, method, url): :rtype: str """ - realm = self._thread_local.chal['realm'] - nonce = self._thread_local.chal['nonce'] - qop = self._thread_local.chal.get('qop') - algorithm = self._thread_local.chal.get('algorithm') - opaque = self._thread_local.chal.get('opaque') + realm = self._thread_local.chal["realm"] + nonce = self._thread_local.chal["nonce"] + qop = self._thread_local.chal.get("qop") + algorithm = self._thread_local.chal.get("algorithm") + opaque = self._thread_local.chal.get("opaque") hash_utf8 = None if algorithm is None: - _algorithm = 'MD5' + _algorithm = "MD5" else: _algorithm = algorithm.upper() # lambdas assume digest modules are imported at the top level - if _algorithm == 'MD5' or _algorithm == 'MD5-SESS': + if _algorithm == "MD5" or _algorithm == "MD5-SESS": + def md5_utf8(x): if isinstance(x, str): - x = x.encode('utf-8') + x = x.encode("utf-8") return hashlib.md5(x).hexdigest() + hash_utf8 = md5_utf8 - elif _algorithm == 'SHA': + elif _algorithm == "SHA": + def sha_utf8(x): if isinstance(x, str): - x = x.encode('utf-8') + x = x.encode("utf-8") return hashlib.sha1(x).hexdigest() + hash_utf8 = sha_utf8 + elif _algorithm == "SHA-256": + + def sha256_utf8(x): + if isinstance(x, str): + x = x.encode("utf-8") + return hashlib.sha256(x).hexdigest() + + hash_utf8 = sha256_utf8 + elif _algorithm == "SHA-512": + + def sha512_utf8(x): + if isinstance(x, str): + x = x.encode("utf-8") + return hashlib.sha512(x).hexdigest() - KD = lambda s, d: hash_utf8("%s:%s" % (s, d)) + hash_utf8 = sha512_utf8 + + KD = lambda s, d: hash_utf8(f"{s}:{d}") # noqa:E731 if hash_utf8 is None: return None @@ -165,10 +184,10 @@ def sha_utf8(x): #: path is request-uri defined in RFC 2616 which should not be empty path = p_parsed.path or "/" if p_parsed.query: - path += '?' + p_parsed.query + path += f"?{p_parsed.query}" - A1 = '%s:%s:%s' % (self.username, realm, self.password) - A2 = '%s:%s' % (method, path) + A1 = f"{self.username}:{realm}:{self.password}" + A2 = f"{method}:{path}" HA1 = hash_utf8(A1) HA2 = hash_utf8(A2) @@ -177,22 +196,20 @@ def sha_utf8(x): self._thread_local.nonce_count += 1 else: self._thread_local.nonce_count = 1 - ncvalue = '%08x' % self._thread_local.nonce_count - s = str(self._thread_local.nonce_count).encode('utf-8') - s += nonce.encode('utf-8') - s += time.ctime().encode('utf-8') + ncvalue = f"{self._thread_local.nonce_count:08x}" + s = str(self._thread_local.nonce_count).encode("utf-8") + s += nonce.encode("utf-8") + s += time.ctime().encode("utf-8") s += os.urandom(8) - cnonce = (hashlib.sha1(s).hexdigest()[:16]) - if _algorithm == 'MD5-SESS': - HA1 = hash_utf8('%s:%s:%s' % (HA1, nonce, cnonce)) + cnonce = hashlib.sha1(s).hexdigest()[:16] + if _algorithm == "MD5-SESS": + HA1 = hash_utf8(f"{HA1}:{nonce}:{cnonce}") if not qop: - respdig = KD(HA1, "%s:%s" % (nonce, HA2)) - elif qop == 'auth' or 'auth' in qop.split(','): - noncebit = "%s:%s:%s:%s:%s" % ( - nonce, ncvalue, cnonce, 'auth', HA2 - ) + respdig = KD(HA1, f"{nonce}:{HA2}") + elif qop == "auth" or "auth" in qop.split(","): + noncebit = f"{nonce}:{ncvalue}:{cnonce}:auth:{HA2}" respdig = KD(HA1, noncebit) else: # XXX handle auth-int. @@ -201,18 +218,20 @@ def sha_utf8(x): self._thread_local.last_nonce = nonce # XXX should the partial digests be encoded too? - base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \ - 'response="%s"' % (self.username, realm, nonce, path, respdig) + base = ( + f'username="{self.username}", realm="{realm}", nonce="{nonce}", ' + f'uri="{path}", response="{respdig}"' + ) if opaque: - base += ', opaque="%s"' % opaque + base += f', opaque="{opaque}"' if algorithm: - base += ', algorithm="%s"' % algorithm + base += f', algorithm="{algorithm}"' if entdig: - base += ', digest="%s"' % entdig + base += f', digest="{entdig}"' if qop: - base += ', qop="auth", nc=%s, cnonce="%s"' % (ncvalue, cnonce) + base += f', qop="auth", nc={ncvalue}, cnonce="{cnonce}"' - return 'Digest %s' % (base) + return f"Digest {base}" def handle_redirect(self, r, **kwargs): """Reset num_401_calls counter on redirects.""" @@ -227,7 +246,7 @@ def handle_401(self, r, **kwargs): """ # If response is not 4xx, do not auth - # See https://github.com/requests/requests/issues/3772 + # See https://github.com/psf/requests/issues/3772 if not 400 <= r.status_code < 500: self._thread_local.num_401_calls = 1 return r @@ -236,13 +255,12 @@ def handle_401(self, r, **kwargs): # Rewind the file position indicator of the body to where # it was to resend the request. r.request.body.seek(self._thread_local.pos) - s_auth = r.headers.get('www-authenticate', '') - - if 'digest' in s_auth.lower() and self._thread_local.num_401_calls < 2: + s_auth = r.headers.get("www-authenticate", "") + if "digest" in s_auth.lower() and self._thread_local.num_401_calls < 2: self._thread_local.num_401_calls += 1 - pat = re.compile(r'digest ', flags=re.IGNORECASE) - self._thread_local.chal = parse_dict_header(pat.sub('', s_auth, count=1)) + pat = re.compile(r"digest ", flags=re.IGNORECASE) + self._thread_local.chal = parse_dict_header(pat.sub("", s_auth, count=1)) # Consume content and release the original connection # to allow our new request to reuse the same one. @@ -252,8 +270,9 @@ def handle_401(self, r, **kwargs): extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) - prep.headers['Authorization'] = self.build_digest_header( - prep.method, prep.url) + prep.headers["Authorization"] = self.build_digest_header( + prep.method, prep.url + ) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep @@ -268,7 +287,7 @@ def __call__(self, r): self.init_per_thread_state() # If we have a saved nonce, skip the 401 if self._thread_local.last_nonce: - r.headers['Authorization'] = self.build_digest_header(r.method, r.url) + r.headers["Authorization"] = self.build_digest_header(r.method, r.url) try: self._thread_local.pos = r.body.tell() except AttributeError: @@ -277,17 +296,19 @@ def __call__(self, r): # file position of the previous body. Ensure it's set to # None. self._thread_local.pos = None - r.register_hook('response', self.handle_401) - r.register_hook('response', self.handle_redirect) + r.register_hook("response", self.handle_401) + r.register_hook("response", self.handle_redirect) self._thread_local.num_401_calls = 1 return r def __eq__(self, other): - return all([ - self.username == getattr(other, 'username', None), - self.password == getattr(other, 'password', None) - ]) + return all( + [ + self.username == getattr(other, "username", None), + self.password == getattr(other, "password", None), + ] + ) def __ne__(self, other): return not self == other diff --git a/requests/certs.py b/src/requests/certs.py similarity index 88% rename from requests/certs.py rename to src/requests/certs.py index d1a378d787..be422c3e91 100644 --- a/requests/certs.py +++ b/src/requests/certs.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- """ requests.certs @@ -14,5 +13,5 @@ """ from certifi import where -if __name__ == '__main__': +if __name__ == "__main__": print(where()) diff --git a/src/requests/compat.py b/src/requests/compat.py new file mode 100644 index 0000000000..7f9d754350 --- /dev/null +++ b/src/requests/compat.py @@ -0,0 +1,106 @@ +""" +requests.compat +~~~~~~~~~~~~~~~ + +This module previously handled import compatibility issues +between Python 2 and Python 3. It remains for backwards +compatibility until the next major version. +""" + +import importlib +import sys + +# ------- +# urllib3 +# ------- +from urllib3 import __version__ as urllib3_version + +# Detect which major version of urllib3 is being used. +try: + is_urllib3_1 = int(urllib3_version.split(".")[0]) == 1 +except (TypeError, AttributeError): + # If we can't discern a version, prefer old functionality. + is_urllib3_1 = True + +# ------------------- +# Character Detection +# ------------------- + + +def _resolve_char_detection(): + """Find supported character detection libraries.""" + chardet = None + for lib in ("chardet", "charset_normalizer"): + if chardet is None: + try: + chardet = importlib.import_module(lib) + except ImportError: + pass + return chardet + + +chardet = _resolve_char_detection() + +# ------- +# Pythons +# ------- + +# Syntax sugar. +_ver = sys.version_info + +#: Python 2.x? +is_py2 = _ver[0] == 2 + +#: Python 3.x? +is_py3 = _ver[0] == 3 + +# json/simplejson module import resolution +has_simplejson = False +try: + import simplejson as json + + has_simplejson = True +except ImportError: + import json + +if has_simplejson: + from simplejson import JSONDecodeError +else: + from json import JSONDecodeError + +# Keep OrderedDict for backwards compatibility. +from collections import OrderedDict +from collections.abc import Callable, Mapping, MutableMapping +from http import cookiejar as cookielib +from http.cookies import Morsel +from io import StringIO + +# -------------- +# Legacy Imports +# -------------- +from urllib.parse import ( + quote, + quote_plus, + unquote, + unquote_plus, + urldefrag, + urlencode, + urljoin, + urlparse, + urlsplit, + urlunparse, +) +from urllib.request import ( + getproxies, + getproxies_environment, + parse_http_list, + proxy_bypass, + proxy_bypass_environment, +) + +builtin_str = str +str = str +bytes = bytes +basestring = (str, bytes) +numeric_types = (int, float) +integer_types = (int,) diff --git a/requests/cookies.py b/src/requests/cookies.py similarity index 76% rename from requests/cookies.py rename to src/requests/cookies.py index ab3c88b9bf..f69d0cda9e 100644 --- a/requests/cookies.py +++ b/src/requests/cookies.py @@ -1,21 +1,18 @@ -# -*- coding: utf-8 -*- - """ requests.cookies ~~~~~~~~~~~~~~~~ -Compatibility code to be able to use `cookielib.CookieJar` with requests. +Compatibility code to be able to use `http.cookiejar.CookieJar` with requests. requests.utils imports from here, so be careful with imports. """ +import calendar import copy import time -import calendar -import collections from ._internal_utils import to_native_string -from .compat import cookielib, urlparse, urlunparse, Morsel +from .compat import Morsel, MutableMapping, cookielib, urlparse, urlunparse try: import threading @@ -23,10 +20,10 @@ import dummy_threading as threading -class MockRequest(object): +class MockRequest: """Wraps a `requests.Request` to mimic a `urllib2.Request`. - The code in `cookielib.CookieJar` expects this interface in order to correctly + The code in `http.cookiejar.CookieJar` expects this interface in order to correctly manage cookie policies, i.e., determine whether a cookie can be set, given the domains of the request and the cookie. @@ -52,16 +49,22 @@ def get_origin_req_host(self): def get_full_url(self): # Only return the response's URL if the user hadn't set the Host # header - if not self._r.headers.get('Host'): + if not self._r.headers.get("Host"): return self._r.url # If they did set it, retrieve it and reconstruct the expected domain - host = to_native_string(self._r.headers['Host'], encoding='utf-8') + host = to_native_string(self._r.headers["Host"], encoding="utf-8") parsed = urlparse(self._r.url) # Reconstruct the URL as we expect it - return urlunparse([ - parsed.scheme, host, parsed.path, parsed.params, parsed.query, - parsed.fragment - ]) + return urlunparse( + [ + parsed.scheme, + host, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + ] + ) def is_unverifiable(self): return True @@ -73,8 +76,10 @@ def get_header(self, name, default=None): return self._r.headers.get(name, self._new_headers.get(name, default)) def add_header(self, key, val): - """cookielib has no legitimate use for this method; add it back if you find one.""" - raise NotImplementedError("Cookie headers should be added with add_unredirected_header()") + """cookiejar has no legitimate use for this method; add it back if you find one.""" + raise NotImplementedError( + "Cookie headers should be added with add_unredirected_header()" + ) def add_unredirected_header(self, name, value): self._new_headers[name] = value @@ -95,15 +100,15 @@ def host(self): return self.get_host() -class MockResponse(object): +class MockResponse: """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`. ...what? Basically, expose the parsed HTTP headers from the server response - the way `cookielib` expects to see them. + the way `http.cookiejar` expects to see them. """ def __init__(self, headers): - """Make a MockResponse for `cookielib` to read. + """Make a MockResponse for `cookiejar` to read. :param headers: a httplib.HTTPMessage or analogous carrying the headers """ @@ -119,12 +124,11 @@ def getheaders(self, name): def extract_cookies_to_jar(jar, request, response): """Extract the cookies from the response into a CookieJar. - :param jar: cookielib.CookieJar (not necessarily a RequestsCookieJar) + :param jar: http.cookiejar.CookieJar (not necessarily a RequestsCookieJar) :param request: our own requests.Request object :param response: urllib3.HTTPResponse object """ - if not (hasattr(response, '_original_response') and - response._original_response): + if not (hasattr(response, "_original_response") and response._original_response): return # the _original_response field is the wrapped httplib.HTTPResponse object, req = MockRequest(request) @@ -141,7 +145,7 @@ def get_cookie_header(jar, request): """ r = MockRequest(request) jar.add_cookie_header(r) - return r.get_new_headers().get('Cookie') + return r.get_new_headers().get("Cookie") def remove_cookie_by_name(cookiejar, name, domain=None, path=None): @@ -169,8 +173,8 @@ class CookieConflictError(RuntimeError): """ -class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): - """Compatibility class; is a cookielib.CookieJar, but exposes a dict +class RequestsCookieJar(cookielib.CookieJar, MutableMapping): + """Compatibility class; is a http.cookiejar.CookieJar, but exposes a dict interface. This is the CookieJar we create by default for requests and sessions that @@ -206,7 +210,9 @@ def set(self, name, value, **kwargs): """ # support client code that unsets cookies by assignment of a None value: if value is None: - remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path')) + remove_cookie_by_name( + self, name, domain=kwargs.get("domain"), path=kwargs.get("path") + ) return if isinstance(value, Morsel): @@ -306,16 +312,15 @@ def get_dict(self, domain=None, path=None): """ dictionary = {} for cookie in iter(self): - if ( - (domain is None or cookie.domain == domain) and - (path is None or cookie.path == path) + if (domain is None or cookie.domain == domain) and ( + path is None or cookie.path == path ): dictionary[cookie.name] = cookie.value return dictionary def __contains__(self, name): try: - return super(RequestsCookieJar, self).__contains__(name) + return super().__contains__(name) except CookieConflictError: return True @@ -336,15 +341,19 @@ def __setitem__(self, name, value): self.set(name, value) def __delitem__(self, name): - """Deletes a cookie given a name. Wraps ``cookielib.CookieJar``'s + """Deletes a cookie given a name. Wraps ``http.cookiejar.CookieJar``'s ``remove_cookie_by_name()``. """ remove_cookie_by_name(self, name) def set_cookie(self, cookie, *args, **kwargs): - if hasattr(cookie.value, 'startswith') and cookie.value.startswith('"') and cookie.value.endswith('"'): - cookie.value = cookie.value.replace('\\"', '') - return super(RequestsCookieJar, self).set_cookie(cookie, *args, **kwargs) + if ( + hasattr(cookie.value, "startswith") + and cookie.value.startswith('"') + and cookie.value.endswith('"') + ): + cookie.value = cookie.value.replace('\\"', "") + return super().set_cookie(cookie, *args, **kwargs) def update(self, other): """Updates this jar with cookies from another CookieJar or dict-like""" @@ -352,7 +361,7 @@ def update(self, other): for cookie in other: self.set_cookie(copy.copy(cookie)) else: - super(RequestsCookieJar, self).update(other) + super().update(other) def _find(self, name, domain=None, path=None): """Requests uses this method internally to get cookie values. @@ -372,7 +381,7 @@ def _find(self, name, domain=None, path=None): if path is None or cookie.path == path: return cookie.value - raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) + raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}") def _find_no_duplicates(self, name, domain=None, path=None): """Both ``__get_item__`` and ``get`` call this function: it's never @@ -391,39 +400,48 @@ def _find_no_duplicates(self, name, domain=None, path=None): if cookie.name == name: if domain is None or cookie.domain == domain: if path is None or cookie.path == path: - if toReturn is not None: # if there are multiple cookies that meet passed in criteria - raise CookieConflictError('There are multiple cookies with name, %r' % (name)) - toReturn = cookie.value # we will eventually return this as long as no cookie conflict + if toReturn is not None: + # if there are multiple cookies that meet passed in criteria + raise CookieConflictError( + f"There are multiple cookies with name, {name!r}" + ) + # we will eventually return this as long as no cookie conflict + toReturn = cookie.value if toReturn: return toReturn - raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) + raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}") def __getstate__(self): """Unlike a normal CookieJar, this class is pickleable.""" state = self.__dict__.copy() # remove the unpickleable RLock object - state.pop('_cookies_lock') + state.pop("_cookies_lock") return state def __setstate__(self, state): """Unlike a normal CookieJar, this class is pickleable.""" self.__dict__.update(state) - if '_cookies_lock' not in self.__dict__: + if "_cookies_lock" not in self.__dict__: self._cookies_lock = threading.RLock() def copy(self): """Return a copy of this RequestsCookieJar.""" new_cj = RequestsCookieJar() + new_cj.set_policy(self.get_policy()) new_cj.update(self) return new_cj + def get_policy(self): + """Return the CookiePolicy instance used.""" + return self._policy + def _copy_cookie_jar(jar): if jar is None: return None - if hasattr(jar, 'copy'): + if hasattr(jar, "copy"): # We're dealing with an instance of RequestsCookieJar return jar.copy() # We're dealing with a generic CookieJar instance @@ -440,31 +458,33 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + "version": 0, + "name": name, + "value": value, + "port": None, + "domain": "", + "path": "/", + "secure": False, + "expires": None, + "discard": True, + "comment": None, + "comment_url": None, + "rest": {"HttpOnly": None}, + "rfc2109": False, + } badargs = set(kwargs) - set(result) if badargs: - err = 'create_cookie() got unexpected keyword arguments: %s' - raise TypeError(err % list(badargs)) + raise TypeError( + f"create_cookie() got unexpected keyword arguments: {list(badargs)}" + ) result.update(kwargs) - result['port_specified'] = bool(result['port']) - result['domain_specified'] = bool(result['domain']) - result['domain_initial_dot'] = result['domain'].startswith('.') - result['path_specified'] = bool(result['path']) + result["port_specified"] = bool(result["port"]) + result["domain_specified"] = bool(result["domain"]) + result["domain_initial_dot"] = result["domain"].startswith(".") + result["path_specified"] = bool(result["path"]) return cookielib.Cookie(**result) @@ -473,30 +493,28 @@ def morsel_to_cookie(morsel): """Convert a Morsel object into a Cookie containing the one k/v pair.""" expires = None - if morsel['max-age']: + if morsel["max-age"]: try: - expires = int(time.time() + int(morsel['max-age'])) + expires = int(time.time() + int(morsel["max-age"])) except ValueError: - raise TypeError('max-age: %s must be integer' % morsel['max-age']) - elif morsel['expires']: - time_template = '%a, %d-%b-%Y %H:%M:%S GMT' - expires = calendar.timegm( - time.strptime(morsel['expires'], time_template) - ) + raise TypeError(f"max-age: {morsel['max-age']} must be integer") + elif morsel["expires"]: + time_template = "%a, %d-%b-%Y %H:%M:%S GMT" + expires = calendar.timegm(time.strptime(morsel["expires"], time_template)) return create_cookie( - comment=morsel['comment'], - comment_url=bool(morsel['comment']), + comment=morsel["comment"], + comment_url=bool(morsel["comment"]), discard=False, - domain=morsel['domain'], + domain=morsel["domain"], expires=expires, name=morsel.key, - path=morsel['path'], + path=morsel["path"], port=None, - rest={'HttpOnly': morsel['httponly']}, + rest={"HttpOnly": morsel["httponly"]}, rfc2109=False, - secure=bool(morsel['secure']), + secure=bool(morsel["secure"]), value=morsel.value, - version=morsel['version'] or 0, + version=morsel["version"] or 0, ) @@ -507,6 +525,7 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): :param cookiejar: (optional) A cookiejar to add the cookies to. :param overwrite: (optional) If False, will not replace cookies already in the jar with new ones. + :rtype: CookieJar """ if cookiejar is None: cookiejar = RequestsCookieJar() @@ -525,13 +544,13 @@ def merge_cookies(cookiejar, cookies): :param cookiejar: CookieJar object to add the cookies to. :param cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar """ if not isinstance(cookiejar, cookielib.CookieJar): - raise ValueError('You can only merge into CookieJar') + raise ValueError("You can only merge into CookieJar") if isinstance(cookies, dict): - cookiejar = cookiejar_from_dict( - cookies, cookiejar=cookiejar, overwrite=False) + cookiejar = cookiejar_from_dict(cookies, cookiejar=cookiejar, overwrite=False) elif isinstance(cookies, cookielib.CookieJar): try: cookiejar.update(cookies) diff --git a/requests/exceptions.py b/src/requests/exceptions.py similarity index 62% rename from requests/exceptions.py rename to src/requests/exceptions.py index a80cad80f1..83986b4898 100644 --- a/requests/exceptions.py +++ b/src/requests/exceptions.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ requests.exceptions ~~~~~~~~~~~~~~~~~~~ @@ -8,6 +6,8 @@ """ from urllib3.exceptions import HTTPError as BaseHTTPError +from .compat import JSONDecodeError as CompatJSONDecodeError + class RequestException(IOError): """There was an ambiguous exception that occurred while handling your @@ -16,13 +16,40 @@ class RequestException(IOError): def __init__(self, *args, **kwargs): """Initialize RequestException with `request` and `response` objects.""" - response = kwargs.pop('response', None) + response = kwargs.pop("response", None) self.response = response - self.request = kwargs.pop('request', None) - if (response is not None and not self.request and - hasattr(response, 'request')): + self.request = kwargs.pop("request", None) + if response is not None and not self.request and hasattr(response, "request"): self.request = self.response.request - super(RequestException, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) + + +class InvalidJSONError(RequestException): + """A JSON error occurred.""" + + +class JSONDecodeError(InvalidJSONError, CompatJSONDecodeError): + """Couldn't decode the text into json""" + + def __init__(self, *args, **kwargs): + """ + Construct the JSONDecodeError instance first with all + args. Then use it's args to construct the IOError so that + the json specific args aren't used as IOError specific args + and the error message from JSONDecodeError is preserved. + """ + CompatJSONDecodeError.__init__(self, *args) + InvalidJSONError.__init__(self, *self.args, **kwargs) + + def __reduce__(self): + """ + The __reduce__ method called when pickling the object must + be the one from the JSONDecodeError (be it json/simplejson) + as it expects all the arguments for instantiation, not just + one like the IOError, and the MRO would by default call the + __reduce__ method from the IOError due to the inheritance order. + """ + return CompatJSONDecodeError.__reduce__(self) class HTTPError(RequestException): @@ -70,11 +97,11 @@ class TooManyRedirects(RequestException): class MissingSchema(RequestException, ValueError): - """The URL schema (e.g. http or https) is missing.""" + """The URL scheme (e.g. http or https) is missing.""" class InvalidSchema(RequestException, ValueError): - """See defaults.py for valid schemas.""" + """The URL scheme provided is either invalid or unsupported.""" class InvalidURL(RequestException, ValueError): @@ -94,11 +121,11 @@ class ChunkedEncodingError(RequestException): class ContentDecodingError(RequestException, BaseHTTPError): - """Failed to decode response content""" + """Failed to decode response content.""" class StreamConsumedError(RequestException, TypeError): - """The content for this response was already consumed""" + """The content for this response was already consumed.""" class RetryError(RequestException): @@ -106,21 +133,19 @@ class RetryError(RequestException): class UnrewindableBodyError(RequestException): - """Requests encountered an error when trying to rewind a body""" + """Requests encountered an error when trying to rewind a body.""" + # Warnings class RequestsWarning(Warning): """Base warning for Requests.""" - pass class FileModeWarning(RequestsWarning, DeprecationWarning): """A file was opened in text mode, but Requests determined its binary length.""" - pass class RequestsDependencyWarning(RequestsWarning): """An imported dependency doesn't match the expected version range.""" - pass diff --git a/src/requests/help.py b/src/requests/help.py new file mode 100644 index 0000000000..8fbcd6560a --- /dev/null +++ b/src/requests/help.py @@ -0,0 +1,134 @@ +"""Module containing bug report helper(s).""" + +import json +import platform +import ssl +import sys + +import idna +import urllib3 + +from . import __version__ as requests_version + +try: + import charset_normalizer +except ImportError: + charset_normalizer = None + +try: + import chardet +except ImportError: + chardet = None + +try: + from urllib3.contrib import pyopenssl +except ImportError: + pyopenssl = None + OpenSSL = None + cryptography = None +else: + import cryptography + import OpenSSL + + +def _implementation(): + """Return a dict with the Python implementation and version. + + Provide both the name and the version of the Python implementation + currently running. For example, on CPython 3.10.3 it will return + {'name': 'CPython', 'version': '3.10.3'}. + + This function works best on CPython and PyPy: in particular, it probably + doesn't work for Jython or IronPython. Future investigation should be done + to work out the correct shape of the code for those platforms. + """ + implementation = platform.python_implementation() + + if implementation == "CPython": + implementation_version = platform.python_version() + elif implementation == "PyPy": + implementation_version = "{}.{}.{}".format( + sys.pypy_version_info.major, + sys.pypy_version_info.minor, + sys.pypy_version_info.micro, + ) + if sys.pypy_version_info.releaselevel != "final": + implementation_version = "".join( + [implementation_version, sys.pypy_version_info.releaselevel] + ) + elif implementation == "Jython": + implementation_version = platform.python_version() # Complete Guess + elif implementation == "IronPython": + implementation_version = platform.python_version() # Complete Guess + else: + implementation_version = "Unknown" + + return {"name": implementation, "version": implementation_version} + + +def info(): + """Generate information for a bug report.""" + try: + platform_info = { + "system": platform.system(), + "release": platform.release(), + } + except OSError: + platform_info = { + "system": "Unknown", + "release": "Unknown", + } + + implementation_info = _implementation() + urllib3_info = {"version": urllib3.__version__} + charset_normalizer_info = {"version": None} + chardet_info = {"version": None} + if charset_normalizer: + charset_normalizer_info = {"version": charset_normalizer.__version__} + if chardet: + chardet_info = {"version": chardet.__version__} + + pyopenssl_info = { + "version": None, + "openssl_version": "", + } + if OpenSSL: + pyopenssl_info = { + "version": OpenSSL.__version__, + "openssl_version": f"{OpenSSL.SSL.OPENSSL_VERSION_NUMBER:x}", + } + cryptography_info = { + "version": getattr(cryptography, "__version__", ""), + } + idna_info = { + "version": getattr(idna, "__version__", ""), + } + + system_ssl = ssl.OPENSSL_VERSION_NUMBER + system_ssl_info = {"version": f"{system_ssl:x}" if system_ssl is not None else ""} + + return { + "platform": platform_info, + "implementation": implementation_info, + "system_ssl": system_ssl_info, + "using_pyopenssl": pyopenssl is not None, + "using_charset_normalizer": chardet is None, + "pyOpenSSL": pyopenssl_info, + "urllib3": urllib3_info, + "chardet": chardet_info, + "charset_normalizer": charset_normalizer_info, + "cryptography": cryptography_info, + "idna": idna_info, + "requests": { + "version": requests_version, + }, + } + + +def main(): + """Pretty-print the bug information as JSON.""" + print(json.dumps(info(), sort_keys=True, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/requests/hooks.py b/src/requests/hooks.py similarity index 79% rename from requests/hooks.py rename to src/requests/hooks.py index 32b32de750..d181ba2ec2 100644 --- a/requests/hooks.py +++ b/src/requests/hooks.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ requests.hooks ~~~~~~~~~~~~~~ @@ -11,21 +9,22 @@ ``response``: The response generated from a Request. """ -HOOKS = ['response'] +HOOKS = ["response"] def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} + # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: - if hasattr(hooks, '__call__'): + if hasattr(hooks, "__call__"): hooks = [hooks] for hook in hooks: _hook_data = hook(hook_data, **kwargs) diff --git a/requests/models.py b/src/requests/models.py similarity index 73% rename from requests/models.py rename to src/requests/models.py index ce4e284e64..c4b25fa079 100644 --- a/requests/models.py +++ b/src/requests/models.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ requests.models ~~~~~~~~~~~~~~~ @@ -7,47 +5,73 @@ This module contains the primary objects that power Requests. """ -import collections import datetime -import sys # Import encoding now, to avoid implicit import later. # Implicit import within threads may cause LookupError when standard library is in a ZIP, -# such as in Embedded Python. See https://github.com/requests/requests/issues/3578. -import encodings.idna +# such as in Embedded Python. See https://github.com/psf/requests/issues/3578. +import encodings.idna # noqa: F401 +from io import UnsupportedOperation +from urllib3.exceptions import ( + DecodeError, + LocationParseError, + ProtocolError, + ReadTimeoutError, + SSLError, +) from urllib3.fields import RequestField from urllib3.filepost import encode_multipart_formdata from urllib3.util import parse_url -from urllib3.exceptions import ( - DecodeError, ReadTimeoutError, ProtocolError, LocationParseError) -from io import UnsupportedOperation -from .hooks import default_hooks -from .structures import CaseInsensitiveDict - -from .auth import HTTPBasicAuth -from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar -from .exceptions import ( - HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, - ContentDecodingError, ConnectionError, StreamConsumedError) from ._internal_utils import to_native_string, unicode_is_ascii -from .utils import ( - guess_filename, get_auth_from_url, requote_uri, - stream_decode_response_unicode, to_key_val_list, parse_header_links, - iter_slices, guess_json_utf, super_len, check_header_validity) +from .auth import HTTPBasicAuth from .compat import ( - cookielib, urlunparse, urlsplit, urlencode, str, bytes, - is_py2, chardet, builtin_str, basestring) + Callable, + JSONDecodeError, + Mapping, + basestring, + builtin_str, + chardet, + cookielib, +) from .compat import json as complexjson +from .compat import urlencode, urlsplit, urlunparse +from .cookies import _copy_cookie_jar, cookiejar_from_dict, get_cookie_header +from .exceptions import ( + ChunkedEncodingError, + ConnectionError, + ContentDecodingError, + HTTPError, + InvalidJSONError, + InvalidURL, +) +from .exceptions import JSONDecodeError as RequestsJSONDecodeError +from .exceptions import MissingSchema +from .exceptions import SSLError as RequestsSSLError +from .exceptions import StreamConsumedError +from .hooks import default_hooks from .status_codes import codes +from .structures import CaseInsensitiveDict +from .utils import ( + check_header_validity, + get_auth_from_url, + guess_filename, + guess_json_utf, + iter_slices, + parse_header_links, + requote_uri, + stream_decode_response_unicode, + super_len, + to_key_val_list, +) #: The set of HTTP status codes that indicate an automatically #: processable redirect. REDIRECT_STATI = ( - codes.moved, # 301 - codes.found, # 302 - codes.other, # 303 + codes.moved, # 301 + codes.found, # 302 + codes.other, # 303 codes.temporary_redirect, # 307 codes.permanent_redirect, # 308 ) @@ -57,7 +81,7 @@ ITER_CHUNK_SIZE = 512 -class RequestEncodingMixin(object): +class RequestEncodingMixin: @property def path_url(self): """Build the path URL to use.""" @@ -68,16 +92,16 @@ def path_url(self): path = p.path if not path: - path = '/' + path = "/" url.append(path) query = p.query if query: - url.append('?') + url.append("?") url.append(query) - return ''.join(url) + return "".join(url) @staticmethod def _encode_params(data): @@ -90,18 +114,21 @@ def _encode_params(data): if isinstance(data, (str, bytes)): return data - elif hasattr(data, 'read'): + elif hasattr(data, "read"): return data - elif hasattr(data, '__iter__'): + elif hasattr(data, "__iter__"): result = [] for k, vs in to_key_val_list(data): - if isinstance(vs, basestring) or not hasattr(vs, '__iter__'): + if isinstance(vs, basestring) or not hasattr(vs, "__iter__"): vs = [vs] for v in vs: if v is not None: result.append( - (k.encode('utf-8') if isinstance(k, str) else k, - v.encode('utf-8') if isinstance(v, str) else v)) + ( + k.encode("utf-8") if isinstance(k, str) else k, + v.encode("utf-8") if isinstance(v, str) else v, + ) + ) return urlencode(result, doseq=True) else: return data @@ -116,7 +143,7 @@ def _encode_files(files, data): The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype) or 4-tuples (filename, fileobj, contentype, custom_headers). """ - if (not files): + if not files: raise ValueError("Files must be provided.") elif isinstance(data, basestring): raise ValueError("Data must not be a string.") @@ -126,7 +153,7 @@ def _encode_files(files, data): files = to_key_val_list(files or {}) for field, val in fields: - if isinstance(val, basestring) or not hasattr(val, '__iter__'): + if isinstance(val, basestring) or not hasattr(val, "__iter__"): val = [val] for v in val: if v is not None: @@ -135,10 +162,15 @@ def _encode_files(files, data): v = str(v) new_fields.append( - (field.decode('utf-8') if isinstance(field, bytes) else field, - v.encode('utf-8') if isinstance(v, str) else v)) + ( + field.decode("utf-8") + if isinstance(field, bytes) + else field, + v.encode("utf-8") if isinstance(v, str) else v, + ) + ) - for (k, v) in files: + for k, v in files: # support for explicit filename ft = None fh = None @@ -155,8 +187,12 @@ def _encode_files(files, data): if isinstance(fp, (str, bytes, bytearray)): fdata = fp - else: + elif hasattr(fp, "read"): fdata = fp.read() + elif fp is None: + continue + else: + fdata = fp rf = RequestField(name=k, data=fdata, filename=fn, headers=fh) rf.make_multipart(content_type=ft) @@ -167,17 +203,17 @@ def _encode_files(files, data): return body, content_type -class RequestHooksMixin(object): +class RequestHooksMixin: def register_hook(self, event, hook): """Properly register a hook.""" if event not in self.hooks: - raise ValueError('Unsupported event specified, with event name "%s"' % (event)) + raise ValueError(f'Unsupported event specified, with event name "{event}"') - if isinstance(hook, collections.Callable): + if isinstance(hook, Callable): self.hooks[event].append(hook) - elif hasattr(hook, '__iter__'): - self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable)) + elif hasattr(hook, "__iter__"): + self.hooks[event].extend(h for h in hook if isinstance(h, Callable)) def deregister_hook(self, event, hook): """Deregister a previously registered hook. @@ -200,9 +236,13 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param json: json for the body to attach to the request (if files or data is not specified). - :param params: dictionary of URL parameters to append to the URL. + :param params: URL parameters to append to the URL. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. :param hooks: dictionary of callback hooks, for internal usage. @@ -210,15 +250,24 @@ class Request(RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> req.prepare() """ - def __init__(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): - + def __init__( + self, + method=None, + url=None, + headers=None, + files=None, + data=None, + params=None, + auth=None, + cookies=None, + hooks=None, + json=None, + ): # Default empty dicts for dict params. data = [] if data is None else data files = [] if files is None else files @@ -227,7 +276,7 @@ def __init__(self, hooks = {} if hooks is None else hooks self.hooks = default_hooks() - for (k, v) in list(hooks.items()): + for k, v in list(hooks.items()): self.register_hook(event=k, hook=v) self.method = method @@ -241,7 +290,7 @@ def __init__(self, self.cookies = cookies def __repr__(self): - return '' % (self.method) + return f"" def prepare(self): """Constructs a :class:`PreparedRequest ` for transmission and returns it.""" @@ -265,13 +314,16 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): """The fully mutable :class:`PreparedRequest ` object, containing the exact bytes that will be sent to the server. - Generated from either a :class:`Request ` object or manually. + Instances are generated from a :class:`Request ` object, and + should not be instantiated manually; doing so may produce undesirable + effects. Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() + >>> r >>> s = requests.Session() @@ -296,9 +348,19 @@ def __init__(self): #: integer denoting starting position of a readable file-like body. self._body_position = None - def prepare(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): + def prepare( + self, + method=None, + url=None, + headers=None, + files=None, + data=None, + params=None, + auth=None, + cookies=None, + hooks=None, + json=None, + ): """Prepares the entire request with the given parameters.""" self.prepare_method(method) @@ -315,7 +377,7 @@ def prepare(self, self.prepare_hooks(hooks) def __repr__(self): - return '' % (self.method) + return f"" def copy(self): p = PreparedRequest() @@ -339,7 +401,7 @@ def _get_idna_encoded_host(host): import idna try: - host = idna.encode(host, uts46=True).decode('utf-8') + host = idna.encode(host, uts46=True).decode("utf-8") except idna.IDNAError: raise UnicodeError return host @@ -350,11 +412,11 @@ def prepare_url(self, url, params): #: We're unable to blindly call unicode/str functions #: as this will include the bytestring indicator (b'') #: on python 3.x. - #: https://github.com/requests/requests/pull/2238 + #: https://github.com/psf/requests/pull/2238 if isinstance(url, bytes): - url = url.decode('utf8') + url = url.decode("utf8") else: - url = unicode(url) if is_py2 else str(url) + url = str(url) # Remove leading whitespaces from url url = url.lstrip() @@ -362,7 +424,7 @@ def prepare_url(self, url, params): # Don't do any URL preparation for non-HTTP schemes like `mailto`, # `data` etc to work around exceptions from `url_parse`, which # handles RFC 3986 only. - if ':' in url and not url.lower().startswith('http'): + if ":" in url and not url.lower().startswith("http"): self.url = url return @@ -373,13 +435,13 @@ def prepare_url(self, url, params): raise InvalidURL(*e.args) if not scheme: - error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?") - error = error.format(to_native_string(url, 'utf8')) - - raise MissingSchema(error) + raise MissingSchema( + f"Invalid URL {url!r}: No scheme supplied. " + f"Perhaps you meant https://{url}?" + ) if not host: - raise InvalidURL("Invalid URL %r: No host supplied" % url) + raise InvalidURL(f"Invalid URL {url!r}: No host supplied") # In general, we want to try IDNA encoding the hostname if the string contains # non-ASCII characters. This allows users to automatically get the correct IDNA @@ -389,33 +451,21 @@ def prepare_url(self, url, params): try: host = self._get_idna_encoded_host(host) except UnicodeError: - raise InvalidURL('URL has an invalid label.') - elif host.startswith(u'*'): - raise InvalidURL('URL has an invalid label.') + raise InvalidURL("URL has an invalid label.") + elif host.startswith(("*", ".")): + raise InvalidURL("URL has an invalid label.") # Carefully reconstruct the network location - netloc = auth or '' + netloc = auth or "" if netloc: - netloc += '@' + netloc += "@" netloc += host if port: - netloc += ':' + str(port) + netloc += f":{port}" # Bare domains aren't valid URLs. if not path: - path = '/' - - if is_py2: - if isinstance(scheme, str): - scheme = scheme.encode('utf-8') - if isinstance(netloc, str): - netloc = netloc.encode('utf-8') - if isinstance(path, str): - path = path.encode('utf-8') - if isinstance(query, str): - query = query.encode('utf-8') - if isinstance(fragment, str): - fragment = fragment.encode('utf-8') + path = "/" if isinstance(params, (str, bytes)): params = to_native_string(params) @@ -423,7 +473,7 @@ def prepare_url(self, url, params): enc_params = self._encode_params(params) if enc_params: if query: - query = '%s&%s' % (query, enc_params) + query = f"{query}&{enc_params}" else: query = enc_params @@ -454,42 +504,51 @@ def prepare_body(self, data, files, json=None): if not data and json is not None: # urllib3 requires a bytes-like body. Python 2's json.dumps # provides this natively, but Python 3 gives a Unicode string. - content_type = 'application/json' - body = complexjson.dumps(json) - if not isinstance(body, bytes): - body = body.encode('utf-8') + content_type = "application/json" - is_stream = all([ - hasattr(data, '__iter__'), - not isinstance(data, (basestring, list, tuple, collections.Mapping)) - ]) + try: + body = complexjson.dumps(json, allow_nan=False) + except ValueError as ve: + raise InvalidJSONError(ve, request=self) - try: - length = super_len(data) - except (TypeError, AttributeError, UnsupportedOperation): - length = None + if not isinstance(body, bytes): + body = body.encode("utf-8") + + is_stream = all( + [ + hasattr(data, "__iter__"), + not isinstance(data, (basestring, list, tuple, Mapping)), + ] + ) if is_stream: + try: + length = super_len(data) + except (TypeError, AttributeError, UnsupportedOperation): + length = None + body = data - if getattr(body, 'tell', None) is not None: + if getattr(body, "tell", None) is not None: # Record the current file position before reading. # This will allow us to rewind a file in the event # of a redirect. try: self._body_position = body.tell() - except (IOError, OSError): + except OSError: # This differentiates from None, allowing us to catch # a failed `tell()` later when trying to rewind the body self._body_position = object() if files: - raise NotImplementedError('Streamed bodies and files are mutually exclusive.') + raise NotImplementedError( + "Streamed bodies and files are mutually exclusive." + ) if length: - self.headers['Content-Length'] = builtin_str(length) + self.headers["Content-Length"] = builtin_str(length) else: - self.headers['Transfer-Encoding'] = 'chunked' + self.headers["Transfer-Encoding"] = "chunked" else: # Multi-part file uploads. if files: @@ -497,16 +556,16 @@ def prepare_body(self, data, files, json=None): else: if data: body = self._encode_params(data) - if isinstance(data, basestring) or hasattr(data, 'read'): + if isinstance(data, basestring) or hasattr(data, "read"): content_type = None else: - content_type = 'application/x-www-form-urlencoded' + content_type = "application/x-www-form-urlencoded" self.prepare_content_length(body) # Add content-type if it wasn't explicitly provided. - if content_type and ('content-type' not in self.headers): - self.headers['Content-Type'] = content_type + if content_type and ("content-type" not in self.headers): + self.headers["Content-Type"] = content_type self.body = body @@ -517,13 +576,16 @@ def prepare_content_length(self, body): if length: # If length exists, set it. Otherwise, we fallback # to Transfer-Encoding: chunked. - self.headers['Content-Length'] = builtin_str(length) - elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None: + self.headers["Content-Length"] = builtin_str(length) + elif ( + self.method not in ("GET", "HEAD") + and self.headers.get("Content-Length") is None + ): # Set Content-Length to 0 for methods that can have a body # but don't provide one. (i.e. not GET or HEAD) - self.headers['Content-Length'] = '0' + self.headers["Content-Length"] = "0" - def prepare_auth(self, auth, url=''): + def prepare_auth(self, auth, url=""): """Prepares the given HTTP auth data.""" # If no Auth is explicitly provided, extract it from the URL first. @@ -563,7 +625,7 @@ def prepare_cookies(self, cookies): cookie_header = get_cookie_header(self._cookies, self) if cookie_header is not None: - self.headers['Cookie'] = cookie_header + self.headers["Cookie"] = cookie_header def prepare_hooks(self, hooks): """Prepares the given hooks.""" @@ -575,14 +637,22 @@ def prepare_hooks(self, hooks): self.register_hook(event, hooks[event]) -class Response(object): +class Response: """The :class:`Response ` object, which contains a server's response to an HTTP request. """ __attrs__ = [ - '_content', 'status_code', 'headers', 'url', 'history', - 'encoding', 'reason', 'cookies', 'elapsed', 'request' + "_content", + "status_code", + "headers", + "url", + "history", + "encoding", + "reason", + "cookies", + "elapsed", + "request", ] def __init__(self): @@ -600,7 +670,7 @@ def __init__(self): #: File-like object representation of response (for advanced usage). #: Use of ``raw`` requires that ``stream=True`` be set on the request. - # This requirement does not apply for use internally to Requests. + #: This requirement does not apply for use internally to Requests. self.raw = None #: Final URL location of Response. @@ -644,21 +714,18 @@ def __getstate__(self): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): setattr(self, name, value) # pickled objects do not have .raw - setattr(self, '_content_consumed', True) - setattr(self, 'raw', None) + setattr(self, "_content_consumed", True) + setattr(self, "raw", None) def __repr__(self): - return '' % (self.status_code) + return f"" def __bool__(self): """Returns True if :attr:`status_code` is less than 400. @@ -704,12 +771,15 @@ def is_redirect(self): """True if this Response is a well-formed HTTP redirect that could have been processed automatically (by :meth:`Session.resolve_redirects`). """ - return ('location' in self.headers and self.status_code in REDIRECT_STATI) + return "location" in self.headers and self.status_code in REDIRECT_STATI @property def is_permanent_redirect(self): """True if this Response one of the permanent versions of redirect.""" - return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect)) + return "location" in self.headers and self.status_code in ( + codes.moved_permanently, + codes.permanent_redirect, + ) @property def next(self): @@ -718,8 +788,13 @@ def next(self): @property def apparent_encoding(self): - """The apparent encoding, provided by the chardet library.""" - return chardet.detect(self.content)['encoding'] + """The apparent encoding, provided by the charset_normalizer or chardet libraries.""" + if chardet is not None: + return chardet.detect(self.content)["encoding"] + else: + # If no character detection library is available, we'll fall back + # to a standard Python utf-8 str. + return "utf-8" def iter_content(self, chunk_size=1, decode_unicode=False): """Iterates over the response data. When stream=True is set on the @@ -740,16 +815,17 @@ def iter_content(self, chunk_size=1, decode_unicode=False): def generate(): # Special case for urllib3. - if hasattr(self.raw, 'stream'): + if hasattr(self.raw, "stream"): try: - for chunk in self.raw.stream(chunk_size, decode_content=True): - yield chunk + yield from self.raw.stream(chunk_size, decode_content=True) except ProtocolError as e: raise ChunkedEncodingError(e) except DecodeError as e: raise ContentDecodingError(e) except ReadTimeoutError as e: raise ConnectionError(e) + except SSLError as e: + raise RequestsSSLError(e) else: # Standard file-like object. while True: @@ -763,7 +839,9 @@ def generate(): if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() elif chunk_size is not None and not isinstance(chunk_size, int): - raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) + raise TypeError( + f"chunk_size must be an int, it is instead a {type(chunk_size)}." + ) # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) @@ -776,7 +854,9 @@ def generate(): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): + def iter_lines( + self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None + ): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. @@ -786,8 +866,9 @@ def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter= pending = None - for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode): - + for chunk in self.iter_content( + chunk_size=chunk_size, decode_unicode=decode_unicode + ): if pending is not None: chunk = pending + chunk @@ -801,8 +882,7 @@ def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter= else: pending = None - for line in lines: - yield line + yield from lines if pending is not None: yield pending @@ -814,13 +894,12 @@ def content(self): if self._content is False: # Read the contents. if self._content_consumed: - raise RuntimeError( - 'The content for this response was already consumed') + raise RuntimeError("The content for this response was already consumed") if self.status_code == 0 or self.raw is None: self._content = None else: - self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes() + self._content = b"".join(self.iter_content(CONTENT_CHUNK_SIZE)) or b"" self._content_consumed = True # don't need to release the connection; that's been handled by urllib3 @@ -832,7 +911,7 @@ def text(self): """Content of the response, in unicode. If Response.encoding is None, encoding will be guessed using - ``chardet``. + ``charset_normalizer`` or ``chardet``. The encoding of the response content is determined based solely on HTTP headers, following RFC 2616 to the letter. If you can take advantage of @@ -845,7 +924,7 @@ def text(self): encoding = self.encoding if not self.content: - return str('') + return "" # Fallback to auto-detected encoding. if self.encoding is None: @@ -853,7 +932,7 @@ def text(self): # Decode unicode from given encoding. try: - content = str(self.content, encoding, errors='replace') + content = str(self.content, encoding, errors="replace") except (LookupError, TypeError): # A LookupError is raised if the encoding was not found which could # indicate a misspelling or similar mistake. @@ -861,75 +940,87 @@ def text(self): # A TypeError can be raised if encoding is None # # So we try blindly encoding. - content = str(self.content, errors='replace') + content = str(self.content, errors="replace") return content def json(self, **kwargs): - r"""Returns the json-encoded content of a response, if any. + r"""Decodes the JSON response body (if any) as a Python object. + + This may return a dictionary, list, etc. depending on what is in the response. :param \*\*kwargs: Optional arguments that ``json.loads`` takes. - :raises ValueError: If the response body does not contain valid json. + :raises requests.exceptions.JSONDecodeError: If the response body does not + contain valid json. """ if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or - # decoding fails, fall back to `self.text` (using chardet to make + # decoding fails, fall back to `self.text` (using charset_normalizer to make # a best guess). encoding = guess_json_utf(self.content) if encoding is not None: try: - return complexjson.loads( - self.content.decode(encoding), **kwargs - ) + return complexjson.loads(self.content.decode(encoding), **kwargs) except UnicodeDecodeError: # Wrong UTF codec detected; usually because it's not UTF-8 # but some other 8-bit codec. This is an RFC violation, # and the server didn't bother to tell us what codec *was* # used. pass - return complexjson.loads(self.text, **kwargs) + except JSONDecodeError as e: + raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) + + try: + return complexjson.loads(self.text, **kwargs) + except JSONDecodeError as e: + # Catch JSON-related errors and raise as requests.JSONDecodeError + # This aliases json.JSONDecodeError and simplejson.JSONDecodeError + raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) @property def links(self): """Returns the parsed header links of the response, if any.""" - header = self.headers.get('link') + header = self.headers.get("link") - # l = MultiDict() - l = {} + resolved_links = {} if header: links = parse_header_links(header) for link in links: - key = link.get('rel') or link.get('url') - l[key] = link + key = link.get("rel") or link.get("url") + resolved_links[key] = link - return l + return resolved_links def raise_for_status(self): - """Raises stored :class:`HTTPError`, if one occurred.""" + """Raises :class:`HTTPError`, if one occurred.""" - http_error_msg = '' + http_error_msg = "" if isinstance(self.reason, bytes): # We attempt to decode utf-8 first because some servers # choose to localize their reason strings. If the string # isn't utf-8, we fall back to iso-8859-1 for all other # encodings. (See PR #3538) try: - reason = self.reason.decode('utf-8') + reason = self.reason.decode("utf-8") except UnicodeDecodeError: - reason = self.reason.decode('iso-8859-1') + reason = self.reason.decode("iso-8859-1") else: reason = self.reason if 400 <= self.status_code < 500: - http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url) + http_error_msg = ( + f"{self.status_code} Client Error: {reason} for url: {self.url}" + ) elif 500 <= self.status_code < 600: - http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url) + http_error_msg = ( + f"{self.status_code} Server Error: {reason} for url: {self.url}" + ) if http_error_msg: raise HTTPError(http_error_msg, response=self) @@ -943,6 +1034,6 @@ def close(self): if not self._content_consumed: self.raw.close() - release_conn = getattr(self.raw, 'release_conn', None) + release_conn = getattr(self.raw, "release_conn", None) if release_conn is not None: release_conn() diff --git a/src/requests/packages.py b/src/requests/packages.py new file mode 100644 index 0000000000..5ab3d8e250 --- /dev/null +++ b/src/requests/packages.py @@ -0,0 +1,23 @@ +import sys + +from .compat import chardet + +# This code exists for backwards compatibility reasons. +# I don't like it either. Just look the other way. :) + +for package in ("urllib3", "idna"): + locals()[package] = __import__(package) + # This traversal is apparently necessary such that the identities are + # preserved (requests.packages.urllib3.* is urllib3.*) + for mod in list(sys.modules): + if mod == package or mod.startswith(f"{package}."): + sys.modules[f"requests.packages.{mod}"] = sys.modules[mod] + +if chardet is not None: + target = chardet.__name__ + for mod in list(sys.modules): + if mod == target or mod.startswith(f"{target}."): + imported_mod = sys.modules[mod] + sys.modules[f"requests.packages.{mod}"] = imported_mod + mod = mod.replace(target, "chardet") + sys.modules[f"requests.packages.{mod}"] = imported_mod diff --git a/requests/sessions.py b/src/requests/sessions.py similarity index 68% rename from requests/sessions.py rename to src/requests/sessions.py index 4ffed46a29..731550de88 100644 --- a/requests/sessions.py +++ b/src/requests/sessions.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- - """ -requests.session -~~~~~~~~~~~~~~~~ +requests.sessions +~~~~~~~~~~~~~~~~~ This module provides a Session object to manage and persist settings across requests (cookies, auth, proxies). @@ -10,39 +8,52 @@ import os import sys import time -from collections import Mapping +from collections import OrderedDict from datetime import timedelta +from ._internal_utils import to_native_string +from .adapters import HTTPAdapter from .auth import _basic_auth_str -from .compat import cookielib, is_py3, OrderedDict, urljoin, urlparse +from .compat import Mapping, cookielib, urljoin, urlparse from .cookies import ( - cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) -from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT -from .hooks import default_hooks, dispatch_hook -from ._internal_utils import to_native_string -from .utils import to_key_val_list, default_headers + RequestsCookieJar, + cookiejar_from_dict, + extract_cookies_to_jar, + merge_cookies, +) from .exceptions import ( - TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) - -from .structures import CaseInsensitiveDict -from .adapters import HTTPAdapter - -from .utils import ( - requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies, - get_auth_from_url, rewind_body + ChunkedEncodingError, + ContentDecodingError, + InvalidSchema, + TooManyRedirects, ) - -from .status_codes import codes +from .hooks import default_hooks, dispatch_hook # formerly defined here, reexposed here for backward compatibility -from .models import REDIRECT_STATI +from .models import ( # noqa: F401 + DEFAULT_REDIRECT_LIMIT, + REDIRECT_STATI, + PreparedRequest, + Request, +) +from .status_codes import codes +from .structures import CaseInsensitiveDict +from .utils import ( # noqa: F401 + DEFAULT_PORTS, + default_headers, + get_auth_from_url, + get_environ_proxies, + get_netrc_auth, + requote_uri, + resolve_proxies, + rewind_body, + should_bypass_proxies, + to_key_val_list, +) # Preferred clock, based on which one is more accurate on a given system. -if sys.platform == 'win32': - try: # Python 3.4+ - preferred_clock = time.perf_counter - except AttributeError: # Earlier than Python 3. - preferred_clock = time.clock +if sys.platform == "win32": + preferred_clock = time.perf_counter else: preferred_clock = time.time @@ -61,8 +72,7 @@ def merge_setting(request_setting, session_setting, dict_class=OrderedDict): # Bypass if not a dictionary (e.g. verify) if not ( - isinstance(session_setting, Mapping) and - isinstance(request_setting, Mapping) + isinstance(session_setting, Mapping) and isinstance(request_setting, Mapping) ): return request_setting @@ -84,17 +94,16 @@ def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict): This is necessary because when request_hooks == {'response': []}, the merge breaks Session hooks entirely. """ - if session_hooks is None or session_hooks.get('response') == []: + if session_hooks is None or session_hooks.get("response") == []: return request_hooks - if request_hooks is None or request_hooks.get('response') == []: + if request_hooks is None or request_hooks.get("response") == []: return session_hooks return merge_setting(request_hooks, session_hooks, dict_class) -class SessionRedirectMixin(object): - +class SessionRedirectMixin: def get_redirect_target(self, resp): """Receives a Response. Returns a redirect URI or ``None``""" # Due to the nature of how requests processes redirects this method will @@ -104,20 +113,61 @@ def get_redirect_target(self, resp): # to cache the redirect location onto the response object as a private # attribute. if resp.is_redirect: - location = resp.headers['location'] + location = resp.headers["location"] # Currently the underlying http module on py3 decode headers # in latin1, but empirical evidence suggests that latin1 is very # rarely used with non-ASCII characters in HTTP headers. # It is more likely to get UTF8 header rather than latin1. # This causes incorrect handling of UTF8 encoded location headers. # To solve this, we re-encode the location in latin1. - if is_py3: - location = location.encode('latin1') - return to_native_string(location, 'utf8') + location = location.encode("latin1") + return to_native_string(location, "utf8") return None - def resolve_redirects(self, resp, req, stream=False, timeout=None, - verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): + def should_strip_auth(self, old_url, new_url): + """Decide whether Authorization header should be removed when redirecting""" + old_parsed = urlparse(old_url) + new_parsed = urlparse(new_url) + if old_parsed.hostname != new_parsed.hostname: + return True + # Special case: allow http -> https redirect when using the standard + # ports. This isn't specified by RFC 7235, but is kept to avoid + # breaking backwards compatibility with older versions of requests + # that allowed any redirects on the same host. + if ( + old_parsed.scheme == "http" + and old_parsed.port in (80, None) + and new_parsed.scheme == "https" + and new_parsed.port in (443, None) + ): + return False + + # Handle default port usage corresponding to scheme. + changed_port = old_parsed.port != new_parsed.port + changed_scheme = old_parsed.scheme != new_parsed.scheme + default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None) + if ( + not changed_scheme + and old_parsed.port in default_port + and new_parsed.port in default_port + ): + return False + + # Standard case: root URI must match + return changed_port or changed_scheme + + def resolve_redirects( + self, + resp, + req, + stream=False, + timeout=None, + verify=True, + cert=None, + proxies=None, + yield_requests=False, + **adapter_kwargs, + ): """Receives a Response. Returns a generator of Responses or Requests.""" hist = [] # keep track of history @@ -138,19 +188,21 @@ def resolve_redirects(self, resp, req, stream=False, timeout=None, resp.raw.read(decode_content=False) if len(resp.history) >= self.max_redirects: - raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=resp) + raise TooManyRedirects( + f"Exceeded {self.max_redirects} redirects.", response=resp + ) # Release the connection back into the pool. resp.close() # Handle redirection without scheme (see: RFC 1808 Section 4) - if url.startswith('//'): + if url.startswith("//"): parsed_rurl = urlparse(resp.url) - url = '%s:%s' % (to_native_string(parsed_rurl.scheme), url) + url = ":".join([to_native_string(parsed_rurl.scheme), url]) # Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2) parsed = urlparse(url) - if parsed.fragment == '' and previous_fragment: + if parsed.fragment == "" and previous_fragment: parsed = parsed._replace(fragment=previous_fragment) elif parsed.fragment: previous_fragment = parsed.fragment @@ -168,19 +220,19 @@ def resolve_redirects(self, resp, req, stream=False, timeout=None, self.rebuild_method(prepared_request, resp) - # https://github.com/requests/requests/issues/1084 - if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect): - # https://github.com/requests/requests/issues/3490 - purged_headers = ('Content-Length', 'Content-Type', 'Transfer-Encoding') + # https://github.com/psf/requests/issues/1084 + if resp.status_code not in ( + codes.temporary_redirect, + codes.permanent_redirect, + ): + # https://github.com/psf/requests/issues/3490 + purged_headers = ("Content-Length", "Content-Type", "Transfer-Encoding") for header in purged_headers: prepared_request.headers.pop(header, None) prepared_request.body = None headers = prepared_request.headers - try: - del headers['Cookie'] - except KeyError: - pass + headers.pop("Cookie", None) # Extract any cookies sent on the response to the cookiejar # in the new request. Because we've mutated our copied prepared @@ -196,9 +248,8 @@ def resolve_redirects(self, resp, req, stream=False, timeout=None, # A failed tell() sets `_body_position` to `object()`. This non-None # value ensures `rewindable` will be True, allowing us to raise an # UnrewindableBodyError, instead of hanging the connection. - rewindable = ( - prepared_request._body_position is not None and - ('Content-Length' in headers or 'Transfer-Encoding' in headers) + rewindable = prepared_request._body_position is not None and ( + "Content-Length" in headers or "Transfer-Encoding" in headers ) # Attempt to rewind consumed file-like object. @@ -211,7 +262,6 @@ def resolve_redirects(self, resp, req, stream=False, timeout=None, if yield_requests: yield req else: - resp = self.send( req, stream=stream, @@ -220,7 +270,7 @@ def resolve_redirects(self, resp, req, stream=False, timeout=None, cert=cert, proxies=proxies, allow_redirects=False, - **adapter_kwargs + **adapter_kwargs, ) extract_cookies_to_jar(self.cookies, prepared_request, resp.raw) @@ -237,22 +287,18 @@ def rebuild_auth(self, prepared_request, response): headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: + if "Authorization" in headers and self.should_strip_auth( + response.request.url, url + ): # If we get redirected to a new host, we should strip out any # authentication headers. - original_parsed = urlparse(response.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] + del headers["Authorization"] # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None if new_auth is not None: prepared_request.prepare_auth(new_auth) - return - def rebuild_proxies(self, prepared_request, proxies): """This method re-evaluates the proxy configuration by considering the environment variables. If we are redirected to a URL covered by @@ -265,32 +311,22 @@ def rebuild_proxies(self, prepared_request, proxies): :rtype: dict """ - proxies = proxies if proxies is not None else {} headers = prepared_request.headers - url = prepared_request.url - scheme = urlparse(url).scheme - new_proxies = proxies.copy() - no_proxy = proxies.get('no_proxy') - - bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy) - if self.trust_env and not bypass_proxy: - environ_proxies = get_environ_proxies(url, no_proxy=no_proxy) - - proxy = environ_proxies.get(scheme, environ_proxies.get('all')) + scheme = urlparse(prepared_request.url).scheme + new_proxies = resolve_proxies(prepared_request, proxies, self.trust_env) - if proxy: - new_proxies.setdefault(scheme, proxy) - - if 'Proxy-Authorization' in headers: - del headers['Proxy-Authorization'] + if "Proxy-Authorization" in headers: + del headers["Proxy-Authorization"] try: username, password = get_auth_from_url(new_proxies[scheme]) except KeyError: username, password = None, None - if username and password: - headers['Proxy-Authorization'] = _basic_auth_str(username, password) + # urllib3 handles proxy authorization for us in the standard adapter. + # Avoid appending this to TLS tunneled requests where it may be leaked. + if not scheme.startswith("https") and username and password: + headers["Proxy-Authorization"] = _basic_auth_str(username, password) return new_proxies @@ -300,19 +336,19 @@ def rebuild_method(self, prepared_request, response): """ method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 - if response.status_code == codes.see_other and method != 'HEAD': - method = 'GET' + # https://tools.ietf.org/html/rfc7231#section-6.4.4 + if response.status_code == codes.see_other and method != "HEAD": + method = "GET" # Do what the browsers do, despite standards... # First, turn 302s into GETs. - if response.status_code == codes.found and method != 'HEAD': - method = 'GET' + if response.status_code == codes.found and method != "HEAD": + method = "GET" # Second, if a POST is responded to with a 301, turn it into a GET. # This bizarre behaviour is explained in Issue 1704. - if response.status_code == codes.moved and method == 'POST': - method = 'GET' + if response.status_code == codes.moved and method == "POST": + method = "GET" prepared_request.method = method @@ -326,24 +362,32 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('http://httpbin.org/get') + ... s.get('https://httpbin.org/get') """ __attrs__ = [ - 'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify', - 'cert', 'prefetch', 'adapters', 'stream', 'trust_env', - 'max_redirects', + "headers", + "cookies", + "auth", + "proxies", + "hooks", + "params", + "verify", + "cert", + "adapters", + "stream", + "trust_env", + "max_redirects", ] def __init__(self): - #: A case-insensitive dictionary of headers to be sent on each #: :class:`Request ` sent from this #: :class:`Session `. @@ -370,6 +414,13 @@ def __init__(self): self.stream = False #: SSL Verification default. + #: Defaults to `True`, requiring requests to verify the TLS certificate at the + #: remote end. + #: If verify is set to `False`, requests will accept any TLS certificate + #: presented by the server, and will ignore hostname mismatches and/or + #: expired certificates, which will make your application vulnerable to + #: man-in-the-middle (MitM) attacks. + #: Only set this to `False` for testing. self.verify = True #: SSL client certificate default, if String, path to ssl client @@ -394,8 +445,8 @@ def __init__(self): # Default connection adapters. self.adapters = OrderedDict() - self.mount('https://', HTTPAdapter()) - self.mount('http://', HTTPAdapter()) + self.mount("https://", HTTPAdapter()) + self.mount("http://", HTTPAdapter()) def __enter__(self): return self @@ -421,7 +472,8 @@ def prepare_request(self, request): # Merge with session cookies merged_cookies = merge_cookies( - merge_cookies(RequestsCookieJar(), self.cookies), cookies) + merge_cookies(RequestsCookieJar(), self.cookies), cookies + ) # Set environment's basic authentication if not explicitly set. auth = request.auth @@ -435,7 +487,9 @@ def prepare_request(self, request): files=request.files, data=request.data, json=request.json, - headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), + headers=merge_setting( + request.headers, self.headers, dict_class=CaseInsensitiveDict + ), params=merge_setting(request.params, self.params), auth=merge_setting(auth, self.auth), cookies=merged_cookies, @@ -443,10 +497,25 @@ def prepare_request(self, request): ) return p - def request(self, method, url, - params=None, data=None, headers=None, cookies=None, files=None, - auth=None, timeout=None, allow_redirects=True, proxies=None, - hooks=None, stream=None, verify=None, cert=None, json=None): + def request( + self, + method, + url, + params=None, + data=None, + headers=None, + cookies=None, + files=None, + auth=None, + timeout=None, + allow_redirects=True, + proxies=None, + hooks=None, + stream=None, + verify=None, + cert=None, + json=None, + ): """Constructs a :class:`Request `, prepares it and sends it. Returns :class:`Response ` object. @@ -454,8 +523,8 @@ def request(self, method, url, :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send - in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the @@ -466,7 +535,7 @@ def request(self, method, url, for multipart encoding upload. :param auth: (optional) Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. - :param timeout: (optional) How long to wait for the server to send + :param timeout: (optional) How many seconds to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) ` tuple. :type timeout: float or tuple @@ -474,11 +543,18 @@ def request(self, method, url, :type allow_redirects: bool :param proxies: (optional) Dictionary mapping protocol or protocol and hostname to the URL of the proxy. + :param hooks: (optional) Dictionary mapping hook name to one event or + list of events, event must be callable. :param stream: (optional) whether to immediately download the response content. Defaults to ``False``. :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path - to a CA bundle to use. Defaults to ``True``. + to a CA bundle to use. Defaults to ``True``. When set to + ``False``, requests will accept any TLS certificate presented by + the server, and will ignore hostname mismatches and/or expired + certificates, which will make your application vulnerable to + man-in-the-middle (MitM) attacks. Setting verify to ``False`` + may be useful during local development or testing. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. :rtype: requests.Response @@ -506,8 +582,8 @@ def request(self, method, url, # Send the request. send_kwargs = { - 'timeout': timeout, - 'allow_redirects': allow_redirects, + "timeout": timeout, + "allow_redirects": allow_redirects, } send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) @@ -522,8 +598,8 @@ def get(self, url, **kwargs): :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) - return self.request('GET', url, **kwargs) + kwargs.setdefault("allow_redirects", True) + return self.request("GET", url, **kwargs) def options(self, url, **kwargs): r"""Sends a OPTIONS request. Returns :class:`Response` object. @@ -533,8 +609,8 @@ def options(self, url, **kwargs): :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) - return self.request('OPTIONS', url, **kwargs) + kwargs.setdefault("allow_redirects", True) + return self.request("OPTIONS", url, **kwargs) def head(self, url, **kwargs): r"""Sends a HEAD request. Returns :class:`Response` object. @@ -544,42 +620,45 @@ def head(self, url, **kwargs): :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', False) - return self.request('HEAD', url, **kwargs) + kwargs.setdefault("allow_redirects", False) + return self.request("HEAD", url, **kwargs) def post(self, url, data=None, json=None, **kwargs): r"""Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ - return self.request('POST', url, data=data, json=json, **kwargs) + return self.request("POST", url, data=data, json=json, **kwargs) def put(self, url, data=None, **kwargs): r"""Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ - return self.request('PUT', url, data=data, **kwargs) + return self.request("PUT", url, data=data, **kwargs) def patch(self, url, data=None, **kwargs): r"""Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ - return self.request('PATCH', url, data=data, **kwargs) + return self.request("PATCH", url, data=data, **kwargs) def delete(self, url, **kwargs): r"""Sends a DELETE request. Returns :class:`Response` object. @@ -589,7 +668,7 @@ def delete(self, url, **kwargs): :rtype: requests.Response """ - return self.request('DELETE', url, **kwargs) + return self.request("DELETE", url, **kwargs) def send(self, request, **kwargs): """Send a given PreparedRequest. @@ -598,19 +677,20 @@ def send(self, request, **kwargs): """ # Set defaults that the hooks can utilize to ensure they always have # the correct parameters to reproduce the previous request. - kwargs.setdefault('stream', self.stream) - kwargs.setdefault('verify', self.verify) - kwargs.setdefault('cert', self.cert) - kwargs.setdefault('proxies', self.proxies) + kwargs.setdefault("stream", self.stream) + kwargs.setdefault("verify", self.verify) + kwargs.setdefault("cert", self.cert) + if "proxies" not in kwargs: + kwargs["proxies"] = resolve_proxies(request, self.proxies, self.trust_env) # It's possible that users might accidentally send a Request object. # Guard against that specific failure case. if isinstance(request, Request): - raise ValueError('You can only send PreparedRequests.') + raise ValueError("You can only send PreparedRequests.") # Set up variables needed for resolve_redirects and dispatching of hooks - allow_redirects = kwargs.pop('allow_redirects', True) - stream = kwargs.get('stream') + allow_redirects = kwargs.pop("allow_redirects", True) + stream = kwargs.get("stream") hooks = request.hooks # Get the appropriate adapter to use @@ -627,22 +707,23 @@ def send(self, request, **kwargs): r.elapsed = timedelta(seconds=elapsed) # Response manipulation hooks - r = dispatch_hook('response', hooks, r, **kwargs) + r = dispatch_hook("response", hooks, r, **kwargs) # Persist cookies if r.history: - # If the hooks create history then we want those cookies too for resp in r.history: extract_cookies_to_jar(self.cookies, resp.request, resp.raw) extract_cookies_to_jar(self.cookies, request, r.raw) - # Redirect resolving generator. - gen = self.resolve_redirects(r, request, **kwargs) - # Resolve redirects if allowed. - history = [resp for resp in gen] if allow_redirects else [] + if allow_redirects: + # Redirect resolving generator. + gen = self.resolve_redirects(r, request, **kwargs) + history = [resp for resp in gen] + else: + history = [] # Shuffle things around if there's history. if history: @@ -655,7 +736,9 @@ def send(self, request, **kwargs): # If redirects aren't being followed, store the response on the Request for Response.next(). if not allow_redirects: try: - r._next = next(self.resolve_redirects(r, request, yield_requests=True, **kwargs)) + r._next = next( + self.resolve_redirects(r, request, yield_requests=True, **kwargs) + ) except StopIteration: pass @@ -673,16 +756,19 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): # Gather clues from the surrounding environment. if self.trust_env: # Set environment's proxies. - no_proxy = proxies.get('no_proxy') if proxies is not None else None + no_proxy = proxies.get("no_proxy") if proxies is not None else None env_proxies = get_environ_proxies(url, no_proxy=no_proxy) - for (k, v) in env_proxies.items(): + for k, v in env_proxies.items(): proxies.setdefault(k, v) - # Look for requests environment configuration and be compatible - # with cURL. + # Look for requests environment configuration + # and be compatible with cURL. if verify is True or verify is None: - verify = (os.environ.get('REQUESTS_CA_BUNDLE') or - os.environ.get('CURL_CA_BUNDLE')) + verify = ( + os.environ.get("REQUESTS_CA_BUNDLE") + or os.environ.get("CURL_CA_BUNDLE") + or verify + ) # Merge all the kwargs. proxies = merge_setting(proxies, self.proxies) @@ -690,8 +776,7 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): verify = merge_setting(verify, self.verify) cert = merge_setting(cert, self.cert) - return {'verify': verify, 'proxies': proxies, 'stream': stream, - 'cert': cert} + return {"proxies": proxies, "stream": stream, "verify": verify, "cert": cert} def get_adapter(self, url): """ @@ -699,13 +784,12 @@ def get_adapter(self, url): :rtype: requests.adapters.BaseAdapter """ - for (prefix, adapter) in self.adapters.items(): - + for prefix, adapter in self.adapters.items(): if url.lower().startswith(prefix.lower()): return adapter # Nothing matches :-/ - raise InvalidSchema("No connection adapters were found for '%s'" % url) + raise InvalidSchema(f"No connection adapters were found for {url!r}") def close(self): """Closes all adapters and as such the session""" @@ -724,7 +808,7 @@ def mount(self, prefix, adapter): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): @@ -736,7 +820,12 @@ def session(): """ Returns a :class:`Session` for context-management. + .. deprecated:: 1.0.0 + + This method has been deprecated since version 1.0.0 and is only kept for + backwards compatibility. New code should use :class:`~requests.sessions.Session` + to create a session. This may be removed at a future date. + :rtype: Session """ - return Session() diff --git a/src/requests/status_codes.py b/src/requests/status_codes.py new file mode 100644 index 0000000000..c7945a2f06 --- /dev/null +++ b/src/requests/status_codes.py @@ -0,0 +1,128 @@ +r""" +The ``codes`` object defines a mapping from common names for HTTP statuses +to their numerical codes, accessible either as attributes or as dictionary +items. + +Example:: + + >>> import requests + >>> requests.codes['temporary_redirect'] + 307 + >>> requests.codes.teapot + 418 + >>> requests.codes['\o/'] + 200 + +Some codes have multiple names, and both upper- and lower-case versions of +the names are allowed. For example, ``codes.ok``, ``codes.OK``, and +``codes.okay`` all correspond to the HTTP status code 200. +""" + +from .structures import LookupDict + +_codes = { + # Informational. + 100: ("continue",), + 101: ("switching_protocols",), + 102: ("processing", "early-hints"), + 103: ("checkpoint",), + 122: ("uri_too_long", "request_uri_too_long"), + 200: ("ok", "okay", "all_ok", "all_okay", "all_good", "\\o/", "✓"), + 201: ("created",), + 202: ("accepted",), + 203: ("non_authoritative_info", "non_authoritative_information"), + 204: ("no_content",), + 205: ("reset_content", "reset"), + 206: ("partial_content", "partial"), + 207: ("multi_status", "multiple_status", "multi_stati", "multiple_stati"), + 208: ("already_reported",), + 226: ("im_used",), + # Redirection. + 300: ("multiple_choices",), + 301: ("moved_permanently", "moved", "\\o-"), + 302: ("found",), + 303: ("see_other", "other"), + 304: ("not_modified",), + 305: ("use_proxy",), + 306: ("switch_proxy",), + 307: ("temporary_redirect", "temporary_moved", "temporary"), + 308: ( + "permanent_redirect", + "resume_incomplete", + "resume", + ), # "resume" and "resume_incomplete" to be removed in 3.0 + # Client Error. + 400: ("bad_request", "bad"), + 401: ("unauthorized",), + 402: ("payment_required", "payment"), + 403: ("forbidden",), + 404: ("not_found", "-o-"), + 405: ("method_not_allowed", "not_allowed"), + 406: ("not_acceptable",), + 407: ("proxy_authentication_required", "proxy_auth", "proxy_authentication"), + 408: ("request_timeout", "timeout"), + 409: ("conflict",), + 410: ("gone",), + 411: ("length_required",), + 412: ("precondition_failed", "precondition"), + 413: ("request_entity_too_large", "content_too_large"), + 414: ("request_uri_too_large", "uri_too_long"), + 415: ("unsupported_media_type", "unsupported_media", "media_type"), + 416: ( + "requested_range_not_satisfiable", + "requested_range", + "range_not_satisfiable", + ), + 417: ("expectation_failed",), + 418: ("im_a_teapot", "teapot", "i_am_a_teapot"), + 421: ("misdirected_request",), + 422: ("unprocessable_entity", "unprocessable", "unprocessable_content"), + 423: ("locked",), + 424: ("failed_dependency", "dependency"), + 425: ("unordered_collection", "unordered", "too_early"), + 426: ("upgrade_required", "upgrade"), + 428: ("precondition_required", "precondition"), + 429: ("too_many_requests", "too_many"), + 431: ("header_fields_too_large", "fields_too_large"), + 444: ("no_response", "none"), + 449: ("retry_with", "retry"), + 450: ("blocked_by_windows_parental_controls", "parental_controls"), + 451: ("unavailable_for_legal_reasons", "legal_reasons"), + 499: ("client_closed_request",), + # Server Error. + 500: ("internal_server_error", "server_error", "/o\\", "✗"), + 501: ("not_implemented",), + 502: ("bad_gateway",), + 503: ("service_unavailable", "unavailable"), + 504: ("gateway_timeout",), + 505: ("http_version_not_supported", "http_version"), + 506: ("variant_also_negotiates",), + 507: ("insufficient_storage",), + 509: ("bandwidth_limit_exceeded", "bandwidth"), + 510: ("not_extended",), + 511: ("network_authentication_required", "network_auth", "network_authentication"), +} + +codes = LookupDict(name="status_codes") + + +def _init(): + for code, titles in _codes.items(): + for title in titles: + setattr(codes, title, code) + if not title.startswith(("\\", "/")): + setattr(codes, title.upper(), code) + + def doc(code): + names = ", ".join(f"``{n}``" for n in _codes[code]) + return "* %d: %s" % (code, names) + + global __doc__ + __doc__ = ( + __doc__ + "\n" + "\n".join(doc(code) for code in sorted(_codes)) + if __doc__ is not None + else None + ) + + +_init() diff --git a/requests/structures.py b/src/requests/structures.py similarity index 84% rename from requests/structures.py rename to src/requests/structures.py index 05d2b3f57b..188e13e482 100644 --- a/requests/structures.py +++ b/src/requests/structures.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ requests.structures ~~~~~~~~~~~~~~~~~~~ @@ -7,16 +5,16 @@ Data structures that power Requests. """ -import collections +from collections import OrderedDict -from .compat import OrderedDict +from .compat import Mapping, MutableMapping -class CaseInsensitiveDict(collections.MutableMapping): +class CaseInsensitiveDict(MutableMapping): """A case-insensitive ``dict``-like object. Implements all methods and operations of - ``collections.MutableMapping`` as well as dict's ``copy``. Also + ``MutableMapping`` as well as dict's ``copy``. Also provides ``lower_items``. All keys are expected to be strings. The structure remembers the @@ -64,14 +62,10 @@ def __len__(self): def lower_items(self): """Like iteritems(), but with all lowercase keys.""" - return ( - (lowerkey, keyval[1]) - for (lowerkey, keyval) - in self._store.items() - ) + return ((lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items()) def __eq__(self, other): - if isinstance(other, collections.Mapping): + if isinstance(other, Mapping): other = CaseInsensitiveDict(other) else: return NotImplemented @@ -91,10 +85,10 @@ class LookupDict(dict): def __init__(self, name=None): self.name = name - super(LookupDict, self).__init__() + super().__init__() def __repr__(self): - return '' % (self.name) + return f"" def __getitem__(self, key): # We allow fall-through here, so values default to None diff --git a/requests/utils.py b/src/requests/utils.py similarity index 63% rename from requests/utils.py rename to src/requests/utils.py index 3f50d485d9..8ab55852cc 100644 --- a/requests/utils.py +++ b/src/requests/utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ requests.utils ~~~~~~~~~~~~~~ @@ -9,7 +7,6 @@ """ import codecs -import collections import contextlib import io import os @@ -20,48 +17,79 @@ import tempfile import warnings import zipfile +from collections import OrderedDict + +from urllib3.util import make_headers, parse_url -from .__version__ import __version__ from . import certs +from .__version__ import __version__ + # to_native_string is unused here, but imported here for backwards compatibility -from ._internal_utils import to_native_string +from ._internal_utils import ( # noqa: F401 + _HEADER_VALIDATORS_BYTE, + _HEADER_VALIDATORS_STR, + HEADER_VALIDATORS, + to_native_string, +) +from .compat import ( + Mapping, + basestring, + bytes, + getproxies, + getproxies_environment, + integer_types, + is_urllib3_1, +) from .compat import parse_http_list as _parse_list_header from .compat import ( - quote, urlparse, bytes, str, OrderedDict, unquote, getproxies, - proxy_bypass, urlunparse, basestring, integer_types, is_py3, - proxy_bypass_environment, getproxies_environment) + proxy_bypass, + proxy_bypass_environment, + quote, + str, + unquote, + urlparse, + urlunparse, +) from .cookies import cookiejar_from_dict -from .structures import CaseInsensitiveDict from .exceptions import ( - InvalidURL, InvalidHeader, FileModeWarning, UnrewindableBodyError) + FileModeWarning, + InvalidHeader, + InvalidURL, + UnrewindableBodyError, +) +from .structures import CaseInsensitiveDict -NETRC_FILES = ('.netrc', '_netrc') +NETRC_FILES = (".netrc", "_netrc") DEFAULT_CA_BUNDLE_PATH = certs.where() +DEFAULT_PORTS = {"http": 80, "https": 443} + +# Ensure that ', ' is used to preserve previous delimiter behavior. +DEFAULT_ACCEPT_ENCODING = ", ".join( + re.split(r",\s*", make_headers(accept_encoding=True)["accept-encoding"]) +) -if sys.platform == 'win32': + +if sys.platform == "win32": # provide a proxy_bypass version on Windows without DNS lookups def proxy_bypass_registry(host): try: - if is_py3: - import winreg - else: - import _winreg as winreg + import winreg except ImportError: return False try: - internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, - r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') + internetSettings = winreg.OpenKey( + winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\CurrentVersion\Internet Settings", + ) # ProxyEnable could be REG_SZ or REG_DWORD, normalizing it - proxyEnable = int(winreg.QueryValueEx(internetSettings, - 'ProxyEnable')[0]) + proxyEnable = int(winreg.QueryValueEx(internetSettings, "ProxyEnable")[0]) # ProxyOverride is almost always a string - proxyOverride = winreg.QueryValueEx(internetSettings, - 'ProxyOverride')[0] - except OSError: + proxyOverride = winreg.QueryValueEx(internetSettings, "ProxyOverride")[0] + except (OSError, ValueError): return False if not proxyEnable or not proxyOverride: return False @@ -69,15 +97,17 @@ def proxy_bypass_registry(host): # make a check value list from the registry entry: replace the # '' string by the localhost entry and the corresponding # canonical entry. - proxyOverride = proxyOverride.split(';') + proxyOverride = proxyOverride.split(";") + # filter out empty strings to avoid re.match return true in the following code. + proxyOverride = filter(None, proxyOverride) # now check if we match one of the registry values. for test in proxyOverride: - if test == '': - if '.' not in host: + if test == "": + if "." not in host: return True - test = test.replace(".", r"\.") # mask dots - test = test.replace("*", r".*") # change glob sequence - test = test.replace("?", r".") # change glob char + test = test.replace(".", r"\.") # mask dots + test = test.replace("*", r".*") # change glob sequence + test = test.replace("?", r".") # change glob char if re.match(test, host, re.I): return True return False @@ -97,7 +127,7 @@ def proxy_bypass(host): # noqa def dict_to_sequence(d): """Returns an internal sequence dictionary update.""" - if hasattr(d, 'items'): + if hasattr(d, "items"): d = d.items() return d @@ -107,37 +137,47 @@ def super_len(o): total_length = None current_position = 0 - if hasattr(o, '__len__'): + if not is_urllib3_1 and isinstance(o, str): + # urllib3 2.x+ treats all strings as utf-8 instead + # of latin-1 (iso-8859-1) like http.client. + o = o.encode("utf-8") + + if hasattr(o, "__len__"): total_length = len(o) - elif hasattr(o, 'len'): + elif hasattr(o, "len"): total_length = o.len - elif hasattr(o, 'fileno'): + elif hasattr(o, "fileno"): try: fileno = o.fileno() - except io.UnsupportedOperation: + except (io.UnsupportedOperation, AttributeError): + # AttributeError is a surprising exception, seeing as how we've just checked + # that `hasattr(o, 'fileno')`. It happens for objects obtained via + # `Tarfile.extractfile()`, per issue 5229. pass else: total_length = os.fstat(fileno).st_size # Having used fstat to determine the file length, we need to # confirm that this file was opened up in binary mode. - if 'b' not in o.mode: - warnings.warn(( - "Requests has determined the content-length for this " - "request using the binary size of the file: however, the " - "file has been opened in text mode (i.e. without the 'b' " - "flag in the mode). This may lead to an incorrect " - "content-length. In Requests 3.0, support will be removed " - "for files in text mode."), - FileModeWarning + if "b" not in o.mode: + warnings.warn( + ( + "Requests has determined the content-length for this " + "request using the binary size of the file: however, the " + "file has been opened in text mode (i.e. without the 'b' " + "flag in the mode). This may lead to an incorrect " + "content-length. In Requests 3.0, support will be removed " + "for files in text mode." + ), + FileModeWarning, ) - if hasattr(o, 'tell'): + if hasattr(o, "tell"): try: current_position = o.tell() - except (OSError, IOError): + except OSError: # This can happen in some weird situations, such as when the file # is actually a special file descriptor like stdin. In this # instance, we don't know what the length is, so set it to zero and @@ -145,8 +185,8 @@ def super_len(o): if total_length is not None: current_position = total_length else: - if hasattr(o, 'seek') and total_length is None: - # StringIO and BytesIO have seek but no useable fileno + if hasattr(o, "seek") and total_length is None: + # StringIO and BytesIO have seek but no usable fileno try: # seek to end of file o.seek(0, 2) @@ -155,7 +195,7 @@ def super_len(o): # seek back to current position to support # partially read file-like objects o.seek(current_position or 0) - except (OSError, IOError): + except OSError: total_length = 0 if total_length is None: @@ -167,20 +207,19 @@ def super_len(o): def get_netrc_auth(url, raise_errors=False): """Returns the Requests tuple auth for a given url from netrc.""" + netrc_file = os.environ.get("NETRC") + if netrc_file is not None: + netrc_locations = (netrc_file,) + else: + netrc_locations = (f"~/{f}" for f in NETRC_FILES) + try: - from netrc import netrc, NetrcParseError + from netrc import NetrcParseError, netrc netrc_path = None - for f in NETRC_FILES: - try: - loc = os.path.expanduser('~/{0}'.format(f)) - except KeyError: - # os.path.expanduser can fail when $HOME is undefined and - # getpwuid fails. See http://bugs.python.org/issue20164 & - # https://github.com/requests/requests/issues/1846 - return - + for f in netrc_locations: + loc = os.path.expanduser(f) if os.path.exists(loc): netrc_path = loc break @@ -190,36 +229,29 @@ def get_netrc_auth(url, raise_errors=False): return ri = urlparse(url) - - # Strip port numbers from netloc. This weird `if...encode`` dance is - # used for Python 3.2, which doesn't support unicode literals. - splitstr = b':' - if isinstance(url, str): - splitstr = splitstr.decode('ascii') - host = ri.netloc.split(splitstr)[0] + host = ri.hostname try: _netrc = netrc(netrc_path).authenticators(host) if _netrc: # Return with login / password - login_i = (0 if _netrc[0] else 1) + login_i = 0 if _netrc[0] else 1 return (_netrc[login_i], _netrc[2]) - except (NetrcParseError, IOError): + except (NetrcParseError, OSError): # If there was a parsing error or a permissions issue reading the file, # we'll just skip netrc auth unless explicitly asked to raise errors. if raise_errors: raise - # AppEngine hackiness. + # App Engine hackiness. except (ImportError, AttributeError): pass def guess_filename(obj): """Tries to guess the filename of the given object.""" - name = getattr(obj, 'name', None) - if (name and isinstance(name, basestring) and name[0] != '<' and - name[-1] != '>'): + name = getattr(obj, "name", None) + if name and isinstance(name, basestring) and name[0] != "<" and name[-1] != ">": return os.path.basename(name) @@ -237,7 +269,11 @@ def extract_zipped_paths(path): archive, member = os.path.split(path) while archive and not os.path.exists(archive): archive, prefix = os.path.split(archive) - member = '/'.join([prefix, member]) + if not prefix: + # If we don't check for an empty prefix after the split (in other words, archive remains unchanged after the split), + # we _can_ end up in an infinite loop on a rare corner case affecting a small number of users + break + member = "/".join([prefix, member]) if not zipfile.is_zipfile(archive): return path @@ -248,13 +284,27 @@ def extract_zipped_paths(path): # we have a valid zip archive and a valid member of that archive tmp = tempfile.gettempdir() - extracted_path = os.path.join(tmp, *member.split('/')) + extracted_path = os.path.join(tmp, member.split("/")[-1]) if not os.path.exists(extracted_path): - extracted_path = zip_file.extract(member, path=tmp) - + # use read + write to avoid the creating nested folders, we only want the file, avoids mkdir racing condition + with atomic_open(extracted_path) as file_handler: + file_handler.write(zip_file.read(member)) return extracted_path +@contextlib.contextmanager +def atomic_open(filename): + """Write a file to the disk in an atomic fashion""" + tmp_descriptor, tmp_name = tempfile.mkstemp(dir=os.path.dirname(filename)) + try: + with os.fdopen(tmp_descriptor, "wb") as tmp_handler: + yield tmp_handler + os.replace(tmp_name, filename) + except BaseException: + os.remove(tmp_name) + raise + + def from_key_val_list(value): """Take an object and test to see if it can be represented as a dictionary. Unless it can not be represented as such, return an @@ -265,7 +315,9 @@ def from_key_val_list(value): >>> from_key_val_list([('key', 'val')]) OrderedDict([('key', 'val')]) >>> from_key_val_list('string') - ValueError: need more than 1 value to unpack + Traceback (most recent call last): + ... + ValueError: cannot encode objects that are not 2-tuples >>> from_key_val_list({'key': 'val'}) OrderedDict([('key', 'val')]) @@ -275,7 +327,7 @@ def from_key_val_list(value): return None if isinstance(value, (str, bytes, bool, int)): - raise ValueError('cannot encode objects that are not 2-tuples') + raise ValueError("cannot encode objects that are not 2-tuples") return OrderedDict(value) @@ -291,7 +343,9 @@ def to_key_val_list(value): >>> to_key_val_list({'key': 'val'}) [('key', 'val')] >>> to_key_val_list('string') - ValueError: cannot encode objects that are not 2-tuples. + Traceback (most recent call last): + ... + ValueError: cannot encode objects that are not 2-tuples :rtype: list """ @@ -299,9 +353,9 @@ def to_key_val_list(value): return None if isinstance(value, (str, bytes, bool, int)): - raise ValueError('cannot encode objects that are not 2-tuples') + raise ValueError("cannot encode objects that are not 2-tuples") - if isinstance(value, collections.Mapping): + if isinstance(value, Mapping): value = value.items() return list(value) @@ -364,10 +418,10 @@ def parse_dict_header(value): """ result = {} for item in _parse_list_header(value): - if '=' not in item: + if "=" not in item: result[item] = None continue - name, value = item.split('=', 1) + name, value = item.split("=", 1) if value[:1] == value[-1:] == '"': value = unquote_header_value(value[1:-1]) result[name] = value @@ -395,8 +449,8 @@ def unquote_header_value(value, is_filename=False): # replace sequence below on a UNC path has the effect of turning # the leading double slash into a single slash and then # _fix_ie_filename() doesn't work correctly. See #458. - if not is_filename or value[:2] != '\\\\': - return value.replace('\\\\', '\\').replace('\\"', '"') + if not is_filename or value[:2] != "\\\\": + return value.replace("\\\\", "\\").replace('\\"', '"') return value @@ -407,11 +461,7 @@ def dict_from_cookiejar(cj): :rtype: dict """ - cookie_dict = {} - - for cookie in cj: - cookie_dict[cookie.name] = cookie.value - + cookie_dict = {cookie.name: cookie.value for cookie in cj} return cookie_dict @@ -431,19 +481,24 @@ def get_encodings_from_content(content): :param content: bytestring to extract encodings from. """ - warnings.warn(( - 'In requests 3.0, get_encodings_from_content will be removed. For ' - 'more information, please see the discussion on issue #2266. (This' - ' warning should only appear once.)'), - DeprecationWarning) + warnings.warn( + ( + "In requests 3.0, get_encodings_from_content will be removed. For " + "more information, please see the discussion on issue #2266. (This" + " warning should only appear once.)" + ), + DeprecationWarning, + ) charset_re = re.compile(r']', flags=re.I) pragma_re = re.compile(r']', flags=re.I) xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]') - return (charset_re.findall(content) + - pragma_re.findall(content) + - xml_re.findall(content)) + return ( + charset_re.findall(content) + + pragma_re.findall(content) + + xml_re.findall(content) + ) def _parse_content_type_header(header): @@ -454,7 +509,7 @@ def _parse_content_type_header(header): parameters """ - tokens = header.split(';') + tokens = header.split(";") content_type, params = tokens[0].strip(), tokens[1:] params_dict = {} items_to_strip = "\"' " @@ -466,8 +521,8 @@ def _parse_content_type_header(header): index_of_equals = param.find("=") if index_of_equals != -1: key = param[:index_of_equals].strip(items_to_strip) - value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key] = value + value = param[index_of_equals + 1 :].strip(items_to_strip) + params_dict[key.lower()] = value return content_type, params_dict @@ -478,34 +533,37 @@ def get_encoding_from_headers(headers): :rtype: str """ - content_type = headers.get('content-type') + content_type = headers.get("content-type") if not content_type: return None content_type, params = _parse_content_type_header(content_type) - if 'charset' in params: - return params['charset'].strip("'\"") + if "charset" in params: + return params["charset"].strip("'\"") - if 'text' in content_type: - return 'ISO-8859-1' + if "text" in content_type: + return "ISO-8859-1" + + if "application/json" in content_type: + # Assume UTF-8 based on RFC 4627: https://www.ietf.org/rfc/rfc4627.txt since the charset was unset + return "utf-8" def stream_decode_response_unicode(iterator, r): - """Stream decodes a iterator.""" + """Stream decodes an iterator.""" if r.encoding is None: - for item in iterator: - yield item + yield from iterator return - decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace') + decoder = codecs.getincrementaldecoder(r.encoding)(errors="replace") for chunk in iterator: rv = decoder.decode(chunk) if rv: yield rv - rv = decoder.decode(b'', final=True) + rv = decoder.decode(b"", final=True) if rv: yield rv @@ -516,7 +574,7 @@ def iter_slices(string, slice_length): if slice_length is None or slice_length <= 0: slice_length = len(string) while pos < len(string): - yield string[pos:pos + slice_length] + yield string[pos : pos + slice_length] pos += slice_length @@ -532,11 +590,14 @@ def get_unicode_from_response(r): :rtype: str """ - warnings.warn(( - 'In requests 3.0, get_unicode_from_response will be removed. For ' - 'more information, please see the discussion on issue #2266. (This' - ' warning should only appear once.)'), - DeprecationWarning) + warnings.warn( + ( + "In requests 3.0, get_unicode_from_response will be removed. For " + "more information, please see the discussion on issue #2266. (This" + " warning should only appear once.)" + ), + DeprecationWarning, + ) tried_encodings = [] @@ -551,14 +612,15 @@ def get_unicode_from_response(r): # Fall back: try: - return str(r.content, encoding, errors='replace') + return str(r.content, encoding, errors="replace") except TypeError: return r.content # The unreserved URI characters (RFC 3986) UNRESERVED_SET = frozenset( - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~") + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~" +) def unquote_unreserved(uri): @@ -567,22 +629,22 @@ def unquote_unreserved(uri): :rtype: str """ - parts = uri.split('%') + parts = uri.split("%") for i in range(1, len(parts)): h = parts[i][0:2] if len(h) == 2 and h.isalnum(): try: c = chr(int(h, 16)) except ValueError: - raise InvalidURL("Invalid percent-escape sequence: '%s'" % h) + raise InvalidURL(f"Invalid percent-escape sequence: '{h}'") if c in UNRESERVED_SET: parts[i] = c + parts[i][2:] else: - parts[i] = '%' + parts[i] + parts[i] = f"%{parts[i]}" else: - parts[i] = '%' + parts[i] - return ''.join(parts) + parts[i] = f"%{parts[i]}" + return "".join(parts) def requote_uri(uri): @@ -615,10 +677,10 @@ def address_in_network(ip, net): :rtype: bool """ - ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0] - netaddr, bits = net.split('/') - netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0] - network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask + ipaddr = struct.unpack("=L", socket.inet_aton(ip))[0] + netaddr, bits = net.split("/") + netmask = struct.unpack("=L", socket.inet_aton(dotted_netmask(int(bits))))[0] + network = struct.unpack("=L", socket.inet_aton(netaddr))[0] & netmask return (ipaddr & netmask) == (network & netmask) @@ -629,8 +691,8 @@ def dotted_netmask(mask): :rtype: str """ - bits = 0xffffffff ^ (1 << 32 - mask) - 1 - return socket.inet_ntoa(struct.pack('>I', bits)) + bits = 0xFFFFFFFF ^ (1 << 32 - mask) - 1 + return socket.inet_ntoa(struct.pack(">I", bits)) def is_ipv4_address(string_ip): @@ -639,7 +701,7 @@ def is_ipv4_address(string_ip): """ try: socket.inet_aton(string_ip) - except socket.error: + except OSError: return False return True @@ -650,9 +712,9 @@ def is_valid_cidr(string_network): :rtype: bool """ - if string_network.count('/') == 1: + if string_network.count("/") == 1: try: - mask = int(string_network.split('/')[1]) + mask = int(string_network.split("/")[1]) except ValueError: return False @@ -660,8 +722,8 @@ def is_valid_cidr(string_network): return False try: - socket.inet_aton(string_network.split('/')[0]) - except socket.error: + socket.inet_aton(string_network.split("/")[0]) + except OSError: return False else: return False @@ -696,21 +758,27 @@ def should_bypass_proxies(url, no_proxy): :rtype: bool """ - get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) + + # Prioritize lowercase environment variables over uppercase + # to keep a consistent behaviour with other http projects (curl, wget). + def get_proxy(key): + return os.environ.get(key) or os.environ.get(key.upper()) # First check whether no_proxy is defined. If it is, check that the URL # we're getting isn't in the no_proxy list. no_proxy_arg = no_proxy if no_proxy is None: - no_proxy = get_proxy('no_proxy') + no_proxy = get_proxy("no_proxy") parsed = urlparse(url) + if parsed.hostname is None: + # URLs don't always have hostnames, e.g. file:/// urls. + return True + if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. - no_proxy = ( - host for host in no_proxy.replace(' ', '').split(',') if host - ) + no_proxy = (host for host in no_proxy.replace(" ", "").split(",") if host) if is_ipv4_address(parsed.hostname): for proxy_ip in no_proxy: @@ -724,7 +792,7 @@ def should_bypass_proxies(url, no_proxy): else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{0}'.format(parsed.port) + host_with_port += f":{parsed.port}" for host in no_proxy: if parsed.hostname.endswith(host) or host_with_port.endswith(host): @@ -732,13 +800,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. - with set_environ('no_proxy', no_proxy_arg): + with set_environ("no_proxy", no_proxy_arg): + # parsed.hostname can be `None` in cases such as a file URI. try: bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): @@ -771,13 +834,13 @@ def select_proxy(url, proxies): proxies = proxies or {} urlparts = urlparse(url) if urlparts.hostname is None: - return proxies.get(urlparts.scheme, proxies.get('all')) + return proxies.get(urlparts.scheme, proxies.get("all")) proxy_keys = [ - urlparts.scheme + '://' + urlparts.hostname, + urlparts.scheme + "://" + urlparts.hostname, urlparts.scheme, - 'all://' + urlparts.hostname, - 'all', + "all://" + urlparts.hostname, + "all", ] proxy = None for proxy_key in proxy_keys: @@ -788,25 +851,54 @@ def select_proxy(url, proxies): return proxy +def resolve_proxies(request, proxies, trust_env=True): + """This method takes proxy information from a request and configuration + input to resolve a mapping of target proxies. This will consider settings + such as NO_PROXY to strip proxy configurations. + + :param request: Request or PreparedRequest + :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs + :param trust_env: Boolean declaring whether to trust environment configs + + :rtype: dict + """ + proxies = proxies if proxies is not None else {} + url = request.url + scheme = urlparse(url).scheme + no_proxy = proxies.get("no_proxy") + new_proxies = proxies.copy() + + if trust_env and not should_bypass_proxies(url, no_proxy=no_proxy): + environ_proxies = get_environ_proxies(url, no_proxy=no_proxy) + + proxy = environ_proxies.get(scheme, environ_proxies.get("all")) + + if proxy: + new_proxies.setdefault(scheme, proxy) + return new_proxies + + def default_user_agent(name="python-requests"): """ Return a string representing the default user agent. :rtype: str """ - return '%s/%s' % (name, __version__) + return f"{name}/{__version__}" def default_headers(): """ :rtype: requests.structures.CaseInsensitiveDict """ - return CaseInsensitiveDict({ - 'User-Agent': default_user_agent(), - 'Accept-Encoding': ', '.join(('gzip', 'deflate')), - 'Accept': '*/*', - 'Connection': 'keep-alive', - }) + return CaseInsensitiveDict( + { + "User-Agent": default_user_agent(), + "Accept-Encoding": DEFAULT_ACCEPT_ENCODING, + "Accept": "*/*", + "Connection": "keep-alive", + } + ) def parse_header_links(value): @@ -819,23 +911,23 @@ def parse_header_links(value): links = [] - replace_chars = ' \'"' + replace_chars = " '\"" value = value.strip(replace_chars) if not value: return links - for val in re.split(', *<', value): + for val in re.split(", *<", value): try: - url, params = val.split(';', 1) + url, params = val.split(";", 1) except ValueError: - url, params = val, '' + url, params = val, "" - link = {'url': url.strip('<> \'"')} + link = {"url": url.strip("<> '\"")} - for param in params.split(';'): + for param in params.split(";"): try: - key, value = param.split('=') + key, value = param.split("=") except ValueError: break @@ -847,7 +939,7 @@ def parse_header_links(value): # Null bytes; no need to recreate these on each call to guess_json_utf -_null = '\x00'.encode('ascii') # encoding to ASCII for Python 3 +_null = "\x00".encode("ascii") # encoding to ASCII for Python 3 _null2 = _null * 2 _null3 = _null * 3 @@ -861,25 +953,25 @@ def guess_json_utf(data): # determine the encoding. Also detect a BOM, if present. sample = data[:4] if sample in (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE): - return 'utf-32' # BOM included + return "utf-32" # BOM included if sample[:3] == codecs.BOM_UTF8: - return 'utf-8-sig' # BOM included, MS style (discouraged) + return "utf-8-sig" # BOM included, MS style (discouraged) if sample[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE): - return 'utf-16' # BOM included + return "utf-16" # BOM included nullcount = sample.count(_null) if nullcount == 0: - return 'utf-8' + return "utf-8" if nullcount == 2: - if sample[::2] == _null2: # 1st and 3rd are null - return 'utf-16-be' + if sample[::2] == _null2: # 1st and 3rd are null + return "utf-16-be" if sample[1::2] == _null2: # 2nd and 4th are null - return 'utf-16-le' + return "utf-16-le" # Did not detect 2 valid UTF-16 ascii-range characters if nullcount == 3: if sample[:3] == _null3: - return 'utf-32-be' + return "utf-32-be" if sample[1:] == _null3: - return 'utf-32-le' + return "utf-32-le" # Did not detect a valid UTF-32 ascii-range character return None @@ -890,15 +982,27 @@ def prepend_scheme_if_needed(url, new_scheme): :rtype: str """ - scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) - - # urlparse is a finicky beast, and sometimes decides that there isn't a - # netloc present. Assume that it's being over-cautious, and switch netloc - # and path if urlparse decided there was no netloc. + parsed = parse_url(url) + scheme, auth, host, port, path, query, fragment = parsed + + # A defect in urlparse determines that there isn't a netloc present in some + # urls. We previously assumed parsing was overly cautious, and swapped the + # netloc and path. Due to a lack of tests on the original defect, this is + # maintained with parse_url for backwards compatibility. + netloc = parsed.netloc if not netloc: netloc, path = path, netloc - return urlunparse((scheme, netloc, path, params, query, fragment)) + if auth: + # parse_url doesn't provide the netloc with auth + # so we'll add it ourselves. + netloc = "@".join([auth, netloc]) + if scheme is None: + scheme = new_scheme + if path is None: + path = "" + + return urlunparse((scheme, netloc, path, "", query, fragment)) def get_auth_from_url(url): @@ -912,35 +1016,39 @@ def get_auth_from_url(url): try: auth = (unquote(parsed.username), unquote(parsed.password)) except (AttributeError, TypeError): - auth = ('', '') + auth = ("", "") return auth -# Moved outside of function to avoid recompile every call -_CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$') -_CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$') - - def check_header_validity(header): - """Verifies that header value is a string which doesn't contain - leading whitespace or return characters. This prevents unintended - header injection. + """Verifies that header parts don't contain leading whitespace + reserved characters, or return characters. :param header: tuple, in the format (name, value). """ name, value = header + _validate_header_part(header, name, 0) + _validate_header_part(header, value, 1) + - if isinstance(value, bytes): - pat = _CLEAN_HEADER_REGEX_BYTE +def _validate_header_part(header, header_part, header_validator_index): + if isinstance(header_part, str): + validator = _HEADER_VALIDATORS_STR[header_validator_index] + elif isinstance(header_part, bytes): + validator = _HEADER_VALIDATORS_BYTE[header_validator_index] else: - pat = _CLEAN_HEADER_REGEX_STR - try: - if not pat.match(value): - raise InvalidHeader("Invalid return character or leading space in header: %s" % name) - except TypeError: - raise InvalidHeader("Value for header {%s: %s} must be of type str or " - "bytes, not %s" % (name, value, type(value))) + raise InvalidHeader( + f"Header part ({header_part!r}) from {header} " + f"must be of type str or bytes, not {type(header_part)}" + ) + + if not validator.match(header_part): + header_kind = "name" if header_validator_index == 0 else "value" + raise InvalidHeader( + f"Invalid leading whitespace, reserved character(s), or return " + f"character(s) in header {header_kind}: {header_part!r}" + ) def urldefragauth(url): @@ -955,21 +1063,24 @@ def urldefragauth(url): if not netloc: netloc, path = path, netloc - netloc = netloc.rsplit('@', 1)[-1] + netloc = netloc.rsplit("@", 1)[-1] - return urlunparse((scheme, netloc, path, params, query, '')) + return urlunparse((scheme, netloc, path, params, query, "")) def rewind_body(prepared_request): """Move file pointer back to its recorded starting position so it can be read again on redirect. """ - body_seek = getattr(prepared_request.body, 'seek', None) - if body_seek is not None and isinstance(prepared_request._body_position, integer_types): + body_seek = getattr(prepared_request.body, "seek", None) + if body_seek is not None and isinstance( + prepared_request._body_position, integer_types + ): try: body_seek(prepared_request._body_position) - except (IOError, OSError): - raise UnrewindableBodyError("An error occurred when rewinding request " - "body for redirect.") + except OSError: + raise UnrewindableBodyError( + "An error occurred when rewinding request body for redirect." + ) else: raise UnrewindableBodyError("Unable to rewind request body for redirect.") diff --git a/tests/__init__.py b/tests/__init__.py index 9be94bcc06..c8561a0801 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,13 +1,14 @@ -# -*- coding: utf-8 -*- - """Requests test package initialisation.""" import warnings -import urllib3 -from urllib3.exceptions import SNIMissingWarning +try: + from urllib3.exceptions import SNIMissingWarning -# urllib3 sets SNIMissingWarning to only go off once, -# while this test suite requires it to always fire -# so that it occurs during test_requests.test_https_warnings -warnings.simplefilter('always', SNIMissingWarning) + # urllib3 1.x sets SNIMissingWarning to only go off once, + # while this test suite requires it to always fire + # so that it occurs during test_requests.test_https_warnings + warnings.simplefilter("always", SNIMissingWarning) +except ImportError: + # urllib3 2.0 removed that warning and errors out instead + SNIMissingWarning = None diff --git a/tests/certs/README.md b/tests/certs/README.md new file mode 100644 index 0000000000..4bf7002e0b --- /dev/null +++ b/tests/certs/README.md @@ -0,0 +1,10 @@ +# Testing Certificates + +This is a collection of certificates useful for testing aspects of Requests' +behaviour. + +The certificates include: + +* [expired](./expired) server certificate with a valid certificate authority +* [mtls](./mtls) provides a valid client certificate with a 2 year validity +* [valid](./valid) has a valid server certificate diff --git a/tests/certs/expired/Makefile b/tests/certs/expired/Makefile new file mode 100644 index 0000000000..d5a51da541 --- /dev/null +++ b/tests/certs/expired/Makefile @@ -0,0 +1,13 @@ +.PHONY: all clean ca server + +ca: + make -C $@ all + +server: + make -C $@ all + +all: ca server + +clean: + make -C ca clean + make -C server clean diff --git a/tests/certs/expired/README.md b/tests/certs/expired/README.md new file mode 100644 index 0000000000..f7234f8820 --- /dev/null +++ b/tests/certs/expired/README.md @@ -0,0 +1,11 @@ +# Expired Certificates and Configuration for Testing + +This has a valid certificate authority in [ca](./ca) and an invalid server +certificate in [server](./server). + +This can all be regenerated with: + +``` +make clean +make all +``` diff --git a/tests/certs/expired/ca/Makefile b/tests/certs/expired/ca/Makefile new file mode 100644 index 0000000000..098193f88d --- /dev/null +++ b/tests/certs/expired/ca/Makefile @@ -0,0 +1,13 @@ +.PHONY: all clean + +root_files = ca-private.key ca.crt + +ca-private.key: + openssl genrsa -out ca-private.key 2048 + +all: ca-private.key + openssl req -x509 -sha256 -days 7300 -key ca-private.key -out ca.crt -config ca.cnf + ln -s ca.crt cacert.pem + +clean: + rm -f cacert.pem ca.crt ca-private.key *.csr diff --git a/tests/certs/expired/ca/ca-private.key b/tests/certs/expired/ca/ca-private.key new file mode 100644 index 0000000000..8aa400e043 --- /dev/null +++ b/tests/certs/expired/ca/ca-private.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQCfZUh82dF/r9GW +89IN2vqOiMMuikIAO3SEI3+uSGCdWT13C+NnrFJ7XF/D6UGy1mvm8KfhSnapWoAk +toyPXSc/GNzJzCwZool7xE+rm/0vbu1XbUcQcqB8nQRLzTChDIGuuD8DHs7bmen1 +9sT5kZy0CIqac383cQxR8W1Fs48xEBJfuBBmyl+bz4ugPci96H4DIuAD2QvP2KKg +Gqs4yyDPSmf86k9+okOsLMQVwGnHety+TPJawCn2aCXl+rmMTSCH2sUEc81cXaVQ +Yxyf6HaqGncCs1O2MzeRbPugEzb5K4ZVM4NYtDMkxrQnZFCALf4XOma1uv5Kh6Qo +FMFHOA6tAgMBAAECgf9YadXLawbJzLx0/smE5fIVHccmCYqSlmgK46XvBjaREO8H +GZEJ8IvP4W09PiUzDbzMXLDCRouLZKevtZJB82nRlfjh9l5/2aho/nsytVO6+8yq +sfK5LNvYQ0Aey7ItosJMJ+bL1ErphHZB+J9Jz3scYaCAC5CFMC+lREVYZEEI9QD4 +P2D5QbmaSeu8jmL/H3fWHjNXWDprue3W/MIf96NZa3qJew45go4TAYYMe5i757KW +Ja40VNfmsgbz4uI9oDXaYL/NkWUaQP1lnh+Mfrm1YnBe2wsLcP/WuM5h0bYzJW/1 +ZeSrZM3fqCMW6SJyrVE1qzqvtw1xQBlrq0B6q0ECgYEA0fi4+ySFGR+mL6k5UjP1 +roREqQgKaLgdhOvD88EnO93Nl6tJ3Qk8LyzPUNbxe1/xTUEKMtglBKOoxCHJJZlg +xXnKBAQUtlmrLFKIGe+UCD+r+wfSpS6Sl7BUDmeCSczG9dPN5vnyZA4ixUke2SCC +k4Eb9Q0AHyNnbXv928r0sfkCgYEAwlZRYmGTVva6cY2YEmMrqbWy4Wxm2Zmdo+Uq +Xu1RZF9a3tGzNbGsyYdeLNY7vVZoVOm1paMJCA8ScNLFtCux2jEPqwqd1OZ8OLhA +1VF3/kYtUSdqwLzWoS1RdD6mZCAHeOE+N0pone4lt3A2o8wtpHsaDA+XSTw2rHLR +LVS+b1UCgYEAtezJ4Ze31pfMdrkpmCa69JVXpBj6Y9c6hGN+aWFuq/k22/WmTuRk +h/9MNR+3JQ1w1l3HB1ytXkKqxBz92hz1csReG3Kpu4EfxYxQriAdY7Q/P4Z8pXAf +xVwayEw439aUgIQef8UKllSFHeiH2NrJKCKSZZT5CQG06HCo+Fn1/4kCgYAYuwtY +TbqGUpefY7l6fYxM6IZ/EWB1SIs7FCq0MdctwsS5nk4EAzxN2SAu7IRlr91PEP7A +uWKo1+Is4WWva/ASKDQqPAuh0EL2pNv7SYbPoPabYTzAkkdt82puNJrQGxNYWrGk +L5/omSnLkkghyBX23IOQDVvfQf5jK6la73HckQKBgAI+iLECAkle9HvnJ3flicau +9FAU1/9pOdM+WogSanhYQ/P2rAwRiyCIkqu62/OoZR5g4kLxWqOOmVvsK3j+gs5F +FtwN7gauq06MAHnWr6qC8ZltzMsGZTVDvqSH2vgV4T1V6ovVpTBPKQ1gWtABEmpm +dyfeA6HHeRAHx8VRGpL6 +-----END PRIVATE KEY----- diff --git a/tests/certs/expired/ca/ca.cnf b/tests/certs/expired/ca/ca.cnf new file mode 100644 index 0000000000..09fcb6de1c --- /dev/null +++ b/tests/certs/expired/ca/ca.cnf @@ -0,0 +1,17 @@ +[req] +default_bits = 2048 +prompt = no +default_md = sha256 +encrypt_key = no +distinguished_name = dn +x509_extensions = v3_ca + +[dn] +C = US # country code +O = Python Software Foundation # organization +OU = python-requests # organization unit/department +CN = Self-Signed Root CA # common name / your cert name + +[v3_ca] +basicConstraints = critical, CA:true +keyUsage = critical, cRLSign, digitalSignature, keyCertSign diff --git a/tests/certs/expired/ca/ca.crt b/tests/certs/expired/ca/ca.crt new file mode 100644 index 0000000000..2c8ebd44ae --- /dev/null +++ b/tests/certs/expired/ca/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDpDCCAoygAwIBAgIUQt0yyZmppkHKNx4aXRrmD5tvjbswDQYJKoZIhvcNAQEL +BQAwajELMAkGA1UEBhMCVVMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3Vu +ZGF0aW9uMRgwFgYDVQQLDA9weXRob24tcmVxdWVzdHMxHDAaBgNVBAMME1NlbGYt +U2lnbmVkIFJvb3QgQ0EwHhcNMjUwMzI5MTM1MTQ1WhcNNDUwMzI0MTM1MTQ1WjBq +MQswCQYDVQQGEwJVUzEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRp +b24xGDAWBgNVBAsMD3B5dGhvbi1yZXF1ZXN0czEcMBoGA1UEAwwTU2VsZi1TaWdu +ZWQgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ9lSHzZ +0X+v0Zbz0g3a+o6Iwy6KQgA7dIQjf65IYJ1ZPXcL42esUntcX8PpQbLWa+bwp+FK +dqlagCS2jI9dJz8Y3MnMLBmiiXvET6ub/S9u7VdtRxByoHydBEvNMKEMga64PwMe +ztuZ6fX2xPmRnLQIippzfzdxDFHxbUWzjzEQEl+4EGbKX5vPi6A9yL3ofgMi4APZ +C8/YoqAaqzjLIM9KZ/zqT36iQ6wsxBXAacd63L5M8lrAKfZoJeX6uYxNIIfaxQRz +zVxdpVBjHJ/odqoadwKzU7YzN5Fs+6ATNvkrhlUzg1i0MyTGtCdkUIAt/hc6ZrW6 +/kqHpCgUwUc4Dq0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8E +BAMCAYYwHQYDVR0OBBYEFAhGiD3+10LBrdMW3+j/ceXMXSqcMA0GCSqGSIb3DQEB +CwUAA4IBAQBRT21cyZ0Jx0JLA2ilYTLvpMsSryGyWrCOXlmRlBt1MAhmxdTRgCmu +UB3UU2pfnrC16UeMVVS411lhzjowFXkXrjAqSUBRcetUIYHlpnGgDdUl4dV/X5kx +HxD9VUBx/QwGeyzFhjzjeN89M2v9kPnhU/kkVfcsafwYiHdC6pwN6zeZNz7JP+GS +rmI+KVpm5C+Nz6ekm3TR8rFgPIsiDTbY3qj/DNYX2+NhpU1DZfm687vhOr3Ekljx +NHNu9++STEjGpirrI8EqQnK+FP2fRJ5D82YZM0d++8tmHKpY0+FRCr8//459sgun +CojmhIobDa2NuF81Jx6Cc7lagCPG3/Ts +-----END CERTIFICATE----- diff --git a/tests/certs/expired/ca/ca.srl b/tests/certs/expired/ca/ca.srl new file mode 100644 index 0000000000..0d6f69d6fe --- /dev/null +++ b/tests/certs/expired/ca/ca.srl @@ -0,0 +1 @@ +4F36C3A7E075BA6452D10EEB81E7F189FF489B83 diff --git a/tests/certs/expired/server/Makefile b/tests/certs/expired/server/Makefile new file mode 100644 index 0000000000..79914ee1db --- /dev/null +++ b/tests/certs/expired/server/Makefile @@ -0,0 +1,16 @@ +.PHONY: all clean + +server.key: + openssl genrsa -out $@ 2048 + +server.csr: server.key + openssl req -key $< -new -out $@ -config cert.cnf + +server.pem: server.csr + openssl x509 -req -CA ../ca/ca.crt -CAkey ../ca/ca-private.key -in server.csr -outform PEM -out server.pem -days 0 -CAcreateserial + openssl x509 -in ../ca/ca.crt -outform PEM >> $@ + +all: server.pem + +clean: + rm -f server.* diff --git a/tests/certs/expired/server/cert.cnf b/tests/certs/expired/server/cert.cnf new file mode 100644 index 0000000000..a773fc679f --- /dev/null +++ b/tests/certs/expired/server/cert.cnf @@ -0,0 +1,24 @@ +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +prompt=no + +[req_distinguished_name] +C = US +ST = DE +O = Python Software Foundation +OU = python-requests +CN = localhost + +[v3_req] +# Extensions to add to a certificate request +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names + +[alt_names] +DNS.1 = *.localhost +DNS.1 = localhost +IP.1 = 127.0.0.1 +IP.2 = ::1 diff --git a/tests/certs/expired/server/server.csr b/tests/certs/expired/server/server.csr new file mode 100644 index 0000000000..d8ba3a5bf2 --- /dev/null +++ b/tests/certs/expired/server/server.csr @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIDHjCCAgYCAQAwbTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkRFMSMwIQYDVQQK +DBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UECwwPcHl0aG9uLXJl +cXVlc3RzMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDcRRwk8IU1YoNu8CHzB5Vh8HP/yLfBtU69LLZq+7rDG31JlR5s +lmcwLLoZ8opUQ5rg8JMRZ7toh5zB4Uc0B4Sg8RhQMSOZYBIJkXdHuqQkciR0vWnN +vD/5CkWEhnj4dxE7xTbDufBlxmwAthC/u72UIsZHavAyLCBqsONa7xuTiogz/d+3 +G+525JrfVr05hhJpT4Ypx5YY+ABkIOuOk/XuudbGm5SquuX6BgjmgaGtDjAuE/Rn +BnliIavCDrG0v3KGbG3Xxt7lFTu+98foForwGCbbQBraty27oZrzAtLltfIHlJRn +jPz0JA9akNDhAihxEsTUhk2d7jFszsd0Ev7DAgMBAAGgbDBqBgkqhkiG9w0BCQ4x +XTBbMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMB +MCwGA1UdEQQlMCOCCWxvY2FsaG9zdIcEfwAAAYcQAAAAAAAAAAAAAAAAAAAAATAN +BgkqhkiG9w0BAQsFAAOCAQEAVVIxJlDrgeG0bOSufgVVRqDQx7XSd15mGlT+CynM +lEFJ3Q9k98T2vRNNGVYYYZAnbdSOW9ACwWGcYm2bzIjbgZV0H2Kz0dLD/GrNuEY+ +O9j6K2toFKc57G7UUkve+N74ldq+hkR4zbb6FQmTlnL2YaPp2dv5TxdMKfHEfPNf +Bg8xpbXdoRc7CYW1ZACme+d2U063GVqQsrIfwGJ+BtE6aNo62T/oEm+G4Wy5iBay +jNv/imwf+JKQ75bTvha9YLUg2scqdYwJj8JlBw7cvkBIHW8GydA3fX4dtV9YBbFi +8RTlWhhLgCXpYbLoDGOqF6f/MuPSIGkV1wVhCUfYA+p+Gw== +-----END CERTIFICATE REQUEST----- diff --git a/tests/certs/expired/server/server.key b/tests/certs/expired/server/server.key new file mode 100644 index 0000000000..c48457b39a --- /dev/null +++ b/tests/certs/expired/server/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDcRRwk8IU1YoNu +8CHzB5Vh8HP/yLfBtU69LLZq+7rDG31JlR5slmcwLLoZ8opUQ5rg8JMRZ7toh5zB +4Uc0B4Sg8RhQMSOZYBIJkXdHuqQkciR0vWnNvD/5CkWEhnj4dxE7xTbDufBlxmwA +thC/u72UIsZHavAyLCBqsONa7xuTiogz/d+3G+525JrfVr05hhJpT4Ypx5YY+ABk +IOuOk/XuudbGm5SquuX6BgjmgaGtDjAuE/RnBnliIavCDrG0v3KGbG3Xxt7lFTu+ +98foForwGCbbQBraty27oZrzAtLltfIHlJRnjPz0JA9akNDhAihxEsTUhk2d7jFs +zsd0Ev7DAgMBAAECggEAUZjCX8a/ufJ3+OE42lQdWO4fsonS1I3LENYe70u4OC2X +QGpenmAq8pQnDpSj/KocliZYfLKeII9YGRRQcaw1S/9z/8TsSJVnqSa7dpVj1+J2 +sc43AxEw65sL/Jdp+bT169vXOTNIpBMYkDzhwH0WMemd5Pfu6c8h5RQI7Pc1knYn +nPNY848qSYCWOUjZS3QmBik/gp9X//yxVCyvxB3xVnb1cpvc952D90Va6nFIWfgN +ix4NgFgAvwIxCFpWI2z7JF8uBdAHPeFAx8pFukQpAzwhaEILlgt3WbvEob9CsdP5 +E39SUkzxiIfVM1du+hRquk9SJ/X56OSLnEjxLarSIQKBgQDyVU00yrv7lG7TJLZ5 +YyfE20A7eUtYdcfatTKfsxKO4KVV+p0mGsNnFhhKziuT5/xtVunbUNFEOaUfmTUP +dCjy4XkDbE/ufxzIm1WTSqknUluVoJRWKmLI5gbDFxu/XGRQNLxQbMK54l3N69PT +EO4kz/jqBYbd4aEmtaSx2R8JowKBgQDosUTgBGDxDE2ODPVoXc+YjReOmVHvGMKK +tA+KDBEs4spxaqhH0OFh6mewx3jA4VHbcIgM50STXrq1VL/QqYclDlY9/VWzkmqp +2Ekc4NoAl3H022pgcbmXx3qapC4Z3hokNFOlbtD9xQf9NMx6c5djTKMYTBrBBWXH +oFhSz6PIYQKBgQCMGqkydmvMffq89CLTd3JMq/4s5GmdUSsk1VHZZuy50kOEvAoT +N7H1bZ7J0Pz83Eji5jb6Z3U1nqZK6Ib20k/CbH1Mb1ifKLp5eOU27Rly9Hiiv15D +munWALe0Hy4Zqs8MWBDv5pGGasuU/F1RUB5/BgaBNoTMz2AeQzJe6Iq7RQKBgCca +Ku3OLpAzNhEp4k9wfEMxaoT/BMK+EWsHiRj0oCo/zi8y8iZnVoiCwHv3eTZIZt4O +Uf6BGof9QjjYjgc9hcVXXGy8Vpt/fkceXmLo8hlpWbAA8yZT1hFIZzT3Y/va09/D +n07MiXgrlQUay0XEiOsZ5Mpfd5t6EblzG4SG+gnhAoGAZ+shbacxhji7cUUYJLrO +9uZJCDCZiyq8yZ+lMzX1kILQwaP6VmSvF4TzKrkrCuD2HzUYcqebko/TvckeTp/a +oYC2put3zt0CHBf/keeeJwjhff19qVyE9mpZwoo7PuS5zmM7pQOLxzCyAM9MdsCz +kmnbborcfh74fkfRcwXm6G8= +-----END PRIVATE KEY----- diff --git a/tests/certs/expired/server/server.pem b/tests/certs/expired/server/server.pem new file mode 100644 index 0000000000..8304c04e49 --- /dev/null +++ b/tests/certs/expired/server/server.pem @@ -0,0 +1,44 @@ +-----BEGIN CERTIFICATE----- +MIIDpzCCAo+gAwIBAgIUTzbDp+B1umRS0Q7rgefxif9Im4EwDQYJKoZIhvcNAQEL +BQAwajELMAkGA1UEBhMCVVMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3Vu +ZGF0aW9uMRgwFgYDVQQLDA9weXRob24tcmVxdWVzdHMxHDAaBgNVBAMME1NlbGYt +U2lnbmVkIFJvb3QgQ0EwHhcNMjUwMjE3MDAzODIyWhcNMjUwMjE3MDAzODIyWjBt +MQswCQYDVQQGEwJVUzELMAkGA1UECAwCREUxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0 +d2FyZSBGb3VuZGF0aW9uMRgwFgYDVQQLDA9weXRob24tcmVxdWVzdHMxEjAQBgNV +BAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANxF +HCTwhTVig27wIfMHlWHwc//It8G1Tr0stmr7usMbfUmVHmyWZzAsuhnyilRDmuDw +kxFnu2iHnMHhRzQHhKDxGFAxI5lgEgmRd0e6pCRyJHS9ac28P/kKRYSGePh3ETvF +NsO58GXGbAC2EL+7vZQixkdq8DIsIGqw41rvG5OKiDP937cb7nbkmt9WvTmGEmlP +hinHlhj4AGQg646T9e651sablKq65foGCOaBoa0OMC4T9GcGeWIhq8IOsbS/coZs +bdfG3uUVO773x+gWivAYJttAGtq3LbuhmvMC0uW18geUlGeM/PQkD1qQ0OECKHES +xNSGTZ3uMWzOx3QS/sMCAwEAAaNCMEAwHQYDVR0OBBYEFFK1WmMqRSzUD2isk8TL +M3JfsVczMB8GA1UdIwQYMBaAFAhGiD3+10LBrdMW3+j/ceXMXSqcMA0GCSqGSIb3 +DQEBCwUAA4IBAQCekjxplL/AI32WtODgw7FpTXNXdyNy8PWEhn3ufL0MqiyseYOZ +bIa0PAecsArlKs5hXJzB7p/hu5CZdvaCButw1jyWQBySCpeJXn3FmGdTkBvhwBHv +y6npmBoy/nbLkIRNRcoLbALlfn/0iGWfmDTRblT7vRNWJmZCZCTA/+ILXJ36ItbF +3JCs3ARF6XWORuZs5Y8cNloOy2brAC/4EHnTfOZBQf8cL8CfHlcNa+nbG1j+wVNO +60e/0v9zTxa2wNzdnLBCW4rqJFJ44aGClxat5tWuypv0snA/0xrqIYWTQGXZPVT6 +rET47dfVbj1QxmW3sAyuy5PskZA9T7pOhcjR +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIUG/CTOPIQbH2BI36TyThUChQyR8wwDQYJKoZIhvcNAQEL +BQAwajELMAkGA1UEBhMCVVMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3Vu +ZGF0aW9uMRgwFgYDVQQLDA9weXRob24tcmVxdWVzdHMxHDAaBgNVBAMME1NlbGYt +U2lnbmVkIFJvb3QgQ0EwHhcNMjUwMjE3MDAzODIyWhcNNDUwMjEyMDAzODIyWjBq +MQswCQYDVQQGEwJVUzEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRp +b24xGDAWBgNVBAsMD3B5dGhvbi1yZXF1ZXN0czEcMBoGA1UEAwwTU2VsZi1TaWdu +ZWQgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ9lSHzZ +0X+v0Zbz0g3a+o6Iwy6KQgA7dIQjf65IYJ1ZPXcL42esUntcX8PpQbLWa+bwp+FK +dqlagCS2jI9dJz8Y3MnMLBmiiXvET6ub/S9u7VdtRxByoHydBEvNMKEMga64PwMe +ztuZ6fX2xPmRnLQIippzfzdxDFHxbUWzjzEQEl+4EGbKX5vPi6A9yL3ofgMi4APZ +C8/YoqAaqzjLIM9KZ/zqT36iQ6wsxBXAacd63L5M8lrAKfZoJeX6uYxNIIfaxQRz +zVxdpVBjHJ/odqoadwKzU7YzN5Fs+6ATNvkrhlUzg1i0MyTGtCdkUIAt/hc6ZrW6 +/kqHpCgUwUc4Dq0CAwEAAaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU +CEaIPf7XQsGt0xbf6P9x5cxdKpwwDQYJKoZIhvcNAQELBQADggEBAHMgyQNA3DQG +0l9eX8RMl4YNwhAXChU/wOTcvD+F4OsJcPqzy6QHFh1AbBBGOI9/HN7du+EiKwGZ +wE+69FEsSbhSv22XidPm+kHPTguWl1eKveeWrrT5MPl9F48s14lKb/8yMEa1/ryG +Iu8NQ6ZL91JbTXdkLoBDya9HZqyXjwcGkXLE8fSqTibJ7EhWS5Q3Ic7WPgUoGAum +b5ygoxqhm+SEyXC2/LAktwmFawkv1SsMeYpT790VIFqJ/TVVnUl+gQ2RjSEl2WLb +UO4Hwq4FZbWF9NrY6JVThLmbcr8eW6+UxWfiXHLw/qTRre4/3367QAUQRt7EuEsb +KOWpOS3fbsI= +-----END CERTIFICATE----- diff --git a/tests/certs/mtls/Makefile b/tests/certs/mtls/Makefile new file mode 100644 index 0000000000..399a906da7 --- /dev/null +++ b/tests/certs/mtls/Makefile @@ -0,0 +1,7 @@ +.PHONY: all clean + +all: + make -C client all + +clean: + make -C client clean diff --git a/tests/certs/mtls/README.md b/tests/certs/mtls/README.md new file mode 100644 index 0000000000..9a3df4623e --- /dev/null +++ b/tests/certs/mtls/README.md @@ -0,0 +1,4 @@ +# Certificate Examples for mTLS + +This has some generated certificates for mTLS utilization. The idea is to be +able to have testing around how Requests handles client certificates. diff --git a/tests/certs/mtls/client/Makefile b/tests/certs/mtls/client/Makefile new file mode 100644 index 0000000000..9c6c388be1 --- /dev/null +++ b/tests/certs/mtls/client/Makefile @@ -0,0 +1,16 @@ +.PHONY: all clean + +client.key: + openssl genrsa -out $@ 2048 + +client.csr: client.key + openssl req -key $< -new -out $@ -config cert.cnf + +client.pem: client.csr + openssl x509 -req -CA ./ca/ca.crt -CAkey ./ca/ca-private.key -in client.csr -outform PEM -out client.pem -days 730 -CAcreateserial + openssl x509 -in ./ca/ca.crt -outform PEM >> $@ + +all: client.pem + +clean: + rm -f client.* diff --git a/tests/certs/mtls/client/ca b/tests/certs/mtls/client/ca new file mode 120000 index 0000000000..85c8e8f2c2 --- /dev/null +++ b/tests/certs/mtls/client/ca @@ -0,0 +1 @@ +../../expired/ca/ \ No newline at end of file diff --git a/tests/certs/mtls/client/cert.cnf b/tests/certs/mtls/client/cert.cnf new file mode 100644 index 0000000000..338e2527ba --- /dev/null +++ b/tests/certs/mtls/client/cert.cnf @@ -0,0 +1,26 @@ +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +prompt=no + +[req_distinguished_name] +C = US +ST = DE +O = Python Software Foundation +OU = python-requests +CN = requests + +[v3_req] +# Extensions to add to a certificate request +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth +subjectAltName = @alt_names + +[alt_names] +DNS.1 = *.localhost +IP.1 = 127.0.0.1 +IP.2 = ::1 +URI.1 = spiffe://trust.python.org/v0/maintainer/sigmavirus24/project/requests/org/psf +URI.2 = spiffe://trust.python.org/v1/maintainer:sigmavirus24/project:requests/org:psf +URI.3 = spiffe://trust.python.org/v1/maintainer=sigmavirus24/project=requests/org=psf diff --git a/tests/certs/mtls/client/client.csr b/tests/certs/mtls/client/client.csr new file mode 100644 index 0000000000..35c8c2f325 --- /dev/null +++ b/tests/certs/mtls/client/client.csr @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEGjCCAwICAQAwbDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkRFMSMwIQYDVQQK +DBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UECwwPcHl0aG9uLXJl +cXVlc3RzMREwDwYDVQQDDAhyZXF1ZXN0czCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAJrdvu5kvCy5g6w67gczETk/u6K0UBmf6Mv0lqXCp3Voyt5fv6SI +/OPWrFV1+m4imNe5bLM7JUoUfkqkmULsjTTh7sxVLjW226vLSYOWWy7PA+OSwUTN +LjydF1dazlPedHPYdmhVShCmyoOd1pUCDQn0/4cEA/WW4mtzZImmoCf/pyAM3XAC +9RBcSSJRywOTe6n9LY6Ko0YUW94ay2M4ClVblDxswDemaAsLFuciKmi53gKx4H/l +areo8p60dgubooiMbcc4E9bzp0oJpfh7xhwKeJtCNEpOik1AiiQIZtwqmkkIHvtI +Go3SZ7WAQU9Eh2r+u3E6aSl+N0PMXK4Y4JsCAwEAAaCCAWcwggFjBgkqhkiG9w0B +CQ4xggFUMIIBUDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIFoDATBgNVHSUEDDAKBggr +BgEFBQcDAjCCAR8GA1UdEQSCARYwggESggsqLmxvY2FsaG9zdIcEfwAAAYcQAAAA +AAAAAAAAAAAAAAAAAYZNc3BpZmZlOi8vdHJ1c3QucHl0aG9uLm9yZy92MC9tYWlu +dGFpbmVyL3NpZ21hdmlydXMyNC9wcm9qZWN0L3JlcXVlc3RzL29yZy9wc2aGTXNw +aWZmZTovL3RydXN0LnB5dGhvbi5vcmcvdjEvbWFpbnRhaW5lcjpzaWdtYXZpcnVz +MjQvcHJvamVjdDpyZXF1ZXN0cy9vcmc6cHNmhk1zcGlmZmU6Ly90cnVzdC5weXRo +b24ub3JnL3YxL21haW50YWluZXI9c2lnbWF2aXJ1czI0L3Byb2plY3Q9cmVxdWVz +dHMvb3JnPXBzZjANBgkqhkiG9w0BAQsFAAOCAQEAMOwYPyq+OpMRQUAgoRjfxTQO +DiqfBzCsjUsPAacLTtebbWBx8y6TkLSb+/Qn3amq3ESo4iBqKpmVwdlAS4P486GD +9f78W3zkZ29jGcnQ+XHb7WvPvzBRoXImE276F9JGqJ+9q39Cbxzh0U2+ofBx2iGY +sSutzU0B/l/FKZRc8thuFoeKqHwVePLGD9p2+2nYI9I08QoGqEokTcFAq0tZ858F +9PdxBZYOKOMpnLZhiJ8qZo23v3ycBXPOjg5ILtQ9EzHoNEA5Mxx/mfNLKJ0NktZD +KXANLWKbXm+w9gTcBLtCPWNeqj5DIGPsHSfq/Bmjbp/o+uOJs6oq3s5rA7WrAA== +-----END CERTIFICATE REQUEST----- diff --git a/tests/certs/mtls/client/client.key b/tests/certs/mtls/client/client.key new file mode 100644 index 0000000000..b6cea90813 --- /dev/null +++ b/tests/certs/mtls/client/client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCa3b7uZLwsuYOs +Ou4HMxE5P7uitFAZn+jL9Jalwqd1aMreX7+kiPzj1qxVdfpuIpjXuWyzOyVKFH5K +pJlC7I004e7MVS41ttury0mDllsuzwPjksFEzS48nRdXWs5T3nRz2HZoVUoQpsqD +ndaVAg0J9P+HBAP1luJrc2SJpqAn/6cgDN1wAvUQXEkiUcsDk3up/S2OiqNGFFve +GstjOApVW5Q8bMA3pmgLCxbnIipoud4CseB/5Wq3qPKetHYLm6KIjG3HOBPW86dK +CaX4e8YcCnibQjRKTopNQIokCGbcKppJCB77SBqN0me1gEFPRIdq/rtxOmkpfjdD +zFyuGOCbAgMBAAECggEABHa5xyNOLTfXrMIyFDELoQvOO71YxbRPQHm3UeXPb9nq +Zwh5fKOaLnMEmp4A7uW+ZBFrKatdwsnebgZaiIxK8ahFesxFvbScllIQt2NBE5NR ++GBFg9cqKwMYJiNu6Qnzb1dg6lbzAJHeKncFNVxOxeey6dBa0NxdgF1eG32bBiwT +yIr6JO7cK5JfcExls5yxdZMZiW08quaeFMQR6Wocod2caDgJUBVbZwivEoNI3ak4 +/kzQaxvRoDMM8uN3LrVH2YUJgURUxSbpOLu8ycZtfg8JRArprUYI0P0OD6iDlly1 +3FhJXqBEY0dWQPmZKP/Elt3ywC1oncW+AqPh9cchiQKBgQDMriOKoBrTmplw0kbo +EEuxTU1A23BqcZfn2+PoUww6UxOvStL/s7iupjl3g4CXw0RgIfyOuJQldLdq7mh6 +W7BpK855fH2CLy1VMdIFY1umHSgJ5Bayd5NmyKdfORUw88PTFFSARYyr5DDqufWg +jI77BMjqHtbyujQKspV+9U66ZwKBgQDBsi5YptfaqMdzh343G5zAtCiGCd3m9NZO +atEEvgq9LKqEVwvJ/FD+3QPAS1MN+ms9KcfJ3G05VUhhEsPGRKibgcE0FNA/VBDO +jS2HK6kZ1M0clC5kHmQfLZxp1q3tA5nW6zqjVFdqYzJsis7G5YxFKhnzui0lgP6V +I1Io+3QvrQKBgQCK6D+sq92o8AnkfICsq6qDCKA+PO68/pyGOUAiAoKQ7qK0W0Z5 +TMIwnRTxHCjgViAIUehx/6hjByQXiPcU2zcNGTLGVgtjl5rfb7FGANlJEg6DL+2L +bwV1QwX75OSR1U136hsy9oByg6oDEvM040+B4gxsf0OHdYEuJWa5w8eLTwKBgCJt +CM+416SFWu2tp0EkJzgYzRsFpermmTBWy8+L91yoE6Zx0iaUMdEadxA2Uwyo9WZp +hpjaFI+cGMEoFKOokE8TQMOA74JR7qrHbNAZcnSk3c+2hohE3oasFKC7By6Y9T69 +kC53TxIZj1y7TwUKx2ODmBk5fcysoJLhNDkUeBIBAoGAQ/YMeMN/pSvTDCPhubbk +9bF0SwoN6DWrzw7gzMMhHlVSPM4fjdBA+wXbhnYF7hiNan7uShtRHVhwyfj+glBz +KIkDxETjxTWx5mbo3tNf8xEMsbHZBe8KPlKxizdAOvShLvkpthTCPiyqn6XTvbs4 +vC0zZrsQWCYT4Wbm7gcOOAU= +-----END PRIVATE KEY----- diff --git a/tests/certs/mtls/client/client.pem b/tests/certs/mtls/client/client.pem new file mode 100644 index 0000000000..0e3091d0e4 --- /dev/null +++ b/tests/certs/mtls/client/client.pem @@ -0,0 +1,44 @@ +-----BEGIN CERTIFICATE----- +MIIDpjCCAo6gAwIBAgIUTzbDp+B1umRS0Q7rgefxif9Im4IwDQYJKoZIhvcNAQEL +BQAwajELMAkGA1UEBhMCVVMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3Vu +ZGF0aW9uMRgwFgYDVQQLDA9weXRob24tcmVxdWVzdHMxHDAaBgNVBAMME1NlbGYt +U2lnbmVkIFJvb3QgQ0EwHhcNMjUwMjE3MDAzODIzWhcNMjcwMjE3MDAzODIzWjBs +MQswCQYDVQQGEwJVUzELMAkGA1UECAwCREUxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0 +d2FyZSBGb3VuZGF0aW9uMRgwFgYDVQQLDA9weXRob24tcmVxdWVzdHMxETAPBgNV +BAMMCHJlcXVlc3RzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmt2+ +7mS8LLmDrDruBzMROT+7orRQGZ/oy/SWpcKndWjK3l+/pIj849asVXX6biKY17ls +szslShR+SqSZQuyNNOHuzFUuNbbbq8tJg5ZbLs8D45LBRM0uPJ0XV1rOU950c9h2 +aFVKEKbKg53WlQINCfT/hwQD9Zbia3NkiaagJ/+nIAzdcAL1EFxJIlHLA5N7qf0t +joqjRhRb3hrLYzgKVVuUPGzAN6ZoCwsW5yIqaLneArHgf+Vqt6jynrR2C5uiiIxt +xzgT1vOnSgml+HvGHAp4m0I0Sk6KTUCKJAhm3CqaSQge+0gajdJntYBBT0SHav67 +cTppKX43Q8xcrhjgmwIDAQABo0IwQDAdBgNVHQ4EFgQU3Ujw+VSuTzPgHqU+KFwO +T4t2MHswHwYDVR0jBBgwFoAUCEaIPf7XQsGt0xbf6P9x5cxdKpwwDQYJKoZIhvcN +AQELBQADggEBAEvrKXWHRRDIf26j2fH9hanx0nh+lxvI4jSWYmK0rJXZA3htEvWn +gcoUspmhmLlgmRMP88lGINMjTsogUubu2j6WF/WuKxAWWvl/hUgK8NzOwOHvByPB +lhO/rSNGdOWGlnaW1TVO4kI8w6c6LwzOCpY8WvOZLW+v7duLhKUdhaJMR9X77Tbt +ohHkyYm0gV79izaFRpA6mdYoyHOR4gyWAKaj942doU794fT4gqQacRNifl/kUbWI +lilktTLyLnmBJgrxHVBxcGe8kwnPafA1k3Gb1w7mY5BSoGKglKjutTfqR3uTjAQb +vskS4SGXmsEIjLrXYWFNHyoC3pCBXWhX6Kc= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIUG/CTOPIQbH2BI36TyThUChQyR8wwDQYJKoZIhvcNAQEL +BQAwajELMAkGA1UEBhMCVVMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3Vu +ZGF0aW9uMRgwFgYDVQQLDA9weXRob24tcmVxdWVzdHMxHDAaBgNVBAMME1NlbGYt +U2lnbmVkIFJvb3QgQ0EwHhcNMjUwMjE3MDAzODIyWhcNNDUwMjEyMDAzODIyWjBq +MQswCQYDVQQGEwJVUzEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRp +b24xGDAWBgNVBAsMD3B5dGhvbi1yZXF1ZXN0czEcMBoGA1UEAwwTU2VsZi1TaWdu +ZWQgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ9lSHzZ +0X+v0Zbz0g3a+o6Iwy6KQgA7dIQjf65IYJ1ZPXcL42esUntcX8PpQbLWa+bwp+FK +dqlagCS2jI9dJz8Y3MnMLBmiiXvET6ub/S9u7VdtRxByoHydBEvNMKEMga64PwMe +ztuZ6fX2xPmRnLQIippzfzdxDFHxbUWzjzEQEl+4EGbKX5vPi6A9yL3ofgMi4APZ +C8/YoqAaqzjLIM9KZ/zqT36iQ6wsxBXAacd63L5M8lrAKfZoJeX6uYxNIIfaxQRz +zVxdpVBjHJ/odqoadwKzU7YzN5Fs+6ATNvkrhlUzg1i0MyTGtCdkUIAt/hc6ZrW6 +/kqHpCgUwUc4Dq0CAwEAAaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU +CEaIPf7XQsGt0xbf6P9x5cxdKpwwDQYJKoZIhvcNAQELBQADggEBAHMgyQNA3DQG +0l9eX8RMl4YNwhAXChU/wOTcvD+F4OsJcPqzy6QHFh1AbBBGOI9/HN7du+EiKwGZ +wE+69FEsSbhSv22XidPm+kHPTguWl1eKveeWrrT5MPl9F48s14lKb/8yMEa1/ryG +Iu8NQ6ZL91JbTXdkLoBDya9HZqyXjwcGkXLE8fSqTibJ7EhWS5Q3Ic7WPgUoGAum +b5ygoxqhm+SEyXC2/LAktwmFawkv1SsMeYpT790VIFqJ/TVVnUl+gQ2RjSEl2WLb +UO4Hwq4FZbWF9NrY6JVThLmbcr8eW6+UxWfiXHLw/qTRre4/3367QAUQRt7EuEsb +KOWpOS3fbsI= +-----END CERTIFICATE----- diff --git a/tests/certs/valid/ca b/tests/certs/valid/ca new file mode 120000 index 0000000000..46f26c3982 --- /dev/null +++ b/tests/certs/valid/ca @@ -0,0 +1 @@ +../expired/ca \ No newline at end of file diff --git a/tests/certs/valid/server/Makefile b/tests/certs/valid/server/Makefile new file mode 100644 index 0000000000..9ce6778c0f --- /dev/null +++ b/tests/certs/valid/server/Makefile @@ -0,0 +1,16 @@ +.PHONY: all clean + +server.key: + openssl genrsa -out $@ 2048 + +server.csr: server.key + openssl req -key $< -config cert.cnf -new -out $@ + +server.pem: server.csr + openssl x509 -req -CA ../ca/ca.crt -CAkey ../ca/ca-private.key -in server.csr -outform PEM -out server.pem -extfile cert.cnf -extensions v3_ca -days 7200 -CAcreateserial + openssl x509 -in ../ca/ca.crt -outform PEM >> $@ + +all: server.pem + +clean: + rm -f server.* diff --git a/tests/certs/valid/server/cert.cnf b/tests/certs/valid/server/cert.cnf new file mode 100644 index 0000000000..f9a01cd8b4 --- /dev/null +++ b/tests/certs/valid/server/cert.cnf @@ -0,0 +1,31 @@ +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +prompt=no + +[req_distinguished_name] +C = US +ST = DE +O = Python Software Foundation +OU = python-requests +CN = localhost + +[v3_req] +# Extensions to add to a certificate request +basicConstraints = critical, CA:FALSE +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = critical, serverAuth +subjectAltName = critical, @alt_names + +[v3_ca] +# Extensions to add to a certificate request +basicConstraints = critical, CA:FALSE +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = critical, serverAuth +subjectAltName = critical, @alt_names + +[alt_names] +DNS.1 = *.localhost +DNS.1 = localhost +IP.1 = 127.0.0.1 +IP.2 = ::1 diff --git a/tests/certs/valid/server/server.csr b/tests/certs/valid/server/server.csr new file mode 100644 index 0000000000..60d082e8ef --- /dev/null +++ b/tests/certs/valid/server/server.csr @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIDKjCCAhICAQAwbTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkRFMSMwIQYDVQQK +DBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UECwwPcHl0aG9uLXJl +cXVlc3RzMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDD3OzTz9TgOiJp/STy/au8G1EDVUuP3HWKAX5ZpkWSTYZfc7FF +pgPQvBq83oh/K1+9Rw0/529N1+KeTp1i9vUBUM2wJ3EzykJP4rMFh+J/Nt6VFfJh +05pTQiHnc85l4U8Qz7fHS6Lc9EG/Yp6yDxt5OeK8QAkNYjvxVhVdpif3GlnICx1e +y1EcPxb9rslERyz0eiL6+BtVbhlSvup/rz2skvaYxoh/pP1RVwbu8VtS0it046Fm +U0TnIdsjrdFjHXNJ2JSs5g6FDB9QEFhYdDOonL4KMcAkXxBLYXpjJ5fj8/X4LFkU +FINry+Q2zdFKYsghCxlW98hmJVJTscVWznqpAgMBAAGgeDB2BgkqhkiG9w0BCQ4x +aTBnMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgWgMBYGA1UdJQEB/wQMMAoG +CCsGAQUFBwMBMC8GA1UdEQEB/wQlMCOCCWxvY2FsaG9zdIcEfwAAAYcQAAAAAAAA +AAAAAAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAQEAn8TdRWYAGL7L8JKpkX2FRMMY +EuFOEeJ2UaQLRbVtb1RhkIkMT6UYulQ16K45dbFe4k8DT59/mHrTl0bJ6pqVPawa +vo7vmKCrSYEz8GrImh1+i6lHaFiqbkAs1mJ6z86dZUofMYwfTPVh64ztamUqwBk7 +Dey+MzWUQvpTVqoaP7cLgzMy+XfcyGuWSoL6pX5XKkt4+A2wiCTsHul1sXOn4vQz +4xY96AUVnkKosXMWXvhbMkncLNH+gs+ZeQI0MekakuMa42N+0BAad3Zx60lifG3N +ugyVUmvVI0Fq6QUlYp3YV6QQj3FDjbgOVsSJNh+2s4SIARJsb/k//Tddzxei7g== +-----END CERTIFICATE REQUEST----- diff --git a/tests/certs/valid/server/server.key b/tests/certs/valid/server/server.key new file mode 100644 index 0000000000..387d5d67f1 --- /dev/null +++ b/tests/certs/valid/server/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDD3OzTz9TgOiJp +/STy/au8G1EDVUuP3HWKAX5ZpkWSTYZfc7FFpgPQvBq83oh/K1+9Rw0/529N1+Ke +Tp1i9vUBUM2wJ3EzykJP4rMFh+J/Nt6VFfJh05pTQiHnc85l4U8Qz7fHS6Lc9EG/ +Yp6yDxt5OeK8QAkNYjvxVhVdpif3GlnICx1ey1EcPxb9rslERyz0eiL6+BtVbhlS +vup/rz2skvaYxoh/pP1RVwbu8VtS0it046FmU0TnIdsjrdFjHXNJ2JSs5g6FDB9Q +EFhYdDOonL4KMcAkXxBLYXpjJ5fj8/X4LFkUFINry+Q2zdFKYsghCxlW98hmJVJT +scVWznqpAgMBAAECggEACY98I+6uJm/QBDpuFkpZmqn+r1n3gUMynZTrFPcvyC9u +krQ0AAFViFfWOkfmg8abOsMAG5FxdmxGTJHrzsvdM748/A9A0FVcHUgkku2KGcmU +3dQfa7UHgG7m9sRJW+G+mUR6ZQkFHyHxH6Vxt6FTJvyzW5sIlhWodWRNUK/unXoe +5QNIh/PM9Mkg91wKVeg8AGdY27zfuUV0KLwhV052hCSwckayi7pjYrH2xDL8fahi +Z/F43N1SI6Q7PTW6izDs2KrlGyexn97jOV+cooSXXaDvivTaB2K4RoidrPt+UAW4 +/e1zw/LbeBvQXX4vsyC9vLO+Av1gYo2PxAZ/ezc5gQKBgQDvY/kgUud6stdMs73N +Qavu9o/Ul4oeQa4fsmfUwj43OjquxpXbfV9sNTLFxFbsl5ADGkpqsR3WhP2090TK +62vkyCBiaYZjyMY6WLDATEvrpfbc74ATEzTU+r9T1rEByq/atbWPZq/zoMo8aAdg +Qk2X7gQO3pTpPeEpetYG42Di+QKBgQDRc9IqPeELQugXEuI0RZ8rsfGRSFKVRelQ +Rz/KpD/S8SsE7ERL0w/w5KgE0IPPbh/SYX2KYxafCqTrPWDNaoguROvUoUJQvf4a +uOMTCRkqdqj8j70zaPj2ohuIIZ3GZHyaXgl/isWtuZ5OeaoY5h3r+4gk5mO41Llz +YikB71SRMQKBgGBNG18BetVFNI9Kj0QO8xeCYIHpJErfqShfIJ3aNiUJa6n7gTV2 +zfg9vlsIjN9IaUqWPPGGprYxcc5m2mm3IwQ57a0pPkLN9dBq9U+mYbQ+Y3ylbCRA +SbST2nvjlflejDezeYJikM21FSYPw0fZ5FUGDuPcbpMVrYp+O7MxrTwhAoGAcjQq +xemTiWZj0iDzwfisP1D5HHRIwyepfaI7wCwquMPS5w5EduuQZ5LloipnlHTBWR7b +Kte4f+N35OREofySYFgoFnoPBKNzp/Jjrf9p/2NP5NYjHaMBDMl7JZDezEwCPNFF +cIukGYN6M+PWwVjHu+Ica7JLcX5b1/QP1ARBIiECgYAsdDa0CoYTVWMeqMq9GjF3 +BElBKCLp6iYqpElWyKTj39LCnLhzRICyYQpblM8zZgtQIvRGT8DYh2HA1Q29yPzt +Rgrz9yfFACdT5gUM5D6sx7L37xbtMQQ7YYOxEAfCUZ0qnMJgpNTJ8/ECkjGJZaif +CXrx6XCdvrM/evZW2JZbyg== +-----END PRIVATE KEY----- diff --git a/tests/certs/valid/server/server.pem b/tests/certs/valid/server/server.pem new file mode 100644 index 0000000000..b4ad4c650d --- /dev/null +++ b/tests/certs/valid/server/server.pem @@ -0,0 +1,46 @@ +-----BEGIN CERTIFICATE----- +MIIEEDCCAvigAwIBAgIUTzbDp+B1umRS0Q7rgefxif9Im4MwDQYJKoZIhvcNAQEL +BQAwajELMAkGA1UEBhMCVVMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3Vu +ZGF0aW9uMRgwFgYDVQQLDA9weXRob24tcmVxdWVzdHMxHDAaBgNVBAMME1NlbGYt +U2lnbmVkIFJvb3QgQ0EwHhcNMjUwMjE3MDAzODIzWhcNNDQxMTA0MDAzODIzWjBt +MQswCQYDVQQGEwJVUzELMAkGA1UECAwCREUxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0 +d2FyZSBGb3VuZGF0aW9uMRgwFgYDVQQLDA9weXRob24tcmVxdWVzdHMxEjAQBgNV +BAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMPc +7NPP1OA6Imn9JPL9q7wbUQNVS4/cdYoBflmmRZJNhl9zsUWmA9C8GrzeiH8rX71H +DT/nb03X4p5OnWL29QFQzbAncTPKQk/iswWH4n823pUV8mHTmlNCIedzzmXhTxDP +t8dLotz0Qb9inrIPG3k54rxACQ1iO/FWFV2mJ/caWcgLHV7LURw/Fv2uyURHLPR6 +Ivr4G1VuGVK+6n+vPayS9pjGiH+k/VFXBu7xW1LSK3TjoWZTROch2yOt0WMdc0nY +lKzmDoUMH1AQWFh0M6icvgoxwCRfEEthemMnl+Pz9fgsWRQUg2vL5DbN0UpiyCEL +GVb3yGYlUlOxxVbOeqkCAwEAAaOBqjCBpzAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB +/wQEAwIFoDAWBgNVHSUBAf8EDDAKBggrBgEFBQcDATAvBgNVHREBAf8EJTAjggls +b2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwHQYDVR0OBBYEFGg5Q9Ll +ADkONmo02kmPg9+aiOxQMB8GA1UdIwQYMBaAFAhGiD3+10LBrdMW3+j/ceXMXSqc +MA0GCSqGSIb3DQEBCwUAA4IBAQA1RlUI34tXrIdsRiALD8iLDFhh816B7qUi+F1j +3dOkTNgYw0CnQ+Vm4wrQjCEVSDQ/9sry5DOfXeGziDpFlZmQ0UuAeM1EJJD5/42l +C/eKquA09+IMEq2U03GPhijrC68sFCfr5wgoB/4HmcZ1c3kLYvRaJhEK7AHDgenY +knKbvuKAiRbCd584eG8HFOW7xv+YqGZ257Ic9flbarkPrazmAJg2P709w7/fP83x +7lnkPZY09jS3lcIpEdtWvzfm+anGF190hKA1yfPdO5bYPvUcEMxXdMtQ6/pbZd5i +F6t95o9CPCqI/lLIn5jAf9Z+Iil3GmKefEZYIGmKJ85HGUqE +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIUG/CTOPIQbH2BI36TyThUChQyR8wwDQYJKoZIhvcNAQEL +BQAwajELMAkGA1UEBhMCVVMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3Vu +ZGF0aW9uMRgwFgYDVQQLDA9weXRob24tcmVxdWVzdHMxHDAaBgNVBAMME1NlbGYt +U2lnbmVkIFJvb3QgQ0EwHhcNMjUwMjE3MDAzODIyWhcNNDUwMjEyMDAzODIyWjBq +MQswCQYDVQQGEwJVUzEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRp +b24xGDAWBgNVBAsMD3B5dGhvbi1yZXF1ZXN0czEcMBoGA1UEAwwTU2VsZi1TaWdu +ZWQgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ9lSHzZ +0X+v0Zbz0g3a+o6Iwy6KQgA7dIQjf65IYJ1ZPXcL42esUntcX8PpQbLWa+bwp+FK +dqlagCS2jI9dJz8Y3MnMLBmiiXvET6ub/S9u7VdtRxByoHydBEvNMKEMga64PwMe +ztuZ6fX2xPmRnLQIippzfzdxDFHxbUWzjzEQEl+4EGbKX5vPi6A9yL3ofgMi4APZ +C8/YoqAaqzjLIM9KZ/zqT36iQ6wsxBXAacd63L5M8lrAKfZoJeX6uYxNIIfaxQRz +zVxdpVBjHJ/odqoadwKzU7YzN5Fs+6ATNvkrhlUzg1i0MyTGtCdkUIAt/hc6ZrW6 +/kqHpCgUwUc4Dq0CAwEAAaMyMDAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU +CEaIPf7XQsGt0xbf6P9x5cxdKpwwDQYJKoZIhvcNAQELBQADggEBAHMgyQNA3DQG +0l9eX8RMl4YNwhAXChU/wOTcvD+F4OsJcPqzy6QHFh1AbBBGOI9/HN7du+EiKwGZ +wE+69FEsSbhSv22XidPm+kHPTguWl1eKveeWrrT5MPl9F48s14lKb/8yMEa1/ryG +Iu8NQ6ZL91JbTXdkLoBDya9HZqyXjwcGkXLE8fSqTibJ7EhWS5Q3Ic7WPgUoGAum +b5ygoxqhm+SEyXC2/LAktwmFawkv1SsMeYpT790VIFqJ/TVVnUl+gQ2RjSEl2WLb +UO4Hwq4FZbWF9NrY6JVThLmbcr8eW6+UxWfiXHLw/qTRre4/3367QAUQRt7EuEsb +KOWpOS3fbsI= +-----END CERTIFICATE----- diff --git a/tests/compat.py b/tests/compat.py index f68e801444..7618aa157b 100644 --- a/tests/compat.py +++ b/tests/compat.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- - -from requests.compat import is_py3 - +import warnings try: import StringIO @@ -13,9 +10,14 @@ except ImportError: cStringIO = None -if is_py3: - def u(s): - return s -else: - def u(s): - return s.decode('unicode-escape') + +def u(s): + warnings.warn( + ( + "This helper function is no longer relevant in Python 3. " + "Usage of this alias should be discontinued as it will be " + "removed in a future release of Requests." + ), + DeprecationWarning, + ) + return s diff --git a/tests/conftest.py b/tests/conftest.py index cd64a7656a..530a4c2a5f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,15 +1,23 @@ -# -*- coding: utf-8 -*- +try: + from http.server import HTTPServer, SimpleHTTPRequestHandler +except ImportError: + from BaseHTTPServer import HTTPServer + from SimpleHTTPServer import SimpleHTTPRequestHandler + +import ssl +import threading import pytest + from requests.compat import urljoin def prepare_url(value): # Issue #1483: Make sure the URL always has a trailing slash - httpbin_url = value.url.rstrip('/') + '/' + httpbin_url = value.url.rstrip("/") + "/" def inner(*suffix): - return urljoin(httpbin_url, '/'.join(suffix)) + return urljoin(httpbin_url, "/".join(suffix)) return inner @@ -22,3 +30,29 @@ def httpbin(httpbin): @pytest.fixture def httpbin_secure(httpbin_secure): return prepare_url(httpbin_secure) + + +@pytest.fixture +def nosan_server(tmp_path_factory): + # delay importing until the fixture in order to make it possible + # to deselect the test via command-line when trustme is not available + import trustme + + tmpdir = tmp_path_factory.mktemp("certs") + ca = trustme.CA() + # only commonName, no subjectAltName + server_cert = ca.issue_cert(common_name="localhost") + ca_bundle = str(tmpdir / "ca.pem") + ca.cert_pem.write_to_path(ca_bundle) + + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + server_cert.configure_cert(context) + server = HTTPServer(("localhost", 0), SimpleHTTPRequestHandler) + server.socket = context.wrap_socket(server.socket, server_side=True) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.start() + + yield "localhost", server.server_address[1], ca_bundle + + server.shutdown() + server_thread.join() diff --git a/tests/test_adapters.py b/tests/test_adapters.py new file mode 100644 index 0000000000..6c55d5a130 --- /dev/null +++ b/tests/test_adapters.py @@ -0,0 +1,8 @@ +import requests.adapters + + +def test_request_url_trims_leading_path_separators(): + """See also https://github.com/psf/requests/issues/6643.""" + a = requests.adapters.HTTPAdapter() + p = requests.Request(method="GET", url="http://127.0.0.1:10000//v:h").prepare() + assert "/v:h" == a.request_url(p, {}) diff --git a/tests/test_help.py b/tests/test_help.py index c11d43f341..5fca6207ef 100644 --- a/tests/test_help.py +++ b/tests/test_help.py @@ -1,40 +1,27 @@ -# -*- encoding: utf-8 - -import sys - -import pytest +from unittest import mock from requests.help import info -@pytest.mark.skipif(sys.version_info[:2] != (2,6), reason="Only run on Python 2.6") -def test_system_ssl_py26(): - """OPENSSL_VERSION_NUMBER isn't provided in Python 2.6, verify we don't - blow up in this case. - """ - assert info()['system_ssl'] == {'version': ''} - - -@pytest.mark.skipif(sys.version_info < (2,7), reason="Only run on Python 2.7+") def test_system_ssl(): """Verify we're actually setting system_ssl when it should be available.""" - assert info()['system_ssl']['version'] != '' + assert info()["system_ssl"]["version"] != "" -class VersionedPackage(object): +class VersionedPackage: def __init__(self, version): self.__version__ = version -def test_idna_without_version_attribute(mocker): +def test_idna_without_version_attribute(): """Older versions of IDNA don't provide a __version__ attribute, verify that if we have such a package, we don't blow up. """ - mocker.patch('requests.help.idna', new=None) - assert info()['idna'] == {'version': ''} + with mock.patch("requests.help.idna", new=None): + assert info()["idna"] == {"version": ""} -def test_idna_with_version_attribute(mocker): +def test_idna_with_version_attribute(): """Verify we're actually setting idna version when it should be available.""" - mocker.patch('requests.help.idna', new=VersionedPackage('2.6')) - assert info()['idna'] == {'version': '2.6'} + with mock.patch("requests.help.idna", new=VersionedPackage("2.6")): + assert info()["idna"] == {"version": "2.6"} diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 014b439182..7445525ec8 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import pytest from requests import hooks @@ -10,14 +8,15 @@ def hook(value): @pytest.mark.parametrize( - 'hooks_list, result', ( - (hook, 'ata'), - ([hook, lambda x: None, hook], 'ta'), - ) + "hooks_list, result", + ( + (hook, "ata"), + ([hook, lambda x: None, hook], "ta"), + ), ) def test_hooks(hooks_list, result): - assert hooks.dispatch_hook('response', {'response': hooks_list}, 'Data') == result + assert hooks.dispatch_hook("response", {"response": hooks_list}, "Data") == result def test_default_hooks(): - assert hooks.default_hooks() == {'response': []} + assert hooks.default_hooks() == {"response": []} diff --git a/tests/test_lowlevel.py b/tests/test_lowlevel.py index 6d6268cd1e..859d07e8a5 100644 --- a/tests/test_lowlevel.py +++ b/tests/test_lowlevel.py @@ -1,34 +1,134 @@ -# -*- coding: utf-8 -*- - -import pytest import threading -import requests +import pytest from tests.testserver.server import Server, consume_socket_content +import requests +from requests.compat import JSONDecodeError + from .utils import override_environ +def echo_response_handler(sock): + """Simple handler that will take request and echo it back to requester.""" + request_content = consume_socket_content(sock, timeout=0.5) + + text_200 = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: %d\r\n\r\n" + b"%s" + ) % (len(request_content), request_content) + sock.send(text_200) + + def test_chunked_upload(): """can safely send generators""" close_server = threading.Event() server = Server.basic_response_server(wait_to_close_event=close_server) - data = iter([b'a', b'b', b'c']) + data = iter([b"a", b"b", b"c"]) with server as (host, port): - url = 'http://{0}:{1}/'.format(host, port) + url = f"http://{host}:{port}/" r = requests.post(url, data=data, stream=True) close_server.set() # release server block assert r.status_code == 200 - assert r.request.headers['Transfer-Encoding'] == 'chunked' + assert r.request.headers["Transfer-Encoding"] == "chunked" + + +def test_chunked_encoding_error(): + """get a ChunkedEncodingError if the server returns a bad response""" + + def incomplete_chunked_response_handler(sock): + request_content = consume_socket_content(sock, timeout=0.5) + + # The server never ends the request and doesn't provide any valid chunks + sock.send( + b"HTTP/1.1 200 OK\r\n" + b"Transfer-Encoding: chunked\r\n" + ) + + return request_content + + close_server = threading.Event() + server = Server(incomplete_chunked_response_handler) + + with server as (host, port): + url = f"http://{host}:{port}/" + with pytest.raises(requests.exceptions.ChunkedEncodingError): + requests.get(url) + close_server.set() # release server block + + +def test_chunked_upload_uses_only_specified_host_header(): + """Ensure we use only the specified Host header for chunked requests.""" + close_server = threading.Event() + server = Server(echo_response_handler, wait_to_close_event=close_server) + + data = iter([b"a", b"b", b"c"]) + custom_host = "sample-host" + + with server as (host, port): + url = f"http://{host}:{port}/" + r = requests.post(url, data=data, headers={"Host": custom_host}, stream=True) + close_server.set() # release server block + + expected_header = b"Host: %s\r\n" % custom_host.encode("utf-8") + assert expected_header in r.content + assert r.content.count(b"Host: ") == 1 + + +def test_chunked_upload_doesnt_skip_host_header(): + """Ensure we don't omit all Host headers with chunked requests.""" + close_server = threading.Event() + server = Server(echo_response_handler, wait_to_close_event=close_server) + + data = iter([b"a", b"b", b"c"]) + + with server as (host, port): + expected_host = f"{host}:{port}" + url = f"http://{host}:{port}/" + r = requests.post(url, data=data, stream=True) + close_server.set() # release server block + + expected_header = b"Host: %s\r\n" % expected_host.encode("utf-8") + assert expected_header in r.content + assert r.content.count(b"Host: ") == 1 + + +def test_conflicting_content_lengths(): + """Ensure we correctly throw an InvalidHeader error if multiple + conflicting Content-Length headers are returned. + """ + + def multiple_content_length_response_handler(sock): + request_content = consume_socket_content(sock, timeout=0.5) + response = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: 16\r\n" + b"Content-Length: 32\r\n\r\n" + b"-- Bad Actor -- Original Content\r\n" + ) + sock.send(response) + + return request_content + + close_server = threading.Event() + server = Server(multiple_content_length_response_handler) + + with server as (host, port): + url = f"http://{host}:{port}/" + with pytest.raises(requests.exceptions.InvalidHeader): + requests.get(url) + close_server.set() def test_digestauth_401_count_reset_on_redirect(): """Ensure we correctly reset num_401_calls after a successful digest auth, followed by a 302 redirect to another digest auth prompt. - See https://github.com/requests/requests/issues/1979. + See https://github.com/psf/requests/issues/1979. """ text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n' b'Content-Length: 0\r\n' @@ -77,7 +177,7 @@ def digest_response_handler(sock): server = Server(digest_response_handler, wait_to_close_event=close_server) with server as (host, port): - url = 'http://{0}:{1}/'.format(host, port) + url = f'http://{host}:{port}/' r = requests.get(url, auth=auth) # Verify server succeeded in authenticating. assert r.status_code == 200 @@ -127,7 +227,7 @@ def digest_failed_response_handler(sock): server = Server(digest_failed_response_handler, wait_to_close_event=close_server) with server as (host, port): - url = 'http://{0}:{1}/'.format(host, port) + url = f'http://{host}:{port}/' r = requests.get(url, auth=auth) # Verify server didn't authenticate us. assert r.status_code == 401 @@ -138,7 +238,7 @@ def digest_failed_response_handler(sock): def test_digestauth_only_on_4xx(): """Ensure we only send digestauth on 4xx challenges. - See https://github.com/requests/requests/issues/3772. + See https://github.com/psf/requests/issues/3772. """ text_200_chal = (b'HTTP/1.1 200 OK\r\n' b'Content-Length: 0\r\n' @@ -164,7 +264,7 @@ def digest_response_handler(sock): server = Server(digest_response_handler, wait_to_close_event=close_server) with server as (host, port): - url = 'http://{0}:{1}/'.format(host, port) + url = f'http://{host}:{port}/' r = requests.get(url, auth=auth) # Verify server didn't receive auth from us. assert r.status_code == 200 @@ -181,17 +281,17 @@ def digest_response_handler(sock): _proxy_combos = [] for prefix, schemes in _schemes_by_var_prefix: for scheme in schemes: - _proxy_combos.append(("{0}_proxy".format(prefix), scheme)) + _proxy_combos.append((f"{prefix}_proxy", scheme)) _proxy_combos += [(var.upper(), scheme) for var, scheme in _proxy_combos] @pytest.mark.parametrize("var,scheme", _proxy_combos) def test_use_proxy_from_environment(httpbin, var, scheme): - url = "{0}://httpbin.org".format(scheme) + url = f"{scheme}://httpbin.org" fake_proxy = Server() # do nothing with the requests; just close the socket with fake_proxy as (host, port): - proxy_url = "socks5://{0}:{1}".format(host, port) + proxy_url = f"socks5://{host}:{port}" kwargs = {var: proxy_url} with override_environ(**kwargs): # fake proxy's lack of response will cause a ConnectionError @@ -206,18 +306,20 @@ def test_use_proxy_from_environment(httpbin, var, scheme): def test_redirect_rfc1808_to_non_ascii_location(): - path = u'š' + path = 'š' expected_path = b'%C5%A1' redirect_request = [] # stores the second request to the server def redirect_resp_handler(sock): consume_socket_content(sock, timeout=0.5) - location = u'//{0}:{1}/{2}'.format(host, port, path) + location = f'//{host}:{port}/{path}' sock.send( - b'HTTP/1.1 301 Moved Permanently\r\n' - b'Content-Length: 0\r\n' - b'Location: ' + location.encode('utf8') + b'\r\n' - b'\r\n' + ( + b'HTTP/1.1 301 Moved Permanently\r\n' + b'Content-Length: 0\r\n' + b'Location: %s\r\n' + b'\r\n' + ) % location.encode('utf8') ) redirect_request.append(consume_socket_content(sock, timeout=0.5)) sock.send(b'HTTP/1.1 200 OK\r\n\r\n') @@ -226,31 +328,24 @@ def redirect_resp_handler(sock): server = Server(redirect_resp_handler, wait_to_close_event=close_server) with server as (host, port): - url = u'http://{0}:{1}'.format(host, port) + url = f'http://{host}:{port}' r = requests.get(url=url, allow_redirects=True) assert r.status_code == 200 assert len(r.history) == 1 assert r.history[0].status_code == 301 assert redirect_request[0].startswith(b'GET /' + expected_path + b' HTTP/1.1') - assert r.url == u'{0}/{1}'.format(url, expected_path.decode('ascii')) + assert r.url == '{}/{}'.format(url, expected_path.decode('ascii')) close_server.set() + def test_fragment_not_sent_with_request(): """Verify that the fragment portion of a URI isn't sent to the server.""" - def response_handler(sock): - req = consume_socket_content(sock, timeout=0.5) - sock.send( - b'HTTP/1.1 200 OK\r\n' - b'Content-Length: '+bytes(len(req))+b'\r\n' - b'\r\n'+req - ) - close_server = threading.Event() - server = Server(response_handler, wait_to_close_event=close_server) + server = Server(echo_response_handler, wait_to_close_event=close_server) with server as (host, port): - url = 'http://{0}:{1}/path/to/thing/#view=edit&token=hunter2'.format(host, port) + url = f'http://{host}:{port}/path/to/thing/#view=edit&token=hunter2' r = requests.get(url) raw_request = r.content @@ -265,9 +360,10 @@ def response_handler(sock): close_server.set() + def test_fragment_update_on_redirect(): """Verify we only append previous fragment if one doesn't exist on new - location. If a new fragment is encounterd in a Location header, it should + location. If a new fragment is encountered in a Location header, it should be added to all subsequent requests. """ @@ -293,17 +389,40 @@ def response_handler(sock): server = Server(response_handler, wait_to_close_event=close_server) with server as (host, port): - url = 'http://{0}:{1}/path/to/thing/#view=edit&token=hunter2'.format(host, port) + url = f'http://{host}:{port}/path/to/thing/#view=edit&token=hunter2' r = requests.get(url) - raw_request = r.content assert r.status_code == 200 assert len(r.history) == 2 assert r.history[0].request.url == url # Verify we haven't overwritten the location with our previous fragment. - assert r.history[1].request.url == 'http://{0}:{1}/get#relevant-section'.format(host, port) + assert r.history[1].request.url == f'http://{host}:{port}/get#relevant-section' # Verify previous fragment is used and not the original. - assert r.url == 'http://{0}:{1}/final-url/#relevant-section'.format(host, port) + assert r.url == f'http://{host}:{port}/final-url/#relevant-section' close_server.set() + + +def test_json_decode_compatibility_for_alt_utf_encodings(): + + def response_handler(sock): + consume_socket_content(sock, timeout=0.5) + sock.send( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 18\r\n\r\n' + b'\xff\xfe{\x00"\x00K0"\x00=\x00"\x00\xab0"\x00\r\n' + ) + + close_server = threading.Event() + server = Server(response_handler, wait_to_close_event=close_server) + + with server as (host, port): + url = f'http://{host}:{port}/' + r = requests.get(url) + r.encoding = None + with pytest.raises(requests.exceptions.JSONDecodeError) as excinfo: + r.json() + assert isinstance(excinfo.value, requests.exceptions.RequestException) + assert isinstance(excinfo.value, JSONDecodeError) + assert r.text not in str(excinfo.value) diff --git a/tests/test_requests.py b/tests/test_requests.py index b374747427..75d2deff2e 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1,45 +1,72 @@ -# -*- coding: utf-8 -*- - """Tests for Requests.""" -from __future__ import division +import collections +import contextlib +import io import json import os import pickle -import collections -import contextlib +import re +import tempfile +import threading import warnings +from unittest import mock -import io -import requests import pytest +import urllib3 +from urllib3.util import Timeout as Urllib3Timeout + +import requests from requests.adapters import HTTPAdapter from requests.auth import HTTPDigestAuth, _basic_auth_str from requests.compat import ( - Morsel, cookielib, getproxies, str, urlparse, - builtin_str, OrderedDict) -from requests.cookies import ( - cookiejar_from_dict, morsel_to_cookie) + JSONDecodeError, + Morsel, + MutableMapping, + builtin_str, + cookielib, + getproxies, + is_urllib3_1, + urlparse, +) +from requests.cookies import cookiejar_from_dict, morsel_to_cookie from requests.exceptions import ( - ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL, - MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects, - ProxyError, InvalidHeader, UnrewindableBodyError, SSLError, InvalidProxyURL) -from requests.models import PreparedRequest -from requests.structures import CaseInsensitiveDict -from requests.sessions import SessionRedirectMixin -from requests.models import urlencode + ChunkedEncodingError, + ConnectionError, + ConnectTimeout, + ContentDecodingError, + InvalidHeader, + InvalidProxyURL, + InvalidSchema, + InvalidURL, + MissingSchema, + ProxyError, + ReadTimeout, + RequestException, + RetryError, +) +from requests.exceptions import SSLError as RequestsSSLError +from requests.exceptions import Timeout, TooManyRedirects, UnrewindableBodyError from requests.hooks import default_hooks +from requests.models import PreparedRequest, urlencode +from requests.sessions import SessionRedirectMixin +from requests.structures import CaseInsensitiveDict -from .compat import StringIO, u +from . import SNIMissingWarning +from .compat import StringIO +from .testserver.server import TLSServer, consume_socket_content from .utils import override_environ -from urllib3.util import Timeout as Urllib3Timeout # Requests to this URL should always fail with a connection timeout (nothing # listening on that port) -TARPIT = 'http://10.255.255.1' +TARPIT = "http://10.255.255.1" + +# This is to avoid waiting the timeout of using TARPIT +INVALID_PROXY = "http://localhost:1" try: from ssl import SSLContext + del SSLContext HAS_MODERN_SSL = True except ImportError: @@ -53,9 +80,9 @@ class TestRequests: + digest_auth_algo = ("MD5", "SHA-256", "SHA-512") def test_entry_points(self): - requests.session requests.session().get requests.session().head @@ -65,100 +92,116 @@ def test_entry_points(self): requests.patch requests.post # Not really an entry point, but people rely on it. - from requests.packages.urllib3.poolmanager import PoolManager + from requests.packages.urllib3.poolmanager import PoolManager # noqa:F401 @pytest.mark.parametrize( - 'exception, url', ( - (MissingSchema, 'hiwpefhipowhefopw'), - (InvalidSchema, 'localhost:3128'), - (InvalidSchema, 'localhost.localdomain:3128/'), - (InvalidSchema, '10.122.1.1:3128/'), - (InvalidURL, 'http://'), - )) + "exception, url", + ( + (MissingSchema, "hiwpefhipowhefopw"), + (InvalidSchema, "localhost:3128"), + (InvalidSchema, "localhost.localdomain:3128/"), + (InvalidSchema, "10.122.1.1:3128/"), + (InvalidURL, "http://"), + (InvalidURL, "http://*example.com"), + (InvalidURL, "http://.example.com"), + ), + ) def test_invalid_url(self, exception, url): with pytest.raises(exception): requests.get(url) def test_basic_building(self): req = requests.Request() - req.url = 'http://kennethreitz.org/' - req.data = {'life': '42'} + req.url = "http://kennethreitz.org/" + req.data = {"life": "42"} pr = req.prepare() assert pr.url == req.url - assert pr.body == 'life=42' + assert pr.body == "life=42" - @pytest.mark.parametrize('method', ('GET', 'HEAD')) + @pytest.mark.parametrize("method", ("GET", "HEAD")) def test_no_content_length(self, httpbin, method): req = requests.Request(method, httpbin(method.lower())).prepare() - assert 'Content-Length' not in req.headers + assert "Content-Length" not in req.headers - @pytest.mark.parametrize('method', ('POST', 'PUT', 'PATCH', 'OPTIONS')) + @pytest.mark.parametrize("method", ("POST", "PUT", "PATCH", "OPTIONS")) def test_no_body_content_length(self, httpbin, method): req = requests.Request(method, httpbin(method.lower())).prepare() - assert req.headers['Content-Length'] == '0' + assert req.headers["Content-Length"] == "0" - @pytest.mark.parametrize('method', ('POST', 'PUT', 'PATCH', 'OPTIONS')) + @pytest.mark.parametrize("method", ("POST", "PUT", "PATCH", "OPTIONS")) def test_empty_content_length(self, httpbin, method): - req = requests.Request(method, httpbin(method.lower()), data='').prepare() - assert req.headers['Content-Length'] == '0' + req = requests.Request(method, httpbin(method.lower()), data="").prepare() + assert req.headers["Content-Length"] == "0" def test_override_content_length(self, httpbin): - headers = { - 'Content-Length': 'not zero' - } - r = requests.Request('POST', httpbin('post'), headers=headers).prepare() - assert 'Content-Length' in r.headers - assert r.headers['Content-Length'] == 'not zero' + headers = {"Content-Length": "not zero"} + r = requests.Request("POST", httpbin("post"), headers=headers).prepare() + assert "Content-Length" in r.headers + assert r.headers["Content-Length"] == "not zero" def test_path_is_not_double_encoded(self): - request = requests.Request('GET', "http://0.0.0.0/get/test case").prepare() + request = requests.Request("GET", "http://0.0.0.0/get/test case").prepare() - assert request.path_url == '/get/test%20case' + assert request.path_url == "/get/test%20case" @pytest.mark.parametrize( - 'url, expected', ( - ('http://example.com/path#fragment', 'http://example.com/path?a=b#fragment'), - ('http://example.com/path?key=value#fragment', 'http://example.com/path?key=value&a=b#fragment') - )) + "url, expected", + ( + ( + "http://example.com/path#fragment", + "http://example.com/path?a=b#fragment", + ), + ( + "http://example.com/path?key=value#fragment", + "http://example.com/path?key=value&a=b#fragment", + ), + ), + ) def test_params_are_added_before_fragment(self, url, expected): - request = requests.Request('GET', url, params={"a": "b"}).prepare() + request = requests.Request("GET", url, params={"a": "b"}).prepare() assert request.url == expected def test_params_original_order_is_preserved_by_default(self): - param_ordered_dict = OrderedDict((('z', 1), ('a', 1), ('k', 1), ('d', 1))) + param_ordered_dict = collections.OrderedDict( + (("z", 1), ("a", 1), ("k", 1), ("d", 1)) + ) session = requests.Session() - request = requests.Request('GET', 'http://example.com/', params=param_ordered_dict) + request = requests.Request( + "GET", "http://example.com/", params=param_ordered_dict + ) prep = session.prepare_request(request) - assert prep.url == 'http://example.com/?z=1&a=1&k=1&d=1' + assert prep.url == "http://example.com/?z=1&a=1&k=1&d=1" def test_params_bytes_are_encoded(self): - request = requests.Request('GET', 'http://example.com', - params=b'test=foo').prepare() - assert request.url == 'http://example.com/?test=foo' + request = requests.Request( + "GET", "http://example.com", params=b"test=foo" + ).prepare() + assert request.url == "http://example.com/?test=foo" def test_binary_put(self): - request = requests.Request('PUT', 'http://example.com', - data=u"ööö".encode("utf-8")).prepare() + request = requests.Request( + "PUT", "http://example.com", data="ööö".encode() + ).prepare() assert isinstance(request.body, bytes) def test_whitespaces_are_removed_from_url(self): # Test for issue #3696 - request = requests.Request('GET', ' http://example.com').prepare() - assert request.url == 'http://example.com/' + request = requests.Request("GET", " http://example.com").prepare() + assert request.url == "http://example.com/" - @pytest.mark.parametrize('scheme', ('http://', 'HTTP://', 'hTTp://', 'HttP://')) + @pytest.mark.parametrize("scheme", ("http://", "HTTP://", "hTTp://", "HttP://")) def test_mixed_case_scheme_acceptable(self, httpbin, scheme): s = requests.Session() s.proxies = getproxies() - parts = urlparse(httpbin('get')) + parts = urlparse(httpbin("get")) url = scheme + parts.netloc + parts.path - r = requests.Request('GET', url) + r = requests.Request("GET", url) r = s.send(r.prepare()) - assert r.status_code == 200, 'failed for scheme {0}'.format(scheme) + assert r.status_code == 200, f"failed for scheme {scheme}" def test_HTTP_200_OK_GET_ALTERNATIVE(self, httpbin): - r = requests.Request('GET', httpbin('get')) + r = requests.Request("GET", httpbin("get")) s = requests.Session() s.proxies = getproxies() @@ -167,103 +210,113 @@ def test_HTTP_200_OK_GET_ALTERNATIVE(self, httpbin): assert r.status_code == 200 def test_HTTP_302_ALLOW_REDIRECT_GET(self, httpbin): - r = requests.get(httpbin('redirect', '1')) + r = requests.get(httpbin("redirect", "1")) assert r.status_code == 200 assert r.history[0].status_code == 302 assert r.history[0].is_redirect def test_HTTP_307_ALLOW_REDIRECT_POST(self, httpbin): - r = requests.post(httpbin('redirect-to'), data='test', params={'url': 'post', 'status_code': 307}) + r = requests.post( + httpbin("redirect-to"), + data="test", + params={"url": "post", "status_code": 307}, + ) assert r.status_code == 200 assert r.history[0].status_code == 307 assert r.history[0].is_redirect - assert r.json()['data'] == 'test' + assert r.json()["data"] == "test" def test_HTTP_307_ALLOW_REDIRECT_POST_WITH_SEEKABLE(self, httpbin): - byte_str = b'test' - r = requests.post(httpbin('redirect-to'), data=io.BytesIO(byte_str), params={'url': 'post', 'status_code': 307}) + byte_str = b"test" + r = requests.post( + httpbin("redirect-to"), + data=io.BytesIO(byte_str), + params={"url": "post", "status_code": 307}, + ) assert r.status_code == 200 assert r.history[0].status_code == 307 assert r.history[0].is_redirect - assert r.json()['data'] == byte_str.decode('utf-8') + assert r.json()["data"] == byte_str.decode("utf-8") def test_HTTP_302_TOO_MANY_REDIRECTS(self, httpbin): try: - requests.get(httpbin('relative-redirect', '50')) + requests.get(httpbin("relative-redirect", "50")) except TooManyRedirects as e: - url = httpbin('relative-redirect', '20') + url = httpbin("relative-redirect", "20") assert e.request.url == url assert e.response.url == url assert len(e.response.history) == 30 else: - pytest.fail('Expected redirect to raise TooManyRedirects but it did not') + pytest.fail("Expected redirect to raise TooManyRedirects but it did not") def test_HTTP_302_TOO_MANY_REDIRECTS_WITH_PARAMS(self, httpbin): s = requests.session() s.max_redirects = 5 try: - s.get(httpbin('relative-redirect', '50')) + s.get(httpbin("relative-redirect", "50")) except TooManyRedirects as e: - url = httpbin('relative-redirect', '45') + url = httpbin("relative-redirect", "45") assert e.request.url == url assert e.response.url == url assert len(e.response.history) == 5 else: - pytest.fail('Expected custom max number of redirects to be respected but was not') + pytest.fail( + "Expected custom max number of redirects to be respected but was not" + ) def test_http_301_changes_post_to_get(self, httpbin): - r = requests.post(httpbin('status', '301')) + r = requests.post(httpbin("status", "301")) assert r.status_code == 200 - assert r.request.method == 'GET' + assert r.request.method == "GET" assert r.history[0].status_code == 301 assert r.history[0].is_redirect def test_http_301_doesnt_change_head_to_get(self, httpbin): - r = requests.head(httpbin('status', '301'), allow_redirects=True) + r = requests.head(httpbin("status", "301"), allow_redirects=True) print(r.content) assert r.status_code == 200 - assert r.request.method == 'HEAD' + assert r.request.method == "HEAD" assert r.history[0].status_code == 301 assert r.history[0].is_redirect def test_http_302_changes_post_to_get(self, httpbin): - r = requests.post(httpbin('status', '302')) + r = requests.post(httpbin("status", "302")) assert r.status_code == 200 - assert r.request.method == 'GET' + assert r.request.method == "GET" assert r.history[0].status_code == 302 assert r.history[0].is_redirect def test_http_302_doesnt_change_head_to_get(self, httpbin): - r = requests.head(httpbin('status', '302'), allow_redirects=True) + r = requests.head(httpbin("status", "302"), allow_redirects=True) assert r.status_code == 200 - assert r.request.method == 'HEAD' + assert r.request.method == "HEAD" assert r.history[0].status_code == 302 assert r.history[0].is_redirect def test_http_303_changes_post_to_get(self, httpbin): - r = requests.post(httpbin('status', '303')) + r = requests.post(httpbin("status", "303")) assert r.status_code == 200 - assert r.request.method == 'GET' + assert r.request.method == "GET" assert r.history[0].status_code == 303 assert r.history[0].is_redirect def test_http_303_doesnt_change_head_to_get(self, httpbin): - r = requests.head(httpbin('status', '303'), allow_redirects=True) + r = requests.head(httpbin("status", "303"), allow_redirects=True) assert r.status_code == 200 - assert r.request.method == 'HEAD' + assert r.request.method == "HEAD" assert r.history[0].status_code == 303 assert r.history[0].is_redirect def test_header_and_body_removal_on_redirect(self, httpbin): - purged_headers = ('Content-Length', 'Content-Type') + purged_headers = ("Content-Length", "Content-Type") ses = requests.Session() - req = requests.Request('POST', httpbin('post'), data={'test': 'data'}) + req = requests.Request("POST", httpbin("post"), data={"test": "data"}) prep = ses.prepare_request(req) resp = ses.send(prep) # Mimic a redirect response resp.status_code = 302 - resp.headers['location'] = 'get' + resp.headers["location"] = "get" # Run request through resolve_redirects next_resp = next(ses.resolve_redirects(resp, prep)) @@ -272,21 +325,21 @@ def test_header_and_body_removal_on_redirect(self, httpbin): assert header not in next_resp.request.headers def test_transfer_enc_removal_on_redirect(self, httpbin): - purged_headers = ('Transfer-Encoding', 'Content-Type') + purged_headers = ("Transfer-Encoding", "Content-Type") ses = requests.Session() - req = requests.Request('POST', httpbin('post'), data=(b'x' for x in range(1))) + req = requests.Request("POST", httpbin("post"), data=(b"x" for x in range(1))) prep = ses.prepare_request(req) - assert 'Transfer-Encoding' in prep.headers + assert "Transfer-Encoding" in prep.headers # Create Response to avoid https://github.com/kevin1024/pytest-httpbin/issues/33 resp = requests.Response() - resp.raw = io.BytesIO(b'the content') + resp.raw = io.BytesIO(b"the content") resp.request = prep - setattr(resp.raw, 'release_conn', lambda *args: args) + setattr(resp.raw, "release_conn", lambda *args: args) # Mimic a redirect response resp.status_code = 302 - resp.headers['location'] = httpbin('get') + resp.headers["location"] = httpbin("get") # Run request through resolve_redirect next_resp = next(ses.resolve_redirects(resp, prep)) @@ -296,94 +349,93 @@ def test_transfer_enc_removal_on_redirect(self, httpbin): def test_fragment_maintained_on_redirect(self, httpbin): fragment = "#view=edit&token=hunter2" - r = requests.get(httpbin('redirect-to?url=get')+fragment) + r = requests.get(httpbin("redirect-to?url=get") + fragment) assert len(r.history) > 0 - assert r.history[0].request.url == httpbin('redirect-to?url=get')+fragment - assert r.url == httpbin('get')+fragment + assert r.history[0].request.url == httpbin("redirect-to?url=get") + fragment + assert r.url == httpbin("get") + fragment def test_HTTP_200_OK_GET_WITH_PARAMS(self, httpbin): - heads = {'User-agent': 'Mozilla/5.0'} + heads = {"User-agent": "Mozilla/5.0"} - r = requests.get(httpbin('user-agent'), headers=heads) + r = requests.get(httpbin("user-agent"), headers=heads) - assert heads['User-agent'] in r.text + assert heads["User-agent"] in r.text assert r.status_code == 200 def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self, httpbin): - heads = {'User-agent': 'Mozilla/5.0'} + heads = {"User-agent": "Mozilla/5.0"} - r = requests.get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads) + r = requests.get( + httpbin("get") + "?test=true", params={"q": "test"}, headers=heads + ) assert r.status_code == 200 def test_set_cookie_on_301(self, httpbin): s = requests.session() - url = httpbin('cookies/set?foo=bar') + url = httpbin("cookies/set?foo=bar") s.get(url) - assert s.cookies['foo'] == 'bar' + assert s.cookies["foo"] == "bar" def test_cookie_sent_on_redirect(self, httpbin): s = requests.session() - s.get(httpbin('cookies/set?foo=bar')) - r = s.get(httpbin('redirect/1')) # redirects to httpbin('get') - assert 'Cookie' in r.json()['headers'] + s.get(httpbin("cookies/set?foo=bar")) + r = s.get(httpbin("redirect/1")) # redirects to httpbin('get') + assert "Cookie" in r.json()["headers"] def test_cookie_removed_on_expire(self, httpbin): s = requests.session() - s.get(httpbin('cookies/set?foo=bar')) - assert s.cookies['foo'] == 'bar' + s.get(httpbin("cookies/set?foo=bar")) + assert s.cookies["foo"] == "bar" s.get( - httpbin('response-headers'), - params={ - 'Set-Cookie': - 'foo=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT' - } + httpbin("response-headers"), + params={"Set-Cookie": "foo=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT"}, ) - assert 'foo' not in s.cookies + assert "foo" not in s.cookies def test_cookie_quote_wrapped(self, httpbin): s = requests.session() s.get(httpbin('cookies/set?foo="bar:baz"')) - assert s.cookies['foo'] == '"bar:baz"' + assert s.cookies["foo"] == '"bar:baz"' def test_cookie_persists_via_api(self, httpbin): s = requests.session() - r = s.get(httpbin('redirect/1'), cookies={'foo': 'bar'}) - assert 'foo' in r.request.headers['Cookie'] - assert 'foo' in r.history[0].request.headers['Cookie'] + r = s.get(httpbin("redirect/1"), cookies={"foo": "bar"}) + assert "foo" in r.request.headers["Cookie"] + assert "foo" in r.history[0].request.headers["Cookie"] def test_request_cookie_overrides_session_cookie(self, httpbin): s = requests.session() - s.cookies['foo'] = 'bar' - r = s.get(httpbin('cookies'), cookies={'foo': 'baz'}) - assert r.json()['cookies']['foo'] == 'baz' + s.cookies["foo"] = "bar" + r = s.get(httpbin("cookies"), cookies={"foo": "baz"}) + assert r.json()["cookies"]["foo"] == "baz" # Session cookie should not be modified - assert s.cookies['foo'] == 'bar' + assert s.cookies["foo"] == "bar" def test_request_cookies_not_persisted(self, httpbin): s = requests.session() - s.get(httpbin('cookies'), cookies={'foo': 'baz'}) + s.get(httpbin("cookies"), cookies={"foo": "baz"}) # Sending a request with cookies should not add cookies to the session assert not s.cookies def test_generic_cookiejar_works(self, httpbin): cj = cookielib.CookieJar() - cookiejar_from_dict({'foo': 'bar'}, cj) + cookiejar_from_dict({"foo": "bar"}, cj) s = requests.session() s.cookies = cj - r = s.get(httpbin('cookies')) + r = s.get(httpbin("cookies")) # Make sure the cookie was sent - assert r.json()['cookies']['foo'] == 'bar' + assert r.json()["cookies"]["foo"] == "bar" # Make sure the session cj is still the custom one assert s.cookies is cj def test_param_cookiejar_works(self, httpbin): cj = cookielib.CookieJar() - cookiejar_from_dict({'foo': 'bar'}, cj) + cookiejar_from_dict({"foo": "bar"}, cj) s = requests.session() - r = s.get(httpbin('cookies'), cookies=cj) + r = s.get(httpbin("cookies"), cookies=cj) # Make sure the cookie was sent - assert r.json()['cookies']['foo'] == 'bar' + assert r.json()["cookies"]["foo"] == "bar" def test_cookielib_cookiejar_on_redirect(self, httpbin): """Tests resolve_redirect doesn't fail when merging cookies @@ -391,18 +443,18 @@ def test_cookielib_cookiejar_on_redirect(self, httpbin): See GH #3579 """ - cj = cookiejar_from_dict({'foo': 'bar'}, cookielib.CookieJar()) + cj = cookiejar_from_dict({"foo": "bar"}, cookielib.CookieJar()) s = requests.Session() - s.cookies = cookiejar_from_dict({'cookie': 'tasty'}) + s.cookies = cookiejar_from_dict({"cookie": "tasty"}) # Prepare request without using Session - req = requests.Request('GET', httpbin('headers'), cookies=cj) + req = requests.Request("GET", httpbin("headers"), cookies=cj) prep_req = req.prepare() # Send request and simulate redirect resp = s.send(prep_req) resp.status_code = 302 - resp.headers['location'] = httpbin('get') + resp.headers["location"] = httpbin("get") redirects = s.resolve_redirects(resp, prep_req) resp = next(redirects) @@ -414,70 +466,69 @@ def test_cookielib_cookiejar_on_redirect(self, httpbin): cookies = {} for c in resp.request._cookies: cookies[c.name] = c.value - assert cookies['foo'] == 'bar' - assert cookies['cookie'] == 'tasty' + assert cookies["foo"] == "bar" + assert cookies["cookie"] == "tasty" def test_requests_in_history_are_not_overridden(self, httpbin): - resp = requests.get(httpbin('redirect/3')) + resp = requests.get(httpbin("redirect/3")) urls = [r.url for r in resp.history] req_urls = [r.request.url for r in resp.history] assert urls == req_urls def test_history_is_always_a_list(self, httpbin): """Show that even with redirects, Response.history is always a list.""" - resp = requests.get(httpbin('get')) + resp = requests.get(httpbin("get")) assert isinstance(resp.history, list) - resp = requests.get(httpbin('redirect/1')) + resp = requests.get(httpbin("redirect/1")) assert isinstance(resp.history, list) assert not isinstance(resp.history, tuple) def test_headers_on_session_with_None_are_not_sent(self, httpbin): """Do not send headers in Session.headers with None values.""" ses = requests.Session() - ses.headers['Accept-Encoding'] = None - req = requests.Request('GET', httpbin('get')) + ses.headers["Accept-Encoding"] = None + req = requests.Request("GET", httpbin("get")) prep = ses.prepare_request(req) - assert 'Accept-Encoding' not in prep.headers + assert "Accept-Encoding" not in prep.headers def test_headers_preserve_order(self, httpbin): """Preserve order when headers provided as OrderedDict.""" ses = requests.Session() - ses.headers = OrderedDict() - ses.headers['Accept-Encoding'] = 'identity' - ses.headers['First'] = '1' - ses.headers['Second'] = '2' - headers = OrderedDict([('Third', '3'), ('Fourth', '4')]) - headers['Fifth'] = '5' - headers['Second'] = '222' - req = requests.Request('GET', httpbin('get'), headers=headers) + ses.headers = collections.OrderedDict() + ses.headers["Accept-Encoding"] = "identity" + ses.headers["First"] = "1" + ses.headers["Second"] = "2" + headers = collections.OrderedDict([("Third", "3"), ("Fourth", "4")]) + headers["Fifth"] = "5" + headers["Second"] = "222" + req = requests.Request("GET", httpbin("get"), headers=headers) prep = ses.prepare_request(req) items = list(prep.headers.items()) - assert items[0] == ('Accept-Encoding', 'identity') - assert items[1] == ('First', '1') - assert items[2] == ('Second', '222') - assert items[3] == ('Third', '3') - assert items[4] == ('Fourth', '4') - assert items[5] == ('Fifth', '5') - - @pytest.mark.parametrize('key', ('User-agent', 'user-agent')) + assert items[0] == ("Accept-Encoding", "identity") + assert items[1] == ("First", "1") + assert items[2] == ("Second", "222") + assert items[3] == ("Third", "3") + assert items[4] == ("Fourth", "4") + assert items[5] == ("Fifth", "5") + + @pytest.mark.parametrize("key", ("User-agent", "user-agent")) def test_user_agent_transfers(self, httpbin, key): + heads = {key: "Mozilla/5.0 (github.com/psf/requests)"} - heads = {key: 'Mozilla/5.0 (github.com/requests/requests)'} - - r = requests.get(httpbin('user-agent'), headers=heads) + r = requests.get(httpbin("user-agent"), headers=heads) assert heads[key] in r.text def test_HTTP_200_OK_HEAD(self, httpbin): - r = requests.head(httpbin('get')) + r = requests.head(httpbin("get")) assert r.status_code == 200 def test_HTTP_200_OK_PUT(self, httpbin): - r = requests.put(httpbin('put')) + r = requests.put(httpbin("put")) assert r.status_code == 200 def test_BASICAUTH_TUPLE_HTTP_200_OK_GET(self, httpbin): - auth = ('user', 'pass') - url = httpbin('basic-auth', 'user', 'pass') + auth = ("user", "pass") + url = httpbin("basic-auth", "user", "pass") r = requests.get(url, auth=auth) assert r.status_code == 200 @@ -491,40 +542,44 @@ def test_BASICAUTH_TUPLE_HTTP_200_OK_GET(self, httpbin): assert r.status_code == 200 @pytest.mark.parametrize( - 'username, password', ( - ('user', 'pass'), - (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8')), + "username, password", + ( + ("user", "pass"), + ("имя".encode(), "пароль".encode()), (42, 42), (None, None), - )) + ), + ) def test_set_basicauth(self, httpbin, username, password): auth = (username, password) - url = httpbin('get') + url = httpbin("get") - r = requests.Request('GET', url, auth=auth) + r = requests.Request("GET", url, auth=auth) p = r.prepare() - assert p.headers['Authorization'] == _basic_auth_str(username, password) + assert p.headers["Authorization"] == _basic_auth_str(username, password) def test_basicauth_encodes_byte_strings(self): """Ensure b'test' formats as the byte string "test" rather than the unicode string "b'test'" in Python 3. """ - auth = (b'\xc5\xafsername', b'test\xc6\xb6') - r = requests.Request('GET', 'http://localhost', auth=auth) + auth = (b"\xc5\xafsername", b"test\xc6\xb6") + r = requests.Request("GET", "http://localhost", auth=auth) p = r.prepare() - assert p.headers['Authorization'] == 'Basic xa9zZXJuYW1lOnRlc3TGtg==' + assert p.headers["Authorization"] == "Basic xa9zZXJuYW1lOnRlc3TGtg==" @pytest.mark.parametrize( - 'url, exception', ( + "url, exception", + ( # Connecting to an unknown domain should raise a ConnectionError - ('http://doesnotexist.google.com', ConnectionError), + ("http://doesnotexist.google.com", ConnectionError), # Connecting to an invalid port should raise a ConnectionError - ('http://localhost:1', ConnectionError), + ("http://localhost:1", ConnectionError), # Inputing a URL that cannot be parsed should raise an InvalidURL error - ('http://fe80::5054:ff:fe5a:fc0', InvalidURL) - )) + ("http://fe80::5054:ff:fe5a:fc0", InvalidURL), + ), + ) def test_errors(self, url, exception): with pytest.raises(exception): requests.get(url, timeout=1) @@ -532,31 +587,101 @@ def test_errors(self, url, exception): def test_proxy_error(self): # any proxy related error (address resolution, no route to host, etc) should result in a ProxyError with pytest.raises(ProxyError): - requests.get('http://localhost:1', proxies={'http': 'non-resolvable-address'}) + requests.get( + "http://localhost:1", proxies={"http": "non-resolvable-address"} + ) def test_proxy_error_on_bad_url(self, httpbin, httpbin_secure): with pytest.raises(InvalidProxyURL): - requests.get(httpbin_secure(), proxies={'https': 'http:/badproxyurl:3128'}) + requests.get(httpbin_secure(), proxies={"https": "http:/badproxyurl:3128"}) with pytest.raises(InvalidProxyURL): - requests.get(httpbin(), proxies={'http': 'http://:8080'}) + requests.get(httpbin(), proxies={"http": "http://:8080"}) with pytest.raises(InvalidProxyURL): - requests.get(httpbin_secure(), proxies={'https': 'https://'}) + requests.get(httpbin_secure(), proxies={"https": "https://"}) with pytest.raises(InvalidProxyURL): - requests.get(httpbin(), proxies={'http': 'http:///example.com:8080'}) + requests.get(httpbin(), proxies={"http": "http:///example.com:8080"}) + + def test_respect_proxy_env_on_send_self_prepared_request(self, httpbin): + with override_environ(http_proxy=INVALID_PROXY): + with pytest.raises(ProxyError): + session = requests.Session() + request = requests.Request("GET", httpbin()) + session.send(request.prepare()) + + def test_respect_proxy_env_on_send_session_prepared_request(self, httpbin): + with override_environ(http_proxy=INVALID_PROXY): + with pytest.raises(ProxyError): + session = requests.Session() + request = requests.Request("GET", httpbin()) + prepared = session.prepare_request(request) + session.send(prepared) + + def test_respect_proxy_env_on_send_with_redirects(self, httpbin): + with override_environ(http_proxy=INVALID_PROXY): + with pytest.raises(ProxyError): + session = requests.Session() + url = httpbin("redirect/1") + print(url) + request = requests.Request("GET", url) + session.send(request.prepare()) + + def test_respect_proxy_env_on_get(self, httpbin): + with override_environ(http_proxy=INVALID_PROXY): + with pytest.raises(ProxyError): + session = requests.Session() + session.get(httpbin()) + + def test_respect_proxy_env_on_request(self, httpbin): + with override_environ(http_proxy=INVALID_PROXY): + with pytest.raises(ProxyError): + session = requests.Session() + session.request(method="GET", url=httpbin()) + + def test_proxy_authorization_preserved_on_request(self, httpbin): + proxy_auth_value = "Bearer XXX" + session = requests.Session() + session.headers.update({"Proxy-Authorization": proxy_auth_value}) + resp = session.request(method="GET", url=httpbin("get")) + sent_headers = resp.json().get("headers", {}) + + assert sent_headers.get("Proxy-Authorization") == proxy_auth_value + + @pytest.mark.parametrize( + "url,has_proxy_auth", + ( + ("http://example.com", True), + ("https://example.com", False), + ), + ) + def test_proxy_authorization_not_appended_to_https_request( + self, url, has_proxy_auth + ): + session = requests.Session() + proxies = { + "http": "http://test:pass@localhost:8080", + "https": "http://test:pass@localhost:8090", + } + req = requests.Request("GET", url) + prep = req.prepare() + session.rebuild_proxies(prep, proxies) + + assert ("Proxy-Authorization" in prep.headers) is has_proxy_auth def test_basicauth_with_netrc(self, httpbin): - auth = ('user', 'pass') - wrong_auth = ('wronguser', 'wrongpass') - url = httpbin('basic-auth', 'user', 'pass') + auth = ("user", "pass") + wrong_auth = ("wronguser", "wrongpass") + url = httpbin("basic-auth", "user", "pass") old_auth = requests.sessions.get_netrc_auth try: + def get_netrc_auth_mock(url): return auth + requests.sessions.get_netrc_auth = get_netrc_auth_mock # Should use netrc and work. @@ -580,94 +705,131 @@ def get_netrc_auth_mock(url): finally: requests.sessions.get_netrc_auth = old_auth - def test_DIGEST_HTTP_200_OK_GET(self, httpbin): + def test_basicauth_with_netrc_leak(self, httpbin): + url1 = httpbin("basic-auth", "user", "pass") + url = url1[len("http://") :] + domain = url.split(":")[0] + url = f"http://example.com:@{url}" + + netrc_file = "" + with tempfile.NamedTemporaryFile(mode="w", delete=False) as fp: + fp.write("machine example.com\n") + fp.write("login wronguser\n") + fp.write("password wrongpass\n") + fp.write(f"machine {domain}\n") + fp.write("login user\n") + fp.write("password pass\n") + fp.close() + netrc_file = fp.name + + old_netrc = os.environ.get("NETRC", "") + os.environ["NETRC"] = netrc_file + + try: + # Should use netrc + # Make sure that we don't use the example.com credentails + # for the request + r = requests.get(url) + assert r.status_code == 200 + finally: + os.environ["NETRC"] = old_netrc + os.unlink(netrc_file) - auth = HTTPDigestAuth('user', 'pass') - url = httpbin('digest-auth', 'auth', 'user', 'pass') + def test_DIGEST_HTTP_200_OK_GET(self, httpbin): + for authtype in self.digest_auth_algo: + auth = HTTPDigestAuth("user", "pass") + url = httpbin("digest-auth", "auth", "user", "pass", authtype, "never") - r = requests.get(url, auth=auth) - assert r.status_code == 200 + r = requests.get(url, auth=auth) + assert r.status_code == 200 - r = requests.get(url) - assert r.status_code == 401 + r = requests.get(url) + assert r.status_code == 401 + print(r.headers["WWW-Authenticate"]) - s = requests.session() - s.auth = HTTPDigestAuth('user', 'pass') - r = s.get(url) - assert r.status_code == 200 + s = requests.session() + s.auth = HTTPDigestAuth("user", "pass") + r = s.get(url) + assert r.status_code == 200 def test_DIGEST_AUTH_RETURNS_COOKIE(self, httpbin): - url = httpbin('digest-auth', 'auth', 'user', 'pass') - auth = HTTPDigestAuth('user', 'pass') - r = requests.get(url) - assert r.cookies['fake'] == 'fake_value' + for authtype in self.digest_auth_algo: + url = httpbin("digest-auth", "auth", "user", "pass", authtype) + auth = HTTPDigestAuth("user", "pass") + r = requests.get(url) + assert r.cookies["fake"] == "fake_value" - r = requests.get(url, auth=auth) - assert r.status_code == 200 + r = requests.get(url, auth=auth) + assert r.status_code == 200 def test_DIGEST_AUTH_SETS_SESSION_COOKIES(self, httpbin): - url = httpbin('digest-auth', 'auth', 'user', 'pass') - auth = HTTPDigestAuth('user', 'pass') - s = requests.Session() - s.get(url, auth=auth) - assert s.cookies['fake'] == 'fake_value' + for authtype in self.digest_auth_algo: + url = httpbin("digest-auth", "auth", "user", "pass", authtype) + auth = HTTPDigestAuth("user", "pass") + s = requests.Session() + s.get(url, auth=auth) + assert s.cookies["fake"] == "fake_value" def test_DIGEST_STREAM(self, httpbin): + for authtype in self.digest_auth_algo: + auth = HTTPDigestAuth("user", "pass") + url = httpbin("digest-auth", "auth", "user", "pass", authtype) - auth = HTTPDigestAuth('user', 'pass') - url = httpbin('digest-auth', 'auth', 'user', 'pass') + r = requests.get(url, auth=auth, stream=True) + assert r.raw.read() != b"" - r = requests.get(url, auth=auth, stream=True) - assert r.raw.read() != b'' - - r = requests.get(url, auth=auth, stream=False) - assert r.raw.read() == b'' + r = requests.get(url, auth=auth, stream=False) + assert r.raw.read() == b"" def test_DIGESTAUTH_WRONG_HTTP_401_GET(self, httpbin): + for authtype in self.digest_auth_algo: + auth = HTTPDigestAuth("user", "wrongpass") + url = httpbin("digest-auth", "auth", "user", "pass", authtype) - auth = HTTPDigestAuth('user', 'wrongpass') - url = httpbin('digest-auth', 'auth', 'user', 'pass') - - r = requests.get(url, auth=auth) - assert r.status_code == 401 + r = requests.get(url, auth=auth) + assert r.status_code == 401 - r = requests.get(url) - assert r.status_code == 401 + r = requests.get(url) + assert r.status_code == 401 - s = requests.session() - s.auth = auth - r = s.get(url) - assert r.status_code == 401 + s = requests.session() + s.auth = auth + r = s.get(url) + assert r.status_code == 401 def test_DIGESTAUTH_QUOTES_QOP_VALUE(self, httpbin): + for authtype in self.digest_auth_algo: + auth = HTTPDigestAuth("user", "pass") + url = httpbin("digest-auth", "auth", "user", "pass", authtype) - auth = HTTPDigestAuth('user', 'pass') - url = httpbin('digest-auth', 'auth', 'user', 'pass') - - r = requests.get(url, auth=auth) - assert '"auth"' in r.request.headers['Authorization'] + r = requests.get(url, auth=auth) + assert '"auth"' in r.request.headers["Authorization"] def test_POSTBIN_GET_POST_FILES(self, httpbin): - - url = httpbin('post') + url = httpbin("post") requests.post(url).raise_for_status() - post1 = requests.post(url, data={'some': 'data'}) + post1 = requests.post(url, data={"some": "data"}) assert post1.status_code == 200 - with open('Pipfile') as f: - post2 = requests.post(url, files={'some': f}) + with open("requirements-dev.txt") as f: + post2 = requests.post(url, files={"some": f}) assert post2.status_code == 200 post4 = requests.post(url, data='[{"some": "json"}]') assert post4.status_code == 200 with pytest.raises(ValueError): - requests.post(url, files=['bad file data']) + requests.post(url, files=["bad file data"]) - def test_POSTBIN_SEEKED_OBJECT_WITH_NO_ITER(self, httpbin): + def test_invalid_files_input(self, httpbin): + url = httpbin("post") + post = requests.post(url, files={"random-file-1": None, "random-file-2": 1}) + assert b'name="random-file-1"' not in post.request.body + assert b'name="random-file-2"' in post.request.body - class TestStream(object): + def test_POSTBIN_SEEKED_OBJECT_WITH_NO_ITER(self, httpbin): + class TestStream: def __init__(self, data): self.data = data.encode() self.length = len(self.data) @@ -678,10 +840,10 @@ def __len__(self): def read(self, size=None): if size: - ret = self.data[self.index:self.index + size] + ret = self.data[self.index : self.index + size] self.index += size else: - ret = self.data[self.index:] + ret = self.data[self.index :] self.index = self.length return ret @@ -696,37 +858,36 @@ def seek(self, offset, where=0): elif where == 2: self.index = self.length + offset - test = TestStream('test') - post1 = requests.post(httpbin('post'), data=test) + test = TestStream("test") + post1 = requests.post(httpbin("post"), data=test) assert post1.status_code == 200 - assert post1.json()['data'] == 'test' + assert post1.json()["data"] == "test" - test = TestStream('test') + test = TestStream("test") test.seek(2) - post2 = requests.post(httpbin('post'), data=test) + post2 = requests.post(httpbin("post"), data=test) assert post2.status_code == 200 - assert post2.json()['data'] == 'st' + assert post2.json()["data"] == "st" def test_POSTBIN_GET_POST_FILES_WITH_DATA(self, httpbin): - - url = httpbin('post') + url = httpbin("post") requests.post(url).raise_for_status() - post1 = requests.post(url, data={'some': 'data'}) + post1 = requests.post(url, data={"some": "data"}) assert post1.status_code == 200 - with open('Pipfile') as f: - post2 = requests.post(url, data={'some': 'data'}, files={'some': f}) + with open("requirements-dev.txt") as f: + post2 = requests.post(url, data={"some": "data"}, files={"some": f}) assert post2.status_code == 200 post4 = requests.post(url, data='[{"some": "json"}]') assert post4.status_code == 200 with pytest.raises(ValueError): - requests.post(url, files=['bad file data']) + requests.post(url, files=["bad file data"]) def test_post_with_custom_mapping(self, httpbin): - class CustomMapping(collections.MutableMapping): + class CustomMapping(MutableMapping): def __init__(self, *args, **kwargs): self.data = dict(*args, **kwargs) @@ -745,172 +906,250 @@ def __iter__(self): def __len__(self): return len(self.data) - data = CustomMapping({'some': 'data'}) - url = httpbin('post') - found_json = requests.post(url, data=data).json().get('form') - assert found_json == {'some': 'data'} + data = CustomMapping({"some": "data"}) + url = httpbin("post") + found_json = requests.post(url, data=data).json().get("form") + assert found_json == {"some": "data"} def test_conflicting_post_params(self, httpbin): - url = httpbin('post') - with open('Pipfile') as f: - pytest.raises(ValueError, "requests.post(url, data='[{\"some\": \"data\"}]', files={'some': f})") - pytest.raises(ValueError, "requests.post(url, data=u('[{\"some\": \"data\"}]'), files={'some': f})") + url = httpbin("post") + with open("requirements-dev.txt") as f: + with pytest.raises(ValueError): + requests.post(url, data='[{"some": "data"}]', files={"some": f}) def test_request_ok_set(self, httpbin): - r = requests.get(httpbin('status', '404')) + r = requests.get(httpbin("status", "404")) assert not r.ok def test_status_raising(self, httpbin): - r = requests.get(httpbin('status', '404')) + r = requests.get(httpbin("status", "404")) with pytest.raises(requests.exceptions.HTTPError): r.raise_for_status() - r = requests.get(httpbin('status', '500')) + r = requests.get(httpbin("status", "500")) assert not r.ok def test_decompress_gzip(self, httpbin): - r = requests.get(httpbin('gzip')) - r.content.decode('ascii') + r = requests.get(httpbin("gzip")) + r.content.decode("ascii") @pytest.mark.parametrize( - 'url, params', ( - ('/get', {'foo': 'føø'}), - ('/get', {'føø': 'føø'}), - ('/get', {'føø': 'føø'}), - ('/get', {'foo': 'foo'}), - ('ø', {'foo': 'foo'}), - )) + "url, params", + ( + ("/get", {"foo": "føø"}), + ("/get", {"føø": "føø"}), + ("/get", {"føø": "føø"}), + ("/get", {"foo": "foo"}), + ("ø", {"foo": "foo"}), + ), + ) def test_unicode_get(self, httpbin, url, params): requests.get(httpbin(url), params=params) def test_unicode_header_name(self, httpbin): requests.put( - httpbin('put'), - headers={str('Content-Type'): 'application/octet-stream'}, - data='\xff') # compat.str is unicode. + httpbin("put"), + headers={"Content-Type": "application/octet-stream"}, + data="\xff", + ) # compat.str is unicode. def test_pyopenssl_redirect(self, httpbin_secure, httpbin_ca_bundle): - requests.get(httpbin_secure('status', '301'), verify=httpbin_ca_bundle) + requests.get(httpbin_secure("status", "301"), verify=httpbin_ca_bundle) def test_invalid_ca_certificate_path(self, httpbin_secure): - INVALID_PATH = '/garbage' + INVALID_PATH = "/garbage" with pytest.raises(IOError) as e: requests.get(httpbin_secure(), verify=INVALID_PATH) - assert str(e.value) == 'Could not find a suitable TLS CA certificate bundle, invalid path: {0}'.format(INVALID_PATH) + assert str( + e.value + ) == "Could not find a suitable TLS CA certificate bundle, invalid path: {}".format( + INVALID_PATH + ) def test_invalid_ssl_certificate_files(self, httpbin_secure): - INVALID_PATH = '/garbage' + INVALID_PATH = "/garbage" with pytest.raises(IOError) as e: requests.get(httpbin_secure(), cert=INVALID_PATH) - assert str(e.value) == 'Could not find the TLS certificate file, invalid path: {0}'.format(INVALID_PATH) + assert str( + e.value + ) == "Could not find the TLS certificate file, invalid path: {}".format( + INVALID_PATH + ) with pytest.raises(IOError) as e: - requests.get(httpbin_secure(), cert=('.', INVALID_PATH)) - assert str(e.value) == 'Could not find the TLS key file, invalid path: {0}'.format(INVALID_PATH) + requests.get(httpbin_secure(), cert=(".", INVALID_PATH)) + assert str(e.value) == ( + f"Could not find the TLS key file, invalid path: {INVALID_PATH}" + ) + + @pytest.mark.parametrize( + "env, expected", + ( + ({}, True), + ({"REQUESTS_CA_BUNDLE": "/some/path"}, "/some/path"), + ({"REQUESTS_CA_BUNDLE": ""}, True), + ({"CURL_CA_BUNDLE": "/some/path"}, "/some/path"), + ({"CURL_CA_BUNDLE": ""}, True), + ({"REQUESTS_CA_BUNDLE": "", "CURL_CA_BUNDLE": ""}, True), + ( + { + "REQUESTS_CA_BUNDLE": "/some/path", + "CURL_CA_BUNDLE": "/curl/path", + }, + "/some/path", + ), + ( + { + "REQUESTS_CA_BUNDLE": "", + "CURL_CA_BUNDLE": "/curl/path", + }, + "/curl/path", + ), + ), + ) + def test_env_cert_bundles(self, httpbin, env, expected): + s = requests.Session() + with mock.patch("os.environ", env): + settings = s.merge_environment_settings( + url=httpbin("get"), proxies={}, stream=False, verify=True, cert=None + ) + assert settings["verify"] == expected def test_http_with_certificate(self, httpbin): - r = requests.get(httpbin(), cert='.') + r = requests.get(httpbin(), cert=".") assert r.status_code == 200 - def test_https_warnings(self, httpbin_secure, httpbin_ca_bundle): + @pytest.mark.skipif( + SNIMissingWarning is None, + reason="urllib3 2.0 removed that warning and errors out instead", + ) + def test_https_warnings(self, nosan_server): """warnings are emitted with requests.get""" + host, port, ca_bundle = nosan_server if HAS_MODERN_SSL or HAS_PYOPENSSL: - warnings_expected = ('SubjectAltNameWarning', ) + warnings_expected = ("SubjectAltNameWarning",) else: - warnings_expected = ('SNIMissingWarning', - 'InsecurePlatformWarning', - 'SubjectAltNameWarning', ) + warnings_expected = ( + "SNIMissingWarning", + "InsecurePlatformWarning", + "SubjectAltNameWarning", + ) - with pytest.warns(None) as warning_records: - warnings.simplefilter('always') - requests.get(httpbin_secure('status', '200'), - verify=httpbin_ca_bundle) + with pytest.warns() as warning_records: + warnings.simplefilter("always") + requests.get(f"https://localhost:{port}/", verify=ca_bundle) - warning_records = [item for item in warning_records - if item.category.__name__ != 'ResourceWarning'] + warning_records = [ + item + for item in warning_records + if item.category.__name__ != "ResourceWarning" + ] - warnings_category = tuple( - item.category.__name__ for item in warning_records) + warnings_category = tuple(item.category.__name__ for item in warning_records) assert warnings_category == warnings_expected def test_certificate_failure(self, httpbin_secure): """ When underlying SSL problems occur, an SSLError is raised. """ - with pytest.raises(SSLError): + with pytest.raises(RequestsSSLError): # Our local httpbin does not have a trusted CA, so this call will # fail if we use our default trust bundle. - requests.get(httpbin_secure('status', '200')) + requests.get(httpbin_secure("status", "200")) def test_urlencoded_get_query_multivalued_param(self, httpbin): - - r = requests.get(httpbin('get'), params=dict(test=['foo', 'baz'])) + r = requests.get(httpbin("get"), params={"test": ["foo", "baz"]}) assert r.status_code == 200 - assert r.url == httpbin('get?test=foo&test=baz') + assert r.url == httpbin("get?test=foo&test=baz") + + def test_form_encoded_post_query_multivalued_element(self, httpbin): + r = requests.Request( + method="POST", url=httpbin("post"), data=dict(test=["foo", "baz"]) + ) + prep = r.prepare() + assert prep.body == "test=foo&test=baz" def test_different_encodings_dont_break_post(self, httpbin): - r = requests.post(httpbin('post'), - data={'stuff': json.dumps({'a': 123})}, - params={'blah': 'asdf1234'}, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) + with open(__file__, "rb") as f: + r = requests.post( + httpbin("post"), + data={"stuff": json.dumps({"a": 123})}, + params={"blah": "asdf1234"}, + files={"file": ("test_requests.py", f)}, + ) assert r.status_code == 200 @pytest.mark.parametrize( - 'data', ( - {'stuff': u('ëlïxr')}, - {'stuff': u('ëlïxr').encode('utf-8')}, - {'stuff': 'elixr'}, - {'stuff': 'elixr'.encode('utf-8')}, - )) + "data", + ( + {"stuff": "ëlïxr"}, + {"stuff": "ëlïxr".encode()}, + {"stuff": "elixr"}, + {"stuff": b"elixr"}, + ), + ) def test_unicode_multipart_post(self, httpbin, data): - r = requests.post(httpbin('post'), - data=data, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) + with open(__file__, "rb") as f: + r = requests.post( + httpbin("post"), + data=data, + files={"file": ("test_requests.py", f)}, + ) assert r.status_code == 200 def test_unicode_multipart_post_fieldnames(self, httpbin): - filename = os.path.splitext(__file__)[0] + '.py' - r = requests.Request( - method='POST', url=httpbin('post'), - data={'stuff'.encode('utf-8'): 'elixr'}, - files={'file': ('test_requests.py', open(filename, 'rb'))}) - prep = r.prepare() + filename = os.path.splitext(__file__)[0] + ".py" + with open(filename, "rb") as f: + r = requests.Request( + method="POST", + url=httpbin("post"), + data={b"stuff": "elixr"}, + files={"file": ("test_requests.py", f)}, + ) + prep = r.prepare() + assert b'name="stuff"' in prep.body - assert b'name="b\'stuff\'"' not in prep.body + assert b"name=\"b'stuff'\"" not in prep.body def test_unicode_method_name(self, httpbin): - files = {'file': open(__file__, 'rb')} - r = requests.request( - method=u('POST'), url=httpbin('post'), files=files) + with open(__file__, "rb") as f: + files = {"file": f} + r = requests.request( + method="POST", + url=httpbin("post"), + files=files, + ) assert r.status_code == 200 def test_unicode_method_name_with_request_object(self, httpbin): - files = {'file': open(__file__, 'rb')} s = requests.Session() - req = requests.Request(u('POST'), httpbin('post'), files=files) - prep = s.prepare_request(req) + with open(__file__, "rb") as f: + files = {"file": f} + req = requests.Request("POST", httpbin("post"), files=files) + prep = s.prepare_request(req) assert isinstance(prep.method, builtin_str) - assert prep.method == 'POST' + assert prep.method == "POST" resp = s.send(prep) assert resp.status_code == 200 def test_non_prepared_request_error(self): s = requests.Session() - req = requests.Request(u('POST'), '/') + req = requests.Request("POST", "/") with pytest.raises(ValueError) as e: s.send(req) - assert str(e.value) == 'You can only send PreparedRequests.' + assert str(e.value) == "You can only send PreparedRequests." def test_custom_content_type(self, httpbin): - r = requests.post( - httpbin('post'), - data={'stuff': json.dumps({'a': 123})}, - files={ - 'file1': ('test_requests.py', open(__file__, 'rb')), - 'file2': ('test_requests', open(__file__, 'rb'), - 'text/py-content-type')}) + with open(__file__, "rb") as f1: + with open(__file__, "rb") as f2: + data = {"stuff": json.dumps({"a": 123})} + files = { + "file1": ("test_requests.py", f1), + "file2": ("test_requests", f2, "text/py-content-type"), + } + r = requests.post(httpbin("post"), data=data, files=files) assert r.status_code == 200 assert b"text/py-content-type" in r.request.body @@ -920,50 +1159,56 @@ def hook(resp, **kwargs): assert kwargs != {} s = requests.Session() - r = requests.Request('GET', httpbin(), hooks={'response': hook}) + r = requests.Request("GET", httpbin(), hooks={"response": hook}) prep = s.prepare_request(r) s.send(prep) def test_session_hooks_are_used_with_no_request_hooks(self, httpbin): - hook = lambda x, *args, **kwargs: x + def hook(*args, **kwargs): + pass + s = requests.Session() - s.hooks['response'].append(hook) - r = requests.Request('GET', httpbin()) + s.hooks["response"].append(hook) + r = requests.Request("GET", httpbin()) prep = s.prepare_request(r) - assert prep.hooks['response'] != [] - assert prep.hooks['response'] == [hook] + assert prep.hooks["response"] != [] + assert prep.hooks["response"] == [hook] def test_session_hooks_are_overridden_by_request_hooks(self, httpbin): - hook1 = lambda x, *args, **kwargs: x - hook2 = lambda x, *args, **kwargs: x + def hook1(*args, **kwargs): + pass + + def hook2(*args, **kwargs): + pass + assert hook1 is not hook2 s = requests.Session() - s.hooks['response'].append(hook2) - r = requests.Request('GET', httpbin(), hooks={'response': [hook1]}) + s.hooks["response"].append(hook2) + r = requests.Request("GET", httpbin(), hooks={"response": [hook1]}) prep = s.prepare_request(r) - assert prep.hooks['response'] == [hook1] + assert prep.hooks["response"] == [hook1] def test_prepared_request_hook(self, httpbin): def hook(resp, **kwargs): resp.hook_working = True return resp - req = requests.Request('GET', httpbin(), hooks={'response': hook}) + req = requests.Request("GET", httpbin(), hooks={"response": hook}) prep = req.prepare() s = requests.Session() s.proxies = getproxies() resp = s.send(prep) - assert hasattr(resp, 'hook_working') + assert hasattr(resp, "hook_working") def test_prepared_from_session(self, httpbin): class DummyAuth(requests.auth.AuthBase): def __call__(self, r): - r.headers['Dummy-Auth-Test'] = 'dummy-auth-test-ok' + r.headers["Dummy-Auth-Test"] = "dummy-auth-test-ok" return r - req = requests.Request('GET', httpbin('headers')) + req = requests.Request("GET", httpbin("headers")) assert not req.auth s = requests.Session() @@ -972,11 +1217,10 @@ def __call__(self, r): prep = s.prepare_request(req) resp = s.send(prep) - assert resp.json()['headers'][ - 'Dummy-Auth-Test'] == 'dummy-auth-test-ok' + assert resp.json()["headers"]["Dummy-Auth-Test"] == "dummy-auth-test-ok" def test_prepare_request_with_bytestring_url(self): - req = requests.Request('GET', b'https://httpbin.org/') + req = requests.Request("GET", b"https://httpbin.org/") s = requests.Session() prep = s.prepare_request(req) assert prep.url == "https://httpbin.org/" @@ -984,61 +1228,63 @@ def test_prepare_request_with_bytestring_url(self): def test_request_with_bytestring_host(self, httpbin): s = requests.Session() resp = s.request( - 'GET', - httpbin('cookies/set?cookie=value'), + "GET", + httpbin("cookies/set?cookie=value"), allow_redirects=False, - headers={'Host': b'httpbin.org'} + headers={"Host": b"httpbin.org"}, ) - assert resp.cookies.get('cookie') == 'value' + assert resp.cookies.get("cookie") == "value" def test_links(self): r = requests.Response() r.headers = { - 'cache-control': 'public, max-age=60, s-maxage=60', - 'connection': 'keep-alive', - 'content-encoding': 'gzip', - 'content-type': 'application/json; charset=utf-8', - 'date': 'Sat, 26 Jan 2013 16:47:56 GMT', - 'etag': '"6ff6a73c0e446c1f61614769e3ceb778"', - 'last-modified': 'Sat, 26 Jan 2013 16:22:39 GMT', - 'link': ('; rel="next", ; ' - ' rel="last"'), - 'server': 'GitHub.com', - 'status': '200 OK', - 'vary': 'Accept', - 'x-content-type-options': 'nosniff', - 'x-github-media-type': 'github.beta', - 'x-ratelimit-limit': '60', - 'x-ratelimit-remaining': '57' + "cache-control": "public, max-age=60, s-maxage=60", + "connection": "keep-alive", + "content-encoding": "gzip", + "content-type": "application/json; charset=utf-8", + "date": "Sat, 26 Jan 2013 16:47:56 GMT", + "etag": '"6ff6a73c0e446c1f61614769e3ceb778"', + "last-modified": "Sat, 26 Jan 2013 16:22:39 GMT", + "link": ( + "; rel="next", ; " + ' rel="last"' + ), + "server": "GitHub.com", + "status": "200 OK", + "vary": "Accept", + "x-content-type-options": "nosniff", + "x-github-media-type": "github.beta", + "x-ratelimit-limit": "60", + "x-ratelimit-remaining": "57", } - assert r.links['next']['rel'] == 'next' + assert r.links["next"]["rel"] == "next" def test_cookie_parameters(self): - key = 'some_cookie' - value = 'some_value' + key = "some_cookie" + value = "some_value" secure = True - domain = 'test.com' - rest = {'HttpOnly': True} + domain = "test.com" + rest = {"HttpOnly": True} jar = requests.cookies.RequestsCookieJar() jar.set(key, value, secure=secure, domain=domain, rest=rest) assert len(jar) == 1 - assert 'some_cookie' in jar + assert "some_cookie" in jar cookie = list(jar)[0] assert cookie.secure == secure assert cookie.domain == domain - assert cookie._rest['HttpOnly'] == rest['HttpOnly'] + assert cookie._rest["HttpOnly"] == rest["HttpOnly"] def test_cookie_as_dict_keeps_len(self): - key = 'some_cookie' - value = 'some_value' + key = "some_cookie" + value = "some_value" - key1 = 'some_cookie1' - value1 = 'some_value1' + key1 = "some_cookie1" + value1 = "some_value1" jar = requests.cookies.RequestsCookieJar() jar.set(key, value) @@ -1054,11 +1300,11 @@ def test_cookie_as_dict_keeps_len(self): assert len(d3) == 2 def test_cookie_as_dict_keeps_items(self): - key = 'some_cookie' - value = 'some_value' + key = "some_cookie" + value = "some_value" - key1 = 'some_cookie1' - value1 = 'some_value1' + key1 = "some_cookie1" + value1 = "some_value1" jar = requests.cookies.RequestsCookieJar() jar.set(key, value) @@ -1068,16 +1314,16 @@ def test_cookie_as_dict_keeps_items(self): d2 = dict(jar.iteritems()) d3 = dict(jar.items()) - assert d1['some_cookie'] == 'some_value' - assert d2['some_cookie'] == 'some_value' - assert d3['some_cookie1'] == 'some_value1' + assert d1["some_cookie"] == "some_value" + assert d2["some_cookie"] == "some_value" + assert d3["some_cookie1"] == "some_value1" def test_cookie_as_dict_keys(self): - key = 'some_cookie' - value = 'some_value' + key = "some_cookie" + value = "some_value" - key1 = 'some_cookie1' - value1 = 'some_value1' + key1 = "some_cookie1" + value1 = "some_value1" jar = requests.cookies.RequestsCookieJar() jar.set(key, value) @@ -1089,11 +1335,11 @@ def test_cookie_as_dict_keys(self): assert list(keys) == list(keys) def test_cookie_as_dict_values(self): - key = 'some_cookie' - value = 'some_value' + key = "some_cookie" + value = "some_value" - key1 = 'some_cookie1' - value1 = 'some_value1' + key1 = "some_cookie1" + value1 = "some_value1" jar = requests.cookies.RequestsCookieJar() jar.set(key, value) @@ -1105,11 +1351,11 @@ def test_cookie_as_dict_values(self): assert list(values) == list(values) def test_cookie_as_dict_items(self): - key = 'some_cookie' - value = 'some_value' + key = "some_cookie" + value = "some_value" - key1 = 'some_cookie1' - value1 = 'some_value1' + key1 = "some_cookie1" + value1 = "some_value1" jar = requests.cookies.RequestsCookieJar() jar.set(key, value) @@ -1121,10 +1367,10 @@ def test_cookie_as_dict_items(self): assert list(items) == list(items) def test_cookie_duplicate_names_different_domains(self): - key = 'some_cookie' - value = 'some_value' - domain1 = 'test1.com' - domain2 = 'test2.com' + key = "some_cookie" + value = "some_value" + domain1 = "test1.com" + domain2 = "test2.com" jar = requests.cookies.RequestsCookieJar() jar.set(key, value, domain=domain1) @@ -1142,9 +1388,9 @@ def test_cookie_duplicate_names_different_domains(self): assert cookie == value def test_cookie_duplicate_names_raises_cookie_conflict_error(self): - key = 'some_cookie' - value = 'some_value' - path = 'some_path' + key = "some_cookie" + value = "some_value" + path = "some_path" jar = requests.cookies.RequestsCookieJar() jar.set(key, value, path=path) @@ -1152,10 +1398,20 @@ def test_cookie_duplicate_names_raises_cookie_conflict_error(self): with pytest.raises(requests.cookies.CookieConflictError): jar.get(key) + def test_cookie_policy_copy(self): + class MyCookiePolicy(cookielib.DefaultCookiePolicy): + pass + + jar = requests.cookies.RequestsCookieJar() + jar.set_policy(MyCookiePolicy()) + assert isinstance(jar.copy().get_policy(), MyCookiePolicy) + def test_time_elapsed_blank(self, httpbin): - r = requests.get(httpbin('get')) + r = requests.get(httpbin("get")) td = r.elapsed - total_seconds = ((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6) + total_seconds = ( + td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6 + ) / 10**6 assert total_seconds > 0.0 def test_empty_response_has_content_none(self): @@ -1164,12 +1420,13 @@ def test_empty_response_has_content_none(self): def test_response_is_iterable(self): r = requests.Response() - io = StringIO.StringIO('abc') + io = StringIO.StringIO("abc") read_ = io.read def read_mock(amt, decode_content=None): return read_(amt) - setattr(io, 'read', read_mock) + + setattr(io, "read", read_mock) r.raw = io assert next(iter(r)) io.close() @@ -1180,24 +1437,24 @@ def test_response_decode_unicode(self): """ r = requests.Response() r._content_consumed = True - r._content = b'the content' - r.encoding = 'ascii' + r._content = b"the content" + r.encoding = "ascii" chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) # also for streaming r = requests.Response() - r.raw = io.BytesIO(b'the content') - r.encoding = 'ascii' + r.raw = io.BytesIO(b"the content") + r.encoding = "ascii" chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) def test_response_reason_unicode(self): # check for unicode HTTP status r = requests.Response() - r.url = u'unicode URL' - r.reason = u'Komponenttia ei löydy'.encode('utf-8') + r.url = "unicode URL" + r.reason = "Komponenttia ei löydy".encode() r.status_code = 404 r.encoding = None assert not r.ok # old behaviour - crashes here @@ -1205,9 +1462,9 @@ def test_response_reason_unicode(self): def test_response_reason_unicode_fallback(self): # check raise_status falls back to ISO-8859-1 r = requests.Response() - r.url = 'some url' - reason = u'Komponenttia ei löydy' - r.reason = reason.encode('latin-1') + r.url = "some url" + reason = "Komponenttia ei löydy" + r.reason = reason.encode("latin-1") r.status_code = 500 r.encoding = None with pytest.raises(requests.exceptions.HTTPError) as e: @@ -1219,22 +1476,41 @@ def test_response_chunk_size_type(self): raise a TypeError. """ r = requests.Response() - r.raw = io.BytesIO(b'the content') + r.raw = io.BytesIO(b"the content") chunks = r.iter_content(1) assert all(len(chunk) == 1 for chunk in chunks) r = requests.Response() - r.raw = io.BytesIO(b'the content') + r.raw = io.BytesIO(b"the content") chunks = r.iter_content(None) - assert list(chunks) == [b'the content'] + assert list(chunks) == [b"the content"] r = requests.Response() - r.raw = io.BytesIO(b'the content') + r.raw = io.BytesIO(b"the content") with pytest.raises(TypeError): chunks = r.iter_content("1024") + @pytest.mark.parametrize( + "exception, args, expected", + ( + (urllib3.exceptions.ProtocolError, tuple(), ChunkedEncodingError), + (urllib3.exceptions.DecodeError, tuple(), ContentDecodingError), + (urllib3.exceptions.ReadTimeoutError, (None, "", ""), ConnectionError), + (urllib3.exceptions.SSLError, tuple(), RequestsSSLError), + ), + ) + def test_iter_content_wraps_exceptions(self, httpbin, exception, args, expected): + r = requests.Response() + r.raw = mock.Mock() + # ReadTimeoutError can't be initialized by mock + # so we'll manually create the instance with args + r.raw.stream.side_effect = exception(*args) + + with pytest.raises(expected): + next(r.iter_content(1024)) + def test_request_and_response_are_pickleable(self, httpbin): - r = requests.get(httpbin('get')) + r = requests.get(httpbin("get")) # verify we can pickle the original request assert pickle.loads(pickle.dumps(r.request)) @@ -1246,7 +1522,7 @@ def test_request_and_response_are_pickleable(self, httpbin): assert r.request.headers == pr.request.headers def test_prepared_request_is_pickleable(self, httpbin): - p = requests.Request('GET', httpbin('get')).prepare() + p = requests.Request("GET", httpbin("get")).prepare() # Verify PreparedRequest can be pickled and unpickled r = pickle.loads(pickle.dumps(p)) @@ -1260,9 +1536,9 @@ def test_prepared_request_is_pickleable(self, httpbin): assert resp.status_code == 200 def test_prepared_request_with_file_is_pickleable(self, httpbin): - files = {'file': open(__file__, 'rb')} - r = requests.Request('POST', httpbin('post'), files=files) - p = r.prepare() + with open(__file__, "rb") as f: + r = requests.Request("POST", httpbin("post"), files={"file": f}) + p = r.prepare() # Verify PreparedRequest can be pickled and unpickled r = pickle.loads(pickle.dumps(p)) @@ -1276,7 +1552,7 @@ def test_prepared_request_with_file_is_pickleable(self, httpbin): assert resp.status_code == 200 def test_prepared_request_with_hook_is_pickleable(self, httpbin): - r = requests.Request('GET', httpbin('get'), hooks=default_hooks()) + r = requests.Request("GET", httpbin("get"), hooks=default_hooks()) p = r.prepare() # Verify PreparedRequest can be pickled @@ -1302,12 +1578,12 @@ def test_http_error(self): response = requests.Response() error = requests.exceptions.HTTPError(response=response) assert error.response == response - error = requests.exceptions.HTTPError('message', response=response) - assert str(error) == 'message' + error = requests.exceptions.HTTPError("message", response=response) + assert str(error) == "message" assert error.response == response def test_session_pickling(self, httpbin): - r = requests.Request('GET', httpbin('get')) + r = requests.Request("GET", httpbin("get")) s = requests.Session() s = pickle.loads(pickle.dumps(s)) @@ -1319,66 +1595,66 @@ def test_session_pickling(self, httpbin): def test_fixes_1329(self, httpbin): """Ensure that header updates are done case-insensitively.""" s = requests.Session() - s.headers.update({'ACCEPT': 'BOGUS'}) - s.headers.update({'accept': 'application/json'}) - r = s.get(httpbin('get')) + s.headers.update({"ACCEPT": "BOGUS"}) + s.headers.update({"accept": "application/json"}) + r = s.get(httpbin("get")) headers = r.request.headers - assert headers['accept'] == 'application/json' - assert headers['Accept'] == 'application/json' - assert headers['ACCEPT'] == 'application/json' + assert headers["accept"] == "application/json" + assert headers["Accept"] == "application/json" + assert headers["ACCEPT"] == "application/json" def test_uppercase_scheme_redirect(self, httpbin): - parts = urlparse(httpbin('html')) + parts = urlparse(httpbin("html")) url = "HTTP://" + parts.netloc + parts.path - r = requests.get(httpbin('redirect-to'), params={'url': url}) + r = requests.get(httpbin("redirect-to"), params={"url": url}) assert r.status_code == 200 assert r.url.lower() == url.lower() def test_transport_adapter_ordering(self): s = requests.Session() - order = ['https://', 'http://'] + order = ["https://", "http://"] assert order == list(s.adapters) - s.mount('http://git', HTTPAdapter()) - s.mount('http://github', HTTPAdapter()) - s.mount('http://github.com', HTTPAdapter()) - s.mount('http://github.com/about/', HTTPAdapter()) + s.mount("http://git", HTTPAdapter()) + s.mount("http://github", HTTPAdapter()) + s.mount("http://github.com", HTTPAdapter()) + s.mount("http://github.com/about/", HTTPAdapter()) order = [ - 'http://github.com/about/', - 'http://github.com', - 'http://github', - 'http://git', - 'https://', - 'http://', + "http://github.com/about/", + "http://github.com", + "http://github", + "http://git", + "https://", + "http://", ] assert order == list(s.adapters) - s.mount('http://gittip', HTTPAdapter()) - s.mount('http://gittip.com', HTTPAdapter()) - s.mount('http://gittip.com/about/', HTTPAdapter()) + s.mount("http://gittip", HTTPAdapter()) + s.mount("http://gittip.com", HTTPAdapter()) + s.mount("http://gittip.com/about/", HTTPAdapter()) order = [ - 'http://github.com/about/', - 'http://gittip.com/about/', - 'http://github.com', - 'http://gittip.com', - 'http://github', - 'http://gittip', - 'http://git', - 'https://', - 'http://', + "http://github.com/about/", + "http://gittip.com/about/", + "http://github.com", + "http://gittip.com", + "http://github", + "http://gittip", + "http://git", + "https://", + "http://", ] assert order == list(s.adapters) s2 = requests.Session() - s2.adapters = {'http://': HTTPAdapter()} - s2.mount('https://', HTTPAdapter()) - assert 'http://' in s2.adapters - assert 'https://' in s2.adapters + s2.adapters = {"http://": HTTPAdapter()} + s2.mount("https://", HTTPAdapter()) + assert "http://" in s2.adapters + assert "https://" in s2.adapters - def test_session_get_adapter_prefix_matching(self, httpbin): - prefix = 'https://example.com' - more_specific_prefix = prefix + '/some/path' + def test_session_get_adapter_prefix_matching(self): + prefix = "https://example.com" + more_specific_prefix = prefix + "/some/path" - url_matching_only_prefix = prefix + '/another/path' - url_matching_more_specific_prefix = more_specific_prefix + '/longer/path' - url_not_matching_prefix = 'https://another.example.com/' + url_matching_only_prefix = prefix + "/another/path" + url_matching_more_specific_prefix = more_specific_prefix + "/longer/path" + url_not_matching_prefix = "https://another.example.com/" s = requests.Session() prefix_adapter = HTTPAdapter() @@ -1387,12 +1663,18 @@ def test_session_get_adapter_prefix_matching(self, httpbin): s.mount(more_specific_prefix, more_specific_prefix_adapter) assert s.get_adapter(url_matching_only_prefix) is prefix_adapter - assert s.get_adapter(url_matching_more_specific_prefix) is more_specific_prefix_adapter - assert s.get_adapter(url_not_matching_prefix) not in (prefix_adapter, more_specific_prefix_adapter) + assert ( + s.get_adapter(url_matching_more_specific_prefix) + is more_specific_prefix_adapter + ) + assert s.get_adapter(url_not_matching_prefix) not in ( + prefix_adapter, + more_specific_prefix_adapter, + ) - def test_session_get_adapter_prefix_matching_mixed_case(self, httpbin): - mixed_case_prefix = 'hTtPs://eXamPle.CoM/MixEd_CAse_PREfix' - url_matching_prefix = mixed_case_prefix + '/full_url' + def test_session_get_adapter_prefix_matching_mixed_case(self): + mixed_case_prefix = "hTtPs://eXamPle.CoM/MixEd_CAse_PREfix" + url_matching_prefix = mixed_case_prefix + "/full_url" s = requests.Session() my_adapter = HTTPAdapter() @@ -1400,9 +1682,11 @@ def test_session_get_adapter_prefix_matching_mixed_case(self, httpbin): assert s.get_adapter(url_matching_prefix) is my_adapter - def test_session_get_adapter_prefix_matching_is_case_insensitive(self, httpbin): - mixed_case_prefix = 'hTtPs://eXamPle.CoM/MixEd_CAse_PREfix' - url_matching_prefix_with_different_case = 'HtTpS://exaMPLe.cOm/MiXeD_caSE_preFIX/another_url' + def test_session_get_adapter_prefix_matching_is_case_insensitive(self): + mixed_case_prefix = "hTtPs://eXamPle.CoM/MixEd_CAse_PREfix" + url_matching_prefix_with_different_case = ( + "HtTpS://exaMPLe.cOm/MiXeD_caSE_preFIX/another_url" + ) s = requests.Session() my_adapter = HTTPAdapter() @@ -1410,155 +1694,261 @@ def test_session_get_adapter_prefix_matching_is_case_insensitive(self, httpbin): assert s.get_adapter(url_matching_prefix_with_different_case) is my_adapter + def test_session_get_adapter_prefix_with_trailing_slash(self): + # from issue #6935 + prefix = "https://example.com/" # trailing slash + url_matching_prefix = "https://example.com/some/path" + url_not_matching_prefix = "https://example.com.other.com/some/path" + + s = requests.Session() + adapter = HTTPAdapter() + s.mount(prefix, adapter) + + assert s.get_adapter(url_matching_prefix) is adapter + assert s.get_adapter(url_not_matching_prefix) is not adapter + + def test_session_get_adapter_prefix_without_trailing_slash(self): + # from issue #6935 + prefix = "https://example.com" # no trailing slash + url_matching_prefix = "https://example.com/some/path" + url_extended_hostname = "https://example.com.other.com/some/path" + + s = requests.Session() + adapter = HTTPAdapter() + s.mount(prefix, adapter) + + assert s.get_adapter(url_matching_prefix) is adapter + assert s.get_adapter(url_extended_hostname) is adapter + def test_header_remove_is_case_insensitive(self, httpbin): # From issue #1321 s = requests.Session() - s.headers['foo'] = 'bar' - r = s.get(httpbin('get'), headers={'FOO': None}) - assert 'foo' not in r.request.headers + s.headers["foo"] = "bar" + r = s.get(httpbin("get"), headers={"FOO": None}) + assert "foo" not in r.request.headers def test_params_are_merged_case_sensitive(self, httpbin): s = requests.Session() - s.params['foo'] = 'bar' - r = s.get(httpbin('get'), params={'FOO': 'bar'}) - assert r.json()['args'] == {'foo': 'bar', 'FOO': 'bar'} + s.params["foo"] = "bar" + r = s.get(httpbin("get"), params={"FOO": "bar"}) + assert r.json()["args"] == {"foo": "bar", "FOO": "bar"} def test_long_authinfo_in_url(self): - url = 'http://{0}:{1}@{2}:9000/path?query#frag'.format( - 'E8A3BE87-9E3F-4620-8858-95478E385B5B', - 'EA770032-DA4D-4D84-8CE9-29C6D910BF1E', - 'exactly-------------sixty-----------three------------characters', + url = "http://{}:{}@{}:9000/path?query#frag".format( + "E8A3BE87-9E3F-4620-8858-95478E385B5B", + "EA770032-DA4D-4D84-8CE9-29C6D910BF1E", + "exactly-------------sixty-----------three------------characters", ) - r = requests.Request('GET', url).prepare() + r = requests.Request("GET", url).prepare() assert r.url == url def test_header_keys_are_native(self, httpbin): - headers = {u('unicode'): 'blah', 'byte'.encode('ascii'): 'blah'} - r = requests.Request('GET', httpbin('get'), headers=headers) + headers = {"unicode": "blah", b"byte": "blah"} + r = requests.Request("GET", httpbin("get"), headers=headers) p = r.prepare() # This is testing that they are builtin strings. A bit weird, but there # we go. - assert 'unicode' in p.headers.keys() - assert 'byte' in p.headers.keys() + assert "unicode" in p.headers.keys() + assert "byte" in p.headers.keys() def test_header_validation(self, httpbin): """Ensure prepare_headers regex isn't flagging valid header contents.""" - headers_ok = {'foo': 'bar baz qux', - 'bar': u'fbbq'.encode('utf8'), - 'baz': '', - 'qux': '1'} - r = requests.get(httpbin('get'), headers=headers_ok) - assert r.request.headers['foo'] == headers_ok['foo'] - - def test_header_value_not_str(self, httpbin): + valid_headers = { + "foo": "bar baz qux", + "bar": b"fbbq", + "baz": "", + "qux": "1", + } + r = requests.get(httpbin("get"), headers=valid_headers) + for key in valid_headers.keys(): + assert valid_headers[key] == r.request.headers[key] + + @pytest.mark.parametrize( + "invalid_header, key", + ( + ({"foo": 3}, "foo"), + ({"bar": {"foo": "bar"}}, "bar"), + ({"baz": ["foo", "bar"]}, "baz"), + ), + ) + def test_header_value_not_str(self, httpbin, invalid_header, key): """Ensure the header value is of type string or bytes as per discussion in GH issue #3386 """ - headers_int = {'foo': 3} - headers_dict = {'bar': {'foo': 'bar'}} - headers_list = {'baz': ['foo', 'bar']} - - # Test for int - with pytest.raises(InvalidHeader) as excinfo: - r = requests.get(httpbin('get'), headers=headers_int) - assert 'foo' in str(excinfo.value) - # Test for dict - with pytest.raises(InvalidHeader) as excinfo: - r = requests.get(httpbin('get'), headers=headers_dict) - assert 'bar' in str(excinfo.value) - # Test for list with pytest.raises(InvalidHeader) as excinfo: - r = requests.get(httpbin('get'), headers=headers_list) - assert 'baz' in str(excinfo.value) + requests.get(httpbin("get"), headers=invalid_header) + assert key in str(excinfo.value) - def test_header_no_return_chars(self, httpbin): + @pytest.mark.parametrize( + "invalid_header", + ( + {"foo": "bar\r\nbaz: qux"}, + {"foo": "bar\n\rbaz: qux"}, + {"foo": "bar\nbaz: qux"}, + {"foo": "bar\rbaz: qux"}, + {"fo\ro": "bar"}, + {"fo\r\no": "bar"}, + {"fo\n\ro": "bar"}, + {"fo\no": "bar"}, + ), + ) + def test_header_no_return_chars(self, httpbin, invalid_header): """Ensure that a header containing return character sequences raise an exception. Otherwise, multiple headers are created from single string. """ - headers_ret = {'foo': 'bar\r\nbaz: qux'} - headers_lf = {'foo': 'bar\nbaz: qux'} - headers_cr = {'foo': 'bar\rbaz: qux'} - - # Test for newline - with pytest.raises(InvalidHeader): - r = requests.get(httpbin('get'), headers=headers_ret) - # Test for line feed with pytest.raises(InvalidHeader): - r = requests.get(httpbin('get'), headers=headers_lf) - # Test for carriage return - with pytest.raises(InvalidHeader): - r = requests.get(httpbin('get'), headers=headers_cr) + requests.get(httpbin("get"), headers=invalid_header) - def test_header_no_leading_space(self, httpbin): + @pytest.mark.parametrize( + "invalid_header", + ( + {" foo": "bar"}, + {"\tfoo": "bar"}, + {" foo": "bar"}, + {"foo": " bar"}, + {"foo": " bar"}, + {"foo": "\tbar"}, + {" ": "bar"}, + ), + ) + def test_header_no_leading_space(self, httpbin, invalid_header): """Ensure headers containing leading whitespace raise InvalidHeader Error before sending. """ - headers_space = {'foo': ' bar'} - headers_tab = {'foo': ' bar'} - - # Test for whitespace with pytest.raises(InvalidHeader): - r = requests.get(httpbin('get'), headers=headers_space) - # Test for tab - with pytest.raises(InvalidHeader): - r = requests.get(httpbin('get'), headers=headers_tab) + requests.get(httpbin("get"), headers=invalid_header) + + def test_header_with_subclass_types(self, httpbin): + """If the subclasses does not behave *exactly* like + the base bytes/str classes, this is not supported. + This test is for backwards compatibility. + """ + + class MyString(str): + pass + + class MyBytes(bytes): + pass + + r_str = requests.get(httpbin("get"), headers={MyString("x-custom"): "myheader"}) + assert r_str.request.headers["x-custom"] == "myheader" + + r_bytes = requests.get( + httpbin("get"), headers={MyBytes(b"x-custom"): b"myheader"} + ) + assert r_bytes.request.headers["x-custom"] == b"myheader" - @pytest.mark.parametrize('files', ('foo', b'foo', bytearray(b'foo'))) + r_mixed = requests.get( + httpbin("get"), headers={MyString("x-custom"): MyBytes(b"myheader")} + ) + assert r_mixed.request.headers["x-custom"] == b"myheader" + + @pytest.mark.parametrize("files", ("foo", b"foo", bytearray(b"foo"))) def test_can_send_objects_with_files(self, httpbin, files): - data = {'a': 'this is a string'} - files = {'b': files} - r = requests.Request('POST', httpbin('post'), data=data, files=files) + data = {"a": "this is a string"} + files = {"b": files} + r = requests.Request("POST", httpbin("post"), data=data, files=files) p = r.prepare() - assert 'multipart/form-data' in p.headers['Content-Type'] + assert "multipart/form-data" in p.headers["Content-Type"] def test_can_send_file_object_with_non_string_filename(self, httpbin): f = io.BytesIO() f.name = 2 - r = requests.Request('POST', httpbin('post'), files={'f': f}) + r = requests.Request("POST", httpbin("post"), files={"f": f}) p = r.prepare() - assert 'multipart/form-data' in p.headers['Content-Type'] + assert "multipart/form-data" in p.headers["Content-Type"] def test_autoset_header_values_are_native(self, httpbin): - data = 'this is a string' - length = '16' - req = requests.Request('POST', httpbin('post'), data=data) + data = "this is a string" + length = "16" + req = requests.Request("POST", httpbin("post"), data=data) p = req.prepare() - assert p.headers['Content-Length'] == length + assert p.headers["Content-Length"] == length def test_nonhttp_schemes_dont_check_URLs(self): test_urls = ( - 'data:image/gif;base64,R0lGODlhAQABAHAAACH5BAUAAAAALAAAAAABAAEAAAICRAEAOw==', - 'file:///etc/passwd', - 'magnet:?xt=urn:btih:be08f00302bc2d1d3cfa3af02024fa647a271431', + "data:image/gif;base64,R0lGODlhAQABAHAAACH5BAUAAAAALAAAAAABAAEAAAICRAEAOw==", + "file:///etc/passwd", + "magnet:?xt=urn:btih:be08f00302bc2d1d3cfa3af02024fa647a271431", ) for test_url in test_urls: - req = requests.Request('GET', test_url) + req = requests.Request("GET", test_url) preq = req.prepare() assert test_url == preq.url - @pytest.mark.xfail(raises=ConnectionError) - def test_auth_is_stripped_on_redirect_off_host(self, httpbin): + def test_auth_is_stripped_on_http_downgrade( + self, httpbin, httpbin_secure, httpbin_ca_bundle + ): r = requests.get( - httpbin('redirect-to'), - params={'url': 'http://www.google.co.uk'}, - auth=('user', 'pass'), + httpbin_secure("redirect-to"), + params={"url": httpbin("get")}, + auth=("user", "pass"), + verify=httpbin_ca_bundle, ) - assert r.history[0].request.headers['Authorization'] - assert not r.request.headers.get('Authorization', '') + assert r.history[0].request.headers["Authorization"] + assert "Authorization" not in r.request.headers def test_auth_is_retained_for_redirect_on_host(self, httpbin): - r = requests.get(httpbin('redirect/1'), auth=('user', 'pass')) - h1 = r.history[0].request.headers['Authorization'] - h2 = r.request.headers['Authorization'] + r = requests.get(httpbin("redirect/1"), auth=("user", "pass")) + h1 = r.history[0].request.headers["Authorization"] + h2 = r.request.headers["Authorization"] assert h1 == h2 + def test_should_strip_auth_host_change(self): + s = requests.Session() + assert s.should_strip_auth( + "http://example.com/foo", "http://another.example.com/" + ) + + def test_should_strip_auth_http_downgrade(self): + s = requests.Session() + assert s.should_strip_auth("https://example.com/foo", "http://example.com/bar") + + def test_should_strip_auth_https_upgrade(self): + s = requests.Session() + assert not s.should_strip_auth( + "http://example.com/foo", "https://example.com/bar" + ) + assert not s.should_strip_auth( + "http://example.com:80/foo", "https://example.com/bar" + ) + assert not s.should_strip_auth( + "http://example.com/foo", "https://example.com:443/bar" + ) + # Non-standard ports should trigger stripping + assert s.should_strip_auth( + "http://example.com:8080/foo", "https://example.com/bar" + ) + assert s.should_strip_auth( + "http://example.com/foo", "https://example.com:8443/bar" + ) + + def test_should_strip_auth_port_change(self): + s = requests.Session() + assert s.should_strip_auth( + "http://example.com:1234/foo", "https://example.com:4321/bar" + ) + + @pytest.mark.parametrize( + "old_uri, new_uri", + ( + ("https://example.com:443/foo", "https://example.com/bar"), + ("http://example.com:80/foo", "http://example.com/bar"), + ("https://example.com/foo", "https://example.com:443/bar"), + ("http://example.com/foo", "http://example.com:80/bar"), + ), + ) + def test_should_strip_auth_default_port(self, old_uri, new_uri): + s = requests.Session() + assert not s.should_strip_auth(old_uri, new_uri) + def test_manual_redirect_with_partial_body_read(self, httpbin): s = requests.Session() - r1 = s.get(httpbin('redirect/2'), allow_redirects=False, stream=True) + r1 = s.get(httpbin("redirect/2"), allow_redirects=False, stream=True) assert r1.is_redirect rg = s.resolve_redirects(r1, r1.request, stream=True) @@ -1576,39 +1966,36 @@ def test_manual_redirect_with_partial_body_read(self, httpbin): assert not r3.is_redirect def test_prepare_body_position_non_stream(self): - data = b'the data' - s = requests.Session() - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + data = b"the data" + prep = requests.Request("GET", "http://example.com", data=data).prepare() assert prep._body_position is None def test_rewind_body(self): - data = io.BytesIO(b'the data') - s = requests.Session() - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + data = io.BytesIO(b"the data") + prep = requests.Request("GET", "http://example.com", data=data).prepare() assert prep._body_position == 0 - assert prep.body.read() == b'the data' + assert prep.body.read() == b"the data" # the data has all been read - assert prep.body.read() == b'' + assert prep.body.read() == b"" # rewind it back requests.utils.rewind_body(prep) - assert prep.body.read() == b'the data' + assert prep.body.read() == b"the data" def test_rewind_partially_read_body(self): - data = io.BytesIO(b'the data') - s = requests.Session() + data = io.BytesIO(b"the data") data.read(4) # read some data - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + prep = requests.Request("GET", "http://example.com", data=data).prepare() assert prep._body_position == 4 - assert prep.body.read() == b'data' + assert prep.body.read() == b"data" # the data has all been read - assert prep.body.read() == b'' + assert prep.body.read() == b"" # rewind it back requests.utils.rewind_body(prep) - assert prep.body.read() == b'data' + assert prep.body.read() == b"data" def test_rewind_body_no_seek(self): class BadFileObj: @@ -1621,15 +2008,14 @@ def tell(self): def __iter__(self): return - data = BadFileObj('the data') - s = requests.Session() - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + data = BadFileObj("the data") + prep = requests.Request("GET", "http://example.com", data=data).prepare() assert prep._body_position == 0 with pytest.raises(UnrewindableBodyError) as e: requests.utils.rewind_body(prep) - assert 'Unable to rewind request body' in str(e) + assert "Unable to rewind request body" in str(e) def test_rewind_body_failed_seek(self): class BadFileObj: @@ -1645,15 +2031,14 @@ def seek(self, pos, whence=0): def __iter__(self): return - data = BadFileObj('the data') - s = requests.Session() - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + data = BadFileObj("the data") + prep = requests.Request("GET", "http://example.com", data=data).prepare() assert prep._body_position == 0 with pytest.raises(UnrewindableBodyError) as e: requests.utils.rewind_body(prep) - assert 'error occurred when rewinding request body' in str(e) + assert "error occurred when rewinding request body" in str(e) def test_rewind_body_failed_tell(self): class BadFileObj: @@ -1666,15 +2051,14 @@ def tell(self): def __iter__(self): return - data = BadFileObj('the data') - s = requests.Session() - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + data = BadFileObj("the data") + prep = requests.Request("GET", "http://example.com", data=data).prepare() assert prep._body_position is not None with pytest.raises(UnrewindableBodyError) as e: requests.utils.rewind_body(prep) - assert 'Unable to rewind request body' in str(e) + assert "Unable to rewind request body" in str(e) def _patch_adapter_gzipped_redirect(self, session, url): adapter = session.get_adapter(url=url) @@ -1684,7 +2068,7 @@ def _patch_adapter_gzipped_redirect(self, session, url): def build_response(*args, **kwargs): resp = org_build_response(*args, **kwargs) if not self._patched_response: - resp.raw.headers['content-encoding'] = 'gzip' + resp.raw.headers["content-encoding"] = "gzip" self._patched_response = True return resp @@ -1692,22 +2076,28 @@ def build_response(*args, **kwargs): def test_redirect_with_wrong_gzipped_header(self, httpbin): s = requests.Session() - url = httpbin('redirect/1') + url = httpbin("redirect/1") self._patch_adapter_gzipped_redirect(s, url) s.get(url) @pytest.mark.parametrize( - 'username, password, auth_str', ( - ('test', 'test', 'Basic dGVzdDp0ZXN0'), - (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8'), 'Basic 0LjQvNGPOtC/0LDRgNC+0LvRjA=='), - )) + "username, password, auth_str", + ( + ("test", "test", "Basic dGVzdDp0ZXN0"), + ( + "имя".encode(), + "пароль".encode(), + "Basic 0LjQvNGPOtC/0LDRgNC+0LvRjA==", + ), + ), + ) def test_basic_auth_str_is_always_native(self, username, password, auth_str): s = _basic_auth_str(username, password) assert isinstance(s, builtin_str) assert s == auth_str def test_requests_history_is_saved(self, httpbin): - r = requests.get(httpbin('redirect/5')) + r = requests.get(httpbin("redirect/5")) total = r.history[-1].history i = 0 for item in r.history: @@ -1715,23 +2105,23 @@ def test_requests_history_is_saved(self, httpbin): i += 1 def test_json_param_post_content_type_works(self, httpbin): - r = requests.post( - httpbin('post'), - json={'life': 42} - ) + r = requests.post(httpbin("post"), json={"life": 42}) assert r.status_code == 200 - assert 'application/json' in r.request.headers['Content-Type'] - assert {'life': 42} == r.json()['json'] + assert "application/json" in r.request.headers["Content-Type"] + assert {"life": 42} == r.json()["json"] def test_json_param_post_should_not_override_data_param(self, httpbin): - r = requests.Request(method='POST', url=httpbin('post'), - data={'stuff': 'elixr'}, - json={'music': 'flute'}) + r = requests.Request( + method="POST", + url=httpbin("post"), + data={"stuff": "elixr"}, + json={"music": "flute"}, + ) prep = r.prepare() - assert 'stuff=elixr' == prep.body + assert "stuff=elixr" == prep.body def test_response_iter_lines(self, httpbin): - r = requests.get(httpbin('stream/4'), stream=True) + r = requests.get(httpbin("stream/4"), stream=True) assert r.status_code == 200 it = r.iter_lines() @@ -1739,7 +2129,7 @@ def test_response_iter_lines(self, httpbin): assert len(list(it)) == 3 def test_response_context_manager(self, httpbin): - with requests.get(httpbin('stream/4'), stream=True) as response: + with requests.get(httpbin("stream/4"), stream=True) as response: assert isinstance(response, requests.Response) assert response.raw.closed @@ -1747,7 +2137,7 @@ def test_response_context_manager(self, httpbin): def test_unconsumed_session_response_closes_connection(self, httpbin): s = requests.session() - with contextlib.closing(s.get(httpbin('stream/4'), stream=True)) as response: + with contextlib.closing(s.get(httpbin("stream/4"), stream=True)) as response: pass assert response._content_consumed is False @@ -1756,35 +2146,35 @@ def test_unconsumed_session_response_closes_connection(self, httpbin): @pytest.mark.xfail def test_response_iter_lines_reentrant(self, httpbin): """Response.iter_lines() is not reentrant safe""" - r = requests.get(httpbin('stream/4'), stream=True) + r = requests.get(httpbin("stream/4"), stream=True) assert r.status_code == 200 next(r.iter_lines()) assert len(list(r.iter_lines())) == 3 - def test_session_close_proxy_clear(self, mocker): + def test_session_close_proxy_clear(self): proxies = { - 'one': mocker.Mock(), - 'two': mocker.Mock(), + "one": mock.Mock(), + "two": mock.Mock(), } session = requests.Session() - mocker.patch.dict(session.adapters['http://'].proxy_manager, proxies) - session.close() - proxies['one'].clear.assert_called_once_with() - proxies['two'].clear.assert_called_once_with() + with mock.patch.dict(session.adapters["http://"].proxy_manager, proxies): + session.close() + proxies["one"].clear.assert_called_once_with() + proxies["two"].clear.assert_called_once_with() - def test_proxy_auth(self, httpbin): + def test_proxy_auth(self): adapter = HTTPAdapter() headers = adapter.proxy_headers("http://user:pass@httpbin.org") - assert headers == {'Proxy-Authorization': 'Basic dXNlcjpwYXNz'} + assert headers == {"Proxy-Authorization": "Basic dXNlcjpwYXNz"} - def test_proxy_auth_empty_pass(self, httpbin): + def test_proxy_auth_empty_pass(self): adapter = HTTPAdapter() headers = adapter.proxy_headers("http://user:@httpbin.org") - assert headers == {'Proxy-Authorization': 'Basic dXNlcjo='} + assert headers == {"Proxy-Authorization": "Basic dXNlcjo="} def test_response_json_when_content_is_None(self, httpbin): - r = requests.get(httpbin('/status/204')) + r = requests.get(httpbin("/status/204")) # Make sure r.content is None r.status_code = 0 r._content = False @@ -1799,7 +2189,7 @@ def test_response_without_release_conn(self): Should work when `release_conn` attr doesn't exist on `response.raw`. """ resp = requests.Response() - resp.raw = StringIO.StringIO('test') + resp.raw = StringIO.StringIO("test") assert not resp.raw.closed resp.close() assert resp.raw.closed @@ -1808,36 +2198,36 @@ def test_empty_stream_with_auth_does_not_set_content_length_header(self, httpbin """Ensure that a byte stream with size 0 will not set both a Content-Length and Transfer-Encoding header. """ - auth = ('user', 'pass') - url = httpbin('post') - file_obj = io.BytesIO(b'') - r = requests.Request('POST', url, auth=auth, data=file_obj) + auth = ("user", "pass") + url = httpbin("post") + file_obj = io.BytesIO(b"") + r = requests.Request("POST", url, auth=auth, data=file_obj) prepared_request = r.prepare() - assert 'Transfer-Encoding' in prepared_request.headers - assert 'Content-Length' not in prepared_request.headers + assert "Transfer-Encoding" in prepared_request.headers + assert "Content-Length" not in prepared_request.headers def test_stream_with_auth_does_not_set_transfer_encoding_header(self, httpbin): """Ensure that a byte stream with size > 0 will not set both a Content-Length and Transfer-Encoding header. """ - auth = ('user', 'pass') - url = httpbin('post') - file_obj = io.BytesIO(b'test data') - r = requests.Request('POST', url, auth=auth, data=file_obj) + auth = ("user", "pass") + url = httpbin("post") + file_obj = io.BytesIO(b"test data") + r = requests.Request("POST", url, auth=auth, data=file_obj) prepared_request = r.prepare() - assert 'Transfer-Encoding' not in prepared_request.headers - assert 'Content-Length' in prepared_request.headers + assert "Transfer-Encoding" not in prepared_request.headers + assert "Content-Length" in prepared_request.headers def test_chunked_upload_does_not_set_content_length_header(self, httpbin): """Ensure that requests with a generator body stream using Transfer-Encoding: chunked, not a Content-Length header. """ - data = (i for i in [b'a', b'b', b'c']) - url = httpbin('post') - r = requests.Request('POST', url, data=data) + data = (i for i in [b"a", b"b", b"c"]) + url = httpbin("post") + r = requests.Request("POST", url, data=data) prepared_request = r.prepare() - assert 'Transfer-Encoding' in prepared_request.headers - assert 'Content-Length' not in prepared_request.headers + assert "Transfer-Encoding" in prepared_request.headers + assert "Content-Length" not in prepared_request.headers def test_custom_redirect_mixin(self, httpbin): """Tests a custom mixin to overwrite ``get_redirect_target``. @@ -1851,23 +2241,24 @@ def test_custom_redirect_mixin(self, httpbin): location = alternate url 3. the custom session catches the edge case and follows the redirect """ - url_final = httpbin('html') - querystring_malformed = urlencode({'location': url_final}) - url_redirect_malformed = httpbin('response-headers?%s' % querystring_malformed) - querystring_redirect = urlencode({'url': url_redirect_malformed}) - url_redirect = httpbin('redirect-to?%s' % querystring_redirect) - urls_test = [url_redirect, - url_redirect_malformed, - url_final, - ] + url_final = httpbin("html") + querystring_malformed = urlencode({"location": url_final}) + url_redirect_malformed = httpbin("response-headers?%s" % querystring_malformed) + querystring_redirect = urlencode({"url": url_redirect_malformed}) + url_redirect = httpbin("redirect-to?%s" % querystring_redirect) + urls_test = [ + url_redirect, + url_redirect_malformed, + url_final, + ] class CustomRedirectSession(requests.Session): def get_redirect_target(self, resp): # default behavior if resp.is_redirect: - return resp.headers['location'] + return resp.headers["location"] # edge case - check to see if 'location' is in headers anyways - location = resp.headers.get('location') + location = resp.headers.get("location") if location and (location != resp.url): return location return None @@ -1884,144 +2275,153 @@ def get_redirect_target(self, resp): class TestCaseInsensitiveDict: - @pytest.mark.parametrize( - 'cid', ( - CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}), - CaseInsensitiveDict([('Foo', 'foo'), ('BAr', 'bar')]), - CaseInsensitiveDict(FOO='foo', BAr='bar'), - )) + "cid", + ( + CaseInsensitiveDict({"Foo": "foo", "BAr": "bar"}), + CaseInsensitiveDict([("Foo", "foo"), ("BAr", "bar")]), + CaseInsensitiveDict(FOO="foo", BAr="bar"), + ), + ) def test_init(self, cid): assert len(cid) == 2 - assert 'foo' in cid - assert 'bar' in cid + assert "foo" in cid + assert "bar" in cid def test_docstring_example(self): cid = CaseInsensitiveDict() - cid['Accept'] = 'application/json' - assert cid['aCCEPT'] == 'application/json' - assert list(cid) == ['Accept'] + cid["Accept"] = "application/json" + assert cid["aCCEPT"] == "application/json" + assert list(cid) == ["Accept"] def test_len(self): - cid = CaseInsensitiveDict({'a': 'a', 'b': 'b'}) - cid['A'] = 'a' + cid = CaseInsensitiveDict({"a": "a", "b": "b"}) + cid["A"] = "a" assert len(cid) == 2 def test_getitem(self): - cid = CaseInsensitiveDict({'Spam': 'blueval'}) - assert cid['spam'] == 'blueval' - assert cid['SPAM'] == 'blueval' + cid = CaseInsensitiveDict({"Spam": "blueval"}) + assert cid["spam"] == "blueval" + assert cid["SPAM"] == "blueval" def test_fixes_649(self): """__setitem__ should behave case-insensitively.""" cid = CaseInsensitiveDict() - cid['spam'] = 'oneval' - cid['Spam'] = 'twoval' - cid['sPAM'] = 'redval' - cid['SPAM'] = 'blueval' - assert cid['spam'] == 'blueval' - assert cid['SPAM'] == 'blueval' - assert list(cid.keys()) == ['SPAM'] + cid["spam"] = "oneval" + cid["Spam"] = "twoval" + cid["sPAM"] = "redval" + cid["SPAM"] = "blueval" + assert cid["spam"] == "blueval" + assert cid["SPAM"] == "blueval" + assert list(cid.keys()) == ["SPAM"] def test_delitem(self): cid = CaseInsensitiveDict() - cid['Spam'] = 'someval' - del cid['sPam'] - assert 'spam' not in cid + cid["Spam"] = "someval" + del cid["sPam"] + assert "spam" not in cid assert len(cid) == 0 def test_contains(self): cid = CaseInsensitiveDict() - cid['Spam'] = 'someval' - assert 'Spam' in cid - assert 'spam' in cid - assert 'SPAM' in cid - assert 'sPam' in cid - assert 'notspam' not in cid + cid["Spam"] = "someval" + assert "Spam" in cid + assert "spam" in cid + assert "SPAM" in cid + assert "sPam" in cid + assert "notspam" not in cid def test_get(self): cid = CaseInsensitiveDict() - cid['spam'] = 'oneval' - cid['SPAM'] = 'blueval' - assert cid.get('spam') == 'blueval' - assert cid.get('SPAM') == 'blueval' - assert cid.get('sPam') == 'blueval' - assert cid.get('notspam', 'default') == 'default' + cid["spam"] = "oneval" + cid["SPAM"] = "blueval" + assert cid.get("spam") == "blueval" + assert cid.get("SPAM") == "blueval" + assert cid.get("sPam") == "blueval" + assert cid.get("notspam", "default") == "default" def test_update(self): cid = CaseInsensitiveDict() - cid['spam'] = 'blueval' - cid.update({'sPam': 'notblueval'}) - assert cid['spam'] == 'notblueval' - cid = CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}) - cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'}) + cid["spam"] = "blueval" + cid.update({"sPam": "notblueval"}) + assert cid["spam"] == "notblueval" + cid = CaseInsensitiveDict({"Foo": "foo", "BAr": "bar"}) + cid.update({"fOO": "anotherfoo", "bAR": "anotherbar"}) assert len(cid) == 2 - assert cid['foo'] == 'anotherfoo' - assert cid['bar'] == 'anotherbar' + assert cid["foo"] == "anotherfoo" + assert cid["bar"] == "anotherbar" def test_update_retains_unchanged(self): - cid = CaseInsensitiveDict({'foo': 'foo', 'bar': 'bar'}) - cid.update({'foo': 'newfoo'}) - assert cid['bar'] == 'bar' + cid = CaseInsensitiveDict({"foo": "foo", "bar": "bar"}) + cid.update({"foo": "newfoo"}) + assert cid["bar"] == "bar" def test_iter(self): - cid = CaseInsensitiveDict({'Spam': 'spam', 'Eggs': 'eggs'}) - keys = frozenset(['Spam', 'Eggs']) + cid = CaseInsensitiveDict({"Spam": "spam", "Eggs": "eggs"}) + keys = frozenset(["Spam", "Eggs"]) assert frozenset(iter(cid)) == keys def test_equality(self): - cid = CaseInsensitiveDict({'SPAM': 'blueval', 'Eggs': 'redval'}) - othercid = CaseInsensitiveDict({'spam': 'blueval', 'eggs': 'redval'}) + cid = CaseInsensitiveDict({"SPAM": "blueval", "Eggs": "redval"}) + othercid = CaseInsensitiveDict({"spam": "blueval", "eggs": "redval"}) assert cid == othercid - del othercid['spam'] + del othercid["spam"] assert cid != othercid - assert cid == {'spam': 'blueval', 'eggs': 'redval'} + assert cid == {"spam": "blueval", "eggs": "redval"} assert cid != object() def test_setdefault(self): - cid = CaseInsensitiveDict({'Spam': 'blueval'}) - assert cid.setdefault('spam', 'notblueval') == 'blueval' - assert cid.setdefault('notspam', 'notblueval') == 'notblueval' + cid = CaseInsensitiveDict({"Spam": "blueval"}) + assert cid.setdefault("spam", "notblueval") == "blueval" + assert cid.setdefault("notspam", "notblueval") == "notblueval" def test_lower_items(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) + cid = CaseInsensitiveDict( + { + "Accept": "application/json", + "user-Agent": "requests", + } + ) keyset = frozenset(lowerkey for lowerkey, v in cid.lower_items()) - lowerkeyset = frozenset(['accept', 'user-agent']) + lowerkeyset = frozenset(["accept", "user-agent"]) assert keyset == lowerkeyset def test_preserve_key_case(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) - keyset = frozenset(['Accept', 'user-Agent']) + cid = CaseInsensitiveDict( + { + "Accept": "application/json", + "user-Agent": "requests", + } + ) + keyset = frozenset(["Accept", "user-Agent"]) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset def test_preserve_last_key_case(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) - cid.update({'ACCEPT': 'application/json'}) - cid['USER-AGENT'] = 'requests' - keyset = frozenset(['ACCEPT', 'USER-AGENT']) + cid = CaseInsensitiveDict( + { + "Accept": "application/json", + "user-Agent": "requests", + } + ) + cid.update({"ACCEPT": "application/json"}) + cid["USER-AGENT"] = "requests" + keyset = frozenset(["ACCEPT", "USER-AGENT"]) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset def test_copy(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) + cid = CaseInsensitiveDict( + { + "Accept": "application/json", + "user-Agent": "requests", + } + ) cid_copy = cid.copy() assert cid == cid_copy - cid['changed'] = True + cid["changed"] = True assert cid != cid_copy @@ -2032,19 +2432,21 @@ def test_expires_valid_str(self): """Test case where we convert expires from string time.""" morsel = Morsel() - morsel['expires'] = 'Thu, 01-Jan-1970 00:00:01 GMT' + morsel["expires"] = "Thu, 01-Jan-1970 00:00:01 GMT" cookie = morsel_to_cookie(morsel) assert cookie.expires == 1 @pytest.mark.parametrize( - 'value, exception', ( + "value, exception", + ( (100, TypeError), - ('woops', ValueError), - )) + ("woops", ValueError), + ), + ) def test_expires_invalid_int(self, value, exception): """Test case where an invalid type is passed for expires.""" morsel = Morsel() - morsel['expires'] = value + morsel["expires"] = value with pytest.raises(exception): morsel_to_cookie(morsel) @@ -2052,7 +2454,7 @@ def test_expires_none(self): """Test case where expires is None.""" morsel = Morsel() - morsel['expires'] = None + morsel["expires"] = None cookie = morsel_to_cookie(morsel) assert cookie.expires is None @@ -2065,7 +2467,7 @@ def test_max_age_valid_int(self): """Test case where a valid max age in seconds is passed.""" morsel = Morsel() - morsel['max-age'] = 60 + morsel["max-age"] = 60 cookie = morsel_to_cookie(morsel) assert isinstance(cookie.expires, int) @@ -2073,34 +2475,31 @@ def test_max_age_invalid_str(self): """Test case where a invalid max age is passed.""" morsel = Morsel() - morsel['max-age'] = 'woops' + morsel["max-age"] = "woops" with pytest.raises(TypeError): morsel_to_cookie(morsel) class TestTimeout: - def test_stream_timeout(self, httpbin): try: - requests.get(httpbin('delay/10'), timeout=2.0) + requests.get(httpbin("delay/10"), timeout=2.0) except requests.exceptions.Timeout as e: - assert 'Read timed out' in e.args[0].args[0] + assert "Read timed out" in e.args[0].args[0] @pytest.mark.parametrize( - 'timeout, error_text', ( - ((3, 4, 5), '(connect, read)'), - ('foo', 'must be an int, float or None'), - )) + "timeout, error_text", + ( + ((3, 4, 5), "(connect, read)"), + ("foo", "must be an int, float or None"), + ), + ) def test_invalid_timeout(self, httpbin, timeout, error_text): with pytest.raises(ValueError) as e: - requests.get(httpbin('get'), timeout=timeout) + requests.get(httpbin("get"), timeout=timeout) assert error_text in str(e) - @pytest.mark.parametrize( - 'timeout', ( - None, - Urllib3Timeout(connect=None, read=None) - )) + @pytest.mark.parametrize("timeout", (None, Urllib3Timeout(connect=None, read=None))) def test_none_timeout(self, httpbin, timeout): """Check that you can set None as a valid timeout value. @@ -2110,53 +2509,47 @@ def test_none_timeout(self, httpbin, timeout): Instead we verify that setting the timeout to None does not prevent the request from succeeding. """ - r = requests.get(httpbin('get'), timeout=timeout) + r = requests.get(httpbin("get"), timeout=timeout) assert r.status_code == 200 @pytest.mark.parametrize( - 'timeout', ( - (None, 0.1), - Urllib3Timeout(connect=None, read=0.1) - )) + "timeout", ((None, 0.1), Urllib3Timeout(connect=None, read=0.1)) + ) def test_read_timeout(self, httpbin, timeout): try: - requests.get(httpbin('delay/10'), timeout=timeout) - pytest.fail('The recv() request should time out.') + requests.get(httpbin("delay/10"), timeout=timeout) + pytest.fail("The recv() request should time out.") except ReadTimeout: pass @pytest.mark.parametrize( - 'timeout', ( - (0.1, None), - Urllib3Timeout(connect=0.1, read=None) - )) + "timeout", ((0.1, None), Urllib3Timeout(connect=0.1, read=None)) + ) def test_connect_timeout(self, timeout): try: requests.get(TARPIT, timeout=timeout) - pytest.fail('The connect() request should time out.') + pytest.fail("The connect() request should time out.") except ConnectTimeout as e: assert isinstance(e, ConnectionError) assert isinstance(e, Timeout) @pytest.mark.parametrize( - 'timeout', ( - (0.1, 0.1), - Urllib3Timeout(connect=0.1, read=0.1) - )) + "timeout", ((0.1, 0.1), Urllib3Timeout(connect=0.1, read=0.1)) + ) def test_total_timeout_connect(self, timeout): try: requests.get(TARPIT, timeout=timeout) - pytest.fail('The connect() request should time out.') + pytest.fail("The connect() request should time out.") except ConnectTimeout: pass def test_encoded_methods(self, httpbin): - """See: https://github.com/requests/requests/issues/2316""" - r = requests.request(b'GET', httpbin('get')) + """See: https://github.com/psf/requests/issues/2316""" + r = requests.request(b"GET", httpbin("get")) assert r.ok -SendCall = collections.namedtuple('SendCall', ('args', 'kwargs')) +SendCall = collections.namedtuple("SendCall", ("args", "kwargs")) class RedirectSession(SessionRedirectMixin): @@ -2180,14 +2573,14 @@ def build_response(self): except IndexError: r.status_code = 200 - r.headers = CaseInsensitiveDict({'Location': '/'}) + r.headers = CaseInsensitiveDict({"Location": "/"}) r.raw = self._build_raw() r.request = request return r def _build_raw(self): - string = StringIO.StringIO('') - setattr(string, 'release_conn', lambda *args: args) + string = StringIO.StringIO("") + setattr(string, "release_conn", lambda *args: args) return string @@ -2195,49 +2588,46 @@ def test_json_encodes_as_bytes(): # urllib3 expects bodies as bytes-like objects body = {"key": "value"} p = PreparedRequest() - p.prepare( - method='GET', - url='https://www.example.com/', - json=body - ) + p.prepare(method="GET", url="https://www.example.com/", json=body) assert isinstance(p.body, bytes) def test_requests_are_updated_each_time(httpbin): session = RedirectSession([303, 307]) - prep = requests.Request('POST', httpbin('post')).prepare() + prep = requests.Request("POST", httpbin("post")).prepare() r0 = session.send(prep) - assert r0.request.method == 'POST' + assert r0.request.method == "POST" assert session.calls[-1] == SendCall((r0.request,), {}) redirect_generator = session.resolve_redirects(r0, prep) default_keyword_args = { - 'stream': False, - 'verify': True, - 'cert': None, - 'timeout': None, - 'allow_redirects': False, - 'proxies': {}, + "stream": False, + "verify": True, + "cert": None, + "timeout": None, + "allow_redirects": False, + "proxies": {}, } for response in redirect_generator: - assert response.request.method == 'GET' + assert response.request.method == "GET" send_call = SendCall((response.request,), default_keyword_args) assert session.calls[-1] == send_call -@pytest.mark.parametrize("var,url,proxy", [ - ('http_proxy', 'http://example.com', 'socks5://proxy.com:9876'), - ('https_proxy', 'https://example.com', 'socks5://proxy.com:9876'), - ('all_proxy', 'http://example.com', 'socks5://proxy.com:9876'), - ('all_proxy', 'https://example.com', 'socks5://proxy.com:9876'), -]) +@pytest.mark.parametrize( + "var,url,proxy", + [ + ("http_proxy", "http://example.com", "socks5://proxy.com:9876"), + ("https_proxy", "https://example.com", "socks5://proxy.com:9876"), + ("all_proxy", "http://example.com", "socks5://proxy.com:9876"), + ("all_proxy", "https://example.com", "socks5://proxy.com:9876"), + ], +) def test_proxy_env_vars_override_default(var, url, proxy): session = requests.Session() prep = PreparedRequest() - prep.prepare(method='GET', url=url) + prep.prepare(method="GET", url=url) - kwargs = { - var: proxy - } + kwargs = {var: proxy} scheme = urlparse(url).scheme with override_environ(**kwargs): proxies = session.rebuild_proxies(prep, {}) @@ -2246,161 +2636,161 @@ def test_proxy_env_vars_override_default(var, url, proxy): @pytest.mark.parametrize( - 'data', ( - (('a', 'b'), ('c', 'd')), - (('c', 'd'), ('a', 'b')), - (('a', 'b'), ('c', 'd'), ('e', 'f')), - )) + "data", + ( + (("a", "b"), ("c", "d")), + (("c", "d"), ("a", "b")), + (("a", "b"), ("c", "d"), ("e", "f")), + ), +) def test_data_argument_accepts_tuples(data): """Ensure that the data argument will accept tuples of strings and properly encode them. """ p = PreparedRequest() p.prepare( - method='GET', - url='http://www.example.com', - data=data, - hooks=default_hooks() + method="GET", url="http://www.example.com", data=data, hooks=default_hooks() ) assert p.body == urlencode(data) @pytest.mark.parametrize( - 'kwargs', ( + "kwargs", + ( None, { - 'method': 'GET', - 'url': 'http://www.example.com', - 'data': 'foo=bar', - 'hooks': default_hooks() - }, - { - 'method': 'GET', - 'url': 'http://www.example.com', - 'data': 'foo=bar', - 'hooks': default_hooks(), - 'cookies': {'foo': 'bar'} + "method": "GET", + "url": "http://www.example.com", + "data": "foo=bar", + "hooks": default_hooks(), }, { - 'method': 'GET', - 'url': u('http://www.example.com/üniçø∂é') + "method": "GET", + "url": "http://www.example.com", + "data": "foo=bar", + "hooks": default_hooks(), + "cookies": {"foo": "bar"}, }, - )) + {"method": "GET", "url": "http://www.example.com/üniçø∂é"}, + ), +) def test_prepared_copy(kwargs): p = PreparedRequest() if kwargs: p.prepare(**kwargs) copy = p.copy() - for attr in ('method', 'url', 'headers', '_cookies', 'body', 'hooks'): + for attr in ("method", "url", "headers", "_cookies", "body", "hooks"): assert getattr(p, attr) == getattr(copy, attr) def test_urllib3_retries(httpbin): from urllib3.util import Retry + s = requests.Session() - s.mount('http://', HTTPAdapter(max_retries=Retry( - total=2, status_forcelist=[500] - ))) + s.mount("http://", HTTPAdapter(max_retries=Retry(total=2, status_forcelist=[500]))) with pytest.raises(RetryError): - s.get(httpbin('status/500')) + s.get(httpbin("status/500")) def test_urllib3_pool_connection_closed(httpbin): s = requests.Session() - s.mount('http://', HTTPAdapter(pool_connections=0, pool_maxsize=0)) + s.mount("http://", HTTPAdapter(pool_connections=0, pool_maxsize=0)) try: - s.get(httpbin('status/200')) + s.get(httpbin("status/200")) except ConnectionError as e: - assert u"Pool is closed." in str(e) + assert "Pool is closed." in str(e) -class TestPreparingURLs(object): +class TestPreparingURLs: @pytest.mark.parametrize( - 'url,expected', + "url,expected", ( - ('http://google.com', 'http://google.com/'), - (u'http://ジェーピーニック.jp', u'http://xn--hckqz9bzb1cyrb.jp/'), - (u'http://xn--n3h.net/', u'http://xn--n3h.net/'), - ( - u'http://ジェーピーニック.jp'.encode('utf-8'), - u'http://xn--hckqz9bzb1cyrb.jp/' - ), - ( - u'http://straße.de/straße', - u'http://xn--strae-oqa.de/stra%C3%9Fe' - ), + ("http://google.com", "http://google.com/"), + ("http://ジェーピーニック.jp", "http://xn--hckqz9bzb1cyrb.jp/"), + ("http://xn--n3h.net/", "http://xn--n3h.net/"), + ("http://ジェーピーニック.jp".encode(), "http://xn--hckqz9bzb1cyrb.jp/"), + ("http://straße.de/straße", "http://xn--strae-oqa.de/stra%C3%9Fe"), ( - u'http://straße.de/straße'.encode('utf-8'), - u'http://xn--strae-oqa.de/stra%C3%9Fe' + "http://straße.de/straße".encode(), + "http://xn--strae-oqa.de/stra%C3%9Fe", ), ( - u'http://Königsgäßchen.de/straße', - u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe' + "http://Königsgäßchen.de/straße", + "http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe", ), ( - u'http://Königsgäßchen.de/straße'.encode('utf-8'), - u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe' + "http://Königsgäßchen.de/straße".encode(), + "http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe", ), + (b"http://xn--n3h.net/", "http://xn--n3h.net/"), ( - b'http://xn--n3h.net/', - u'http://xn--n3h.net/' + b"http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/", + "http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/", ), ( - b'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/', - u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/' + "http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/", + "http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/", ), - ( - u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/', - u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/' - ) - ) + ), ) def test_preparing_url(self, url, expected): - r = requests.Request('GET', url=url) + def normalize_percent_encode(x): + # Helper function that normalizes equivalent + # percent-encoded bytes before comparisons + for c in re.findall(r"%[a-fA-F0-9]{2}", x): + x = x.replace(c, c.upper()) + return x + + r = requests.Request("GET", url=url) p = r.prepare() - assert p.url == expected + assert normalize_percent_encode(p.url) == expected @pytest.mark.parametrize( - 'url', + "url", ( b"http://*.google.com", b"http://*", - u"http://*.google.com", - u"http://*", - u"http://☃.net/" - ) + "http://*.google.com", + "http://*", + "http://☃.net/", + ), ) def test_preparing_bad_url(self, url): - r = requests.Request('GET', url=url) + r = requests.Request("GET", url=url) with pytest.raises(requests.exceptions.InvalidURL): r.prepare() + @pytest.mark.parametrize("url, exception", (("http://:1", InvalidURL),)) + def test_redirecting_to_bad_url(self, httpbin, url, exception): + with pytest.raises(exception): + requests.get(httpbin("redirect-to"), params={"url": url}) + @pytest.mark.parametrize( - 'input, expected', + "input, expected", ( ( b"http+unix://%2Fvar%2Frun%2Fsocket/path%7E", - u"http+unix://%2Fvar%2Frun%2Fsocket/path~", + "http+unix://%2Fvar%2Frun%2Fsocket/path~", ), ( - u"http+unix://%2Fvar%2Frun%2Fsocket/path%7E", - u"http+unix://%2Fvar%2Frun%2Fsocket/path~", + "http+unix://%2Fvar%2Frun%2Fsocket/path%7E", + "http+unix://%2Fvar%2Frun%2Fsocket/path~", ), ( b"mailto:user@example.org", - u"mailto:user@example.org", + "mailto:user@example.org", ), ( - u"mailto:user@example.org", - u"mailto:user@example.org", + "mailto:user@example.org", + "mailto:user@example.org", ), ( b"data:SSDimaUgUHl0aG9uIQ==", - u"data:SSDimaUgUHl0aG9uIQ==", - ) - ) + "data:SSDimaUgUHl0aG9uIQ==", + ), + ), ) def test_url_mutation(self, input, expected): """ @@ -2409,40 +2799,242 @@ def test_url_mutation(self, input, expected): any URL whose scheme doesn't begin with "http" is left alone, and those whose scheme *does* begin with "http" are mutated. """ - r = requests.Request('GET', url=input) + r = requests.Request("GET", url=input) p = r.prepare() assert p.url == expected @pytest.mark.parametrize( - 'input, params, expected', + "input, params, expected", ( ( b"http+unix://%2Fvar%2Frun%2Fsocket/path", {"key": "value"}, - u"http+unix://%2Fvar%2Frun%2Fsocket/path?key=value", + "http+unix://%2Fvar%2Frun%2Fsocket/path?key=value", ), ( - u"http+unix://%2Fvar%2Frun%2Fsocket/path", + "http+unix://%2Fvar%2Frun%2Fsocket/path", {"key": "value"}, - u"http+unix://%2Fvar%2Frun%2Fsocket/path?key=value", + "http+unix://%2Fvar%2Frun%2Fsocket/path?key=value", ), ( b"mailto:user@example.org", {"key": "value"}, - u"mailto:user@example.org", + "mailto:user@example.org", ), ( - u"mailto:user@example.org", + "mailto:user@example.org", {"key": "value"}, - u"mailto:user@example.org", + "mailto:user@example.org", ), - ) + ), ) def test_parameters_for_nonstandard_schemes(self, input, params, expected): """ Setting parameters for nonstandard schemes is allowed if those schemes begin with "http", and is forbidden otherwise. """ - r = requests.Request('GET', url=input, params=params) + r = requests.Request("GET", url=input, params=params) p = r.prepare() assert p.url == expected + + def test_post_json_nan(self, httpbin): + data = {"foo": float("nan")} + with pytest.raises(requests.exceptions.InvalidJSONError): + requests.post(httpbin("post"), json=data) + + def test_json_decode_compatibility(self, httpbin): + r = requests.get(httpbin("bytes/20")) + with pytest.raises(requests.exceptions.JSONDecodeError) as excinfo: + r.json() + assert isinstance(excinfo.value, RequestException) + assert isinstance(excinfo.value, JSONDecodeError) + assert r.text not in str(excinfo.value) + + def test_json_decode_persists_doc_attr(self, httpbin): + r = requests.get(httpbin("bytes/20")) + with pytest.raises(requests.exceptions.JSONDecodeError) as excinfo: + r.json() + assert excinfo.value.doc == r.text + + def test_status_code_425(self): + r1 = requests.codes.get("TOO_EARLY") + r2 = requests.codes.get("too_early") + r3 = requests.codes.get("UNORDERED") + r4 = requests.codes.get("unordered") + r5 = requests.codes.get("UNORDERED_COLLECTION") + r6 = requests.codes.get("unordered_collection") + + assert r1 == 425 + assert r2 == 425 + assert r3 == 425 + assert r4 == 425 + assert r5 == 425 + assert r6 == 425 + + def test_different_connection_pool_for_tls_settings_verify_True(self): + def response_handler(sock): + consume_socket_content(sock, timeout=0.5) + sock.send( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: 18\r\n\r\n" + b'\xff\xfe{\x00"\x00K0"\x00=\x00"\x00\xab0"\x00\r\n' + ) + + s = requests.Session() + close_server = threading.Event() + server = TLSServer( + handler=response_handler, + wait_to_close_event=close_server, + requests_to_handle=3, + cert_chain="tests/certs/expired/server/server.pem", + keyfile="tests/certs/expired/server/server.key", + ) + + with server as (host, port): + url = f"https://{host}:{port}" + r1 = s.get(url, verify=False) + assert r1.status_code == 200 + + # Cannot verify self-signed certificate + with pytest.raises(requests.exceptions.SSLError): + s.get(url) + + close_server.set() + assert 2 == len(s.adapters["https://"].poolmanager.pools) + + def test_different_connection_pool_for_tls_settings_verify_bundle_expired_cert( + self, + ): + def response_handler(sock): + consume_socket_content(sock, timeout=0.5) + sock.send( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: 18\r\n\r\n" + b'\xff\xfe{\x00"\x00K0"\x00=\x00"\x00\xab0"\x00\r\n' + ) + + s = requests.Session() + close_server = threading.Event() + server = TLSServer( + handler=response_handler, + wait_to_close_event=close_server, + requests_to_handle=3, + cert_chain="tests/certs/expired/server/server.pem", + keyfile="tests/certs/expired/server/server.key", + ) + + with server as (host, port): + url = f"https://{host}:{port}" + r1 = s.get(url, verify=False) + assert r1.status_code == 200 + + # Has right trust bundle, but certificate expired + with pytest.raises(requests.exceptions.SSLError): + s.get(url, verify="tests/certs/expired/ca/ca.crt") + + close_server.set() + assert 2 == len(s.adapters["https://"].poolmanager.pools) + + def test_different_connection_pool_for_tls_settings_verify_bundle_unexpired_cert( + self, + ): + def response_handler(sock): + consume_socket_content(sock, timeout=0.5) + sock.send( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: 18\r\n\r\n" + b'\xff\xfe{\x00"\x00K0"\x00=\x00"\x00\xab0"\x00\r\n' + ) + + s = requests.Session() + close_server = threading.Event() + server = TLSServer( + handler=response_handler, + wait_to_close_event=close_server, + requests_to_handle=3, + cert_chain="tests/certs/valid/server/server.pem", + keyfile="tests/certs/valid/server/server.key", + ) + + with server as (host, port): + url = f"https://{host}:{port}" + r1 = s.get(url, verify=False) + assert r1.status_code == 200 + + r2 = s.get(url, verify="tests/certs/valid/ca/ca.crt") + assert r2.status_code == 200 + + close_server.set() + assert 2 == len(s.adapters["https://"].poolmanager.pools) + + def test_different_connection_pool_for_mtls_settings(self): + client_cert = None + + def response_handler(sock): + nonlocal client_cert + client_cert = sock.getpeercert() + consume_socket_content(sock, timeout=0.5) + sock.send( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: 18\r\n\r\n" + b'\xff\xfe{\x00"\x00K0"\x00=\x00"\x00\xab0"\x00\r\n' + ) + + s = requests.Session() + close_server = threading.Event() + server = TLSServer( + handler=response_handler, + wait_to_close_event=close_server, + requests_to_handle=2, + cert_chain="tests/certs/expired/server/server.pem", + keyfile="tests/certs/expired/server/server.key", + mutual_tls=True, + cacert="tests/certs/expired/ca/ca.crt", + ) + + cert = ( + "tests/certs/mtls/client/client.pem", + "tests/certs/mtls/client/client.key", + ) + with server as (host, port): + url = f"https://{host}:{port}" + r1 = s.get(url, verify=False, cert=cert) + assert r1.status_code == 200 + with pytest.raises(requests.exceptions.SSLError): + s.get(url, cert=cert) + close_server.set() + + assert client_cert is not None + + +def test_content_length_for_bytes_data(httpbin): + data = "This is a string containing multi-byte UTF-8 ☃️" + encoded_data = data.encode("utf-8") + length = str(len(encoded_data)) + req = requests.Request("POST", httpbin("post"), data=encoded_data) + p = req.prepare() + + assert p.headers["Content-Length"] == length + + +@pytest.mark.skipif( + is_urllib3_1, + reason="urllib3 2.x encodes all strings to utf-8, urllib3 1.x uses latin-1", +) +def test_content_length_for_string_data_counts_bytes(httpbin): + data = "This is a string containing multi-byte UTF-8 ☃️" + length = str(len(data.encode("utf-8"))) + req = requests.Request("POST", httpbin("post"), data=data) + p = req.prepare() + + assert p.headers["Content-Length"] == length + + +def test_json_decode_errors_are_serializable_deserializable(): + json_decode_error = requests.exceptions.JSONDecodeError( + "Extra data", + '{"responseCode":["706"],"data":null}{"responseCode":["706"],"data":null}', + 36, + ) + deserialized_error = pickle.loads(pickle.dumps(json_decode_error)) + assert repr(json_decode_error) == repr(deserialized_error) diff --git a/tests/test_structures.py b/tests/test_structures.py index e4d2459fe7..e2fd5baaf2 100644 --- a/tests/test_structures.py +++ b/tests/test_structures.py @@ -1,26 +1,25 @@ -# -*- coding: utf-8 -*- - import pytest from requests.structures import CaseInsensitiveDict, LookupDict class TestCaseInsensitiveDict: - @pytest.fixture(autouse=True) def setup(self): """CaseInsensitiveDict instance with "Accept" header.""" self.case_insensitive_dict = CaseInsensitiveDict() - self.case_insensitive_dict['Accept'] = 'application/json' + self.case_insensitive_dict["Accept"] = "application/json" def test_list(self): - assert list(self.case_insensitive_dict) == ['Accept'] + assert list(self.case_insensitive_dict) == ["Accept"] - possible_keys = pytest.mark.parametrize('key', ('accept', 'ACCEPT', 'aCcEpT', 'Accept')) + possible_keys = pytest.mark.parametrize( + "key", ("accept", "ACCEPT", "aCcEpT", "Accept") + ) @possible_keys def test_getitem(self, key): - assert self.case_insensitive_dict[key] == 'application/json' + assert self.case_insensitive_dict[key] == "application/json" @possible_keys def test_delitem(self, key): @@ -28,7 +27,9 @@ def test_delitem(self, key): assert key not in self.case_insensitive_dict def test_lower_items(self): - assert list(self.case_insensitive_dict.lower_items()) == [('accept', 'application/json')] + assert list(self.case_insensitive_dict.lower_items()) == [ + ("accept", "application/json") + ] def test_repr(self): assert repr(self.case_insensitive_dict) == "{'Accept': 'application/json'}" @@ -39,32 +40,33 @@ def test_copy(self): assert copy == self.case_insensitive_dict @pytest.mark.parametrize( - 'other, result', ( - ({'AccePT': 'application/json'}, True), + "other, result", + ( + ({"AccePT": "application/json"}, True), ({}, False), - (None, False) - ) + (None, False), + ), ) def test_instance_equality(self, other, result): assert (self.case_insensitive_dict == other) is result class TestLookupDict: - @pytest.fixture(autouse=True) def setup(self): """LookupDict instance with "bad_gateway" attribute.""" - self.lookup_dict = LookupDict('test') + self.lookup_dict = LookupDict("test") self.lookup_dict.bad_gateway = 502 def test_repr(self): assert repr(self.lookup_dict) == "" get_item_parameters = pytest.mark.parametrize( - 'key, value', ( - ('bad_gateway', 502), - ('not_a_key', None) - ) + "key, value", + ( + ("bad_gateway", 502), + ("not_a_key", None), + ), ) @get_item_parameters diff --git a/tests/test_testserver.py b/tests/test_testserver.py index 3c770759c3..c73a3f1f59 100644 --- a/tests/test_testserver.py +++ b/tests/test_testserver.py @@ -1,16 +1,14 @@ -# -*- coding: utf-8 -*- - -import threading import socket +import threading import time import pytest -import requests from tests.testserver.server import Server +import requests -class TestTestServer: +class TestTestServer: def test_basic(self): """messages are sent and received properly""" question = b"success?" @@ -44,36 +42,37 @@ def test_server_closes(self): def test_text_response(self): """the text_response_server sends the given text""" server = Server.text_response_server( - "HTTP/1.1 200 OK\r\n" + - "Content-Length: 6\r\n" + - "\r\nroflol" + "HTTP/1.1 200 OK\r\n" "Content-Length: 6\r\n" "\r\nroflol" ) with server as (host, port): - r = requests.get('http://{0}:{1}'.format(host, port)) + r = requests.get(f"http://{host}:{port}") assert r.status_code == 200 - assert r.text == u'roflol' - assert r.headers['Content-Length'] == '6' + assert r.text == "roflol" + assert r.headers["Content-Length"] == "6" def test_basic_response(self): """the basic response server returns an empty http response""" with Server.basic_response_server() as (host, port): - r = requests.get('http://{0}:{1}'.format(host, port)) + r = requests.get(f"http://{host}:{port}") assert r.status_code == 200 - assert r.text == u'' - assert r.headers['Content-Length'] == '0' + assert r.text == "" + assert r.headers["Content-Length"] == "0" def test_basic_waiting_server(self): """the server waits for the block_server event to be set before closing""" block_server = threading.Event() - with Server.basic_response_server(wait_to_close_event=block_server) as (host, port): + with Server.basic_response_server(wait_to_close_event=block_server) as ( + host, + port, + ): sock = socket.socket() sock.connect((host, port)) - sock.sendall(b'send something') + sock.sendall(b"send something") time.sleep(2.5) - sock.sendall(b'still alive') + sock.sendall(b"still alive") block_server.set() # release server block def test_multiple_requests(self): @@ -83,7 +82,7 @@ def test_multiple_requests(self): server = Server.basic_response_server(requests_to_handle=requests_to_handle) with server as (host, port): - server_url = 'http://{0}:{1}'.format(host, port) + server_url = f"http://{host}:{port}" for _ in range(requests_to_handle): r = requests.get(server_url) assert r.status_code == 200 @@ -97,8 +96,8 @@ def test_request_recovery(self): """can check the requests content""" # TODO: figure out why this sometimes fails when using pytest-xdist. server = Server.basic_response_server(requests_to_handle=2) - first_request = b'put your hands up in the air' - second_request = b'put your hand down in the floor' + first_request = b"put your hands up in the air" + second_request = b"put your hand down in the floor" with server as address: sock1 = socket.socket() @@ -123,15 +122,15 @@ def test_requests_after_timeout_are_not_received(self): sock = socket.socket() sock.connect(address) time.sleep(1.5) - sock.sendall(b'hehehe, not received') + sock.sendall(b"hehehe, not received") sock.close() - assert server.handler_results[0] == b'' + assert server.handler_results[0] == b"" def test_request_recovery_with_bigger_timeout(self): """a biggest timeout can be specified""" server = Server.basic_response_server(request_timeout=3) - data = b'bananadine' + data = b"bananadine" with server as address: sock = socket.socket() diff --git a/tests/test_utils.py b/tests/test_utils.py index f39cd67bc6..f9a287af1b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,40 +1,63 @@ -# -*- coding: utf-8 -*- - -import os import copy import filecmp -from io import BytesIO +import os +import tarfile import zipfile from collections import deque +from io import BytesIO +from unittest import mock import pytest + from requests import compat +from requests._internal_utils import unicode_is_ascii from requests.cookies import RequestsCookieJar from requests.structures import CaseInsensitiveDict from requests.utils import ( - address_in_network, dotted_netmask, extract_zipped_paths, - get_auth_from_url, _parse_content_type_header, get_encoding_from_headers, - get_encodings_from_content, get_environ_proxies, - guess_filename, guess_json_utf, is_ipv4_address, - is_valid_cidr, iter_slices, parse_dict_header, - parse_header_links, prepend_scheme_if_needed, - requote_uri, select_proxy, should_bypass_proxies, super_len, - to_key_val_list, to_native_string, - unquote_header_value, unquote_unreserved, - urldefragauth, add_dict_to_cookiejar, set_environ) -from requests._internal_utils import unicode_is_ascii + _parse_content_type_header, + add_dict_to_cookiejar, + address_in_network, + dotted_netmask, + extract_zipped_paths, + get_auth_from_url, + get_encoding_from_headers, + get_encodings_from_content, + get_environ_proxies, + get_netrc_auth, + guess_filename, + guess_json_utf, + is_ipv4_address, + is_valid_cidr, + iter_slices, + parse_dict_header, + parse_header_links, + prepend_scheme_if_needed, + requote_uri, + select_proxy, + set_environ, + should_bypass_proxies, + super_len, + to_key_val_list, + to_native_string, + unquote_header_value, + unquote_unreserved, + urldefragauth, +) from .compat import StringIO, cStringIO class TestSuperLen: - @pytest.mark.parametrize( - 'stream, value', ( - (StringIO.StringIO, 'Test'), - (BytesIO, b'Test'), - pytest.mark.skipif('cStringIO is None')((cStringIO, 'Test')), - )) + "stream, value", + ( + (StringIO.StringIO, "Test"), + (BytesIO, b"Test"), + pytest.param( + cStringIO, "Test", marks=pytest.mark.skipif("cStringIO is None") + ), + ), + ) def test_io_streams(self, stream, value): """Ensures that we properly deal with different kinds of IO streams.""" assert super_len(stream()) == 0 @@ -43,13 +66,14 @@ def test_io_streams(self, stream, value): def test_super_len_correctly_calculates_len_of_partially_read_file(self): """Ensure that we handle partially consumed file like objects.""" s = StringIO.StringIO() - s.write('foobarbogus') + s.write("foobarbogus") assert super_len(s) == 0 - @pytest.mark.parametrize('error', [IOError, OSError]) + @pytest.mark.parametrize("error", [IOError, OSError]) def test_super_len_handles_files_raising_weird_errors_in_tell(self, error): """If tell() raises errors, assume the cursor is at position zero.""" - class BoomFile(object): + + class BoomFile: def __len__(self): return 5 @@ -58,10 +82,11 @@ def tell(self): assert super_len(BoomFile()) == 0 - @pytest.mark.parametrize('error', [IOError, OSError]) + @pytest.mark.parametrize("error", [IOError, OSError]) def test_super_len_tell_ioerror(self, error): """Ensure that if tell gives an IOError super_len doesn't fail""" - class NoLenBoomFile(object): + + class NoLenBoomFile: def tell(self): raise error() @@ -71,40 +96,54 @@ def seek(self, offset, whence): assert super_len(NoLenBoomFile()) == 0 def test_string(self): - assert super_len('Test') == 4 + assert super_len("Test") == 4 @pytest.mark.parametrize( - 'mode, warnings_num', ( - ('r', 1), - ('rb', 0), - )) + "mode, warnings_num", + ( + ("r", 1), + ("rb", 0), + ), + ) def test_file(self, tmpdir, mode, warnings_num, recwarn): - file_obj = tmpdir.join('test.txt') - file_obj.write('Test') + file_obj = tmpdir.join("test.txt") + file_obj.write("Test") with file_obj.open(mode) as fd: assert super_len(fd) == 4 assert len(recwarn) == warnings_num + def test_tarfile_member(self, tmpdir): + file_obj = tmpdir.join("test.txt") + file_obj.write("Test") + + tar_obj = str(tmpdir.join("test.tar")) + with tarfile.open(tar_obj, "w") as tar: + tar.add(str(file_obj), arcname="test.txt") + + with tarfile.open(tar_obj) as tar: + member = tar.extractfile("test.txt") + assert super_len(member) == 4 + def test_super_len_with__len__(self): - foo = [1,2,3,4] + foo = [1, 2, 3, 4] len_foo = super_len(foo) assert len_foo == 4 def test_super_len_with_no__len__(self): - class LenFile(object): + class LenFile: def __init__(self): self.len = 5 assert super_len(LenFile()) == 5 def test_super_len_with_tell(self): - foo = StringIO.StringIO('12345') + foo = StringIO.StringIO("12345") assert super_len(foo) == 5 foo.read(2) assert super_len(foo) == 3 def test_super_len_with_fileno(self): - with open(__file__, 'rb') as f: + with open(__file__, "rb") as f: length = super_len(f) file_data = f.read() assert length == len(file_data) @@ -114,38 +153,58 @@ def test_super_len_with_no_matches(self): assert super_len(object()) == 0 -class TestToKeyValList: +class TestGetNetrcAuth: + def test_works(self, tmp_path, monkeypatch): + netrc_path = tmp_path / ".netrc" + monkeypatch.setenv("NETRC", str(netrc_path)) + with open(netrc_path, "w") as f: + f.write("machine example.com login aaaa password bbbb\n") + auth = get_netrc_auth("http://example.com/thing") + assert auth == ("aaaa", "bbbb") + + def test_not_vulnerable_to_bad_url_parsing(self, tmp_path, monkeypatch): + netrc_path = tmp_path / ".netrc" + monkeypatch.setenv("NETRC", str(netrc_path)) + with open(netrc_path, "w") as f: + f.write("machine example.com login aaaa password bbbb\n") + auth = get_netrc_auth("http://example.com:@evil.com/'") + assert auth is None + +class TestToKeyValList: @pytest.mark.parametrize( - 'value, expected', ( - ([('key', 'val')], [('key', 'val')]), - ((('key', 'val'), ), [('key', 'val')]), - ({'key': 'val'}, [('key', 'val')]), - (None, None) - )) + "value, expected", + ( + ([("key", "val")], [("key", "val")]), + ((("key", "val"),), [("key", "val")]), + ({"key": "val"}, [("key", "val")]), + (None, None), + ), + ) def test_valid(self, value, expected): assert to_key_val_list(value) == expected def test_invalid(self): with pytest.raises(ValueError): - to_key_val_list('string') + to_key_val_list("string") class TestUnquoteHeaderValue: - @pytest.mark.parametrize( - 'value, expected', ( + "value, expected", + ( (None, None), - ('Test', 'Test'), - ('"Test"', 'Test'), - ('"Test\\\\"', 'Test\\'), - ('"\\\\Comp\\Res"', '\\Comp\\Res'), - )) + ("Test", "Test"), + ('"Test"', "Test"), + ('"Test\\\\"', "Test\\"), + ('"\\\\Comp\\Res"', "\\Comp\\Res"), + ), + ) def test_valid(self, value, expected): assert unquote_header_value(value) == expected def test_is_filename(self): - assert unquote_header_value('"\\\\Comp\\Res"', True) == '\\\\Comp\\Res' + assert unquote_header_value('"\\\\Comp\\Res"', True) == "\\\\Comp\\Res" class TestGetEnvironProxies: @@ -153,146 +212,162 @@ class TestGetEnvironProxies: in no_proxy variable. """ - @pytest.fixture(autouse=True, params=['no_proxy', 'NO_PROXY']) + @pytest.fixture(autouse=True, params=["no_proxy", "NO_PROXY"]) def no_proxy(self, request, monkeypatch): - monkeypatch.setenv(request.param, '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') + monkeypatch.setenv( + request.param, "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" + ) @pytest.mark.parametrize( - 'url', ( - 'http://192.168.0.1:5000/', - 'http://192.168.0.1/', - 'http://172.16.1.1/', - 'http://172.16.1.1:5000/', - 'http://localhost.localdomain:5000/v1.0/', - )) + "url", + ( + "http://192.168.0.1:5000/", + "http://192.168.0.1/", + "http://172.16.1.1/", + "http://172.16.1.1:5000/", + "http://localhost.localdomain:5000/v1.0/", + ), + ) def test_bypass(self, url): assert get_environ_proxies(url, no_proxy=None) == {} @pytest.mark.parametrize( - 'url', ( - 'http://192.168.1.1:5000/', - 'http://192.168.1.1/', - 'http://www.requests.com/', - )) + "url", + ( + "http://192.168.1.1:5000/", + "http://192.168.1.1/", + "http://www.requests.com/", + ), + ) def test_not_bypass(self, url): assert get_environ_proxies(url, no_proxy=None) != {} @pytest.mark.parametrize( - 'url', ( - 'http://192.168.1.1:5000/', - 'http://192.168.1.1/', - 'http://www.requests.com/', - )) + "url", + ( + "http://192.168.1.1:5000/", + "http://192.168.1.1/", + "http://www.requests.com/", + ), + ) def test_bypass_no_proxy_keyword(self, url): - no_proxy = '192.168.1.1,requests.com' + no_proxy = "192.168.1.1,requests.com" assert get_environ_proxies(url, no_proxy=no_proxy) == {} @pytest.mark.parametrize( - 'url', ( - 'http://192.168.0.1:5000/', - 'http://192.168.0.1/', - 'http://172.16.1.1/', - 'http://172.16.1.1:5000/', - 'http://localhost.localdomain:5000/v1.0/', - )) + "url", + ( + "http://192.168.0.1:5000/", + "http://192.168.0.1/", + "http://172.16.1.1/", + "http://172.16.1.1:5000/", + "http://localhost.localdomain:5000/v1.0/", + ), + ) def test_not_bypass_no_proxy_keyword(self, url, monkeypatch): # This is testing that the 'no_proxy' argument overrides the # environment variable 'no_proxy' - monkeypatch.setenv('http_proxy', 'http://proxy.example.com:3128/') - no_proxy = '192.168.1.1,requests.com' + monkeypatch.setenv("http_proxy", "http://proxy.example.com:3128/") + no_proxy = "192.168.1.1,requests.com" assert get_environ_proxies(url, no_proxy=no_proxy) != {} class TestIsIPv4Address: - def test_valid(self): - assert is_ipv4_address('8.8.8.8') + assert is_ipv4_address("8.8.8.8") - @pytest.mark.parametrize('value', ('8.8.8.8.8', 'localhost.localdomain')) + @pytest.mark.parametrize("value", ("8.8.8.8.8", "localhost.localdomain")) def test_invalid(self, value): assert not is_ipv4_address(value) class TestIsValidCIDR: - def test_valid(self): - assert is_valid_cidr('192.168.1.0/24') + assert is_valid_cidr("192.168.1.0/24") @pytest.mark.parametrize( - 'value', ( - '8.8.8.8', - '192.168.1.0/a', - '192.168.1.0/128', - '192.168.1.0/-1', - '192.168.1.999/24', - )) + "value", + ( + "8.8.8.8", + "192.168.1.0/a", + "192.168.1.0/128", + "192.168.1.0/-1", + "192.168.1.999/24", + ), + ) def test_invalid(self, value): assert not is_valid_cidr(value) class TestAddressInNetwork: - def test_valid(self): - assert address_in_network('192.168.1.1', '192.168.1.0/24') + assert address_in_network("192.168.1.1", "192.168.1.0/24") def test_invalid(self): - assert not address_in_network('172.16.0.1', '192.168.1.0/24') + assert not address_in_network("172.16.0.1", "192.168.1.0/24") class TestGuessFilename: - @pytest.mark.parametrize( - 'value', (1, type('Fake', (object,), {'name': 1})()), + "value", + (1, type("Fake", (object,), {"name": 1})()), ) def test_guess_filename_invalid(self, value): assert guess_filename(value) is None @pytest.mark.parametrize( - 'value, expected_type', ( - (b'value', compat.bytes), - (b'value'.decode('utf-8'), compat.str) - )) + "value, expected_type", + ( + (b"value", compat.bytes), + (b"value".decode("utf-8"), compat.str), + ), + ) def test_guess_filename_valid(self, value, expected_type): - obj = type('Fake', (object,), {'name': value})() + obj = type("Fake", (object,), {"name": value})() result = guess_filename(obj) assert result == value assert isinstance(result, expected_type) class TestExtractZippedPaths: - @pytest.mark.parametrize( - 'path', ( - '/', + "path", + ( + "/", __file__, pytest.__file__, - '/etc/invalid/location', - )) + "/etc/invalid/location", + ), + ) def test_unzipped_paths_unchanged(self, path): assert path == extract_zipped_paths(path) def test_zipped_paths_extracted(self, tmpdir): - zipped_py = tmpdir.join('test.zip') - with zipfile.ZipFile(zipped_py.strpath, 'w') as f: + zipped_py = tmpdir.join("test.zip") + with zipfile.ZipFile(zipped_py.strpath, "w") as f: f.write(__file__) _, name = os.path.splitdrive(__file__) - zipped_path = os.path.join(zipped_py.strpath, name.lstrip(r'\/')) + zipped_path = os.path.join(zipped_py.strpath, name.lstrip(r"\/")) extracted_path = extract_zipped_paths(zipped_path) assert extracted_path != zipped_path assert os.path.exists(extracted_path) assert filecmp.cmp(extracted_path, __file__) + def test_invalid_unc_path(self): + path = r"\\localhost\invalid\location" + assert extract_zipped_paths(path) == path -class TestContentEncodingDetection: +class TestContentEncodingDetection: def test_none(self): - encodings = get_encodings_from_content('') + encodings = get_encodings_from_content("") assert not len(encodings) @pytest.mark.parametrize( - 'content', ( + "content", + ( # HTML5 meta charset attribute '', # HTML4 pragma directive @@ -301,242 +376,283 @@ def test_none(self): '', # XHTML 1.x served as XML '', - )) + ), + ) def test_pragmas(self, content): encodings = get_encodings_from_content(content) assert len(encodings) == 1 - assert encodings[0] == 'UTF-8' + assert encodings[0] == "UTF-8" def test_precedence(self): - content = ''' + content = """ - '''.strip() - assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML'] + """.strip() + assert get_encodings_from_content(content) == ["HTML5", "HTML4", "XML"] class TestGuessJSONUTF: - @pytest.mark.parametrize( - 'encoding', ( - 'utf-32', 'utf-8-sig', 'utf-16', 'utf-8', 'utf-16-be', 'utf-16-le', - 'utf-32-be', 'utf-32-le' - )) + "encoding", + ( + "utf-32", + "utf-8-sig", + "utf-16", + "utf-8", + "utf-16-be", + "utf-16-le", + "utf-32-be", + "utf-32-le", + ), + ) def test_encoded(self, encoding): - data = '{}'.encode(encoding) + data = "{}".encode(encoding) assert guess_json_utf(data) == encoding def test_bad_utf_like_encoding(self): - assert guess_json_utf(b'\x00\x00\x00\x00') is None + assert guess_json_utf(b"\x00\x00\x00\x00") is None @pytest.mark.parametrize( - ('encoding', 'expected'), ( - ('utf-16-be', 'utf-16'), - ('utf-16-le', 'utf-16'), - ('utf-32-be', 'utf-32'), - ('utf-32-le', 'utf-32') - )) + ("encoding", "expected"), + ( + ("utf-16-be", "utf-16"), + ("utf-16-le", "utf-16"), + ("utf-32-be", "utf-32"), + ("utf-32-le", "utf-32"), + ), + ) def test_guess_by_bom(self, encoding, expected): - data = u'\ufeff{}'.encode(encoding) + data = "\ufeff{}".encode(encoding) assert guess_json_utf(data) == expected USER = PASSWORD = "%!*'();:@&=+$,/?#[] " -ENCODED_USER = compat.quote(USER, '') -ENCODED_PASSWORD = compat.quote(PASSWORD, '') +ENCODED_USER = compat.quote(USER, "") +ENCODED_PASSWORD = compat.quote(PASSWORD, "") @pytest.mark.parametrize( - 'url, auth', ( - ( - 'http://' + ENCODED_USER + ':' + ENCODED_PASSWORD + '@' + - 'request.com/url.html#test', - (USER, PASSWORD) - ), - ( - 'http://user:pass@complex.url.com/path?query=yes', - ('user', 'pass') - ), - ( - 'http://user:pass%20pass@complex.url.com/path?query=yes', - ('user', 'pass pass') - ), + "url, auth", + ( ( - 'http://user:pass pass@complex.url.com/path?query=yes', - ('user', 'pass pass') + f"http://{ENCODED_USER}:{ENCODED_PASSWORD}@request.com/url.html#test", + (USER, PASSWORD), ), + ("http://user:pass@complex.url.com/path?query=yes", ("user", "pass")), ( - 'http://user%25user:pass@complex.url.com/path?query=yes', - ('user%user', 'pass') + "http://user:pass%20pass@complex.url.com/path?query=yes", + ("user", "pass pass"), ), + ("http://user:pass pass@complex.url.com/path?query=yes", ("user", "pass pass")), ( - 'http://user:pass%23pass@complex.url.com/path?query=yes', - ('user', 'pass#pass') + "http://user%25user:pass@complex.url.com/path?query=yes", + ("user%user", "pass"), ), ( - 'http://complex.url.com/path?query=yes', - ('', '') + "http://user:pass%23pass@complex.url.com/path?query=yes", + ("user", "pass#pass"), ), - )) + ("http://complex.url.com/path?query=yes", ("", "")), + ), +) def test_get_auth_from_url(url, auth): assert get_auth_from_url(url) == auth @pytest.mark.parametrize( - 'uri, expected', ( + "uri, expected", + ( ( # Ensure requoting doesn't break expectations - 'http://example.com/fiz?buz=%25ppicture', - 'http://example.com/fiz?buz=%25ppicture', + "http://example.com/fiz?buz=%25ppicture", + "http://example.com/fiz?buz=%25ppicture", ), ( # Ensure we handle unquoted percent signs in redirects - 'http://example.com/fiz?buz=%ppicture', - 'http://example.com/fiz?buz=%25ppicture', + "http://example.com/fiz?buz=%ppicture", + "http://example.com/fiz?buz=%25ppicture", ), - )) + ), +) def test_requote_uri_with_unquoted_percents(uri, expected): - """See: https://github.com/requests/requests/issues/2356""" + """See: https://github.com/psf/requests/issues/2356""" assert requote_uri(uri) == expected @pytest.mark.parametrize( - 'uri, expected', ( + "uri, expected", + ( ( # Illegal bytes - 'http://example.com/?a=%--', - 'http://example.com/?a=%--', + "http://example.com/?a=%--", + "http://example.com/?a=%--", ), ( # Reserved characters - 'http://example.com/?a=%300', - 'http://example.com/?a=00', - ) - )) + "http://example.com/?a=%300", + "http://example.com/?a=00", + ), + ), +) def test_unquote_unreserved(uri, expected): assert unquote_unreserved(uri) == expected @pytest.mark.parametrize( - 'mask, expected', ( - (8, '255.0.0.0'), - (24, '255.255.255.0'), - (25, '255.255.255.128'), - )) + "mask, expected", + ( + (8, "255.0.0.0"), + (24, "255.255.255.0"), + (25, "255.255.255.128"), + ), +) def test_dotted_netmask(mask, expected): assert dotted_netmask(mask) == expected -http_proxies = {'http': 'http://http.proxy', - 'http://some.host': 'http://some.host.proxy'} -all_proxies = {'all': 'socks5://http.proxy', - 'all://some.host': 'socks5://some.host.proxy'} -mixed_proxies = {'http': 'http://http.proxy', - 'http://some.host': 'http://some.host.proxy', - 'all': 'socks5://http.proxy'} +http_proxies = { + "http": "http://http.proxy", + "http://some.host": "http://some.host.proxy", +} +all_proxies = { + "all": "socks5://http.proxy", + "all://some.host": "socks5://some.host.proxy", +} +mixed_proxies = { + "http": "http://http.proxy", + "http://some.host": "http://some.host.proxy", + "all": "socks5://http.proxy", +} + + @pytest.mark.parametrize( - 'url, expected, proxies', ( - ('hTTp://u:p@Some.Host/path', 'http://some.host.proxy', http_proxies), - ('hTTp://u:p@Other.Host/path', 'http://http.proxy', http_proxies), - ('hTTp:///path', 'http://http.proxy', http_proxies), - ('hTTps://Other.Host', None, http_proxies), - ('file:///etc/motd', None, http_proxies), - - ('hTTp://u:p@Some.Host/path', 'socks5://some.host.proxy', all_proxies), - ('hTTp://u:p@Other.Host/path', 'socks5://http.proxy', all_proxies), - ('hTTp:///path', 'socks5://http.proxy', all_proxies), - ('hTTps://Other.Host', 'socks5://http.proxy', all_proxies), - - ('http://u:p@other.host/path', 'http://http.proxy', mixed_proxies), - ('http://u:p@some.host/path', 'http://some.host.proxy', mixed_proxies), - ('https://u:p@other.host/path', 'socks5://http.proxy', mixed_proxies), - ('https://u:p@some.host/path', 'socks5://http.proxy', mixed_proxies), - ('https://', 'socks5://http.proxy', mixed_proxies), + "url, expected, proxies", + ( + ("hTTp://u:p@Some.Host/path", "http://some.host.proxy", http_proxies), + ("hTTp://u:p@Other.Host/path", "http://http.proxy", http_proxies), + ("hTTp:///path", "http://http.proxy", http_proxies), + ("hTTps://Other.Host", None, http_proxies), + ("file:///etc/motd", None, http_proxies), + ("hTTp://u:p@Some.Host/path", "socks5://some.host.proxy", all_proxies), + ("hTTp://u:p@Other.Host/path", "socks5://http.proxy", all_proxies), + ("hTTp:///path", "socks5://http.proxy", all_proxies), + ("hTTps://Other.Host", "socks5://http.proxy", all_proxies), + ("http://u:p@other.host/path", "http://http.proxy", mixed_proxies), + ("http://u:p@some.host/path", "http://some.host.proxy", mixed_proxies), + ("https://u:p@other.host/path", "socks5://http.proxy", mixed_proxies), + ("https://u:p@some.host/path", "socks5://http.proxy", mixed_proxies), + ("https://", "socks5://http.proxy", mixed_proxies), # XXX: unsure whether this is reasonable behavior - ('file:///etc/motd', 'socks5://http.proxy', all_proxies), - )) + ("file:///etc/motd", "socks5://http.proxy", all_proxies), + ), +) def test_select_proxies(url, expected, proxies): """Make sure we can select per-host proxies correctly.""" assert select_proxy(url, proxies) == expected @pytest.mark.parametrize( - 'value, expected', ( - ('foo="is a fish", bar="as well"', {'foo': 'is a fish', 'bar': 'as well'}), - ('key_without_value', {'key_without_value': None}) - )) + "value, expected", + ( + ('foo="is a fish", bar="as well"', {"foo": "is a fish", "bar": "as well"}), + ("key_without_value", {"key_without_value": None}), + ), +) def test_parse_dict_header(value, expected): assert parse_dict_header(value) == expected @pytest.mark.parametrize( - 'value, expected', ( - ( - 'application/xml', - ('application/xml', {}) - ), + "value, expected", + ( + ("application/xml", ("application/xml", {})), ( - 'application/json ; charset=utf-8', - ('application/json', {'charset': 'utf-8'}) + "application/json ; charset=utf-8", + ("application/json", {"charset": "utf-8"}), ), ( - 'text/plain', - ('text/plain', {}) + "application/json ; Charset=utf-8", + ("application/json", {"charset": "utf-8"}), ), + ("text/plain", ("text/plain", {})), ( - 'multipart/form-data; boundary = something ; boundary2=\'something_else\' ; no_equals ', - ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + "multipart/form-data; boundary = something ; boundary2='something_else' ; no_equals ", + ( + "multipart/form-data", + { + "boundary": "something", + "boundary2": "something_else", + "no_equals": True, + }, + ), ), ( - 'multipart/form-data; boundary = something ; boundary2="something_else" ; no_equals ', - ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + 'multipart/form-data; boundary = something ; boundary2="something_else" ; no_equals ', + ( + "multipart/form-data", + { + "boundary": "something", + "boundary2": "something_else", + "no_equals": True, + }, + ), ), ( - 'multipart/form-data; boundary = something ; \'boundary2=something_else\' ; no_equals ', - ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + "multipart/form-data; boundary = something ; 'boundary2=something_else' ; no_equals ", + ( + "multipart/form-data", + { + "boundary": "something", + "boundary2": "something_else", + "no_equals": True, + }, + ), ), ( 'multipart/form-data; boundary = something ; "boundary2=something_else" ; no_equals ', - ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ( + "multipart/form-data", + { + "boundary": "something", + "boundary2": "something_else", + "no_equals": True, + }, + ), ), - ( - 'application/json ; ; ', - ('application/json', {}) - ) - )) + ("application/json ; ; ", ("application/json", {})), + ), +) def test__parse_content_type_header(value, expected): assert _parse_content_type_header(value) == expected @pytest.mark.parametrize( - 'value, expected', ( - ( - CaseInsensitiveDict(), - None - ), - ( - CaseInsensitiveDict({'content-type': 'application/json; charset=utf-8'}), - 'utf-8' - ), + "value, expected", + ( + (CaseInsensitiveDict(), None), ( - CaseInsensitiveDict({'content-type': 'text/plain'}), - 'ISO-8859-1' + CaseInsensitiveDict({"content-type": "application/json; charset=utf-8"}), + "utf-8", ), - )) + (CaseInsensitiveDict({"content-type": "text/plain"}), "ISO-8859-1"), + ), +) def test_get_encoding_from_headers(value, expected): assert get_encoding_from_headers(value) == expected @pytest.mark.parametrize( - 'value, length', ( - ('', 0), - ('T', 1), - ('Test', 4), - ('Cont', 0), - ('Other', -5), - ('Content', None), - )) + "value, length", + ( + ("", 0), + ("T", 1), + ("Test", 4), + ("Cont", 0), + ("Other", -5), + ("Content", None), + ), +) def test_iter_slices(value, length): if length is None or (length <= 0 and len(value) > 0): # Reads all content at once @@ -546,181 +662,198 @@ def test_iter_slices(value, length): @pytest.mark.parametrize( - 'value, expected', ( + "value, expected", + ( ( '; rel=front; type="image/jpeg"', - [{'url': 'http:/.../front.jpeg', 'rel': 'front', 'type': 'image/jpeg'}] - ), - ( - '', - [{'url': 'http:/.../front.jpeg'}] - ), - ( - ';', - [{'url': 'http:/.../front.jpeg'}] + [{"url": "http:/.../front.jpeg", "rel": "front", "type": "image/jpeg"}], ), + ("", [{"url": "http:/.../front.jpeg"}]), + (";", [{"url": "http:/.../front.jpeg"}]), ( '; type="image/jpeg",;', [ - {'url': 'http:/.../front.jpeg', 'type': 'image/jpeg'}, - {'url': 'http://.../back.jpeg'} - ] - ), - ( - '', - [] + {"url": "http:/.../front.jpeg", "type": "image/jpeg"}, + {"url": "http://.../back.jpeg"}, + ], ), - )) + ("", []), + ), +) def test_parse_header_links(value, expected): assert parse_header_links(value) == expected @pytest.mark.parametrize( - 'value, expected', ( - ('example.com/path', 'http://example.com/path'), - ('//example.com/path', 'http://example.com/path'), - )) + "value, expected", + ( + ("example.com/path", "http://example.com/path"), + ("//example.com/path", "http://example.com/path"), + ("example.com:80", "http://example.com:80"), + ( + "http://user:pass@example.com/path?query", + "http://user:pass@example.com/path?query", + ), + ("http://user@example.com/path?query", "http://user@example.com/path?query"), + ), +) def test_prepend_scheme_if_needed(value, expected): - assert prepend_scheme_if_needed(value, 'http') == expected + assert prepend_scheme_if_needed(value, "http") == expected @pytest.mark.parametrize( - 'value, expected', ( - ('T', 'T'), - (b'T', 'T'), - (u'T', 'T'), - )) + "value, expected", + ( + ("T", "T"), + (b"T", "T"), + ("T", "T"), + ), +) def test_to_native_string(value, expected): assert to_native_string(value) == expected @pytest.mark.parametrize( - 'url, expected', ( - ('http://u:p@example.com/path?a=1#test', 'http://example.com/path?a=1'), - ('http://example.com/path', 'http://example.com/path'), - ('//u:p@example.com/path', '//example.com/path'), - ('//example.com/path', '//example.com/path'), - ('example.com/path', '//example.com/path'), - ('scheme:u:p@example.com/path', 'scheme://example.com/path'), - )) + "url, expected", + ( + ("http://u:p@example.com/path?a=1#test", "http://example.com/path?a=1"), + ("http://example.com/path", "http://example.com/path"), + ("//u:p@example.com/path", "//example.com/path"), + ("//example.com/path", "//example.com/path"), + ("example.com/path", "//example.com/path"), + ("scheme:u:p@example.com/path", "scheme://example.com/path"), + ), +) def test_urldefragauth(url, expected): assert urldefragauth(url) == expected @pytest.mark.parametrize( - 'url, expected', ( - ('http://192.168.0.1:5000/', True), - ('http://192.168.0.1/', True), - ('http://172.16.1.1/', True), - ('http://172.16.1.1:5000/', True), - ('http://localhost.localdomain:5000/v1.0/', True), - ('http://google.com:6000/', True), - ('http://172.16.1.12/', False), - ('http://172.16.1.12:5000/', False), - ('http://google.com:5000/v1.0/', False), - )) + "url, expected", + ( + ("http://192.168.0.1:5000/", True), + ("http://192.168.0.1/", True), + ("http://172.16.1.1/", True), + ("http://172.16.1.1:5000/", True), + ("http://localhost.localdomain:5000/v1.0/", True), + ("http://google.com:6000/", True), + ("http://172.16.1.12/", False), + ("http://172.16.1.12:5000/", False), + ("http://google.com:5000/v1.0/", False), + ("file:///some/path/on/disk", True), + ), +) def test_should_bypass_proxies(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not """ - monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') - monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') + monkeypatch.setenv( + "no_proxy", + "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000", + ) + monkeypatch.setenv( + "NO_PROXY", + "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000", + ) assert should_bypass_proxies(url, no_proxy=None) == expected @pytest.mark.parametrize( - 'url, expected', ( - ('http://172.16.1.1/', '172.16.1.1'), - ('http://172.16.1.1:5000/', '172.16.1.1'), - ('http://user:pass@172.16.1.1', '172.16.1.1'), - ('http://user:pass@172.16.1.1:5000', '172.16.1.1'), - ('http://hostname/', 'hostname'), - ('http://hostname:5000/', 'hostname'), - ('http://user:pass@hostname', 'hostname'), - ('http://user:pass@hostname:5000', 'hostname'), - )) -def test_should_bypass_proxies_pass_only_hostname(url, expected, mocker): + "url, expected", + ( + ("http://172.16.1.1/", "172.16.1.1"), + ("http://172.16.1.1:5000/", "172.16.1.1"), + ("http://user:pass@172.16.1.1", "172.16.1.1"), + ("http://user:pass@172.16.1.1:5000", "172.16.1.1"), + ("http://hostname/", "hostname"), + ("http://hostname:5000/", "hostname"), + ("http://user:pass@hostname", "hostname"), + ("http://user:pass@hostname:5000", "hostname"), + ), +) +def test_should_bypass_proxies_pass_only_hostname(url, expected): """The proxy_bypass function should be called with a hostname or IP without a port number or auth credentials. """ - proxy_bypass = mocker.patch('requests.utils.proxy_bypass') - should_bypass_proxies(url, no_proxy=None) - proxy_bypass.assert_called_once_with(expected) + with mock.patch("requests.utils.proxy_bypass") as proxy_bypass: + should_bypass_proxies(url, no_proxy=None) + proxy_bypass.assert_called_once_with(expected) @pytest.mark.parametrize( - 'cookiejar', ( + "cookiejar", + ( compat.cookielib.CookieJar(), - RequestsCookieJar() - )) + RequestsCookieJar(), + ), +) def test_add_dict_to_cookiejar(cookiejar): """Ensure add_dict_to_cookiejar works for non-RequestsCookieJar CookieJars """ - cookiedict = {'test': 'cookies', - 'good': 'cookies'} + cookiedict = {"test": "cookies", "good": "cookies"} cj = add_dict_to_cookiejar(cookiejar, cookiedict) - cookies = dict((cookie.name, cookie.value) for cookie in cj) + cookies = {cookie.name: cookie.value for cookie in cj} assert cookiedict == cookies @pytest.mark.parametrize( - 'value, expected', ( - (u'test', True), - (u'æíöû', False), - (u'ジェーピーニック', False), - ) + "value, expected", + ( + ("test", True), + ("æíöû", False), + ("ジェーピーニック", False), + ), ) def test_unicode_is_ascii(value, expected): assert unicode_is_ascii(value) is expected @pytest.mark.parametrize( - 'url, expected', ( - ('http://192.168.0.1:5000/', True), - ('http://192.168.0.1/', True), - ('http://172.16.1.1/', True), - ('http://172.16.1.1:5000/', True), - ('http://localhost.localdomain:5000/v1.0/', True), - ('http://172.16.1.12/', False), - ('http://172.16.1.12:5000/', False), - ('http://google.com:5000/v1.0/', False), - )) -def test_should_bypass_proxies_no_proxy( - url, expected, monkeypatch): + "url, expected", + ( + ("http://192.168.0.1:5000/", True), + ("http://192.168.0.1/", True), + ("http://172.16.1.1/", True), + ("http://172.16.1.1:5000/", True), + ("http://localhost.localdomain:5000/v1.0/", True), + ("http://172.16.1.12/", False), + ("http://172.16.1.12:5000/", False), + ("http://google.com:5000/v1.0/", False), + ), +) +def test_should_bypass_proxies_no_proxy(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not using the 'no_proxy' argument """ - no_proxy = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' + no_proxy = "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" # Test 'no_proxy' argument assert should_bypass_proxies(url, no_proxy=no_proxy) == expected -@pytest.mark.skipif(os.name != 'nt', reason='Test only on Windows') +@pytest.mark.skipif(os.name != "nt", reason="Test only on Windows") @pytest.mark.parametrize( - 'url, expected, override', ( - ('http://192.168.0.1:5000/', True, None), - ('http://192.168.0.1/', True, None), - ('http://172.16.1.1/', True, None), - ('http://172.16.1.1:5000/', True, None), - ('http://localhost.localdomain:5000/v1.0/', True, None), - ('http://172.16.1.22/', False, None), - ('http://172.16.1.22:5000/', False, None), - ('http://google.com:5000/v1.0/', False, None), - ('http://mylocalhostname:5000/v1.0/', True, ''), - ('http://192.168.0.1/', False, ''), - )) -def test_should_bypass_proxies_win_registry(url, expected, override, - monkeypatch): + "url, expected, override", + ( + ("http://192.168.0.1:5000/", True, None), + ("http://192.168.0.1/", True, None), + ("http://172.16.1.1/", True, None), + ("http://172.16.1.1:5000/", True, None), + ("http://localhost.localdomain:5000/v1.0/", True, None), + ("http://172.16.1.22/", False, None), + ("http://172.16.1.22:5000/", False, None), + ("http://google.com:5000/v1.0/", False, None), + ("http://mylocalhostname:5000/v1.0/", True, ""), + ("http://192.168.0.1/", False, ""), + ), +) +def test_should_bypass_proxies_win_registry(url, expected, override, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not with Windows registry settings """ if override is None: - override = '192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1' - if compat.is_py3: - import winreg - else: - import _winreg as winreg + override = "192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1" + import winreg class RegHandle: def Close(self): @@ -734,30 +867,65 @@ def OpenKey(key, subkey): def QueryValueEx(key, value_name): if key is ie_settings: - if value_name == 'ProxyEnable': + if value_name == "ProxyEnable": # this could be a string (REG_SZ) or a 32-bit number (REG_DWORD) proxyEnableValues.rotate() return [proxyEnableValues[0]] - elif value_name == 'ProxyOverride': + elif value_name == "ProxyOverride": return [override] - monkeypatch.setenv('http_proxy', '') - monkeypatch.setenv('https_proxy', '') - monkeypatch.setenv('ftp_proxy', '') - monkeypatch.setenv('no_proxy', '') - monkeypatch.setenv('NO_PROXY', '') - monkeypatch.setattr(winreg, 'OpenKey', OpenKey) - monkeypatch.setattr(winreg, 'QueryValueEx', QueryValueEx) + monkeypatch.setenv("http_proxy", "") + monkeypatch.setenv("https_proxy", "") + monkeypatch.setenv("ftp_proxy", "") + monkeypatch.setenv("no_proxy", "") + monkeypatch.setenv("NO_PROXY", "") + monkeypatch.setattr(winreg, "OpenKey", OpenKey) + monkeypatch.setattr(winreg, "QueryValueEx", QueryValueEx) assert should_bypass_proxies(url, None) == expected +@pytest.mark.skipif(os.name != "nt", reason="Test only on Windows") +def test_should_bypass_proxies_win_registry_bad_values(monkeypatch): + """Tests for function should_bypass_proxies to check if proxy + can be bypassed or not with Windows invalid registry settings. + """ + import winreg + + class RegHandle: + def Close(self): + pass + + ie_settings = RegHandle() + + def OpenKey(key, subkey): + return ie_settings + + def QueryValueEx(key, value_name): + if key is ie_settings: + if value_name == "ProxyEnable": + # Invalid response; Should be an int or int-y value + return [""] + elif value_name == "ProxyOverride": + return ["192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1"] + + monkeypatch.setenv("http_proxy", "") + monkeypatch.setenv("https_proxy", "") + monkeypatch.setenv("no_proxy", "") + monkeypatch.setenv("NO_PROXY", "") + monkeypatch.setattr(winreg, "OpenKey", OpenKey) + monkeypatch.setattr(winreg, "QueryValueEx", QueryValueEx) + assert should_bypass_proxies("http://172.16.1.1/", None) is False + + @pytest.mark.parametrize( - 'env_name, value', ( - ('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain'), - ('no_proxy', None), - ('a_new_key', '192.168.0.0/24,127.0.0.1,localhost.localdomain'), - ('a_new_key', None), - )) + "env_name, value", + ( + ("no_proxy", "192.168.0.0/24,127.0.0.1,localhost.localdomain"), + ("no_proxy", None), + ("a_new_key", "192.168.0.0/24,127.0.0.1,localhost.localdomain"), + ("a_new_key", None), + ), +) def test_set_environ(env_name, value): """Tests set_environ will set environ values and will restore the environ.""" environ_copy = copy.deepcopy(os.environ) @@ -771,7 +939,39 @@ def test_set_environ_raises_exception(): """Tests set_environ will raise exceptions in context when the value parameter is None.""" with pytest.raises(Exception) as exception: - with set_environ('test1', None): - raise Exception('Expected exception') + with set_environ("test1", None): + raise Exception("Expected exception") + + assert "Expected exception" in str(exception.value) + - assert 'Expected exception' in str(exception.value) +@pytest.mark.skipif(os.name != "nt", reason="Test only on Windows") +def test_should_bypass_proxies_win_registry_ProxyOverride_value(monkeypatch): + """Tests for function should_bypass_proxies to check if proxy + can be bypassed or not with Windows ProxyOverride registry value ending with a semicolon. + """ + import winreg + + class RegHandle: + def Close(self): + pass + + ie_settings = RegHandle() + + def OpenKey(key, subkey): + return ie_settings + + def QueryValueEx(key, value_name): + if key is ie_settings: + if value_name == "ProxyEnable": + return [1] + elif value_name == "ProxyOverride": + return [ + "192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1;<-loopback>;" + ] + + monkeypatch.setenv("NO_PROXY", "") + monkeypatch.setenv("no_proxy", "") + monkeypatch.setattr(winreg, "OpenKey", OpenKey) + monkeypatch.setattr(winreg, "QueryValueEx", QueryValueEx) + assert should_bypass_proxies("http://example.com/", None) is False diff --git a/tests/testserver/server.py b/tests/testserver/server.py index 6a1dcaa5b2..da1b65608e 100644 --- a/tests/testserver/server.py +++ b/tests/testserver/server.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- - -import threading -import socket import select +import socket +import ssl +import threading def consume_socket_content(sock, timeout=0.5): chunks = 65536 - content = b'' + content = b"" while True: more_to_read = select.select([sock], [], [], timeout)[0] @@ -25,10 +24,18 @@ def consume_socket_content(sock, timeout=0.5): class Server(threading.Thread): """Dummy server using for unit testing""" + WAIT_EVENT_TIMEOUT = 5 - def __init__(self, handler=None, host='localhost', port=0, requests_to_handle=1, wait_to_close_event=None): - super(Server, self).__init__() + def __init__( + self, + handler=None, + host="localhost", + port=0, + requests_to_handle=1, + wait_to_close_event=None, + ): + super().__init__() self.handler = handler or consume_socket_content self.handler_results = [] @@ -45,19 +52,16 @@ def __init__(self, handler=None, host='localhost', port=0, requests_to_handle=1, def text_response_server(cls, text, request_timeout=0.5, **kwargs): def text_response_handler(sock): request_content = consume_socket_content(sock, timeout=request_timeout) - sock.send(text.encode('utf-8')) + sock.send(text.encode("utf-8")) return request_content - return Server(text_response_handler, **kwargs) @classmethod def basic_response_server(cls, **kwargs): return cls.text_response_server( - "HTTP/1.1 200 OK\r\n" + - "Content-Length: 0\r\n\r\n", - **kwargs + "HTTP/1.1 200 OK\r\n" + "Content-Length: 0\r\n\r\n", **kwargs ) def run(self): @@ -71,20 +75,20 @@ def run(self): if self.wait_to_close_event: self.wait_to_close_event.wait(self.WAIT_EVENT_TIMEOUT) finally: - self.ready_event.set() # just in case of exception + self.ready_event.set() # just in case of exception self._close_server_sock_ignore_errors() self.stop_event.set() def _create_socket_and_bind(self): sock = socket.socket() sock.bind((self.host, self.port)) - sock.listen(0) + sock.listen() return sock def _close_server_sock_ignore_errors(self): try: self.server_sock.close() - except IOError: + except OSError: pass def _handle_requests(self): @@ -96,20 +100,24 @@ def _handle_requests(self): handler_result = self.handler(sock) self.handler_results.append(handler_result) + sock.close() def _accept_connection(self): try: - ready, _, _ = select.select([self.server_sock], [], [], self.WAIT_EVENT_TIMEOUT) + ready, _, _ = select.select( + [self.server_sock], [], [], self.WAIT_EVENT_TIMEOUT + ) if not ready: return None return self.server_sock.accept()[0] - except (select.error, socket.error): + except OSError: return None def __enter__(self): self.start() - self.ready_event.wait(self.WAIT_EVENT_TIMEOUT) + if not self.ready_event.wait(self.WAIT_EVENT_TIMEOUT): + raise RuntimeError("Timeout waiting for server to be ready.") return self.host, self.port def __exit__(self, exc_type, exc_value, traceback): @@ -124,4 +132,45 @@ def __exit__(self, exc_type, exc_value, traceback): # ensure server thread doesn't get stuck waiting for connections self._close_server_sock_ignore_errors() self.join() - return False # allow exceptions to propagate + return False # allow exceptions to propagate + + +class TLSServer(Server): + def __init__( + self, + *, + handler=None, + host="localhost", + port=0, + requests_to_handle=1, + wait_to_close_event=None, + cert_chain=None, + keyfile=None, + mutual_tls=False, + cacert=None, + ): + super().__init__( + handler=handler, + host=host, + port=port, + requests_to_handle=requests_to_handle, + wait_to_close_event=wait_to_close_event, + ) + self.cert_chain = cert_chain + self.keyfile = keyfile + self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.ssl_context.load_cert_chain(self.cert_chain, keyfile=self.keyfile) + self.mutual_tls = mutual_tls + self.cacert = cacert + if mutual_tls: + # For simplicity, we're going to assume that the client cert is + # issued by the same CA as our Server certificate + self.ssl_context.verify_mode = ssl.CERT_OPTIONAL + self.ssl_context.load_verify_locations(self.cacert) + + def _create_socket_and_bind(self): + sock = socket.socket() + sock = self.ssl_context.wrap_socket(sock, server_side=True) + sock.bind((self.host, self.port)) + sock.listen() + return sock diff --git a/tests/utils.py b/tests/utils.py index 9b797fd4e4..6cb75bfb6a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import contextlib import os diff --git a/tox.ini b/tox.ini index 38bf3ac44a..70c2855123 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,18 @@ [tox] -envlist = py26,py27,py34,py35,py36 +envlist = py{39,310,311,312,313,314}-{default, use_chardet_on_py3} [testenv] - +deps = -rrequirements-dev.txt +extras = + security + socks commands = - pip install -e .[socks] - python setup.py test + pytest {posargs:tests} + +[testenv:default] + +[testenv:use_chardet_on_py3] +extras = + security + socks + use_chardet_on_py3