Merge pull request #389 from FoamyGuy/requests_timeout

Requests timeout

Author: FoamyGuy

adabot/__init__.py

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: 2023 Tim Cocks
+#
+# SPDX-License-Identifier: MIT
+
+"""AdaBot is a friendly helper bot that works across the web to make people's
+lives better."""
+
+REQUESTS_TIMEOUT = 30

adabot/arduino_libraries.py

@@ -11,7 +11,7 @@
 
 import requests
 
-from adabot import github_requests as gh_reqs
+from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT
 
 logger = logging.getLogger(__name__)
 ch = logging.StreamHandler(stream=sys.stdout)
@@ -86,7 +86,8 @@ def is_arduino_library(repo):
         + repo["name"]
         + "/"
         + repo["default_branch"]
-        + "/library.properties"
+        + "/library.properties",
+        timeout=REQUESTS_TIMEOUT,
     )
     return lib_prop_file.ok
 
@@ -116,7 +117,8 @@ def validate_library_properties(repo):
         + repo["name"]
         + "/"
         + repo["default_branch"]
-        + "/library.properties"
+        + "/library.properties",
+        timeout=REQUESTS_TIMEOUT,
     )
     if not lib_prop_file.ok:
         # print("{} skipped".format(repo["name"]))
@@ -193,7 +195,8 @@ def validate_actions(repo):
         + repo["name"]
         + "/"
         + repo["default_branch"]
-        + "/.github/workflows/githubci.yml"
+        + "/.github/workflows/githubci.yml",
+        timeout=REQUESTS_TIMEOUT,
     )
     return repo_has_actions.ok
 
@@ -309,7 +312,10 @@ def main(verbosity=1, output_file=None):  # pylint: disable=missing-function-doc
         logger.setLevel("CRITICAL")
 
     try:
-        reply = requests.get("http://downloads.arduino.cc/libraries/library_index.json")
+        reply = requests.get(
+            "http://downloads.arduino.cc/libraries/library_index.json",
+            timeout=REQUESTS_TIMEOUT,
+        )
         if not reply.ok:
             logging.error(
                 "Could not fetch http://downloads.arduino.cc/libraries/library_index.json"

adabot/circuitpython_libraries.py

@@ -17,7 +17,7 @@
 import github as pygithub
 import requests
 
-from adabot import github_requests as gh_reqs
+from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT
 from adabot import pypi_requests as pypi
 from adabot.lib import circuitpython_library_validators as cirpy_lib_vals
 from adabot.lib import common_funcs
@@ -223,7 +223,7 @@ def run_library_checks(validators, kw_args, error_depth):
     resp = requests.get(
         "https://raw.githubusercontent.com/adafruit/"
         "CircuitPython_Community_Bundle/main/.gitmodules",
-        timeout=30,
+        timeout=REQUESTS_TIMEOUT,
     )
     community_bundle_submodules = resp.text
     community_library_count = community_bundle_submodules.count("submodule")

adabot/circuitpython_library_download_stats.py

@@ -17,7 +17,7 @@
 from google.cloud import bigquery
 import google.oauth2.service_account
 
-from adabot import github_requests as gh_reqs
+from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT
 from adabot.lib import common_funcs
 
 # Setup ArgumentParser
@@ -60,7 +60,7 @@
 def retrieve_piwheels_stats():
     """Get data dump of piwheels download stats"""
     stats = {}
-    response = requests.get(PIWHEELS_PACKAGES_URL)
+    response = requests.get(PIWHEELS_PACKAGES_URL, timeout=REQUESTS_TIMEOUT)
     if response.ok:
         packages = response.json()
         stats = {

adabot/circuitpython_library_patches.py

@@ -13,6 +13,7 @@
 import sh
 from sh.contrib import git
 
+from adabot import REQUESTS_TIMEOUT
 from adabot.lib import common_funcs
 
 
@@ -100,7 +101,8 @@ def get_patches(run_local):
     return_list = []
     if not run_local:
         contents = requests.get(
-            "https://api.github.com/repos/adafruit/adabot/contents/patches"
+            "https://api.github.com/repos/adafruit/adabot/contents/patches",
+            timeout=REQUESTS_TIMEOUT
         )
         if contents.ok:
             for patch in contents.json():

adabot/circuitpython_library_release.py

@@ -48,7 +48,7 @@ def make_release(new_tag, logger, test_run=False):
 
 def get_pypi_name():
     """
-    return the shorthand pypi project name
+    return the shorthand project name used for pypi, docs, etc.
     """
     data = toml.load("pyproject.toml")
 
@@ -126,7 +126,7 @@ def get_release_info():
     }
 
 
-def get_compare_url(tag_name):
+def get_compare_url(tag_name, compare_to_tag_name="main"):
     """
     Get the URL to the GitHub compare page for the latest release compared
     to current main.
@@ -138,7 +138,9 @@ def get_compare_url(tag_name):
     if not remote_url.startswith("https"):
         return "Sorry, Unknown Remotes"
 
-    compare_url = remote_url.replace(".git", f"/compare/{tag_name}...main")
+    compare_url = remote_url.replace(
+        ".git", f"/compare/{tag_name}...{compare_to_tag_name}"
+    )
     return compare_url
 
 

adabot/lib/assign_hacktober_label.py

@@ -11,7 +11,7 @@
 import datetime
 import requests
 
-from adabot import github_requests as gh_reqs
+from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT
 from adabot.lib import common_funcs
 
 cli_args = argparse.ArgumentParser(description="Hacktoberfest Label Assigner")
@@ -73,7 +73,9 @@ def get_open_issues(repo):
         )
 
         if response.links.get("next"):
-            response = requests.get(response.links["next"]["url"])
+            response = requests.get(
+                response.links["next"]["url"], timeout=REQUESTS_TIMEOUT
+            )
         else:
             break
 

adabot/lib/circuitpython_library_validators.py

@@ -21,7 +21,7 @@
 import parse
 
 import github as pygithub
-from adabot import github_requests as gh_reqs
+from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT
 from adabot.lib import common_funcs
 from adabot.lib import assign_hacktober_label as hacktober
 
@@ -231,7 +231,7 @@ def rtd_yml_base(self):
                 "%20if%20cookiecutter.sphinx_docs%20in%20%5B'y'%2C%20'yes'%5D%20%25"
                 "%7D.readthedocs.yaml%7B%25%20endif%20%25%7D"
             )
-            rtd_yml = requests.get(rtd_yml_dl_url)
+            rtd_yml = requests.get(rtd_yml_dl_url, timeout=REQUESTS_TIMEOUT)
             if rtd_yml.ok:
                 try:
                     self._rtd_yaml_base = yaml.safe_load(rtd_yml.text)
@@ -255,7 +255,7 @@ def pcc_versions(self):
                 "circuitpython/main/%7B%7B%20cookiecutter.__dirname%20%7D%7D/.pre-"
                 "commit-config.yaml"
             )
-            pcc_yml = requests.get(pcc_yml_dl_url)
+            pcc_yml = requests.get(pcc_yml_dl_url, timeout=REQUESTS_TIMEOUT)
             if pcc_yml.ok:
                 try:
                     pcc_yaml_base = yaml.safe_load(pcc_yml.text)
@@ -463,7 +463,7 @@ def _filter_file_diffs(filenames):
     def _validate_readme(self, download_url):
         # We use requests because file contents are hosted by
         # githubusercontent.com, not the API domain.
-        contents = requests.get(download_url, timeout=30)
+        contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT)
         if not contents.ok:
             return [ERROR_README_DOWNLOAD_FAILED]
 
@@ -509,7 +509,7 @@ def _validate_py_for_u_modules(self, download_url):
         """
         # We use requests because file contents are hosted by
         # githubusercontent.com, not the API domain.
-        contents = requests.get(download_url, timeout=30)
+        contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT)
         if not contents.ok:
             return [ERROR_PYFILE_DOWNLOAD_FAILED]
 
@@ -547,7 +547,7 @@ def _validate_actions_build_yml(self, actions_build_info):
         """
 
         download_url = actions_build_info["download_url"]
-        contents = requests.get(download_url, timeout=30)
+        contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT)
         if not contents.ok:
             return [ERROR_PYFILE_DOWNLOAD_FAILED]
 
@@ -557,7 +557,7 @@ def _validate_actions_build_yml(self, actions_build_info):
 
     def _validate_pre_commit_config_yaml(self, file_info):
         download_url = file_info["download_url"]
-        contents = requests.get(download_url, timeout=30)
+        contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT)
         if not contents.ok:
             return [ERROR_PYFILE_DOWNLOAD_FAILED]
 
@@ -594,15 +594,15 @@ def _validate_pre_commit_config_yaml(self, file_info):
     def _validate_pyproject_toml(self, file_info):
         """Check pyproject.toml for pypi compatibility"""
         download_url = file_info["download_url"]
-        contents = requests.get(download_url, timeout=30)
+        contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT)
         if not contents.ok:
             return [ERROR_TOMLFILE_DOWNLOAD_FAILED]
         return []
 
     def _validate_requirements_txt(self, repo, file_info, check_blinka=True):
         """Check requirements.txt for pypi compatibility"""
         download_url = file_info["download_url"]
-        contents = requests.get(download_url, timeout=30)
+        contents = requests.get(download_url, timeout=REQUESTS_TIMEOUT)
         if not contents.ok:
             return [ERROR_PYFILE_DOWNLOAD_FAILED]
 
@@ -717,7 +717,9 @@ def validate_contents(self, repo):
                 if ".readthedocs.yaml" in files:
                     filename = ".readthedocs.yaml"
                 file_info = content_list[files.index(filename)]
-                rtd_contents = requests.get(file_info["download_url"])
+                rtd_contents = requests.get(
+                    file_info["download_url"], timeout=REQUESTS_TIMEOUT
+                )
                 if rtd_contents.ok:
                     try:
                         rtd_yml = yaml.safe_load(rtd_contents.text)
@@ -735,7 +737,9 @@ def validate_contents(self, repo):
             if len(self._pcc_versions) or self.pcc_versions != "":
                 filename = ".pre-commit-config.yaml"
                 file_info = content_list[files.index(filename)]
-                pcc_contents = requests.get(file_info["download_url"])
+                pcc_contents = requests.get(
+                    file_info["download_url"], timeout=REQUESTS_TIMEOUT
+                )
                 if pcc_contents.ok:
                     try:
                         pcc_yml = yaml.safe_load(pcc_contents.text)
@@ -888,7 +892,8 @@ def validate_readthedocs(self, repo):
             return []
         if not self.rtd_subprojects:
             rtd_response = requests.get(
-                "https://readthedocs.org/api/v2/project/74557/subprojects/", timeout=15
+                "https://readthedocs.org/api/v2/project/74557/subprojects/",
+                timeout=REQUESTS_TIMEOUT,
             )
             if not rtd_response.ok:
                 return [ERROR_RTD_SUBPROJECT_FAILED]
@@ -937,7 +942,7 @@ def validate_readthedocs(self, repo):
             url = f"https://readthedocs.org/api/v3/projects/{rtd_slug}/builds/"
             rtd_token = os.environ["RTD_TOKEN"]
             headers = {"Authorization": f"token {rtd_token}"}
-            response = requests.get(url, headers=headers)
+            response = requests.get(url, headers=headers, timeout=REQUESTS_TIMEOUT)
             json_response = response.json()
 
             error_message = json_response.get("detail")
@@ -981,7 +986,7 @@ def validate_core_driver_page(self, repo):
                     "https://raw.githubusercontent.com/adafruit/Adafruit_CircuitPython_Bundle/"
                     "main/docs/drivers.rst"
                 ),
-                timeout=15,
+                timeout=REQUESTS_TIMEOUT,
             )
             if not driver_page.ok:
                 return [ERROR_DRIVERS_PAGE_DOWNLOAD_FAILED]

adabot/lib/common_funcs.py

@@ -12,7 +12,7 @@
 import os
 import re
 import requests
-from adabot import github_requests as gh_reqs
+from adabot import github_requests as gh_reqs, REQUESTS_TIMEOUT
 from adabot import pypi_requests as pypi
 
 CORE_REPO_URL = "/repos/adafruit/circuitpython"
@@ -96,7 +96,7 @@ def get_bundle_submodules():
     # master branch of the bundle is the canonical source of the bundle release.
     result = requests.get(
         "https://raw.githubusercontent.com/adafruit/Adafruit_CircuitPython_Bundle/main/.gitmodules",
-        timeout=15,
+        timeout=REQUESTS_TIMEOUT,
     )
     if result.status_code != 200:
         # output_handler("Failed to access bundle .gitmodules file from GitHub!", quiet=True)

adabot/pypi_requests.py

@@ -9,6 +9,8 @@
 
 import requests
 
+from adabot import REQUESTS_TIMEOUT
+
 
 def _fix_url(url):
     if url.startswith("/"):
@@ -18,4 +20,4 @@ def _fix_url(url):
 
 def get(url, **kwargs):
     """Process a GET request from pypi.org"""
-    return requests.get(_fix_url(url), timeout=30, **kwargs)
+    return requests.get(_fix_url(url), timeout=REQUESTS_TIMEOUT, **kwargs)

tools/docs_status.py

@@ -21,10 +21,12 @@
 import requests
 from github.Repository import Repository
 from github.ContentFile import ContentFile
+
 from iterate_libraries import (
     iter_remote_bundle_with_func,
     RemoteLibFunc_IterResult,
 )
+from adabot import REQUESTS_TIMEOUT
 
 
 def check_docs_status(
@@ -64,7 +66,7 @@ def check_docs_status(
     # GET the latest documentation build runs
     url = f"https://readthedocs.org/api/v3/projects/{rtd_slug}/builds/"
     headers = {"Authorization": f"token {rtd_token}"}
-    response = requests.get(url, headers=headers)
+    response = requests.get(url, headers=headers, timeout=REQUESTS_TIMEOUT)
     json_response: dict[str, Any] = response.json()
 
     # Return the results of the latest run

tools/file_compare.py

@@ -20,6 +20,7 @@
 import requests
 from requests.structures import CaseInsensitiveDict
 
+from adabot import REQUESTS_TIMEOUT
 from adabot.lib.common_funcs import list_repos
 
 
@@ -54,9 +55,9 @@ def compare(git_file: str, token: Optional[str] = None) -> list:
             headers = CaseInsensitiveDict()
             headers["Authorization"] = f"token {token}"
 
-            resp = requests.get(url, headers=headers)
+            resp = requests.get(url, headers=headers, timeout=REQUESTS_TIMEOUT)
         else:
-            resp = requests.get(url)
+            resp = requests.get(url, timeout=REQUESTS_TIMEOUT)
 
         if resp.status_code != 200:
             print(name)

tools/find_text.py

@@ -19,6 +19,7 @@
 
 import requests
 
+from adabot import REQUESTS_TIMEOUT
 from adabot.lib.common_funcs import list_repos
 
 argumentList = sys.argv[1:]
@@ -129,7 +130,9 @@ def prettyprint(info, results):
 
 for repo in all_repos:
     INFO = "getting {} for: {}".format(FILE, repo["name"])
-    response = requests.get(URL_TEMPLATE.format(repo["name"], FILE))
+    response = requests.get(
+        URL_TEMPLATE.format(repo["name"], FILE), timeout=REQUESTS_TIMEOUT
+    )
     result = []
     if response.status_code == 404:
         RESULTS["file_not_found"].append(repo["html_url"])