fetch.py

import requests
from requests import Response
from retrying import retry

from config import Req, retry_max_number, retry_min_random_wait, retry_max_random_wait, fetch_timeout


def need_retry(exception):
    result = isinstance(exception, (requests.ConnectionError,
                                    requests.ReadTimeout,
                                    requests.exceptions.ConnectTimeout,
                                    requests.exceptions.ReadTimeout,
                                    requests.exceptions.Timeout))
    if result:
        print('Exception:{} occurred, retrying...'.format(type(exception)))
    return result


def fetch(url, method=Req.GET, **kwargs):
    @retry(stop_max_attempt_number=retry_max_number, wait_random_min=retry_min_random_wait,
           wait_random_max=retry_max_random_wait, retry_on_exception=need_retry)
    def _fetch(url, **kwargs) -> Response:
        # kwargs.update({'verify': False})
        kwargs.update({'timeout': fetch_timeout})
        response = requests.post(url, **kwargs) if method == Req.POST else requests.get(url, **kwargs)
        if response.status_code != 200:
            raise requests.ConnectionError('Expected status code 200, but got {}'.format(response.status_code))
        return response

    try:
        resp = _fetch(url, **kwargs)
        return resp
    except Exception as e:
        print('Something got wrong, error msg:{}'.format(e))
        return None