Implementing retry for requests in Python
you can use urllib3.util.retry
module in combination with requests
to have something as follow:
from urllib3.util.retry import Retry
import requests
from requests.adapters import HTTPAdapter
def retry_session(retries, session=None, backoff_factor=0.3):
session = session or requests.Session()
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
method_whitelist=False,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
Usage:
session = retry_session(retries=5)
session.post(url=endpoint, data=json.dumps(x), headers=headers)
NB: You can also inherit from Retry
class and customize the retry behavior and retry intervals.
I found that the default behaviour of Retries did not apply to POST. To do so required the addition of method_whitelist, e.g. below:
def retry_session(retries=5):
session = Session()
retries = Retry(total=retries,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504],
method_whitelist=frozenset(['GET', 'POST']))
session.mount('https://', HTTPAdapter(max_retries=retries))
session.mount('http://', HTTPAdapter(max_retries=retries))
return session