diff options
| -rw-r--r-- | project/boost/download.py | 11 | ||||
| -rw-r--r-- | project/utils.py | 22 |
2 files changed, 30 insertions, 3 deletions
diff --git a/project/boost/download.py b/project/boost/download.py index ae18e94..13a89f4 100644 --- a/project/boost/download.py +++ b/project/boost/download.py @@ -25,14 +25,19 @@ import urllib.request from project.boost.archive import Archive, PermanentStorage, TemporaryStorage from project.boost.version import Version -from project.utils import normalize_path, setup_logging +from project.utils import normalize_path, retry, setup_logging + + +@retry(urllib.request.URLError) +def _download_try_url_retry(url): + with urllib.request.urlopen(url, timeout=20) as request: + return request.read() def _download_try_url(url): logging.info('Trying URL: %s', url) try: - with urllib.request.urlopen(url, timeout=20) as request: - return request.read() + return _download_try_url_retry(url) except urllib.request.URLError as e: logging.error("Couldn't download from this mirror, an error occured:") logging.exception(e) diff --git a/project/utils.py b/project/utils.py index 5874eab..76557b1 100644 --- a/project/utils.py +++ b/project/utils.py @@ -4,10 +4,12 @@ # Distributed under the MIT License. from contextlib import contextmanager +import functools import logging import os.path import subprocess import tempfile +import time def normalize_path(s): @@ -80,3 +82,23 @@ def env(name): if name not in os.environ: raise RuntimeError(f'undefined environment variable: {name}') return os.environ[name] + + +def retry(exc_type, timeout=5, retries=3, backoff=2): + def wrapper(func): + @functools.wraps(func) + def func2(*args, **kwargs): + current_timeout = timeout + for retry_n in range(retries): + try: + return func(*args, **kwargs) + except exc_type as e: + logging.exception(e) + if retry_n < retries: + logging.error('Retrying after %d seconds', current_timeout) + time.sleep(current_timeout) + current_timeout *= backoff + continue + raise + return func2 + return wrapper |