aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/project
diff options
context:
space:
mode:
authorEgor Tensin <Egor.Tensin@gmail.com>2021-05-03 11:07:21 +0300
committerEgor Tensin <Egor.Tensin@gmail.com>2021-05-03 11:15:12 +0300
commitc50da6df065985db1cd2d92f386c17a1731150ad (patch)
tree5c2fc0e3865dd2bf8af188b861c88d720579c3a8 /project
parentproject.boost: Bintray was shut down (diff)
downloadcmake-common-c50da6df065985db1cd2d92f386c17a1731150ad.tar.gz
cmake-common-c50da6df065985db1cd2d92f386c17a1731150ad.zip
project.boost.download: add --no-retry parameter
This is to facilitate testing mostly, but still required substantion refactoring.
Diffstat (limited to 'project')
-rw-r--r--project/boost/download.py108
-rw-r--r--project/ci/boost.py4
2 files changed, 66 insertions, 46 deletions
diff --git a/project/boost/download.py b/project/boost/download.py
index df92e0a..708e457 100644
--- a/project/boost/download.py
+++ b/project/boost/download.py
@@ -29,47 +29,9 @@ from project.boost.version import Version
from project.utils import normalize_path, mkdir_parent, retry, setup_logging
-@retry(urllib.request.URLError)
-def _download_try_url_retry(url):
- with urllib.request.urlopen(url, timeout=20) as request:
- return request.read()
-
-
-def _download_try_url(url):
- logging.info('Trying URL: %s', url)
- try:
- return _download_try_url_retry(url)
- except urllib.request.URLError as e:
- logging.error("Couldn't download from this mirror, an error occured:")
- logging.exception(e)
-
-
-@contextmanager
-def _download_try_all_urls(version, storage):
- urls = version.get_download_urls()
- for url in urls:
- reply = _download_try_url(url)
- if reply is None:
- continue
- with storage.write_archive(version, reply) as path:
- yield path
- return
- raise RuntimeError("Couldn't download Boost from any of the mirrors")
-
-
-@contextmanager
-def _download_if_necessary(version, storage):
- path = storage.get_archive(version)
- if path is not None:
- logging.info('Using existing Boost archive: %s', path)
- yield path
- return
- with _download_try_all_urls(version, storage) as path:
- yield path
-
-
-class DownloadParameters:
- def __init__(self, version, unpack_dir=None, cache_dir=None, dest_path=None):
+class Download:
+ def __init__(self, version, unpack_dir=None, cache_dir=None,
+ dest_path=None, no_retry=False):
if unpack_dir is None:
if cache_dir is None:
unpack_dir = '.'
@@ -89,18 +51,74 @@ class DownloadParameters:
else:
self.storage = PermanentStorage(cache_dir)
self.dest_path = dest_path
+ self.no_retry = no_retry
@staticmethod
def from_args(args):
- return DownloadParameters(**vars(args))
+ return Download(**vars(args))
def rename_if_necessary(self, boost_dir):
if self.dest_path is not None:
os.rename(boost_dir.path, self.dest_path)
+ @staticmethod
+ def _download_url(url):
+ with urllib.request.urlopen(url, timeout=20) as request:
+ return request.read()
+
+ @staticmethod
+ @retry(urllib.request.URLError)
+ def _download_url_retry(url):
+ return Download._download_url(url)
+
+ def _try_url(self, url):
+ logging.info('Trying URL: %s', url)
+ try:
+ if self.no_retry:
+ return self._download_url(url)
+ return self._download_url_retry(url)
+ except urllib.request.URLError as e:
+ logging.error("Couldn't download from this mirror, an error occured:")
+ logging.exception(e)
+
+ @contextmanager
+ def _try_primary_url(self):
+ urls = self.version.get_download_urls()
+ for url in urls:
+ return self._try_url(url)
+
+ def _try_urls(self):
+ urls = self.version.get_download_urls()
+ for url in urls:
+ reply = self._try_url(url)
+ if self.no_retry:
+ break
+ if reply is not None:
+ break
+ if reply is None:
+ raise RuntimeError("Couldn't download Boost from any of the mirrors")
+ return reply
+
+ @contextmanager
+ def _download_from_cdn(self):
+ reply = self._try_urls()
+ with self.storage.write_archive(self.version, reply) as path:
+ yield path
+ return
+
+ @contextmanager
+ def download_if_necessary(self):
+ path = self.storage.get_archive(self.version)
+ if path is not None:
+ logging.info('Using existing Boost archive: %s', path)
+ yield path
+ return
+ with self._download_from_cdn() as path:
+ yield path
+
def download(params):
- with _download_if_necessary(params.version, params.storage) as path:
+ with params.download_if_necessary() as path:
archive = Archive(params.version, path)
boost_dir = archive.unpack(params.unpack_dir)
params.rename_if_necessary(boost_dir)
@@ -120,6 +138,8 @@ def _parse_args(argv=None):
parser.add_argument('--cache', metavar='DIR', dest='cache_dir',
type=normalize_path,
help='download directory (temporary file unless specified)')
+ parser.add_argument('--no-retry', action='store_true',
+ help=argparse.SUPPRESS)
parser.add_argument('version', metavar='VERSION',
type=Version.from_string,
help='Boost version (in the MAJOR.MINOR.PATCH format)')
@@ -133,7 +153,7 @@ def _parse_args(argv=None):
def _main(argv=None):
args = _parse_args(argv)
with setup_logging():
- download(DownloadParameters.from_args(args))
+ download(Download.from_args(args))
if __name__ == '__main__':
diff --git a/project/ci/boost.py b/project/ci/boost.py
index a68e761..6861603 100644
--- a/project/ci/boost.py
+++ b/project/ci/boost.py
@@ -7,7 +7,7 @@ import argparse
import sys
from project.boost.build import BuildParameters, build
-from project.boost.download import DownloadParameters, download
+from project.boost.download import Download, download
from project.ci.dirs import Dirs
from project.linkage import Linkage
from project.utils import setup_logging
@@ -51,7 +51,7 @@ def build_ci(dirs, argv=None):
version = dirs.get_boost_version()
build_dir = dirs.get_build_dir()
boost_dir = dirs.get_boost_dir()
- params = DownloadParameters(version, cache_dir=build_dir, dest_path=boost_dir)
+ params = Download(version, cache_dir=build_dir, dest_path=boost_dir)
download(params)
params = BuildParameters(boost_dir,