project.boost.download: add --no-retry parameter

This is to facilitate testing mostly, but still required substantion
refactoring.

2 files changed, 66 insertions, 46 deletions

diff --git a/project/boost/download.py b/project/boost/download.py
index df92e0a..708e457 100644
--- a/project/boost/download.py
+++ b/project/boost/download.py
@@ -29,47 +29,9 @@ from project.boost.version import Version
 from project.utils import normalize_path, mkdir_parent, retry, setup_logging
 
 
-@retry(urllib.request.URLError)
-def _download_try_url_retry(url):
-    with urllib.request.urlopen(url, timeout=20) as request:
-        return request.read()
-
-
-def _download_try_url(url):
-    logging.info('Trying URL: %s', url)
-    try:
-        return _download_try_url_retry(url)
-    except urllib.request.URLError as e:
-        logging.error("Couldn't download from this mirror, an error occured:")
-        logging.exception(e)
-
-
-@contextmanager
-def _download_try_all_urls(version, storage):
-    urls = version.get_download_urls()
-    for url in urls:
-        reply = _download_try_url(url)
-        if reply is None:
-            continue
-        with storage.write_archive(version, reply) as path:
-            yield path
-            return
-    raise RuntimeError("Couldn't download Boost from any of the mirrors")
-
-
-@contextmanager
-def _download_if_necessary(version, storage):
-    path = storage.get_archive(version)
-    if path is not None:
-        logging.info('Using existing Boost archive: %s', path)
-        yield path
-        return
-    with _download_try_all_urls(version, storage) as path:
-        yield path
-
-
-class DownloadParameters:
-    def __init__(self, version, unpack_dir=None, cache_dir=None, dest_path=None):
+class Download:
+    def __init__(self, version, unpack_dir=None, cache_dir=None,
+                 dest_path=None, no_retry=False):
         if unpack_dir is None:
             if cache_dir is None:
                 unpack_dir = '.'
@@ -89,18 +51,74 @@ class DownloadParameters:
         else:
             self.storage = PermanentStorage(cache_dir)
         self.dest_path = dest_path
+        self.no_retry = no_retry
 
     @staticmethod
     def from_args(args):
-        return DownloadParameters(**vars(args))
+        return Download(**vars(args))
 
     def rename_if_necessary(self, boost_dir):
         if self.dest_path is not None:
             os.rename(boost_dir.path, self.dest_path)
 
+    @staticmethod
+    def _download_url(url):
+        with urllib.request.urlopen(url, timeout=20) as request:
+            return request.read()
+
+    @staticmethod
+    @retry(urllib.request.URLError)
+    def _download_url_retry(url):
+        return Download._download_url(url)
+
+    def _try_url(self, url):
+        logging.info('Trying URL: %s', url)
+        try:
+            if self.no_retry:
+                return self._download_url(url)
+            return self._download_url_retry(url)
+        except urllib.request.URLError as e:
+            logging.error("Couldn't download from this mirror, an error occured:")
+            logging.exception(e)
+
+    @contextmanager
+    def _try_primary_url(self):
+        urls = self.version.get_download_urls()
+        for url in urls:
+            return self._try_url(url)
+
+    def _try_urls(self):
+        urls = self.version.get_download_urls()
+        for url in urls:
+            reply = self._try_url(url)
+            if self.no_retry:
+                break
+            if reply is not None:
+                break
+        if reply is None:
+            raise RuntimeError("Couldn't download Boost from any of the mirrors")
+        return reply
+
+    @contextmanager
+    def _download_from_cdn(self):
+        reply = self._try_urls()
+        with self.storage.write_archive(self.version, reply) as path:
+            yield path
+            return
+
+    @contextmanager
+    def download_if_necessary(self):
+        path = self.storage.get_archive(self.version)
+        if path is not None:
+            logging.info('Using existing Boost archive: %s', path)
+            yield path
+            return
+        with self._download_from_cdn() as path:
+            yield path
+
 
 def download(params):
-    with _download_if_necessary(params.version, params.storage) as path:
+    with params.download_if_necessary() as path:
         archive = Archive(params.version, path)
         boost_dir = archive.unpack(params.unpack_dir)
         params.rename_if_necessary(boost_dir)
@@ -120,6 +138,8 @@ def _parse_args(argv=None):
     parser.add_argument('--cache', metavar='DIR', dest='cache_dir',
                         type=normalize_path,
                         help='download directory (temporary file unless specified)')
+    parser.add_argument('--no-retry', action='store_true',
+                        help=argparse.SUPPRESS)
     parser.add_argument('version', metavar='VERSION',
                         type=Version.from_string,
                         help='Boost version (in the MAJOR.MINOR.PATCH format)')
@@ -133,7 +153,7 @@ def _parse_args(argv=None):
 def _main(argv=None):
     args = _parse_args(argv)
     with setup_logging():
-        download(DownloadParameters.from_args(args))
+        download(Download.from_args(args))
 
 
 if __name__ == '__main__':
diff --git a/project/ci/boost.py b/project/ci/boost.py
index a68e761..6861603 100644
--- a/project/ci/boost.py
+++ b/project/ci/boost.py
@@ -7,7 +7,7 @@ import argparse
 import sys
 
 from project.boost.build import BuildParameters, build
-from project.boost.download import DownloadParameters, download
+from project.boost.download import Download, download
 from project.ci.dirs import Dirs
 from project.linkage import Linkage
 from project.utils import setup_logging
@@ -51,7 +51,7 @@ def build_ci(dirs, argv=None):
         version = dirs.get_boost_version()
         build_dir = dirs.get_build_dir()
         boost_dir = dirs.get_boost_dir()
-        params = DownloadParameters(version, cache_dir=build_dir, dest_path=boost_dir)
+        params = Download(version, cache_dir=build_dir, dest_path=boost_dir)
         download(params)
 
         params = BuildParameters(boost_dir,