diff options
| -rw-r--r-- | .github/actions/download-boost/action.yml | 15 | ||||
| -rw-r--r-- | .github/workflows/boost_download.yml | 1 | ||||
| -rw-r--r-- | project/boost/download.py | 108 | ||||
| -rw-r--r-- | project/ci/boost.py | 4 |
4 files changed, 80 insertions, 48 deletions
diff --git a/.github/actions/download-boost/action.yml b/.github/actions/download-boost/action.yml index 95f5793..7e15075 100644 --- a/.github/actions/download-boost/action.yml +++ b/.github/actions/download-boost/action.yml @@ -2,8 +2,12 @@ name: Download Boost description: Download & unpack Boost inputs: boost-version: - description: 'Boost version' + description: Boost version required: true + no-retry: + description: Don't retry downloads + required: false + default: 0 runs: using: composite steps: @@ -31,10 +35,17 @@ runs: $python = 'python' $cache_dir = $env:BOOST_CACHE_DIR + $no_retry = '${{ inputs.no-retry }}' -eq '1' + + $args = @() + if ($no_retry) { + $args += '--no-retry' + } + if ($env:CI_HOST_CYGWIN) { $python = 'python3' $cache_dir = cygpath.exe -ua $cache_dir } - & $python -m project.boost.download --cache $cache_dir -- $env:BOOST_VERSION + & $python -m project.boost.download --cache $cache_dir $args -- $env:BOOST_VERSION shell: pwsh diff --git a/.github/workflows/boost_download.yml b/.github/workflows/boost_download.yml index d5dc717..a718f60 100644 --- a/.github/workflows/boost_download.yml +++ b/.github/workflows/boost_download.yml @@ -35,6 +35,7 @@ jobs: uses: ./.github/actions/download-boost with: boost-version: '${{ matrix.boost-version }}' + no-retry: 1 - name: Check that Boost was downloaded uses: ./.github/actions/check-boost-download diff --git a/project/boost/download.py b/project/boost/download.py index df92e0a..708e457 100644 --- a/project/boost/download.py +++ b/project/boost/download.py @@ -29,47 +29,9 @@ from project.boost.version import Version from project.utils import normalize_path, mkdir_parent, retry, setup_logging -@retry(urllib.request.URLError) -def _download_try_url_retry(url): - with urllib.request.urlopen(url, timeout=20) as request: - return request.read() - - -def _download_try_url(url): - logging.info('Trying URL: %s', url) - try: - return _download_try_url_retry(url) - except urllib.request.URLError as e: - logging.error("Couldn't download from this mirror, an error occured:") - logging.exception(e) - - -@contextmanager -def _download_try_all_urls(version, storage): - urls = version.get_download_urls() - for url in urls: - reply = _download_try_url(url) - if reply is None: - continue - with storage.write_archive(version, reply) as path: - yield path - return - raise RuntimeError("Couldn't download Boost from any of the mirrors") - - -@contextmanager -def _download_if_necessary(version, storage): - path = storage.get_archive(version) - if path is not None: - logging.info('Using existing Boost archive: %s', path) - yield path - return - with _download_try_all_urls(version, storage) as path: - yield path - - -class DownloadParameters: - def __init__(self, version, unpack_dir=None, cache_dir=None, dest_path=None): +class Download: + def __init__(self, version, unpack_dir=None, cache_dir=None, + dest_path=None, no_retry=False): if unpack_dir is None: if cache_dir is None: unpack_dir = '.' @@ -89,18 +51,74 @@ class DownloadParameters: else: self.storage = PermanentStorage(cache_dir) self.dest_path = dest_path + self.no_retry = no_retry @staticmethod def from_args(args): - return DownloadParameters(**vars(args)) + return Download(**vars(args)) def rename_if_necessary(self, boost_dir): if self.dest_path is not None: os.rename(boost_dir.path, self.dest_path) + @staticmethod + def _download_url(url): + with urllib.request.urlopen(url, timeout=20) as request: + return request.read() + + @staticmethod + @retry(urllib.request.URLError) + def _download_url_retry(url): + return Download._download_url(url) + + def _try_url(self, url): + logging.info('Trying URL: %s', url) + try: + if self.no_retry: + return self._download_url(url) + return self._download_url_retry(url) + except urllib.request.URLError as e: + logging.error("Couldn't download from this mirror, an error occured:") + logging.exception(e) + + @contextmanager + def _try_primary_url(self): + urls = self.version.get_download_urls() + for url in urls: + return self._try_url(url) + + def _try_urls(self): + urls = self.version.get_download_urls() + for url in urls: + reply = self._try_url(url) + if self.no_retry: + break + if reply is not None: + break + if reply is None: + raise RuntimeError("Couldn't download Boost from any of the mirrors") + return reply + + @contextmanager + def _download_from_cdn(self): + reply = self._try_urls() + with self.storage.write_archive(self.version, reply) as path: + yield path + return + + @contextmanager + def download_if_necessary(self): + path = self.storage.get_archive(self.version) + if path is not None: + logging.info('Using existing Boost archive: %s', path) + yield path + return + with self._download_from_cdn() as path: + yield path + def download(params): - with _download_if_necessary(params.version, params.storage) as path: + with params.download_if_necessary() as path: archive = Archive(params.version, path) boost_dir = archive.unpack(params.unpack_dir) params.rename_if_necessary(boost_dir) @@ -120,6 +138,8 @@ def _parse_args(argv=None): parser.add_argument('--cache', metavar='DIR', dest='cache_dir', type=normalize_path, help='download directory (temporary file unless specified)') + parser.add_argument('--no-retry', action='store_true', + help=argparse.SUPPRESS) parser.add_argument('version', metavar='VERSION', type=Version.from_string, help='Boost version (in the MAJOR.MINOR.PATCH format)') @@ -133,7 +153,7 @@ def _parse_args(argv=None): def _main(argv=None): args = _parse_args(argv) with setup_logging(): - download(DownloadParameters.from_args(args)) + download(Download.from_args(args)) if __name__ == '__main__': diff --git a/project/ci/boost.py b/project/ci/boost.py index a68e761..6861603 100644 --- a/project/ci/boost.py +++ b/project/ci/boost.py @@ -7,7 +7,7 @@ import argparse import sys from project.boost.build import BuildParameters, build -from project.boost.download import DownloadParameters, download +from project.boost.download import Download, download from project.ci.dirs import Dirs from project.linkage import Linkage from project.utils import setup_logging @@ -51,7 +51,7 @@ def build_ci(dirs, argv=None): version = dirs.get_boost_version() build_dir = dirs.get_build_dir() boost_dir = dirs.get_boost_dir() - params = DownloadParameters(version, cache_dir=build_dir, dest_path=boost_dir) + params = Download(version, cache_dir=build_dir, dest_path=boost_dir) download(params) params = BuildParameters(boost_dir, |