#!/usr/bin/env python3 # Copyright (c) 2019 Egor Tensin # This file is part of the "cmake-common" project. # For details, see https://github.com/egor-tensin/cmake-common. # Distributed under the MIT License. '''Download & build Boost. This script downloads and builds the Boost libraries. It's main purpose is to: 1) provide a cross-platform way to download & unpack the Boost distribution archive, 2) set the correct --stagedir parameter value to avoid name clashes. Please pick a command below. You can execute `%(prog)s COMMAND --help` to view its usage message. A simple usage example: $ %(prog)s download 1.71.0 ... $ %(prog)s build -- boost_1_71_0/ --with-filesystem --with-program_options ... ''' import abc import argparse from collections import namedtuple from contextlib import contextmanager from enum import Enum from functools import total_ordering import logging import os.path import platform import re import shutil import subprocess import sys import tempfile import urllib.request @contextmanager def _chdir(path): cwd = os.getcwd() os.chdir(path) try: yield finally: os.chdir(cwd) def _setup_logging(): logging.basicConfig( format='%(asctime)s | %(levelname)s | %(message)s', level=logging.INFO) def _on_windows(): return platform.system() == 'Windows' def _on_linux(): return not _on_windows() def _run_executable(cmd_line): logging.info('Running executable: %s', cmd_line) return subprocess.run(cmd_line, check=True) class Platform(Enum): X86 = 'x86' X64 = 'x64' WIN32 = 'Win32' def __str__(self): return self.value @staticmethod def all(): return (Platform.X86, Platform.X64) def get_address_model(self): if self is Platform.X86: return 32 if self is Platform.X64: return 64 if self is Platform.WIN32: return 32 raise NotImplementedError(f'unsupported platform: {self}') def _parse_platform(s): try: return Platform(s) except ValueError: raise argparse.ArgumentTypeError(f'invalid platform: {s}') class Configuration(Enum): # AFAIK, Boost only supports debug/release, MinSizeRel and RelWithDebInfo # are for compatibility with CMake, they map to "release". # The libraries will still reside in stage/PLATFORM/CONFIGURATION/lib, even # if CONFIGURATION is MinSizeRel/RelWithDebInfo. DEBUG = 'Debug' MINSIZEREL = 'MinSizeRel' RELWITHDEBINFO = 'RelWithDebInfo' RELEASE = 'Release' def normalize(self): '''Roughly maps CMake's CMAKE_BUILD_TYPE to Boost's variant.''' if self is Configuration.MINSIZEREL: return Configuration.RELEASE if self is Configuration.RELWITHDEBINFO: return Configuration.RELEASE return self @staticmethod def all(): return (Configuration.DEBUG, Configuration.RELEASE) def __str__(self): return self.value def _parse_configuration(s): try: return Configuration(s) except ValueError: raise argparse.ArgumentTypeError(f'invalid configuration: {s}') class Linkage(Enum): STATIC = 'static' SHARED = 'shared' @staticmethod def all(): return tuple(Linkage) def __str__(self): return self.value def _parse_linkage(s): try: return Linkage(s) except ValueError: raise argparse.ArgumentTypeError(f'invalid linkage: {s}') _Version = namedtuple('_Version', ['major', 'minor', 'patch']) @total_ordering class BoostVersion: def __init__(self, major, minor, patch): self._impl = _Version(major, minor, patch) @property def major(self): return self._impl.major @property def minor(self): return self._impl.minor @property def patch(self): return self._impl.patch def __lt__(self, other): return self._impl < other._impl def __eq__(self, other): return self._impl == other._impl @staticmethod def from_string(s): result = re.match(r'^(\d+)\.(\d+)\.(\d+)$', s) if result is None: raise ValueError(f'invalid Boost version: {s}') major = int(result.group(1)) minor = int(result.group(2)) patch = int(result.group(3)) return BoostVersion(major, minor, patch) def __str__(self): return f'{self.major}.{self.minor}.{self.patch}' @property def archive_ext(self): return '.tar.gz' def dir_path(self, parent_dir): return os.path.join(parent_dir, self.dir_name) @property def dir_name(self): return f'boost_{self.major}_{self.minor}_{self.patch}' @property def archive_name(self): return f'{self.dir_name}{self.archive_ext}' def _get_bintray_url(self): return f'https://dl.bintray.com/boostorg/release/{self}/source/{self.archive_name}' def _get_sourceforge_url(self): return f'https://sourceforge.net/projects/boost/files/boost/{self}/{self.archive_name}/download' def get_download_urls(self): if self._impl < _Version(1, 63, 0): # For versions older than 1.63.0, SourceForge is the only option: return [self._get_sourceforge_url()] # Otherwise, Bintray is preferred (the official website links to it). return [self._get_bintray_url(), self._get_sourceforge_url()] class BoostArchive: def __init__(self, version, path): self.version = version self.path = path @property def dir_name(self): return self.version.dir_name def unpack(self, dest_dir): path = os.path.join(dest_dir, self.dir_name) if os.path.exists(path): raise RuntimeError(f'Boost directory already exists: {path}') logging.info('Unpacking Boost to: %s', path) shutil.unpack_archive(self.path, dest_dir) return BoostDir(path) class ArchiveStorage(abc.ABC): @contextmanager def download(self, version): path = self.get_archive(version) if path is not None: logging.info('Using existing Boost archive: %s', path) yield BoostArchive(version, path) return urls = version.get_download_urls() for url in urls: logging.info('Trying URL: %s', url) try: with urllib.request.urlopen(url) as request: with self.write_archive(version, request.read()) as path: yield BoostArchive(version, path) return except urllib.request.URLError as e: logging.error("Couldn't download from this mirror, an error occured:") logging.exception(e) raise RuntimeError("Couldn't download Boost from any of the mirrors") @abc.abstractmethod def get_archive(self, version): pass @contextmanager @abc.abstractmethod def write_archive(self, version, contents): pass class CacheStorage(ArchiveStorage): def __init__(self, cache_dir): self._dir = cache_dir def _archive_path(self, version): return os.path.join(self._dir, version.archive_name) def get_archive(self, version): path = self._archive_path(version) if os.path.exists(path): return path return None @contextmanager def write_archive(self, version, contents): path = self._archive_path(version) logging.info('Writing Boost archive: %s', path) if os.path.exists(path): raise RuntimeError(f'cannot download Boost, file already exists: {path}') with open(path, mode='w+b') as dest: dest.write(contents) yield path class TempStorage(ArchiveStorage): def __init__(self, temp_dir): self._dir = temp_dir def get_archive(self, version): return None @contextmanager def write_archive(self, version, contents): with tempfile.NamedTemporaryFile(prefix=f'boost_{version}_', suffix=version.archive_ext, dir=self._dir, delete=False) as dest: path = dest.name logging.info('Writing Boost archive: %s', path) dest.write(contents) try: yield path finally: logging.info('Removing temporary Boost archive: %s', path) os.remove(path) class BoostDir: def __init__(self, path): if not os.path.isdir(path): raise RuntimeError(f"Boost directory doesn't exist: {path}") self.path = path def _go(self): return _chdir(self.path) def build(self, params): with self._go(): self._bootstrap_if_required() self._b2(params) def _bootstrap_if_required(self): if os.path.isfile(self._b2_path()): logging.info('Not going to bootstrap, b2 is already there') return self.bootstrap() def bootstrap(self): with self._go(): _run_executable(self._bootstrap_path()) def _b2(self, params): for b2_params in params.enum_b2_args(): _run_executable([self._b2_path()] + b2_params) @staticmethod def _bootstrap_path(): return os.path.join('.', BoostDir._bootstrap_name()) @staticmethod def _bootstrap_name(): ext = '.sh' if _on_windows(): ext = '.bat' return f'bootstrap{ext}' @staticmethod def _b2_path(): return os.path.join('.', BoostDir._b2_name()) @staticmethod def _b2_name(): ext = '' if _on_windows(): ext = '.exe' return f'b2{ext}' class BuildParameters: def __init__(self, args): self.platforms = args.platforms or Platform.all() self.configurations = args.configurations or Configuration.all() self.link = args.link or Linkage.all() self.runtime_link = args.runtime_link self.stage_dir = 'stage' self.build_dir = args.build_dir self.boost_dir = args.boost_dir self.b2_args = args.b2_args def enum_b2_args(self): with self._create_build_dir() as build_dir: for platform in self.platforms: for configuration in self.configurations: for link, runtime_link in self._linkage_options(): yield self._build_params(build_dir, platform, configuration, link, runtime_link) def _linkage_options(self): for link in self.link: runtime_link = self.runtime_link if runtime_link is Linkage.STATIC: if link is Linkage.SHARED: logging.warning("Cannot link the runtime statically to a dynamic library, going to link dynamically") runtime_link = Linkage.SHARED elif _on_linux(): logging.warning("Cannot link to the GNU C Library (which is assumed) statically, going to link dynamically") runtime_link = Linkage.SHARED yield link, runtime_link @contextmanager def _create_build_dir(self): if self.build_dir is not None: logging.info('Build directory: %s', self.build_dir) yield self.build_dir return with tempfile.TemporaryDirectory(dir=os.path.dirname(self.boost_dir)) as build_dir: logging.info('Build directory: %s', build_dir) try: yield build_dir finally: logging.info('Removing build directory: %s', build_dir) return def _build_params(self, build_dir, platform, configuration, link, runtime_link): params = [] params.append(self._build_dir(build_dir)) params.append(self._stagedir(platform, configuration)) params.append(self._link(link)) params.append(self._runtime_link(runtime_link)) params.append(self._address_model(platform)) params.append(self._variant(configuration)) params += self.b2_args return params @staticmethod def _build_dir(build_dir): return f'--build-dir={build_dir}' def _stagedir(self, platform, configuration): # Having different --stagedir values for every configuration/platform # combination is unnecessary on Windows. # Even for older Boost versions (when the binaries weren't tagged with # their target platform) only a single --stagedir for every platform # would suffice. # For newer versions, just a single --stagedir would do, as the # binaries are tagged with the target platform, as well as their # configuration (a.k.a. "variant" in Boost's terminology). # Still, uniformity helps. platform = str(platform) configuration = str(configuration) return f'--stagedir={os.path.join(self.stage_dir, platform, configuration)}' @staticmethod def _link(link): return f'link={link}' @staticmethod def _runtime_link(runtime_link): return f'runtime-link={runtime_link}' @staticmethod def _address_model(platform): return f'address-model={platform.get_address_model()}' @staticmethod def _variant(configuration): return f'variant={str(configuration.normalize()).lower()}' def _parse_dir(s): return os.path.abspath(os.path.normpath(s)) def _parse_args(argv=None): if argv is None: argv = sys.argv[1:] logging.info('Command line arguments: %s', argv) parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) subparsers = parser.add_subparsers(dest='command') download = subparsers.add_parser('download', help='download & bootstrap Boost') download.add_argument('--cache', metavar='DIR', dest='cache_dir', type=_parse_dir, help='download directory (temporary file unless specified)') download.add_argument('--unpack', metavar='DIR', dest='unpack_dir', type=_parse_dir, default='.', help='directory to unpack the archive to') download.add_argument('boost_version', metavar='VERSION', type=BoostVersion.from_string, help='Boost version (in the MAJOR.MINOR.PATCH format)') build = subparsers.add_parser('build', help='build the libraries') # These are used to put the built libraries into proper stage/ # subdirectories (to avoid name clashes). build.add_argument('--platform', metavar='PLATFORM', nargs='*', dest='platforms', default=[], type=_parse_platform, help=f'target platform ({"/".join(map(str, Platform))})') build.add_argument('--configuration', metavar='CONFIGURATION', nargs='*', dest='configurations', default=[], type=_parse_configuration, help=f'target configuration ({"/".join(map(str, Configuration))})') # This is needed because the default behaviour on Linux and Windows is # different: static & dynamic libs are built on Linux, but only static libs # are built on Windows by default. build.add_argument('--link', metavar='LINKAGE', nargs='*', default=[], type=_parse_linkage, help=f'how the libraries are linked ({"/".join(map(str, Linkage))})') # This is used to omit runtime-link=static I'd have to otherwise use a lot, # plus the script validates the link= and runtime-link= combinations. build.add_argument('--runtime-link', metavar='LINKAGE', type=_parse_linkage, default=Linkage.STATIC, help=f'how the libraries link to the runtime ({"/".join(map(str, Linkage))})') build.add_argument('--build', metavar='DIR', dest='build_dir', type=_parse_dir, help='Boost build directory (temporary directory unless specified)') build.add_argument('boost_dir', metavar='DIR', type=_parse_dir, help='root Boost directory') build.add_argument('b2_args', nargs='*', metavar='B2_ARG', default=[], help='additional b2 arguments, to be passed verbatim') args = parser.parse_args(argv) if args.command is None: parser.error("please specify a command") return args def build(args): build_params = BuildParameters(args) boost_dir = BoostDir(args.boost_dir) boost_dir.build(build_params) def download(args): storage = TempStorage(args.unpack_dir) if args.cache_dir is not None: storage = CacheStorage(args.cache_dir) with storage.download(args.boost_version) as archive: boost_dir = archive.unpack(args.unpack_dir) boost_dir.bootstrap() def main(argv=None): args = _parse_args(argv) if args.command == 'download': download(args) elif args.command == 'build': build(args) else: raise NotImplementedError(f'unsupported command: {args.command}') def _main(argv=None): _setup_logging() try: main(argv) except Exception as e: logging.exception(e) raise if __name__ == '__main__': _main()