From dd2c5b58c4fe77d7ce35f3abb6e1bb399560a2db Mon Sep 17 00:00:00 2001 From: Egor Tensin Date: Sun, 17 Jan 2021 13:54:57 +0300 Subject: GIANT CLUSTERFUCK OF A COMMIT OK, this is epic. I was basically just trying to a) support Clang and b) add more test coverage. _THREE MONTHS_ and a few hundred CI runs later, this is what I came up with. I don't know how it ended up being what it is, but here we go. Some highlights of the changes: 1) CI builds has been moved to GitHub Actions, 2) the entire notion of a toolchain has been reworked; it now supports Clang on all platforms. * .github: this directory contains the GitHub Actions workflow scripts/actions. In the process, I created like 6 external GitHub actions, but it's still pretty massive. An upside is that it covers much more platform/toolchain combinations _and_ check a lot of the expected post-conditions. TODO: .ci/Makefile is obsolete now, as well as .travis.yml and .appveyor.yml. * common.cmake: added Clang support. In the process, a great deal has been learned about how CMake works; in particular, static runtime support has been reworked to be more robust. * project: the entire notion of a "toolchain" has been reworked. Instead of a measly --mingw parameter, there's now a separate --toolset parameter, which allows you to choose between GCC, Clang, MSVC, etc. Both Boost and CMake build scripts were enhanced greatly to support Clang and other toolchains in a more robust way. --- project/boost/build.py | 34 ++-- project/boost/directory.py | 18 +- project/boost/download.py | 5 +- project/boost/toolchain.py | 455 +++++++++++++++++++++++++++++++++++++++++---- project/ci/boost.py | 11 +- project/ci/cmake.py | 8 +- project/cmake/build.py | 53 ++++-- project/cmake/toolchain.py | 281 +++++++++++++++++++++++----- project/mingw.py | 16 +- project/toolchain.py | 45 +++++ 10 files changed, 787 insertions(+), 139 deletions(-) create mode 100644 project/toolchain.py (limited to 'project') diff --git a/project/boost/build.py b/project/boost/build.py index a9fe8da..3a073f3 100644 --- a/project/boost/build.py +++ b/project/boost/build.py @@ -5,8 +5,8 @@ R'''Build Boost. -This script builds the Boost libraries. It's main utility is setting the -correct --stagedir parameter value to avoid name clashes. +This script builds the Boost libraries. It main utility is setting the correct +--stagedir parameter value to avoid name clashes. Usage example: @@ -46,6 +46,7 @@ import sys import tempfile from project.boost.directory import BoostDir +from project.toolchain import ToolchainType from project.boost.toolchain import Toolchain from project.configuration import Configuration from project.linkage import Linkage @@ -60,14 +61,14 @@ DEFAULT_CONFIGURATIONS = (Configuration.DEBUG, Configuration.RELEASE,) # binaries from a CI, etc. and run them everywhere): DEFAULT_LINK = (Linkage.STATIC,) DEFAULT_RUNTIME_LINK = Linkage.STATIC -# Shut compilers up: -COMMON_B2_ARGS = ['-d0'] +B2_QUIET = ['-d0'] +B2_VERBOSE = ['-d2', '--debug-configuration'] class BuildParameters: def __init__(self, boost_dir, build_dir=None, platforms=None, configurations=None, link=None, runtime_link=None, - mingw=False, b2_args=None): + toolset=None, verbose=False, b2_args=None): boost_dir = normalize_path(boost_dir) if build_dir is not None: @@ -76,10 +77,12 @@ class BuildParameters: configurations = configurations or DEFAULT_CONFIGURATIONS link = link or DEFAULT_LINK runtime_link = runtime_link or DEFAULT_RUNTIME_LINK + toolset = toolset or ToolchainType.AUTO + verbosity = B2_VERBOSE if verbose else B2_QUIET if b2_args: - b2_args = COMMON_B2_ARGS + b2_args + b2_args = verbosity + b2_args else: - b2_args = COMMON_B2_ARGS + b2_args = verbosity self.boost_dir = boost_dir self.build_dir = build_dir @@ -88,17 +91,20 @@ class BuildParameters: self.configurations = configurations self.link = link self.runtime_link = runtime_link - self.mingw = mingw + self.toolset = toolset self.b2_args = b2_args @staticmethod def from_args(args): return BuildParameters(**vars(args)) + def get_bootstrap_args(self): + return self.toolset.get_bootstrap_args() + def enum_b2_args(self): with self._create_build_dir() as build_dir: for platform in self.platforms: - with Toolchain.detect(platform, mingw=self.mingw) as toolchain: + with Toolchain.detect(self.toolset, platform) as toolchain: for configuration in self.configurations: for link, runtime_link in self._linkage_options(): yield self._build_params(build_dir, toolchain, @@ -138,9 +144,9 @@ class BuildParameters: params.append(self._stagedir(toolchain, configuration)) params.append('--layout=system') params += toolchain.get_b2_args() + params.append(self._variant(configuration)) params.append(self._link(link)) params.append(self._runtime_link(runtime_link)) - params.append(self._variant(configuration)) params += self.b2_args return params @@ -204,8 +210,10 @@ def _parse_args(argv=None): type=Linkage.parse, default=DEFAULT_RUNTIME_LINK, help=f'how the libraries link to the runtime ({linkage_options})') - parser.add_argument('--mingw', action='store_true', - help='build using MinGW-w64') + toolset_options = '/'.join(map(str, ToolchainType.all())) + parser.add_argument('--toolset', metavar='TOOLSET', + type=ToolchainType.parse, default=ToolchainType.AUTO, + help=f'toolset to use ({toolset_options})') parser.add_argument('--build', metavar='DIR', dest='build_dir', type=normalize_path, @@ -214,6 +222,8 @@ def _parse_args(argv=None): type=normalize_path, help='root Boost directory') + parser.add_argument('-v', '--verbose', action='store_true', + help='verbose b2 invocation (quiet by default)') parser.add_argument('b2_args', metavar='B2_ARG', nargs='*', default=[], help='additional b2 arguments, to be passed verbatim') diff --git a/project/boost/directory.py b/project/boost/directory.py index 17448b6..9b35194 100644 --- a/project/boost/directory.py +++ b/project/boost/directory.py @@ -6,6 +6,7 @@ import logging import os.path +from project.boost.toolchain import BootstrapToolchain from project.utils import cd, run from project.os import on_windows @@ -21,18 +22,19 @@ class BoostDir: def build(self, params): with self._go(): - self._bootstrap_if_required() + self._bootstrap_if_required(params) self._b2(params) - def _bootstrap_if_required(self): + def _bootstrap_if_required(self, params): if os.path.isfile(self._b2_path()): logging.info('Not going to bootstrap, b2 is already there') return - self.bootstrap() + self.bootstrap(params) - def bootstrap(self): + def bootstrap(self, params): with self._go(): - run(self._bootstrap_path()) + toolchain = BootstrapToolchain.detect(params.toolset) + run([self._bootstrap_path()] + self._bootstrap_args(toolchain)) def _b2(self, params): for b2_params in params.enum_b2_args(): @@ -49,6 +51,12 @@ class BoostDir: ext = '.bat' return f'bootstrap{ext}' + @staticmethod + def _bootstrap_args(toolchain): + if on_windows(): + return toolchain.get_bootstrap_bat_args() + return toolchain.get_bootstrap_sh_args() + @staticmethod def _b2_path(): return os.path.join('.', BoostDir._b2_name()) diff --git a/project/boost/download.py b/project/boost/download.py index c3451a8..ca113a6 100644 --- a/project/boost/download.py +++ b/project/boost/download.py @@ -5,8 +5,8 @@ R'''Download & bootstrap Boost. -This script downloads and bootstraps a Boost distribution. It's main utility -is that it's supposed to be cross-platform. +This script downloads and unpacks a Boost distribution archive. Its main +utility is that it's supposed to be cross-platform. Usage examples: @@ -97,7 +97,6 @@ def download(params): with _download_if_necessary(params.version, params.storage) as path: archive = Archive(params.version, path) boost_dir = archive.unpack(params.unpack_dir) - boost_dir.bootstrap() params.rename_if_necessary(boost_dir) diff --git a/project/boost/toolchain.py b/project/boost/toolchain.py index a77c073..cce06a4 100644 --- a/project/boost/toolchain.py +++ b/project/boost/toolchain.py @@ -3,78 +3,455 @@ # For details, see https://github.com/egor-tensin/cmake-common. # Distributed under the MIT License. -R'''Compiler detection. - -It is assumed that Boost.Build is good enough to detect both GCC on Linux and -MSVC on Windows. From that point on, it's just a matter of setting the correct -address-model= value. - -But I also frequently use MinGW-w64, and the most convinient way to use it that -I know is making a "user config" and passing it to b2 using the --user-config -parameter. -''' +# Hate speech +# ----------- +# +# Is there a person who doesn't hate Boost.Build? I'm not sure, I'm definitely +# _not_ one of these people. Maybe it's the lack of adoption (meaning that +# learning it is useless outside of Boost), maybe it's the incomprehensible +# syntax. Maybe it's the absolutely insane compiler-specific configuration +# files (tools/build/src/tools/*.jam), which are impossible to figure out. +# Maybe it's the fact that the implementation switched from C to C++ while some +# half-baked Python implementation has been there since at least 2015 (see the +# marvelous memo "Status: mostly ported." at the top of tools/build/src/build_system.py). +# +# What I hate the most though is how its various subtle, implicit and invisible +# decision-making heuristics changed thoughout the release history of Boost. +# You have a config and a compiler that will happily build version 1.65.0? +# Great! Want to use the same config and the same compiler to build version +# 1.72.0? Well, too fucking bad, it doesn't work anymore. This I really do +# hate the most. +# +# Three kinds of toolsets +# ----------------------- +# +# b2 accepts the toolset= parameter. What about building b2 itself though? +# Well, this is what the bootstrap.{sh,bat} scripts do. They also accept a +# toolset argument, but it is _completely_ different to that of b2. That's +# sort of OK, since e.g. cross-compiling b2 is something we rarely want to do +# (and hence there must typically be a native toolset available). +# +# bootstrap.sh and bootstrap.bat are completely different (of course!), and +# accept different arguments for their toolset parameters. +# +# Config file insanity +# -------------------- +# +# Say, we're building Boost on Windows using the GCC from a MinGW-w64 +# distribution. We can pass toolset=gcc and all the required flags on the +# command line no problem. What if we want to make a user configuration file +# so that 1) the command line is less polluted, and 2) it can possibly be +# shared? Well, if we put +# +# using gcc : : : value... ; +# +# there, Boost 1.65.0 will happily build everything, while Boost 1.72.0 will +# complain about "duplicate initialization of gcc". This is because when we +# ran `bootstrap.bat gcc` earlier, it wrote `using gcc ;` in project-config.jam. +# And while Boost 1.65.0 detects that toolset=gcc means we're going to use the +# MinGW GCC, and magically turns toolset=gcc to toolset=gcc-mingw, Boost 1.72.0 +# does no such thing, and chokes on the "duplicate" GCC declaration. +# +# We also cannot put +# +# using gcc : custom : : ; +# +# without the executable path, since Boost insists that `g++ -dumpversion` must +# equal to "custom" (which makes total sense, lol). So we have to force it, +# and do provide the path. +# +# Windows & Clang +# --------------- +# +# Building Boost using Clang on Windows is a sad story. As of 2020, there're +# three main ways to install the native Clang toolchain on Windows: +# +# * download the installer from llvm.org (`choco install llvm` does this) +# a.k.a. the upstream, +# * install it as part of a MSYS2 installation (`pacman -S mingw-w64-x86_64-clang`), +# * install as part of a Visual Studio installation. +# +# Using the latter method, you can switch a project to use the LLVM toolset +# using Visual Studio, but that's stupid. The former two, on the other hand, +# give us the the required clang/clang++/clang-cl executables, so everything +# seems to be fine. +# +# Except it's not fine. Let's start with the fact that prior to 1.66.0, +# toolset=clang is completely broken on Windows. It's just an alias for +# clang-linux, and it's hardcoded to require the ar & ranlib executables to +# create static libraries. Which is fine on Linux, since, and I'm quoting the +# source, "ar is always available". But it's not fine on Windows, since +# ar/ranlib are not, in fact, available there by default. Sure, you can +# install some kind of MinGW toolchain, and it might even work, but what the +# hell, honestly? +# +# Luckily, both the upstream distribution and the MSYS2 mingw-w64-x86_64-llvm +# package come with the llvm-ar and llvm-ranlib utilities. So we can put +# something like this in the config: +# +# using clang : custom : clang++.exe : llvm-ar llvm-ranlib.exe ; +# +# and later call +# +# b2 toolset=clang-custom --user-config=path/to/config.jam ... +# +# But, as I mentioned, prior to 1.66.0, toolset=clang is _hardcoded_ to use ar +# & ranlib, these exact utility names. So either get them as part of some +# MinGW distribution or build Boost using another toolset. +# +# Now, it's all fine, but building stuff on Windows adds another thing into the +# equation: debug runtimes. When you build Boost using MSVC, for example, it +# picks one of the appropriate /MT[d] or /MD[d] flags to build the Boost +# libraries with. Emulating these flags with toolset=clang is complicated and +# inconvenient. Luckily, there's the clang-cl.exe executable, which aims to +# provide command line interface compatible with that of cl.exe. +# +# Boost.Build even supports toolset=clang-win, which should use clang-cl.exe. +# But alas, it's completely broken prior to 1.69.0. It just doesn't work at +# all. So, if you want to build w/ clang-cl.exe, either use Boost 1.69.0 or +# later, or build using another toolset. +# +# Cygwin & Clang +# -------------- +# +# Now, a few words about Clang on Cygwin. When building 1.65.0, I encountered +# the following error: +# +# /usr/include/w32api/synchapi.h:127:26: error: conflicting types for 'Sleep' +# WINBASEAPI VOID WINAPI Sleep (DWORD dwMilliseconds); +# ^ +# ./boost/smart_ptr/detail/yield_k.hpp:64:29: note: previous declaration is here +# extern "C" void __stdcall Sleep( unsigned long ms ); +# ^ +# +# GCC doesn't emit an error here because /usr/include is in a pre-configured +# "system" include directories list, and the declaration there take precedence, +# I guess? The root of the problem BTW is that sizeof(unsigned long) is +# +# * 4 for MSVC and MinGW-born GCCs, +# * 8 for Clang (and, strangely, Cygwin GCC; why don't we get runtime +# errors?). +# +# The fix is to add `define=BOOST_USE_WINDOWS_H`. I don't even know what's the +# point of not having it as a default. import abc from contextlib import contextmanager import logging +import os.path +import shutil import project.mingw +import project.os +from project.toolchain import ToolchainType from project.utils import temp_file +class BootstrapToolchain(abc.ABC): + @abc.abstractmethod + def get_bootstrap_bat_args(self): + pass + + @abc.abstractmethod + def get_bootstrap_sh_args(self): + pass + + @staticmethod + def detect(hint): + if hint is ToolchainType.AUTO: + return BootstrapAuto() + if hint is ToolchainType.MSVC: + return BootstrapMSVC() + if hint is ToolchainType.GCC: + return BootstrapGCC() + if hint is ToolchainType.MINGW: + return BootstrapMinGW() + if hint is ToolchainType.CLANG: + return BootstrapClang() + if hint is ToolchainType.CLANG_CL: + return BootstrapClangCL() + raise NotImplementedError(f'unrecognized toolset: {hint}') + + +class BootstrapAuto(BootstrapToolchain): + # Let Boost.Build do the detection. Most commonly it means GCC on + # Linux-likes and MSVC on Windows. + + def get_bootstrap_bat_args(self): + return [] + + def get_bootstrap_sh_args(self): + return [] + + +class BootstrapMSVC(BootstrapAuto): + # bootstrap.bat picks up MSVC by default. + pass + + +class BootstrapGCC(BootstrapToolchain): + def get_bootstrap_bat_args(self): + return ['gcc'] + + def get_bootstrap_sh_args(self): + return ['--with-toolset=gcc'] + + +def _gcc_or_auto(): + if shutil.which('gcc') is not None: + return ['gcc'] + return [] + + +class BootstrapMinGW(BootstrapToolchain): + def get_bootstrap_bat_args(self): + # On Windows, prefer GCC if it's available. + return _gcc_or_auto() + + def get_bootstrap_sh_args(self): + return [] + + +class BootstrapClang(BootstrapToolchain): + def get_bootstrap_bat_args(self): + # As of 1.74.0, bootstrap.bat isn't really aware of Clang, so try GCC, + # then auto-detect. + return _gcc_or_auto() + + def get_bootstrap_sh_args(self): + # bootstrap.sh, on the other hand, is very much aware of Clang, and + # it can build b2 using this compiler. + return ['--with-toolset=clang'] + + +class BootstrapClangCL(BootstrapClang): + # There's no point in building b2 using clang-cl; clang though, presumably + # installed alongside clang-cl, should still be used if possible. + pass + + class Toolchain(abc.ABC): def __init__(self, platform): self.platform = platform - @abc.abstractmethod def get_b2_args(self): - pass + return [ + # Always pass the address-model explicitly. + f'address-model={self.platform.get_address_model()}' + ] @staticmethod @contextmanager - def detect(platform, mingw=False): - if mingw: + def detect(hint, platform): + if hint is ToolchainType.AUTO: + yield Auto(platform) + elif hint is ToolchainType.MSVC: + yield MSVC(platform) + elif hint is ToolchainType.GCC: + with GCC.setup(platform) as toolchain: + yield toolchain + elif hint is ToolchainType.MINGW: with MinGW.setup(platform) as toolchain: yield toolchain + elif hint is ToolchainType.CLANG: + with Clang.setup(platform) as toolchain: + yield toolchain + elif hint is ToolchainType.CLANG_CL: + yield ClangCL(platform) else: - yield Native(platform) + raise NotImplementedError(f'unrecognized toolset: {hint}') - @staticmethod - def _format_user_config(tag, compiler, **kwargs): - features = (f'<{k}>{v}' for k, v in kwargs.items()) - features = ' '.join(features) - return f'using gcc : {tag} : {compiler} : {features} ;' +class Auto(Toolchain): + # Let Boost.Build do the detection. Most commonly it means GCC on + # Linux-likes and MSVC on Windows. + pass -class Native(Toolchain): + +class MSVC(Auto): def get_b2_args(self): - return [f'address-model={self.platform.get_address_model()}'] + return super().get_b2_args() + [ + 'toolset=msvc', + ] + +def _full_exe_name(exe): + if project.os.on_linux(): + # There's no PATHEXT on Linux. + return exe + # b2 on Windows/Cygwin doesn't like it when the executable name doesn't + # include the extension. + dir_path = os.path.dirname(exe) or None + path = shutil.which(exe, path=dir_path) + if not path: + raise RuntimeError(f"executable '{exe}' could not be found") + if project.os.on_cygwin(): + # On Cygwin, shutil.which('gcc') == '/usr/bin/gcc' and shutil.which('gcc.exe') + # == '/usr/bin/gcc.exe'; we want the latter version. shutil.which('clang++') + # == '/usr/bin/clang++' is fine though, since it _is_ the complete path + # (clang++ is a symlink). + if os.path.exists(path) and os.path.exists(path + '.exe'): + path += '.exe' + if dir_path: + # If it was found in a specific directory, include the directory in the + # result. shutil.which returns the executable name prefixed with the + # path argument. + return path + # If it was found in PATH, just return the basename (which includes the + # extension). + return os.path.basename(path) -class MinGW(Toolchain): - TAG = 'custom' - def __init__(self, platform, config_path): +class BoostBuildToolset: + CUSTOM = 'custom' + + def __init__(self, compiler, path, options): + if not compiler: + raise RuntimeError('compiler type is required (like gcc, clang, etc.)') + self.compiler = compiler + self.version = BoostBuildToolset.CUSTOM + path = path or '' + path = path and _full_exe_name(path) + self.path = path + options = options or [] + self.options = options + + @property + def toolset_id(self): + if self.version: + return f'{self.compiler}-{self.version}' + return self.compiler + + @property + def b2_arg(self): + return f'toolset={self.toolset_id}' + + def _format_using_options(self): + return ''.join(f'\n <{name}>{val}' for name, val in self.options) + + def format_using(self): + version = self.version and f'{self.version} ' + path = self.path and f'{self.path} ' + return f'''using {self.compiler} : {version}: {path}:{self._format_using_options()} +;''' + + +class ConfigFile(Toolchain): + def __init__(self, platform, config_path, toolset): super().__init__(platform) self.config_path = config_path - - def get_b2_args(self): - return [f'--user-config={self.config_path}', f'toolset=gcc-{MinGW.TAG}'] + self.toolset = toolset @staticmethod - def _format_mingw_user_config(platform): - compiler = project.mingw.get_gxx(platform) - features = { - 'target-os': 'windows', - 'address-model': platform.get_address_model(), - } - return Toolchain._format_user_config(MinGW.TAG, compiler, **features) + @abc.abstractmethod + def get_toolset(platform): + pass @staticmethod + @abc.abstractmethod + def format_config(toolset): + pass + + @classmethod @contextmanager - def setup(platform): - config = MinGW._format_mingw_user_config(platform) + def setup(cls, platform): + toolset = cls.get_toolset(platform) + config = cls.format_config(toolset) logging.info('Using user config:\n%s', config) - tmp = temp_file(config, mode='w', prefix='mingw_w64_', suffix='.jam') + tmp = temp_file(config, mode='w', prefix='user_config_', suffix='.jam') with tmp as path: - yield MinGW(platform, path) + yield cls(platform, path, toolset) + + def get_b2_args(self): + # All the required options and the toolset definition should be in the + # user configuration file. + return super().get_b2_args() + [ + f'--user-config={self.config_path}', + self.toolset.b2_arg, + ] + + +class GCC(ConfigFile): + # Force GCC. We don't care whether it's a native Linux GCC or a + # MinGW-flavoured GCC on Windows. + COMPILER = 'gcc' + + @staticmethod + def get_options(): + return [ + # TODO: this is a petty attempt to get rid of build warnings in + # older Boost versions. Revise and expand this list or remove it? + # warning: 'template class std::auto_ptr' is deprecated + ('cxxflags', '-Wno-deprecated-declarations'), + # warning: unnecessary parentheses in declaration of 'assert_arg' + ('cxxflags', '-Wno-parentheses'), + ] + + @staticmethod + def get_toolset(platform): + return BoostBuildToolset(GCC.COMPILER, 'g++', GCC.get_options()) + + @staticmethod + def format_config(toolset): + return toolset.format_using() + + +class MinGW(GCC): + # It's important that Boost.Build is actually smart enough to detect the + # GCC prefix (like "x86_64-w64-mingw32" and prepend it to other tools like + # "ar"). + + @staticmethod + def get_toolset(platform): + path = project.mingw.get_gxx(platform) + return BoostBuildToolset(MinGW.COMPILER, path, MinGW.get_options()) + + +class Clang(ConfigFile): + COMPILER = 'clang' + + @staticmethod + def get_toolset(platform): + options = [ + ('cxxflags', '-DBOOST_USE_WINDOWS_H'), + # TODO: this is a petty attempt to get rid of build warnings in + # older Boost versions. Revise and expand this list or remove it? + # warning: unused typedef 'boost_concept_check464' [-Wunused-local-typedef] + ('cxxflags', '-Wno-unused-local-typedef'), + # error: constant expression evaluates to -105 which cannot be narrowed to type 'boost::re_detail::cpp_regex_traits_implementation::char_class_type' (aka 'unsigned int') + ('cxxflags', '-Wno-c++11-narrowing'), + ] + GCC.get_options() + if project.os.on_windows(): + # Prefer LLVM binutils: + if shutil.which('llvm-ar') is not None: + options.append(('archiver', 'llvm-ar')) + if shutil.which('llvm-ranlib') is not None: + options.append(('ranlib', 'llvm-ranlib')) + return BoostBuildToolset(Clang.COMPILER, 'clang++', options) + + @staticmethod + def format_config(toolset): + # To make clang.exe/clang++.exe work on Windows, some tweaks are + # required. I borrowed them from CMake's Windows-Clang.cmake [1]. + # Adding them globally to Boost.Build options is described in [2]. + # + # [1]: https://github.com/Kitware/CMake/blob/v3.18.4/Modules/Platform/Windows-Clang.cmake + # [2]: https://stackoverflow.com/questions/2715106/how-to-create-a-new-variant-in-bjam + return f'''project : requirements + windows:_MT + windows,debug:_DEBUG + windows,static,debug:"-Xclang -flto-visibility-public-std -Xclang --dependent-lib=libcmtd" + windows,static,release:"-Xclang -flto-visibility-public-std -Xclang --dependent-lib=libcmt" + windows,shared,debug:"-D_DLL -Xclang --dependent-lib=msvcrtd" + windows,shared,release:"-D_DLL -Xclang --dependent-lib=msvcrt" +; +{toolset.format_using()} +''' + + +class ClangCL(Toolchain): + def get_b2_args(self): + return super().get_b2_args() + [ + f'toolset=clang-win', + 'define=BOOST_USE_WINDOWS_H', + ] diff --git a/project/ci/boost.py b/project/ci/boost.py index c17c9db..3da5c9b 100644 --- a/project/ci/boost.py +++ b/project/ci/boost.py @@ -7,8 +7,9 @@ import argparse import logging import sys -from project.boost.download import DownloadParameters, download from project.boost.build import BuildParameters, build +from project.boost.download import DownloadParameters, download +from project.boost.toolchain import ToolchainType from project.linkage import Linkage @@ -27,8 +28,10 @@ def _parse_args(dirs, argv=None): parser.add_argument('--runtime-link', metavar='LINKAGE', type=Linkage.parse, help='how the libraries link to the runtime') - parser.add_argument('--mingw', action='store_true', - help='build using MinGW-w64') + parser.add_argument('--toolset', metavar='TOOLSET', + type=ToolchainType.parse, + help='toolset to use') + parser.add_argument('b2_args', metavar='B2_ARG', nargs='*', default=[], help='additional b2 arguments, to be passed verbatim') @@ -50,6 +53,6 @@ def build_ci(dirs, argv=None): configurations=(dirs.get_configuration(),), link=args.link, runtime_link=args.runtime_link, - mingw=args.mingw, + toolset=args.toolset, b2_args=args.b2_args) build(params) diff --git a/project/ci/cmake.py b/project/ci/cmake.py index 4a58749..df2b55a 100644 --- a/project/ci/cmake.py +++ b/project/ci/cmake.py @@ -8,6 +8,7 @@ import logging import sys from project.cmake.build import BuildParameters, build +from project.toolchain import ToolchainType def _parse_args(dirs, argv=None): @@ -23,8 +24,9 @@ def _parse_args(dirs, argv=None): help='install directory') parser.add_argument('--boost', metavar='DIR', dest='boost_dir', help='set Boost directory path') - parser.add_argument('--mingw', action='store_true', - help='build using MinGW-w64') + parser.add_argument('--toolset', metavar='TOOLSET', + type=ToolchainType.parse, + help=f'toolset to use') parser.add_argument('cmake_args', nargs='*', metavar='CMAKE_ARG', default=[], help='additional CMake arguments, to be passed verbatim') return parser.parse_args(argv) @@ -39,6 +41,6 @@ def build_ci(dirs, argv=None): platform=dirs.get_platform(), configuration=dirs.get_configuration(), boost_dir=args.boost_dir or dirs.get_boost_dir(), - mingw=args.mingw, + toolset=args.toolset, cmake_args=dirs.get_cmake_args() + args.cmake_args) build(params) diff --git a/project/cmake/build.py b/project/cmake/build.py index e683eff..6bc7772 100644 --- a/project/cmake/build.py +++ b/project/cmake/build.py @@ -30,10 +30,13 @@ import tempfile from project.cmake.toolchain import Toolchain from project.configuration import Configuration from project.platform import Platform +from project.toolchain import ToolchainType from project.utils import normalize_path, run, setup_logging +DEFAULT_PLATFORM = Platform.native() DEFAULT_CONFIGURATION = Configuration.DEBUG +DEFAULT_TOOLSET = ToolchainType.AUTO def run_cmake(cmake_args): @@ -41,15 +44,18 @@ def run_cmake(cmake_args): class GenerationPhase: - def __init__(self, src_dir, build_dir, install_dir=None, - platform=None, configuration=DEFAULT_CONFIGURATION, - boost_dir=None, mingw=False, cmake_args=None): + def __init__(self, src_dir, build_dir, install_dir=None, platform=None, + configuration=None, boost_dir=None, toolset=None, + cmake_args=None): src_dir = normalize_path(src_dir) build_dir = normalize_path(build_dir) if install_dir is not None: install_dir = normalize_path(install_dir) + platform = platform or DEFAULT_PLATFORM + configuration = configuration or DEFAULT_CONFIGURATION if boost_dir is not None: boost_dir = normalize_path(boost_dir) + toolset = toolset or DEFAULT_TOOLSET cmake_args = cmake_args or [] self.src_dir = src_dir @@ -58,7 +64,7 @@ class GenerationPhase: self.platform = platform self.configuration = configuration self.boost_dir = boost_dir - self.mingw = mingw + self.toolset = toolset self.cmake_args = cmake_args def _cmake_args(self, toolchain): @@ -87,44 +93,47 @@ class GenerationPhase: platform = Platform.native() return os.path.join(boost_dir, 'stage', str(platform), str(configuration), 'lib') - def run(self): - with Toolchain.detect(self.platform, self.build_dir, mingw=self.mingw) as toolchain: - run_cmake(self._cmake_args(toolchain)) + def run(self, toolchain): + run_cmake(self._cmake_args(toolchain)) class BuildPhase: - def __init__(self, build_dir, install_dir=None, - configuration=DEFAULT_CONFIGURATION): + def __init__(self, build_dir, install_dir=None, configuration=None): build_dir = normalize_path(build_dir) + configuration = configuration or DEFAULT_CONFIGURATION self.build_dir = build_dir self.install_dir = install_dir self.configuration = configuration - def _cmake_args(self): + def _cmake_args(self, toolchain): result = ['--build', self.build_dir] result += ['--config', str(self.configuration)] if self.install_dir is not None: result += ['--target', 'install'] + result += ['--'] + toolchain.get_build_args() return result - def run(self): - run_cmake(self._cmake_args()) + def run(self, toolchain): + run_cmake(self._cmake_args(toolchain)) class BuildParameters: def __init__(self, src_dir, build_dir=None, install_dir=None, - platform=None, configuration=DEFAULT_CONFIGURATION, - boost_dir=None, mingw=False, cmake_args=None): + platform=None, configuration=None, boost_dir=None, + toolset=None, cmake_args=None): src_dir = normalize_path(src_dir) if build_dir is not None: build_dir = normalize_path(build_dir) if install_dir is not None: install_dir = normalize_path(install_dir) + platform = platform or DEFAULT_PLATFORM + configuration = configuration or DEFAULT_CONFIGURATION if boost_dir is not None: boost_dir = normalize_path(boost_dir) + toolset = toolset or DEFAULT_TOOLSET cmake_args = cmake_args or [] self.src_dir = src_dir @@ -133,7 +142,7 @@ class BuildParameters: self.platform = platform self.configuration = configuration self.boost_dir = boost_dir - self.mingw = mingw + self.toolset = toolset self.cmake_args = cmake_args @staticmethod @@ -158,17 +167,19 @@ class BuildParameters: def build(params): with params.create_build_dir() as build_dir: + toolchain = Toolchain.detect(params.toolset, params.platform, build_dir) + gen_phase = GenerationPhase(params.src_dir, build_dir, install_dir=params.install_dir, platform=params.platform, configuration=params.configuration, boost_dir=params.boost_dir, - mingw=params.mingw, + toolset=params.toolset, cmake_args=params.cmake_args) - gen_phase.run() + gen_phase.run(toolchain) build_phase = BuildPhase(build_dir, install_dir=params.install_dir, configuration=params.configuration) - build_phase.run() + build_phase.run(toolchain) def _parse_args(argv=None): @@ -201,8 +212,10 @@ def _parse_args(argv=None): type=normalize_path, help='set Boost directory path') - parser.add_argument('--mingw', action='store_true', - help='build using MinGW-w64') + toolset_options = '/'.join(map(str, ToolchainType.all())) + parser.add_argument('--toolset', metavar='TOOLSET', + type=ToolchainType.parse, default=ToolchainType.AUTO, + help=f'toolset to use ({toolset_options})') parser.add_argument('src_dir', metavar='DIR', type=normalize_path, diff --git a/project/cmake/toolchain.py b/project/cmake/toolchain.py index 073cd1b..7c96628 100644 --- a/project/cmake/toolchain.py +++ b/project/cmake/toolchain.py @@ -3,13 +3,127 @@ # For details, see https://github.com/egor-tensin/cmake-common. # Distributed under the MIT License. +# Default generator +# ----------------- +# +# As of CMake 3.18, the default generator (unless set explicitly) is: +# * the newest Visual Studio or "NMake Makefiles" on Windows, +# * "Unix Makefiles" otherwise. +# This is regardless of whether any executables like gcc, cl or make are +# available [1]. +# +# Makefile generators +# ------------------- +# +# CMake has a number of "... Makefiles" generators. "Unix Makefiles" uses +# gmake/make/smake, whichever is found first, and cc/c++ for compiler +# detection [2]. "MinGW Makefiles" looks for mingw32-make.exe in a number of +# well-known locations, uses gcc/g++ directly, and is aware of windres [3]. In +# addition, "Unix Makefiles" uses /bin/sh as the SHELL value in the Makefile, +# while the MinGW version uses cmd.exe. I don't think it matters on Windows +# though, since the non-existent /bin/sh is ignored anyway [4]. "NMake +# Makefiles" is similar, except it defaults to using cl [5]. +# +# It's important to _not_ use the -A parameter with any of the Makefile +# generators - it's an error. This goes for "NMake Makefiles" also. "NMake +# Makefiles" doesn't attempt to search for installed Visual Studio compilers, +# you need to use it from one of the Visual Studio-provided shells. +# +# Visual Studio generators +# ------------------------ +# +# These are special. They ignore the CMAKE__COMPILER parameters and use +# cl by default [9]. They support specifying the toolset to use via the -T +# parameter (the "Platform Toolset" value in the project's properties) since +# 3.18 [10]. The toolset list varies between Visual Studio versions, and I'm +# too lazy to learn exactly which version supports which toolsets. +# +# cmake --build uses msbuild with Visual Studio generators. You can pass the +# path to a different cl.exe by doing something like +# +# msbuild ... /p:CLToolExe=another-cl.exe /p:CLToolPath=C:\parent\dir +# +# It's important that the generators for Visual Studio 2017 or older use Win32 +# Win32 as the default platform [12]. Because of that, we need to pass the -A +# parameter. +# +# mingw32-make vs make +# -------------------- +# +# No idea what the actual differences are. The explanation in the FAQ [6] +# about how GNU make "is lacking in some functionality and has modified +# functionality due to the lack of POSIX on Win32" isn't terribly helpful. +# +# It's important that you can install either on Windows (`choco install make` +# for GNU make and `choco install mingw` to install a MinGW-w64 distribution +# with mingw32-make.exe included). Personally, I don't see any difference +# between using either make.exe or mingw32-make.exe w/ CMake on Windows. But, +# since MinGW-w64 distributions do include mingw32-make.exe and not make.exe, +# we'll try to detect that. +# +# Cross-compilation +# ----------------- +# +# If you want to e.g. build x86 binary on x64 and vice versa, the easiest way +# seems to be to make a CMake "toolchain file", which initializes the proper +# compiler flags (like -m64/-m32, etc.). Such file could look like this: +# +# set(CMAKE_C_COMPILER gcc) +# set(CMAKE_C_FLAGS -m32) +# set(CMAKE_CXX_COMPILER g++) +# set(CMAKE_CXX_FLAGS -m32) +# +# You can then pass the path to it using the CMAKE_TOOLCHAIN_FILE parameter. +# +# If you use the Visual Studio generators, just use the -A parameter, like "-A +# Win32". +# +# As a side note, if you want to cross-compile between x86 and x64 using GCC on +# Ubuntu, you need to install the gcc-multilib package. +# +# Windows & Clang +# --------------- +# +# Using Clang on Windows is no easy task, of course. Prior to 3.15, there was +# no support for building things using the clang++.exe executable, only +# clang-cl.exe was supported [7]. If you specified -DCMAKE_CXX_COMPILER=clang++, +# CMake would stil pass MSVC-style command line options to the compiler (like +# /MD, /nologo, etc.), which clang++ doesn't like [8]. +# +# So, in summary, you can only use clang++ since 3.15. clang-cl doesn't work +# with Visual Studio generators unless you specify the proper toolset using the +# -T parameter. You can set the ClToolExe property using msbuild, but while +# that might work in practice, clang-cl.exe needs to map some unsupported +# options for everything to work properly. For an example of how this is done, +# see the LLVM.Cpp.Common.* files at [11]. +# +# I recommend using Clang (either clang-cl or clang++ since 3.15) using the +# "NMake Makefiles" generator. +# +# References +# ---------- +# +# [1]: cmake::EvaluateDefaultGlobalGenerator +# https://github.com/Kitware/CMake/blob/v3.18.4/Source/cmake.cxx +# [2]: https://github.com/Kitware/CMake/blob/v3.18.4/Source/cmGlobalUnixMakefileGenerator3.cxx +# [3]: https://github.com/Kitware/CMake/blob/v3.18.4/Source/cmGlobalMinGWMakefileGenerator.cxx +# [4]: https://www.gnu.org/software/make/manual/html_node/Choosing-the-Shell.html +# [5]: https://github.com/Kitware/CMake/blob/v3.18.4/Source/cmGlobalNMakeMakefileGenerator.cxx +# [6]: http://mingw.org/wiki/FAQ +# [7]: https://cmake.org/cmake/help/v3.15/release/3.15.html#compilers +# [8]: https://github.com/Kitware/CMake/blob/v3.14.7/Modules/Platform/Windows-Clang.cmake +# [9]: https://gitlab.kitware.com/cmake/cmake/-/issues/19174 +# [10]: https://cmake.org/cmake/help/v3.8/release/3.8.html +# [11]: https://github.com/llvm/llvm-project/tree/e408935bb5339e20035d84307c666fbdd15e99e0/llvm/tools/msbuild +# [12]: https://cmake.org/cmake/help/v3.18/generator/Visual%20Studio%2015%202017.html + import abc -from contextlib import contextmanager import os.path +import shutil import project.mingw -from project.platform import Platform from project.os import on_windows +from project.toolchain import ToolchainType class Toolchain(abc.ABC): @@ -17,89 +131,164 @@ class Toolchain(abc.ABC): def get_cmake_args(self): pass - @staticmethod - @contextmanager - def detect(platform, build_dir, mingw=False): - if mingw: - with MinGW.setup(platform, build_dir) as toolchain: - yield toolchain - return - - if on_windows(): - # MSVC is assumed. - if platform is None: - yield Native() - return - yield MSVC(platform) - return + @abc.abstractmethod + def get_build_args(self): + pass - with GCC.setup(platform, build_dir) as toolchain: - yield toolchain - return + @staticmethod + def detect(hint, platform, build_dir): + if hint is ToolchainType.AUTO: + if on_windows(): + # We need to specify the -A parameter. This might break if + # none of the Visual Studio generators are available, but the + # NMake one is, although I don't know how this can be possible + # normally. + hint = ToolchainType.MSVC + else: + # Same thing for the -m32/-m64 flags. + hint = ToolchainType.GCC + if hint is ToolchainType.MSVC: + return MSVC(platform) + if hint is ToolchainType.GCC: + return GCC.setup(platform, build_dir) + if hint is ToolchainType.MINGW: + return MinGW.setup(platform, build_dir) + if hint is ToolchainType.CLANG: + return Clang.setup(platform, build_dir) + if hint is ToolchainType.CLANG_CL: + return ClangCL.setup(platform, build_dir) + raise NotImplementedError(f'unrecognized toolset: {hint}') -class Native(Toolchain): +class Auto(Toolchain): def get_cmake_args(self): return [] + def get_build_args(self): + return [] -class MSVC(Toolchain): + +class MSVC(Auto): def __init__(self, platform): self.platform = platform def get_cmake_args(self): + # This doesn't actually specify the generator of course, but I don't + # want to implement VS detection logic. return ['-A', self.platform.get_cmake_arch()] + def get_build_args(self): + return ['/m'] + -class File(Toolchain): +class Makefile(Toolchain): def __init__(self, path): self.path = path @staticmethod - def _get_path(build_dir): + def _get_config_path(build_dir): return os.path.join(build_dir, 'custom_toolchain.cmake') + def _get_makefile_generator(self): + if on_windows(): + if shutil.which('mingw32-make'): + return 'MinGW Makefiles' + return 'Unix Makefiles' + # On Linux/Cygwin, make all the way: + return 'Unix Makefiles' + + @classmethod + def write_config(cls, build_dir, contents): + path = Makefile._get_config_path(build_dir) + with open(path, mode='w') as file: + file.write(contents) + return cls(path) + def get_cmake_args(self): - return ['-D', f'CMAKE_TOOLCHAIN_FILE={self.path}'] + return [ + '-D', f'CMAKE_TOOLCHAIN_FILE={self.path}', + # The Visual Studio generator is the default on Windows, override + # it: + '-G', self._get_makefile_generator(), + ] + + def get_build_args(self): + return [] -class GCC(File): +class GCC(Makefile): @staticmethod def _format(platform): return f''' -set(CMAKE_C_COMPILER gcc) -set(CMAKE_C_FLAGS -m{platform.get_address_model()}) -set(CMAKE_CXX_COMIPLER g++) -set(CMAKE_CXX_FLAGS -m{platform.get_address_model()}) +set(CMAKE_C_COMPILER gcc) +set(CMAKE_C_FLAGS -m{platform.get_address_model()}) +set(CMAKE_CXX_COMPILER g++) +set(CMAKE_CXX_FLAGS -m{platform.get_address_model()}) ''' @staticmethod - @contextmanager def setup(platform, build_dir): - if platform is None: - yield Native() - return - path = File._get_path(build_dir) - with open(path, mode='w') as file: - file.write(GCC._format(platform)) - yield GCC(path) + return GCC.write_config(build_dir, GCC._format(platform)) -class MinGW(File): +class MinGW(Makefile): @staticmethod def _format(platform): return f''' set(CMAKE_C_COMPILER {project.mingw.get_gcc(platform)}) set(CMAKE_CXX_COMPILER {project.mingw.get_gxx(platform)}) -set(CMAKE_RC_COMILER {project.mingw.get_windres(platform)}) +set(CMAKE_AR {project.mingw.get_ar(platform)}) +set(CMAKE_RANLIB {project.mingw.get_ranlib(platform)}) +set(CMAKE_RC_COMPILER {project.mingw.get_windres(platform)}) set(CMAKE_SYSTEM_NAME Windows) ''' @staticmethod - @contextmanager def setup(platform, build_dir): - platform = platform or Platform.native() - path = File._get_path(build_dir) - with open(path, mode='w') as file: - file.write(MinGW._format(platform)) - yield MinGW(path) + return MinGW.write_config(build_dir, MinGW._format(platform)) + + +class Clang(Makefile): + @staticmethod + def _format(platform): + return f''' +if(CMAKE_VERSION VERSION_LESS "3.15" AND WIN32) + set(CMAKE_C_COMPILER clang-cl) + set(CMAKE_CXX_COMPILER clang-cl) + set(CMAKE_C_FLAGS -m{platform.get_address_model()}) + set(CMAKE_CXX_FLAGS -m{platform.get_address_model()}) +else() + set(CMAKE_C_COMPILER clang) + set(CMAKE_CXX_COMPILER clang++) + set(CMAKE_C_FLAGS -m{platform.get_address_model()}) + set(CMAKE_CXX_FLAGS -m{platform.get_address_model()}) +endif() +''' + + def _get_makefile_generator(self): + if on_windows(): + # MinGW utilities like make might be unavailable, but NMake can + # very much be there. + if shutil.which('nmake'): + return 'NMake Makefiles' + return super()._get_makefile_generator() + + @staticmethod + def setup(platform, build_dir): + return Clang.write_config(build_dir, Clang._format(platform)) + + +class ClangCL(Clang): + @staticmethod + def _format(platform): + return f''' +set(CMAKE_C_COMPILER clang-cl) +set(CMAKE_CXX_COMPILER clang-cl) +set(CMAKE_C_FLAGS -m{platform.get_address_model()}) +set(CMAKE_CXX_FLAGS -m{platform.get_address_model()}) +set(CMAKE_SYSTEM_NAME Windows) +''' + + @staticmethod + def setup(platform, build_dir): + return ClangCL.write_config(build_dir, ClangCL._format(platform)) diff --git a/project/mingw.py b/project/mingw.py index 1e136cd..731cee9 100644 --- a/project/mingw.py +++ b/project/mingw.py @@ -3,8 +3,6 @@ # For details, see https://github.com/egor-tensin/cmake-common. # Distributed under the MIT License. -from project.os import on_windows_like - def _get_compiler_prefix(platform): target_arch = platform.get_address_model() @@ -17,11 +15,7 @@ def _get_compiler_prefix(platform): def _get(platform, what): prefix = _get_compiler_prefix(platform) - ext = '' - if on_windows_like(): - # Boost.Build wants the .exe extension at the end on Cygwin. - ext = '.exe' - path = f'{prefix}-w64-mingw32-{what}{ext}' + path = f'{prefix}-w64-mingw32-{what}' return path @@ -33,5 +27,13 @@ def get_gxx(platform): return _get(platform, 'g++') +def get_ar(platform): + return _get(platform, 'gcc-ar') + + +def get_ranlib(platform): + return _get(platform, 'gcc-ranlib') + + def get_windres(platform): return _get(platform, 'windres') diff --git a/project/toolchain.py b/project/toolchain.py new file mode 100644 index 0000000..d931c6b --- /dev/null +++ b/project/toolchain.py @@ -0,0 +1,45 @@ +# Copyright (c) 2020 Egor Tensin +# This file is part of the "cmake-common" project. +# For details, see https://github.com/egor-tensin/cmake-common. +# Distributed under the MIT License. + +'''Supported platform/build system/compiler combinations include, but are not +limited to: + +* Linux / make / Clang, +* Linux / make / GCC, +* Linux / make / MinGW-w64, +* Windows / make / Clang (clang.exe & clang++.exe), +* Windows / make / Clang (clang-cl.exe, Boost 1.69.0 or higher only), +* Windows / make / MinGW-w64, +* Windows / msbuild / MSVC, +* Cygwin / make / Clang, +* Cygwin / make / GCC, +* Cygwin / make / MinGW-w64. +''' + +import argparse +from enum import Enum + + +class ToolchainType(Enum): + AUTO = 'auto' # This most commonly means GCC on Linux and MSVC on Windows. + MSVC = 'msvc' # Force MSVC. + GCC = 'gcc' # Force GCC. + MINGW = 'mingw' # As in MinGW-w64; GCC with the PLATFORM-w64-mingw32 prefix. + CLANG = 'clang' + CLANG_CL = 'clang-cl' + + def __str__(self): + return self.value + + @staticmethod + def all(): + return tuple(ToolchainType) + + @staticmethod + def parse(s): + try: + return ToolchainType(s) + except ValueError: + raise argparse.ArgumentTypeError(f'invalid toolset: {s}') -- cgit v1.2.3