From dd2c5b58c4fe77d7ce35f3abb6e1bb399560a2db Mon Sep 17 00:00:00 2001
From: Egor Tensin <Egor.Tensin@gmail.com>
Date: Sun, 17 Jan 2021 13:54:57 +0300
Subject: GIANT CLUSTERFUCK OF A COMMIT
OK, this is epic. I was basically just trying to a) support Clang and
b) add more test coverage. _THREE MONTHS_ and a few hundred CI runs
later, this is what I came up with. I don't know how it ended up being
what it is, but here we go.
Some highlights of the changes:
1) CI builds has been moved to GitHub Actions,
2) the entire notion of a toolchain has been reworked; it now supports
Clang on all platforms.
* .github: this directory contains the GitHub Actions workflow
scripts/actions. In the process, I created like 6 external GitHub
actions, but it's still pretty massive. An upside is that it covers
much more platform/toolchain combinations _and_ check a lot of the
expected post-conditions. TODO: .ci/Makefile is obsolete now, as well
as .travis.yml and .appveyor.yml.
* common.cmake: added Clang support. In the process, a great deal has
been learned about how CMake works; in particular, static runtime
support has been reworked to be more robust.
* project: the entire notion of a "toolchain" has been reworked.
Instead of a measly --mingw parameter, there's now a separate --toolset
parameter, which allows you to choose between GCC, Clang, MSVC, etc.
Both Boost and CMake build scripts were enhanced greatly to support
Clang and other toolchains in a more robust way.
---
project/cmake/build.py | 53 +++++----
project/cmake/toolchain.py | 281 +++++++++++++++++++++++++++++++++++++--------
2 files changed, 268 insertions(+), 66 deletions(-)
(limited to 'project/cmake')
diff --git a/project/cmake/build.py b/project/cmake/build.py
index e683eff..6bc7772 100644
--- a/project/cmake/build.py
+++ b/project/cmake/build.py
@@ -30,10 +30,13 @@ import tempfile
from project.cmake.toolchain import Toolchain
from project.configuration import Configuration
from project.platform import Platform
+from project.toolchain import ToolchainType
from project.utils import normalize_path, run, setup_logging
+DEFAULT_PLATFORM = Platform.native()
DEFAULT_CONFIGURATION = Configuration.DEBUG
+DEFAULT_TOOLSET = ToolchainType.AUTO
def run_cmake(cmake_args):
@@ -41,15 +44,18 @@ def run_cmake(cmake_args):
class GenerationPhase:
- def __init__(self, src_dir, build_dir, install_dir=None,
- platform=None, configuration=DEFAULT_CONFIGURATION,
- boost_dir=None, mingw=False, cmake_args=None):
+ def __init__(self, src_dir, build_dir, install_dir=None, platform=None,
+ configuration=None, boost_dir=None, toolset=None,
+ cmake_args=None):
src_dir = normalize_path(src_dir)
build_dir = normalize_path(build_dir)
if install_dir is not None:
install_dir = normalize_path(install_dir)
+ platform = platform or DEFAULT_PLATFORM
+ configuration = configuration or DEFAULT_CONFIGURATION
if boost_dir is not None:
boost_dir = normalize_path(boost_dir)
+ toolset = toolset or DEFAULT_TOOLSET
cmake_args = cmake_args or []
self.src_dir = src_dir
@@ -58,7 +64,7 @@ class GenerationPhase:
self.platform = platform
self.configuration = configuration
self.boost_dir = boost_dir
- self.mingw = mingw
+ self.toolset = toolset
self.cmake_args = cmake_args
def _cmake_args(self, toolchain):
@@ -87,44 +93,47 @@ class GenerationPhase:
platform = Platform.native()
return os.path.join(boost_dir, 'stage', str(platform), str(configuration), 'lib')
- def run(self):
- with Toolchain.detect(self.platform, self.build_dir, mingw=self.mingw) as toolchain:
- run_cmake(self._cmake_args(toolchain))
+ def run(self, toolchain):
+ run_cmake(self._cmake_args(toolchain))
class BuildPhase:
- def __init__(self, build_dir, install_dir=None,
- configuration=DEFAULT_CONFIGURATION):
+ def __init__(self, build_dir, install_dir=None, configuration=None):
build_dir = normalize_path(build_dir)
+ configuration = configuration or DEFAULT_CONFIGURATION
self.build_dir = build_dir
self.install_dir = install_dir
self.configuration = configuration
- def _cmake_args(self):
+ def _cmake_args(self, toolchain):
result = ['--build', self.build_dir]
result += ['--config', str(self.configuration)]
if self.install_dir is not None:
result += ['--target', 'install']
+ result += ['--'] + toolchain.get_build_args()
return result
- def run(self):
- run_cmake(self._cmake_args())
+ def run(self, toolchain):
+ run_cmake(self._cmake_args(toolchain))
class BuildParameters:
def __init__(self, src_dir, build_dir=None, install_dir=None,
- platform=None, configuration=DEFAULT_CONFIGURATION,
- boost_dir=None, mingw=False, cmake_args=None):
+ platform=None, configuration=None, boost_dir=None,
+ toolset=None, cmake_args=None):
src_dir = normalize_path(src_dir)
if build_dir is not None:
build_dir = normalize_path(build_dir)
if install_dir is not None:
install_dir = normalize_path(install_dir)
+ platform = platform or DEFAULT_PLATFORM
+ configuration = configuration or DEFAULT_CONFIGURATION
if boost_dir is not None:
boost_dir = normalize_path(boost_dir)
+ toolset = toolset or DEFAULT_TOOLSET
cmake_args = cmake_args or []
self.src_dir = src_dir
@@ -133,7 +142,7 @@ class BuildParameters:
self.platform = platform
self.configuration = configuration
self.boost_dir = boost_dir
- self.mingw = mingw
+ self.toolset = toolset
self.cmake_args = cmake_args
@staticmethod
@@ -158,17 +167,19 @@ class BuildParameters:
def build(params):
with params.create_build_dir() as build_dir:
+ toolchain = Toolchain.detect(params.toolset, params.platform, build_dir)
+
gen_phase = GenerationPhase(params.src_dir, build_dir,
install_dir=params.install_dir,
platform=params.platform,
configuration=params.configuration,
boost_dir=params.boost_dir,
- mingw=params.mingw,
+ toolset=params.toolset,
cmake_args=params.cmake_args)
- gen_phase.run()
+ gen_phase.run(toolchain)
build_phase = BuildPhase(build_dir, install_dir=params.install_dir,
configuration=params.configuration)
- build_phase.run()
+ build_phase.run(toolchain)
def _parse_args(argv=None):
@@ -201,8 +212,10 @@ def _parse_args(argv=None):
type=normalize_path,
help='set Boost directory path')
- parser.add_argument('--mingw', action='store_true',
- help='build using MinGW-w64')
+ toolset_options = '/'.join(map(str, ToolchainType.all()))
+ parser.add_argument('--toolset', metavar='TOOLSET',
+ type=ToolchainType.parse, default=ToolchainType.AUTO,
+ help=f'toolset to use ({toolset_options})')
parser.add_argument('src_dir', metavar='DIR',
type=normalize_path,
diff --git a/project/cmake/toolchain.py b/project/cmake/toolchain.py
index 073cd1b..7c96628 100644
--- a/project/cmake/toolchain.py
+++ b/project/cmake/toolchain.py
@@ -3,13 +3,127 @@
# For details, see https://github.com/egor-tensin/cmake-common.
# Distributed under the MIT License.
+# Default generator
+# -----------------
+#
+# As of CMake 3.18, the default generator (unless set explicitly) is:
+# * the newest Visual Studio or "NMake Makefiles" on Windows,
+# * "Unix Makefiles" otherwise.
+# This is regardless of whether any executables like gcc, cl or make are
+# available [1].
+#
+# Makefile generators
+# -------------------
+#
+# CMake has a number of "... Makefiles" generators. "Unix Makefiles" uses
+# gmake/make/smake, whichever is found first, and cc/c++ for compiler
+# detection [2]. "MinGW Makefiles" looks for mingw32-make.exe in a number of
+# well-known locations, uses gcc/g++ directly, and is aware of windres [3]. In
+# addition, "Unix Makefiles" uses /bin/sh as the SHELL value in the Makefile,
+# while the MinGW version uses cmd.exe. I don't think it matters on Windows
+# though, since the non-existent /bin/sh is ignored anyway [4]. "NMake
+# Makefiles" is similar, except it defaults to using cl [5].
+#
+# It's important to _not_ use the -A parameter with any of the Makefile
+# generators - it's an error. This goes for "NMake Makefiles" also. "NMake
+# Makefiles" doesn't attempt to search for installed Visual Studio compilers,
+# you need to use it from one of the Visual Studio-provided shells.
+#
+# Visual Studio generators
+# ------------------------
+#
+# These are special. They ignore the CMAKE_<LANG>_COMPILER parameters and use
+# cl by default [9]. They support specifying the toolset to use via the -T
+# parameter (the "Platform Toolset" value in the project's properties) since
+# 3.18 [10]. The toolset list varies between Visual Studio versions, and I'm
+# too lazy to learn exactly which version supports which toolsets.
+#
+# cmake --build uses msbuild with Visual Studio generators. You can pass the
+# path to a different cl.exe by doing something like
+#
+# msbuild ... /p:CLToolExe=another-cl.exe /p:CLToolPath=C:\parent\dir
+#
+# It's important that the generators for Visual Studio 2017 or older use Win32
+# Win32 as the default platform [12]. Because of that, we need to pass the -A
+# parameter.
+#
+# mingw32-make vs make
+# --------------------
+#
+# No idea what the actual differences are. The explanation in the FAQ [6]
+# about how GNU make "is lacking in some functionality and has modified
+# functionality due to the lack of POSIX on Win32" isn't terribly helpful.
+#
+# It's important that you can install either on Windows (`choco install make`
+# for GNU make and `choco install mingw` to install a MinGW-w64 distribution
+# with mingw32-make.exe included). Personally, I don't see any difference
+# between using either make.exe or mingw32-make.exe w/ CMake on Windows. But,
+# since MinGW-w64 distributions do include mingw32-make.exe and not make.exe,
+# we'll try to detect that.
+#
+# Cross-compilation
+# -----------------
+#
+# If you want to e.g. build x86 binary on x64 and vice versa, the easiest way
+# seems to be to make a CMake "toolchain file", which initializes the proper
+# compiler flags (like -m64/-m32, etc.). Such file could look like this:
+#
+# set(CMAKE_C_COMPILER gcc)
+# set(CMAKE_C_FLAGS -m32)
+# set(CMAKE_CXX_COMPILER g++)
+# set(CMAKE_CXX_FLAGS -m32)
+#
+# You can then pass the path to it using the CMAKE_TOOLCHAIN_FILE parameter.
+#
+# If you use the Visual Studio generators, just use the -A parameter, like "-A
+# Win32".
+#
+# As a side note, if you want to cross-compile between x86 and x64 using GCC on
+# Ubuntu, you need to install the gcc-multilib package.
+#
+# Windows & Clang
+# ---------------
+#
+# Using Clang on Windows is no easy task, of course. Prior to 3.15, there was
+# no support for building things using the clang++.exe executable, only
+# clang-cl.exe was supported [7]. If you specified -DCMAKE_CXX_COMPILER=clang++,
+# CMake would stil pass MSVC-style command line options to the compiler (like
+# /MD, /nologo, etc.), which clang++ doesn't like [8].
+#
+# So, in summary, you can only use clang++ since 3.15. clang-cl doesn't work
+# with Visual Studio generators unless you specify the proper toolset using the
+# -T parameter. You can set the ClToolExe property using msbuild, but while
+# that might work in practice, clang-cl.exe needs to map some unsupported
+# options for everything to work properly. For an example of how this is done,
+# see the LLVM.Cpp.Common.* files at [11].
+#
+# I recommend using Clang (either clang-cl or clang++ since 3.15) using the
+# "NMake Makefiles" generator.
+#
+# References
+# ----------
+#
+# [1]: cmake::EvaluateDefaultGlobalGenerator
+# https://github.com/Kitware/CMake/blob/v3.18.4/Source/cmake.cxx
+# [2]: https://github.com/Kitware/CMake/blob/v3.18.4/Source/cmGlobalUnixMakefileGenerator3.cxx
+# [3]: https://github.com/Kitware/CMake/blob/v3.18.4/Source/cmGlobalMinGWMakefileGenerator.cxx
+# [4]: https://www.gnu.org/software/make/manual/html_node/Choosing-the-Shell.html
+# [5]: https://github.com/Kitware/CMake/blob/v3.18.4/Source/cmGlobalNMakeMakefileGenerator.cxx
+# [6]: http://mingw.org/wiki/FAQ
+# [7]: https://cmake.org/cmake/help/v3.15/release/3.15.html#compilers
+# [8]: https://github.com/Kitware/CMake/blob/v3.14.7/Modules/Platform/Windows-Clang.cmake
+# [9]: https://gitlab.kitware.com/cmake/cmake/-/issues/19174
+# [10]: https://cmake.org/cmake/help/v3.8/release/3.8.html
+# [11]: https://github.com/llvm/llvm-project/tree/e408935bb5339e20035d84307c666fbdd15e99e0/llvm/tools/msbuild
+# [12]: https://cmake.org/cmake/help/v3.18/generator/Visual%20Studio%2015%202017.html
+
import abc
-from contextlib import contextmanager
import os.path
+import shutil
import project.mingw
-from project.platform import Platform
from project.os import on_windows
+from project.toolchain import ToolchainType
class Toolchain(abc.ABC):
@@ -17,89 +131,164 @@ class Toolchain(abc.ABC):
def get_cmake_args(self):
pass
- @staticmethod
- @contextmanager
- def detect(platform, build_dir, mingw=False):
- if mingw:
- with MinGW.setup(platform, build_dir) as toolchain:
- yield toolchain
- return
-
- if on_windows():
- # MSVC is assumed.
- if platform is None:
- yield Native()
- return
- yield MSVC(platform)
- return
+ @abc.abstractmethod
+ def get_build_args(self):
+ pass
- with GCC.setup(platform, build_dir) as toolchain:
- yield toolchain
- return
+ @staticmethod
+ def detect(hint, platform, build_dir):
+ if hint is ToolchainType.AUTO:
+ if on_windows():
+ # We need to specify the -A parameter. This might break if
+ # none of the Visual Studio generators are available, but the
+ # NMake one is, although I don't know how this can be possible
+ # normally.
+ hint = ToolchainType.MSVC
+ else:
+ # Same thing for the -m32/-m64 flags.
+ hint = ToolchainType.GCC
+ if hint is ToolchainType.MSVC:
+ return MSVC(platform)
+ if hint is ToolchainType.GCC:
+ return GCC.setup(platform, build_dir)
+ if hint is ToolchainType.MINGW:
+ return MinGW.setup(platform, build_dir)
+ if hint is ToolchainType.CLANG:
+ return Clang.setup(platform, build_dir)
+ if hint is ToolchainType.CLANG_CL:
+ return ClangCL.setup(platform, build_dir)
+ raise NotImplementedError(f'unrecognized toolset: {hint}')
-class Native(Toolchain):
+class Auto(Toolchain):
def get_cmake_args(self):
return []
+ def get_build_args(self):
+ return []
-class MSVC(Toolchain):
+
+class MSVC(Auto):
def __init__(self, platform):
self.platform = platform
def get_cmake_args(self):
+ # This doesn't actually specify the generator of course, but I don't
+ # want to implement VS detection logic.
return ['-A', self.platform.get_cmake_arch()]
+ def get_build_args(self):
+ return ['/m']
+
-class File(Toolchain):
+class Makefile(Toolchain):
def __init__(self, path):
self.path = path
@staticmethod
- def _get_path(build_dir):
+ def _get_config_path(build_dir):
return os.path.join(build_dir, 'custom_toolchain.cmake')
+ def _get_makefile_generator(self):
+ if on_windows():
+ if shutil.which('mingw32-make'):
+ return 'MinGW Makefiles'
+ return 'Unix Makefiles'
+ # On Linux/Cygwin, make all the way:
+ return 'Unix Makefiles'
+
+ @classmethod
+ def write_config(cls, build_dir, contents):
+ path = Makefile._get_config_path(build_dir)
+ with open(path, mode='w') as file:
+ file.write(contents)
+ return cls(path)
+
def get_cmake_args(self):
- return ['-D', f'CMAKE_TOOLCHAIN_FILE={self.path}']
+ return [
+ '-D', f'CMAKE_TOOLCHAIN_FILE={self.path}',
+ # The Visual Studio generator is the default on Windows, override
+ # it:
+ '-G', self._get_makefile_generator(),
+ ]
+
+ def get_build_args(self):
+ return []
-class GCC(File):
+class GCC(Makefile):
@staticmethod
def _format(platform):
return f'''
-set(CMAKE_C_COMPILER gcc)
-set(CMAKE_C_FLAGS -m{platform.get_address_model()})
-set(CMAKE_CXX_COMIPLER g++)
-set(CMAKE_CXX_FLAGS -m{platform.get_address_model()})
+set(CMAKE_C_COMPILER gcc)
+set(CMAKE_C_FLAGS -m{platform.get_address_model()})
+set(CMAKE_CXX_COMPILER g++)
+set(CMAKE_CXX_FLAGS -m{platform.get_address_model()})
'''
@staticmethod
- @contextmanager
def setup(platform, build_dir):
- if platform is None:
- yield Native()
- return
- path = File._get_path(build_dir)
- with open(path, mode='w') as file:
- file.write(GCC._format(platform))
- yield GCC(path)
+ return GCC.write_config(build_dir, GCC._format(platform))
-class MinGW(File):
+class MinGW(Makefile):
@staticmethod
def _format(platform):
return f'''
set(CMAKE_C_COMPILER {project.mingw.get_gcc(platform)})
set(CMAKE_CXX_COMPILER {project.mingw.get_gxx(platform)})
-set(CMAKE_RC_COMILER {project.mingw.get_windres(platform)})
+set(CMAKE_AR {project.mingw.get_ar(platform)})
+set(CMAKE_RANLIB {project.mingw.get_ranlib(platform)})
+set(CMAKE_RC_COMPILER {project.mingw.get_windres(platform)})
set(CMAKE_SYSTEM_NAME Windows)
'''
@staticmethod
- @contextmanager
def setup(platform, build_dir):
- platform = platform or Platform.native()
- path = File._get_path(build_dir)
- with open(path, mode='w') as file:
- file.write(MinGW._format(platform))
- yield MinGW(path)
+ return MinGW.write_config(build_dir, MinGW._format(platform))
+
+
+class Clang(Makefile):
+ @staticmethod
+ def _format(platform):
+ return f'''
+if(CMAKE_VERSION VERSION_LESS "3.15" AND WIN32)
+ set(CMAKE_C_COMPILER clang-cl)
+ set(CMAKE_CXX_COMPILER clang-cl)
+ set(CMAKE_C_FLAGS -m{platform.get_address_model()})
+ set(CMAKE_CXX_FLAGS -m{platform.get_address_model()})
+else()
+ set(CMAKE_C_COMPILER clang)
+ set(CMAKE_CXX_COMPILER clang++)
+ set(CMAKE_C_FLAGS -m{platform.get_address_model()})
+ set(CMAKE_CXX_FLAGS -m{platform.get_address_model()})
+endif()
+'''
+
+ def _get_makefile_generator(self):
+ if on_windows():
+ # MinGW utilities like make might be unavailable, but NMake can
+ # very much be there.
+ if shutil.which('nmake'):
+ return 'NMake Makefiles'
+ return super()._get_makefile_generator()
+
+ @staticmethod
+ def setup(platform, build_dir):
+ return Clang.write_config(build_dir, Clang._format(platform))
+
+
+class ClangCL(Clang):
+ @staticmethod
+ def _format(platform):
+ return f'''
+set(CMAKE_C_COMPILER clang-cl)
+set(CMAKE_CXX_COMPILER clang-cl)
+set(CMAKE_C_FLAGS -m{platform.get_address_model()})
+set(CMAKE_CXX_FLAGS -m{platform.get_address_model()})
+set(CMAKE_SYSTEM_NAME Windows)
+'''
+
+ @staticmethod
+ def setup(platform, build_dir):
+ return ClangCL.write_config(build_dir, ClangCL._format(platform))
--
cgit v1.2.3