cmake: Fix CMakeToolchain (fixes #8293)

Instead of guessing the internal compiler
variables, Meson now runns CMake once to
determine what they actually are.

mesonbuild/cmake/interpreter.py

@@ -892,7 +892,7 @@ class CMakeInterpreter:
         self.trace = CMakeTraceParser(cmake_exe.version(), self.build_dir, permissive=True)
 
         preload_file = mesondata['cmake/data/preload.cmake'].write_to_private(self.env)
-        toolchain = CMakeToolchain(self.env, self.for_machine, CMakeExecScope.SUBPROJECT, self.build_dir.parent, preload_file)
+        toolchain = CMakeToolchain(cmake_exe, self.env, self.for_machine, CMakeExecScope.SUBPROJECT, self.build_dir, preload_file)
         toolchain_file = toolchain.write()
 
         # TODO: drop this check once the deprecated `cmake_args` kwarg is removed

mesonbuild/cmake/toolchain.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from pathlib import Path
+from .traceparser import CMakeTraceParser
 from ..envconfig import CMakeSkipCompilerTest
 from ..mesonlib import MachineChoice
 from .common import language_map
@@ -24,53 +25,49 @@ from enum import Enum
 from textwrap import dedent
 
 if T.TYPE_CHECKING:
+    from .executor import CMakeExecutor
     from ..envconfig import MachineInfo, Properties, CMakeVariables
     from ..environment import Environment
-    from ..compilers import Compiler
-
-
-_MESON_TO_CMAKE_MAPPING = {
-    'arm':          'ARMCC',
-    'armclang':     'ARMClang',
-    'clang':        'Clang',
-    'clang-cl':     'MSVC',
-    'flang':        'Flang',
-    'g95':          'G95',
-    'gcc':          'GNU',
-    'intel':        'Intel',
-    'intel-cl':     'MSVC',
-    'msvc':         'MSVC',
-    'pathscale':    'PathScale',
-    'pgi':          'PGI',
-    'sun':          'SunPro',
-}
 
 class CMakeExecScope(Enum):
     SUBPROJECT = 'subproject'
     DEPENDENCY = 'dependency'
 
 class CMakeToolchain:
-    def __init__(self, env: 'Environment', for_machine: MachineChoice, exec_scope: CMakeExecScope, out_dir: Path, preload_file: T.Optional[Path] = None) -> None:
+    def __init__(self, cmakebin: 'CMakeExecutor', env: 'Environment', for_machine: MachineChoice, exec_scope: CMakeExecScope, build_dir: Path, preload_file: T.Optional[Path] = None) -> None:
         self.env            = env
+        self.cmakebin       = cmakebin
         self.for_machine    = for_machine
         self.exec_scope     = exec_scope
         self.preload_file   = preload_file
-        self.toolchain_file = out_dir / 'CMakeMesonToolchainFile.cmake'
+        self.build_dir      = build_dir
-        self.toolchain_file = self.toolchain_file.resolve()
+        self.build_dir      = self.build_dir.resolve()
+        self.toolchain_file = build_dir / 'CMakeMesonToolchainFile.cmake'
+        self.cmcache_file   = build_dir / 'CMakeCache.txt'
         self.minfo          = self.env.machines[self.for_machine]
         self.properties     = self.env.properties[self.for_machine]
         self.compilers      = self.env.coredata.compilers[self.for_machine]
         self.cmakevars      = self.env.cmakevars[self.for_machine]
+        self.cmakestate     = self.env.coredata.cmake_cache[self.for_machine]
 
         self.variables = self.get_defaults()
         self.variables.update(self.cmakevars.get_variables())
 
+        # Determine whether CMake the compiler test should be skipped
+        skip_status = self.properties.get_cmake_skip_compiler_test()
+        self.skip_check = skip_status == CMakeSkipCompilerTest.ALWAYS
+        if skip_status == CMakeSkipCompilerTest.DEP_ONLY and self.exec_scope == CMakeExecScope.DEPENDENCY:
+            self.skip_check = True
+        if not self.properties.get_cmake_defaults():
+            self.skip_check = False
+
         assert self.toolchain_file.is_absolute()
 
     def write(self) -> Path:
         if not self.toolchain_file.parent.exists():
             self.toolchain_file.parent.mkdir(parents=True)
         self.toolchain_file.write_text(self.generate())
+        self.cmcache_file.write_text(self.generate_cache())
         mlog.cmd_ci_include(self.toolchain_file.as_posix())
         return self.toolchain_file
 
@@ -80,6 +77,16 @@ class CMakeToolchain:
             args += ['-DMESON_PRELOAD_FILE=' + self.preload_file.as_posix()]
         return args
 
+    @staticmethod
+    def _print_vars(vars: T.Dict[str, T.List[str]]) -> str:
+        res = ''
+        for key, value in vars.items():
+            res += 'set(' + key
+            for i in value:
+                res += f' "{i}"'
+            res += ')\n'
+        return res
+
     def generate(self) -> str:
         res = dedent('''\
             ######################################
@@ -100,14 +107,19 @@ class CMakeToolchain:
         for key, value in self.variables.items():
             self.variables[key] = [x.replace('\\', '/') for x in value]
 
+        # Set compiler
+        if self.skip_check:
+            self.update_cmake_compiler_state()
+            res += '# CMake compiler state variables\n'
+            for lang, vars in self.cmakestate:
+                res += f'# -- Variables for language {lang}\n'
+                res += self._print_vars(vars)
+                res += '\n'
+            res += '\n'
+
         # Set variables from the current machine config
         res += '# Variables from meson\n'
-        for key, value in self.variables.items():
+        res += self._print_vars(self.variables)
-            res += 'set(' + key
-            for i in value:
-                res += f' "{i}"'
-
-            res += ')\n'
         res += '\n'
 
         # Add the user provided toolchain file
@@ -121,6 +133,15 @@ class CMakeToolchain:
 
         return res
 
+    def generate_cache(self) -> str:
+        if not self.skip_check:
+            return ''
+
+        res = ''
+        for name, v in self.cmakestate.cmake_cache.items():
+            res += f'{name}:{v.type}={";".join(v.value)}\n'
+        return res
+
     def get_defaults(self) -> T.Dict[str, T.List[str]]:
         defaults = {}  # type: T.Dict[str, T.List[str]]
 
@@ -151,11 +172,6 @@ class CMakeToolchain:
         if sys_root:
             defaults['CMAKE_SYSROOT'] = [sys_root]
 
-        # Determine whether CMake the compiler test should be skipped
-        skip_check = self.properties.get_cmake_skip_compiler_test() == CMakeSkipCompilerTest.ALWAYS
-        if self.properties.get_cmake_skip_compiler_test() == CMakeSkipCompilerTest.DEP_ONLY and self.exec_scope == CMakeExecScope.DEPENDENCY:
-            skip_check = True
-
         def make_abs(exe: str) -> str:
             if Path(exe).is_absolute():
                 return exe
@@ -168,9 +184,6 @@ class CMakeToolchain:
         # Set the compiler variables
         for lang, comp_obj in self.compilers.items():
             exe_list = [make_abs(x) for x in comp_obj.get_exelist()]
-            comp_id = CMakeToolchain.meson_compiler_to_cmake_id(comp_obj)
-            comp_version = comp_obj.version.upper()
-
             prefix = 'CMAKE_{}_'.format(language_map.get(lang, lang.upper()))
 
             if not exe_list:
@@ -183,35 +196,50 @@ class CMakeToolchain:
             if comp_obj.get_id() == 'clang-cl':
                 defaults['CMAKE_LINKER'] = comp_obj.get_linker_exelist()
 
-            # Setting the variables after this check cause CMake to skip
-            # validating the compiler
-            if not skip_check:
-                continue
-
-            defaults[prefix + 'COMPILER_ID']      = [comp_id]
-            defaults[prefix + 'COMPILER_VERSION'] = [comp_version]
-            #defaults[prefix + 'COMPILER_LOADED']  = ['1']
-            defaults[prefix + 'COMPILER_FORCED']  = ['1']
-            defaults[prefix + 'COMPILER_WORKS']   = ['TRUE']
-            #defaults[prefix + 'ABI_COMPILED']     = ['TRUE']
-
         return defaults
 
-    @staticmethod
+    def update_cmake_compiler_state(self) -> None:
-    def meson_compiler_to_cmake_id(cobj: 'Compiler') -> str:
+        # Check if all variables are already cached
-        """Translate meson compiler's into CMAKE compiler ID's.
+        if self.cmakestate.languages.issuperset(self.compilers.keys()):
+            return
 
-        Most of these can be handled by a simple table lookup, with a few
+        # Generate the CMakeLists.txt
-        exceptions.
+        mlog.debug('CMake Toolchain: Calling CMake once to generate the compiler state')
+        languages     = list(self.compilers.keys())
+        lang_ids      = [language_map.get(x, x.upper()) for x in languages]
+        cmake_content = dedent(f'''
+            cmake_minimum_required(VERSION 3.7)
+            project(CompInfo {' '.join(lang_ids)})
+        ''')
 
-        Clang and Apple's Clang are both identified as "clang" by meson. To make
+        build_dir = Path(self.env.scratch_dir) / '__CMake_compiler_info__'
-        things more complicated gcc and vanilla clang both use Apple's ld64 on
+        build_dir.mkdir(parents=True, exist_ok=True)
-        macOS. The only way to know for sure is to do an isinstance() check.
+        cmake_file = build_dir / 'CMakeLists.txt'
-        """
+        cmake_file.write_text(cmake_content)
-        from ..compilers import (AppleClangCCompiler, AppleClangCPPCompiler,
+
-                                AppleClangObjCCompiler, AppleClangObjCPPCompiler)
+        # Generate the temporary toolchain file
-        if isinstance(cobj, (AppleClangCCompiler, AppleClangCPPCompiler,
+        temp_toolchain_file = build_dir / 'CMakeMesonTempToolchainFile.cmake'
-                            AppleClangObjCCompiler, AppleClangObjCPPCompiler)):
+        temp_toolchain_file.write_text(CMakeToolchain._print_vars(self.variables))
-            return 'AppleClang'
+
-        # If no mapping, try GNU and hope that the build files don't care
+        # Configure
-        return _MESON_TO_CMAKE_MAPPING.get(cobj.get_id(), 'GNU')
+        trace = CMakeTraceParser(self.cmakebin.version(), build_dir)
+        self.cmakebin.set_exec_mode(print_cmout=False, always_capture_stderr=trace.requires_stderr())
+        cmake_args = [*trace.trace_args(), '-DCMAKE_TOOLCHAIN_FILE=' + temp_toolchain_file.as_posix(), '.']
+        rc, _, raw_trace = self.cmakebin.call(cmake_args, build_dir=build_dir, disable_cache=True)
+
+        if rc != 0:
+            mlog.warning('CMake Toolchain: Failed to determine CMake compilers state')
+            return
+
+        # Parse output
+        trace.parse(raw_trace)
+        self.cmakestate.cmake_cache = {**trace.cache}
+
+        vars_by_file = {k.name: v for (k, v) in trace.vars_by_file.items()}
+
+        for lang in languages:
+            lang_cmake = language_map.get(lang, lang.upper())
+            file_name  = f'CMake{lang_cmake}Compiler.cmake'
+            vars = vars_by_file.setdefault(file_name, {})
+            vars[f'CMAKE_{lang_cmake}_COMPILER_FORCED'] = ['1']
+            self.cmakestate.update(lang, vars)

mesonbuild/coredata.py

@@ -36,6 +36,7 @@ if T.TYPE_CHECKING:
     from .compilers.compilers import Compiler, CompileResult  # noqa: F401
     from .environment import Environment
     from .mesonlib import OptionOverrideProxy
+    from .cmake.traceparser import CMakeCacheEntry
 
     OptionDictType = T.Union[T.Dict[str, 'UserOption[T.Any]'], OptionOverrideProxy]
     KeyedOptionDictType = T.Union[T.Dict['OptionKey', 'UserOption[T.Any]'], OptionOverrideProxy]
@@ -374,6 +375,34 @@ class DependencyCache:
     def clear(self) -> None:
         self.__cache.clear()
 
+
+class CMakeStateCache:
+    """Class that stores internal CMake compiler states.
+
+    This cache is used to reduce the startup overhead of CMake by caching
+    all internal CMake compiler variables.
+    """
+
+    def __init__(self) -> None:
+        self.__cache: T.Dict[str, T.Dict[str, T.List[str]]] = {}
+        self.cmake_cache: T.Dict[str, 'CMakeCacheEntry'] = {}
+
+    def __iter__(self) -> T.Iterator[T.Tuple[str, T.Dict[str, T.List[str]]]]:
+        return iter(self.__cache.items())
+
+    def items(self) -> T.Iterator[T.Tuple[str, T.Dict[str, T.List[str]]]]:
+        return iter(self.__cache.items())
+
+    def update(self, language: str, variables: T.Dict[str, T.List[str]]):
+        if language not in self.__cache:
+            self.__cache[language] = {}
+        self.__cache[language].update(variables)
+
+    @property
+    def languages(self) -> T.Set[str]:
+        return set(self.__cache.keys())
+
+
 # Can't bind this near the class method it seems, sadly.
 _V = T.TypeVar('_V')
 
@@ -414,6 +443,9 @@ class CoreData:
 
         self.compiler_check_cache = OrderedDict()  # type: T.Dict[CompilerCheckCacheKey, compiler.CompileResult]
 
+        # CMake cache
+        self.cmake_cache: PerMachine[CMakeStateCache] = PerMachine(CMakeStateCache(), CMakeStateCache())
+
         # Only to print a warning if it changes between Meson invocations.
         self.config_files = self.__load_config_files(options, scratch_dir, 'native')
         self.builtin_options_libdir_cross_fixup()

mesonbuild/dependencies/base.py

@@ -1146,7 +1146,7 @@ class CMakeDependency(ExternalDependency):
         gen_list += CMakeDependency.class_cmake_generators
 
         temp_parser = CMakeTraceParser(self.cmakebin.version(), self._get_build_dir())
-        toolchain = CMakeToolchain(self.env, self.for_machine, CMakeExecScope.DEPENDENCY, self._get_build_dir())
+        toolchain = CMakeToolchain(self.cmakebin, self.env, self.for_machine, CMakeExecScope.DEPENDENCY, self._get_build_dir())
         toolchain.write()
 
         for i in gen_list:
@@ -1346,7 +1346,7 @@ class CMakeDependency(ExternalDependency):
 
         # Map the components
         comp_mapped = self._map_component_list(modules, components)
-        toolchain = CMakeToolchain(self.env, self.for_machine, CMakeExecScope.DEPENDENCY, self._get_build_dir())
+        toolchain = CMakeToolchain(self.cmakebin, self.env, self.for_machine, CMakeExecScope.DEPENDENCY, self._get_build_dir())
         toolchain.write()
 
         for i in gen_list: