From f02ffc007c103189a3f80204676268f9a4429daa Mon Sep 17 00:00:00 2001 From: Marvin Scholz Date: Fri, 11 Feb 2022 03:48:11 +0100 Subject: [PATCH 1/5] clike: add more reliable ways to check underscore prefix Fix #5482 --- mesonbuild/compilers/mixins/clike.py | 79 +++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 6 deletions(-) diff --git a/mesonbuild/compilers/mixins/clike.py b/mesonbuild/compilers/mixins/clike.py index 269ce5ca6..7044b4e54 100644 --- a/mesonbuild/compilers/mixins/clike.py +++ b/mesonbuild/compilers/mixins/clike.py @@ -900,9 +900,10 @@ class CLikeCompiler(Compiler): return self.compiles(t, env, extra_args=extra_args, dependencies=dependencies) - def symbols_have_underscore_prefix(self, env: 'Environment') -> bool: + def _symbols_have_underscore_prefix_searchbin(self, env: 'Environment') -> bool: ''' - Check if the compiler prefixes an underscore to global C symbols + Check if symbols have underscore prefix by compiling a small test binary + and then searching the binary for the string, ''' symbol_name = b'meson_uscore_prefix' code = '''#ifdef __cplusplus @@ -914,7 +915,7 @@ class CLikeCompiler(Compiler): #endif ''' args = self.get_compiler_check_args(CompileCheckMode.COMPILE) - n = 'symbols_have_underscore_prefix' + n = '_symbols_have_underscore_prefix_searchbin' with self._build_wrapper(code, env, extra_args=args, mode='compile', want_output=True, temp_dir=env.scratch_dir) as p: if p.returncode != 0: raise RuntimeError(f'BUG: Unable to compile {n!r} check: {p.stdout}') @@ -925,13 +926,79 @@ class CLikeCompiler(Compiler): # Check if the underscore form of the symbol is somewhere # in the output file. if b'_' + symbol_name in line: - mlog.debug("Symbols have underscore prefix: YES") + mlog.debug("Underscore prefix check found prefixed function in binary") return True # Else, check if the non-underscored form is present elif symbol_name in line: - mlog.debug("Symbols have underscore prefix: NO") + mlog.debug("Underscore prefix check found non-prefixed function in binary") return False - raise RuntimeError(f'BUG: {n!r} check failed unexpectedly') + raise RuntimeError(f'BUG: {n!r} check did not find symbol string in binary') + + def _symbols_have_underscore_prefix_define(self, env: 'Environment') -> T.Optional[bool]: + ''' + Check if symbols have underscore prefix by querying the + __USER_LABEL_PREFIX__ define that most compilers provide + for this. Return if functions have underscore prefix or None + if it was not possible to determine, like when the compiler + does not set the define or the define has an unexpected value. + ''' + delim = '"MESON_HAVE_UNDERSCORE_DELIMITER" ' + code = f''' + #ifndef __USER_LABEL_PREFIX__ + #define MESON_UNDERSCORE_PREFIX unsupported + #else + #define MESON_UNDERSCORE_PREFIX __USER_LABEL_PREFIX__ + #endif + {delim}MESON_UNDERSCORE_PREFIX + ''' + with self._build_wrapper(code, env, mode='preprocess', want_output=False, temp_dir=env.scratch_dir) as p: + if p.returncode != 0: + raise RuntimeError(f'BUG: Unable to preprocess _symbols_have_underscore_prefix_define check: {p.stdout}') + symbol_prefix = p.stdout.partition(delim)[-1].rstrip() + + mlog.debug(f'Queried compiler for function prefix: __USER_LABEL_PREFIX__ is "{symbol_prefix!s}"') + if symbol_prefix == '_': + return True + elif symbol_prefix == '': + return False + else: + return None + + def _symbols_have_underscore_prefix_list(self, env: 'Environment') -> T.Optional[bool]: + ''' + Check if symbols have underscore prefix by consulting a hardcoded + list of cases where we know the results. + Return if functions have underscore prefix or None if unknown. + ''' + m = env.machines[self.for_machine] + # Darwin always uses the underscore prefix, not matter what + if m.is_darwin(): + return True + # Windows uses the underscore prefix on x86 (32bit) only + if m.is_windows() or m.is_cygwin(): + return m.cpu_family == 'x86' + return None + + + def symbols_have_underscore_prefix(self, env: 'Environment') -> bool: + ''' + Check if the compiler prefixes an underscore to global C symbols + ''' + # First, try to query the compiler directly + result = self._symbols_have_underscore_prefix_define(env) + if result is not None: + return result + + # Else, try to consult a hardcoded list of cases we know + # absolutely have an underscore prefix + result = self._symbols_have_underscore_prefix_list(env) + if result is not None: + return result + + # As a last resort, try search in a compiled binary, which is the + # most unreliable way of checking this, see #5482 + return self._symbols_have_underscore_prefix_searchbin(env) + def _get_patterns(self, env: 'Environment', prefixes: T.List[str], suffixes: T.List[str], shared: bool = False) -> T.List[str]: patterns = [] # type: T.List[str] From 1b03441bc46819f098e935370294fa88a544d7d0 Mon Sep 17 00:00:00 2001 From: Marvin Scholz Date: Fri, 11 Feb 2022 03:49:17 +0100 Subject: [PATCH 2/5] visualstudio: do not query underscore define with MSVC MSVC does not has the builtin define to check for the symbol prefix, so do not try to query it at all, to save some time. --- mesonbuild/compilers/mixins/visualstudio.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mesonbuild/compilers/mixins/visualstudio.py b/mesonbuild/compilers/mixins/visualstudio.py index 5118e41eb..2f2645375 100644 --- a/mesonbuild/compilers/mixins/visualstudio.py +++ b/mesonbuild/compilers/mixins/visualstudio.py @@ -383,6 +383,23 @@ class VisualStudioLikeCompiler(Compiler, metaclass=abc.ABCMeta): def get_argument_syntax(self) -> str: return 'msvc' + def symbols_have_underscore_prefix(self, env: 'Environment') -> bool: + ''' + Check if the compiler prefixes an underscore to global C symbols. + + This overrides the Clike method, as for MSVC checking the + underscore prefix based on the compiler define never works, + so do not even try. + ''' + # Try to consult a hardcoded list of cases we know + # absolutely have an underscore prefix + result = self._symbols_have_underscore_prefix_list(env) + if result is not None: + return result + + # As a last resort, try search in a compiled binary + return self._symbols_have_underscore_prefix_searchbin(env) + class MSVCCompiler(VisualStudioLikeCompiler): From 89620dc8e7f04c35dba6df10d4d6dc3e23facd91 Mon Sep 17 00:00:00 2001 From: Marvin Scholz Date: Wed, 30 Mar 2022 17:05:23 +0200 Subject: [PATCH 3/5] clike: print stderr instead of stdout for debugging When something goes wrong with running the compiler in _symbols_have_underscore_prefix_searchbin, print stderr instead, as it actually contains helpful output while stdout is usually empty in this case. --- mesonbuild/compilers/mixins/clike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/compilers/mixins/clike.py b/mesonbuild/compilers/mixins/clike.py index 7044b4e54..162005fe6 100644 --- a/mesonbuild/compilers/mixins/clike.py +++ b/mesonbuild/compilers/mixins/clike.py @@ -918,7 +918,7 @@ class CLikeCompiler(Compiler): n = '_symbols_have_underscore_prefix_searchbin' with self._build_wrapper(code, env, extra_args=args, mode='compile', want_output=True, temp_dir=env.scratch_dir) as p: if p.returncode != 0: - raise RuntimeError(f'BUG: Unable to compile {n!r} check: {p.stdout}') + raise RuntimeError(f'BUG: Unable to compile {n!r} check: {p.stderr}') if not os.path.isfile(p.output_name): raise RuntimeError(f'BUG: Can\'t find compiled test code for {n!r} check') with open(p.output_name, 'rb') as o: From 9e597ce905aa2b426e87f590a671b6726c09f075 Mon Sep 17 00:00:00 2001 From: Marvin Scholz Date: Wed, 30 Mar 2022 17:47:02 +0200 Subject: [PATCH 4/5] unittests: add underscore prefix tests Tests the two new detection methods for the underscore prefix against what is detected in the binary. Contrary to the in-the-wild failures, we can trust the binary here as we do not get any additional flags that influence the binary contents in this test environment. --- unittests/allplatformstests.py | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py index 8a7fcd754..25d630514 100644 --- a/unittests/allplatformstests.py +++ b/unittests/allplatformstests.py @@ -1546,6 +1546,43 @@ class AllPlatformTests(BasePlatformTests): self.build() self.run_tests() + def get_convincing_fake_env_and_cc(self) -> None: + ''' + Return a fake env and C compiler with the fake env + machine info properly detected using that compiler. + ''' + env = get_fake_env('', self.builddir, self.prefix) + cc = detect_c_compiler(env, MachineChoice.HOST) + # Detect machine info + env.machines.host = mesonbuild.environment.detect_machine_info({'c':cc}) + return (env, cc) + + def test_underscore_prefix_detection_list(self) -> None: + ''' + Test the underscore detection hardcoded lookup list + against what was detected in the binary. + ''' + env, cc = self.get_convincing_fake_env_and_cc() + expected_uscore = cc._symbols_have_underscore_prefix_searchbin(env) + list_uscore = cc._symbols_have_underscore_prefix_list(env) + if list_uscore is not None: + self.assertEqual(list_uscore, expected_uscore) + else: + raise SkipTest('No match in underscore prefix list for this platform.') + + def test_underscore_prefix_detection_define(self) -> None: + ''' + Test the underscore detection based on compiler-defined preprocessor macro + against what was detected in the binary. + ''' + env, cc = self.get_convincing_fake_env_and_cc() + expected_uscore = cc._symbols_have_underscore_prefix_searchbin(env) + define_uscore = cc._symbols_have_underscore_prefix_define(env) + if define_uscore is not None: + self.assertEqual(define_uscore, expected_uscore) + else: + raise SkipTest('Did not find the underscore prefix define __USER_LABEL_PREFIX__') + @skipIfNoPkgconfig def test_pkgconfig_static(self): ''' From 34ea8fdf981b5e10c702e9adfcab7f9e316a129b Mon Sep 17 00:00:00 2001 From: Marvin Scholz Date: Wed, 30 Mar 2022 23:30:39 +0200 Subject: [PATCH 5/5] unittests: move get_convincing_fake_env_and_cc to run_tests.py --- run_tests.py | 13 ++++++++++++- unittests/allplatformstests.py | 17 +++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/run_tests.py b/run_tests.py index dc7a8f9ec..fcd933789 100755 --- a/run_tests.py +++ b/run_tests.py @@ -40,7 +40,7 @@ from mesonbuild import mesonlib from mesonbuild import mesonmain from mesonbuild import mtest from mesonbuild import mlog -from mesonbuild.environment import Environment, detect_ninja +from mesonbuild.environment import Environment, detect_ninja, detect_machine_info from mesonbuild.coredata import backendlist, version as meson_version from mesonbuild.mesonlib import OptionKey, setup_vsenv @@ -153,6 +153,17 @@ def get_fake_env(sdir='', bdir=None, prefix='', opts=None): env.machines.host.cpu_family = 'x86_64' # Used on macOS inside find_library return env +def get_convincing_fake_env_and_cc(bdir, prefix): + ''' + Return a fake env and C compiler with the fake env + machine info properly detected using that compiler. + Useful for running compiler checks in the unit tests. + ''' + env = get_fake_env('', bdir, prefix) + cc = compilers.detect_c_compiler(env, mesonlib.MachineChoice.HOST) + # Detect machine info + env.machines.host = detect_machine_info({'c':cc}) + return (env, cc) Backend = Enum('Backend', 'ninja vs xcode') diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py index 25d630514..67c50e76d 100644 --- a/unittests/allplatformstests.py +++ b/unittests/allplatformstests.py @@ -60,7 +60,7 @@ from mesonbuild.scripts import destdir_join from mesonbuild.wrap.wrap import PackageDefinition, WrapException from run_tests import ( - Backend, exe_suffix, get_fake_env + Backend, exe_suffix, get_fake_env, get_convincing_fake_env_and_cc ) from .baseplatformtests import BasePlatformTests @@ -1546,23 +1546,12 @@ class AllPlatformTests(BasePlatformTests): self.build() self.run_tests() - def get_convincing_fake_env_and_cc(self) -> None: - ''' - Return a fake env and C compiler with the fake env - machine info properly detected using that compiler. - ''' - env = get_fake_env('', self.builddir, self.prefix) - cc = detect_c_compiler(env, MachineChoice.HOST) - # Detect machine info - env.machines.host = mesonbuild.environment.detect_machine_info({'c':cc}) - return (env, cc) - def test_underscore_prefix_detection_list(self) -> None: ''' Test the underscore detection hardcoded lookup list against what was detected in the binary. ''' - env, cc = self.get_convincing_fake_env_and_cc() + env, cc = get_convincing_fake_env_and_cc(self.builddir, self.prefix) expected_uscore = cc._symbols_have_underscore_prefix_searchbin(env) list_uscore = cc._symbols_have_underscore_prefix_list(env) if list_uscore is not None: @@ -1575,7 +1564,7 @@ class AllPlatformTests(BasePlatformTests): Test the underscore detection based on compiler-defined preprocessor macro against what was detected in the binary. ''' - env, cc = self.get_convincing_fake_env_and_cc() + env, cc = get_convincing_fake_env_and_cc(self.builddir, self.prefix) expected_uscore = cc._symbols_have_underscore_prefix_searchbin(env) define_uscore = cc._symbols_have_underscore_prefix_define(env) if define_uscore is not None: