project tests: log the reason why a test is skipped

We expose a reason after the string 'MESON_SKIP_TEST', but it is
actually ignored when running the test, so it is only useful as
documentation and really might as well be a comment.

Make it even more useful by actually printing that string after the
'[SKIPPED]' message.

Also, sometimes a test can be skipped for multiple reasons, and it would
be useful to know which one occurred.

run_project_tests.py

@@ -1173,6 +1173,12 @@ class LogRunFuture:
 
 RunFutureUnion = T.Union[TestRunFuture, LogRunFuture]
 
+def test_emits_skip_msg(line: str) -> bool:
+    for prefix in {'Problem encountered', 'Assert failed', 'Failed to configure the CMake subproject'}:
+        if f'{prefix}: MESON_SKIP_TEST' in line:
+            return True
+    return False
+
 def _run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]],
                log_name_base: str,
                failfast: bool,
@@ -1281,10 +1287,19 @@ def _run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]],
         if result is None:
             # skipped due to skipped category skip or 'tools:' or 'skip_on_env:'
             is_skipped = True
+            skip_reason = 'not run because preconditions were not met'
             skip_as_expected = True
         else:
             # skipped due to test outputting 'MESON_SKIP_TEST'
-            is_skipped = 'MESON_SKIP_TEST' in result.stdo
+            for l in result.stdo.splitlines():
+                if test_emits_skip_msg(l):
+                    is_skipped = True
+                    offset = l.index('MESON_SKIP_TEST') + 16
+                    skip_reason = l[offset:].strip()
+                    break
+            else:
+                is_skipped = False
+                skip_reason = ''
             if not skip_dont_care(t):
                 skip_as_expected = (is_skipped == t.skip_expected)
             else:
@@ -1295,6 +1310,7 @@ def _run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]],
 
         if is_skipped and skip_as_expected:
             f.update_log(TestStatus.SKIP)
+            safe_print(bold('Reason:'), skip_reason)
             current_test = ET.SubElement(current_suite, 'testcase', {'name': testname, 'classname': t.category})
             ET.SubElement(current_test, 'skipped', {})
             continue

run_single_test.py

@@ -13,7 +13,7 @@ import pathlib
 import typing as T
 
 from mesonbuild import mlog
-from run_project_tests import TestDef, load_test_json, run_test, BuildStep
+from run_project_tests import TestDef, load_test_json, run_test, BuildStep, test_emits_skip_msg
 from run_project_tests import setup_commands, detect_system_compiler, print_tool_versions
 
 if T.TYPE_CHECKING:
@@ -59,7 +59,21 @@ def main() -> None:
     results = [run_test(t, t.args, should_fail(t.path), args.use_tmpdir) for t in tests]
     failed = False
     for test, result in zip(tests, results):
-        if (result is None) or ('MESON_SKIP_TEST' in result.stdo):
+        if result is None:
+            is_skipped = True
+            skip_reason = 'not run because preconditions were not met'
+        else:
+            for l in result.stdo.splitlines():
+                if test_emits_skip_msg(l):
+                    is_skipped = True
+                    offset = l.index('MESON_SKIP_TEST') + 16
+                    skip_reason = l[offset:].strip()
+                    break
+            else:
+                is_skipped = False
+                skip_reason = ''
+
+        if is_skipped:
             msg = mlog.yellow('SKIP:')
         elif result.msg:
             msg = mlog.red('FAIL:')
@@ -67,6 +81,8 @@ def main() -> None:
         else:
             msg = mlog.green('PASS:')
         mlog.log(msg, *test.display_name())
+        if skip_reason:
+            mlog.log(mlog.bold('Reason:'), skip_reason)
         if result is not None and result.msg and 'MESON_SKIP_TEST' not in result.stdo:
             mlog.log('reason:', result.msg)
             if result.step is BuildStep.configure: