Virtualization
/
meson
mirror of https://gitlab.com/qemu-project/meson.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #7197 from jon-turney/test-output-check-mandatory 

Make the expected output check mandatory for failing-meson and warning-meson tests
This commit is contained in:
Jussi Pakkanen 2020-05-26 02:52:12 +03:00 committed by GitHub
parent 6f199db95b 6b1e1ffa77
commit 1e7f1ad4c9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 115 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
run_project_tests.py
View File

@ -603,18 +603,16 @@ def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args,
return testresult
def gather_tests(testdir: Path) -> T.List[TestDef]:
def gather_tests(testdir: Path, stdout_mandatory: bool) -> T.List[TestDef]:
tests = [t.name for t in testdir.iterdir() if t.is_dir()]
tests = [t for t in tests if not t.startswith('.')] # Filter non-tests files (dot files, etc)
test_defs = [TestDef(testdir / t, None, []) for t in tests]
all_tests = [] # type: T.List[TestDef]
for t in test_defs:
test_def = {}
test_def_file = t.path / 'test.json'
if not test_def_file.is_file():
all_tests += [t]
continue
test_def = json.loads(test_def_file.read_text())
if test_def_file.is_file():
test_def = json.loads(test_def_file.read_text())
# Handle additional environment variables
env = {} # type: T.Dict[str, str]
@ -632,6 +630,8 @@ def gather_tests(testdir: Path) -> T.List[TestDef]:
# Handle expected output
stdout = test_def.get('stdout', [])
if stdout_mandatory and not stdout:
raise RuntimeError("{} must contain a non-empty stdout key".format(test_def_file))
# Handle the do_not_set_opts list
do_not_set_opts = test_def.get('do_not_set_opts', []) # type: T.List[str]
@ -906,45 +906,50 @@ def detect_tests_to_run(only: T.List[str], use_tmp: bool) -> T.List[T.Tuple[str,
shutil.which('pgfortran') or
shutil.which('ifort'))
# Name, subdirectory, skip condition.
class TestCategory:
def __init__(self, category: str, subdir: str, skip: bool = False, stdout_mandatory: bool = False):
self.category = category # category name
self.subdir = subdir # subdirectory
self.skip = skip # skip condition
self.stdout_mandatory = stdout_mandatory # expected stdout is mandatory for tests in this categroy
all_tests = [
('cmake', 'cmake', not shutil.which('cmake') or (os.environ.get('compiler') == 'msvc2015' and under_ci)),
('common', 'common', False),
('warning-meson', 'warning', False),
('failing-meson', 'failing', False),
('failing-build', 'failing build', False),
('failing-test', 'failing test', False),
('keyval', 'keyval', False),
('platform-osx', 'osx', not mesonlib.is_osx()),
('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()),
('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()),
('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()),
('C#', 'csharp', skip_csharp(backend)),
('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))),
('rust', 'rust', should_skip_rust(backend)),
('d', 'd', backend is not Backend.ninja or not have_d_compiler()),
('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or not have_objc_compiler(options.use_tmpdir)),
('objective c++', 'objcpp', backend not in (Backend.ninja, Backend.xcode) or not have_objcpp_compiler(options.use_tmpdir)),
('fortran', 'fortran', skip_fortran or backend != Backend.ninja),
('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')),
TestCategory('cmake', 'cmake', not shutil.which('cmake') or (os.environ.get('compiler') == 'msvc2015' and under_ci)),
TestCategory('common', 'common'),
TestCategory('warning-meson', 'warning', stdout_mandatory=True),
TestCategory('failing-meson', 'failing', stdout_mandatory=True),
TestCategory('failing-build', 'failing build'),
TestCategory('failing-test', 'failing test'),
TestCategory('keyval', 'keyval'),
TestCategory('platform-osx', 'osx', not mesonlib.is_osx()),
TestCategory('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()),
TestCategory('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()),
TestCategory('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()),
TestCategory('C#', 'csharp', skip_csharp(backend)),
TestCategory('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))),
TestCategory('rust', 'rust', should_skip_rust(backend)),
TestCategory('d', 'd', backend is not Backend.ninja or not have_d_compiler()),
TestCategory('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or not have_objc_compiler(options.use_tmpdir)),
TestCategory('objective c++', 'objcpp', backend not in (Backend.ninja, Backend.xcode) or not have_objcpp_compiler(options.use_tmpdir)),
TestCategory('fortran', 'fortran', skip_fortran or backend != Backend.ninja),
TestCategory('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')),
# CUDA tests on Windows: use Ninja backend: python run_project_tests.py --only cuda --backend ninja
('cuda', 'cuda', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('nvcc')),
('python3', 'python3', backend is not Backend.ninja),
('python', 'python', backend is not Backend.ninja),
('fpga', 'fpga', shutil.which('yosys') is None),
('frameworks', 'frameworks', False),
('nasm', 'nasm', False),
('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja),
TestCategory('cuda', 'cuda', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('nvcc')),
TestCategory('python3', 'python3', backend is not Backend.ninja),
TestCategory('python', 'python', backend is not Backend.ninja),
TestCategory('fpga', 'fpga', shutil.which('yosys') is None),
TestCategory('frameworks', 'frameworks'),
TestCategory('nasm', 'nasm'),
TestCategory('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja),
]
names = [t[0] for t in all_tests]
assert names == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests names'
categories = [t.category for t in all_tests]
assert categories == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests categories'
if only:
ind = [names.index(o) for o in only]
all_tests = [all_tests[i] for i in ind]
gathered_tests = [(name, gather_tests(Path('test cases', subdir)), skip) for name, subdir, skip in all_tests]
all_tests = [t for t in all_tests if t.category in only]
gathered_tests = [(t.category, gather_tests(Path('test cases', t.subdir), t.stdout_mandatory), t.skip) for t in all_tests]
return gathered_tests
def run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]],

7
test cases/failing/100 fallback consistency/test.json Normal file
View File

@ -0,0 +1,7 @@
{
"stdout": [
{
"line": "test cases/failing/100 fallback consistency/meson.build:7:0: ERROR: Inconsistency: Subproject has overridden the dependency with another variable than 'dep2'"
}
]
}

7
test cases/failing/101 no native compiler/test.json Normal file
View File

@ -0,0 +1,7 @@
{
"stdout": [
{
"line": "test cases/failing/101 no native compiler/meson.build:12:0: ERROR: No host machine compiler for \"main.c\""
}
]
}

7
test cases/failing/102 subdir parse error/test.json Normal file
View File

@ -0,0 +1,7 @@
{
"stdout": [
{
"line": "test cases/failing/102 subdir parse error/subdir/meson.build:1:0: ERROR: Plusassignment target must be an id."
}
]
}

7
test cases/failing/103 invalid option file/test.json Normal file
View File

@ -0,0 +1,7 @@
{
"stdout": [
{
"line": "test cases/failing/103 invalid option file/meson_options.txt:1:0: ERROR: lexer"
}
]
}

7
test cases/failing/104 no lang/test.json Normal file
View File

@ -0,0 +1,7 @@
{
"stdout": [
{
"line": "test cases/failing/104 no lang/meson.build:2:0: ERROR: No host machine compiler for \"main.c\""
}
]
}

7
test cases/failing/105 no glib-compile-resources/test.json Normal file
View File

@ -0,0 +1,7 @@
{
"stdout": [
{
"line": "test cases/failing/105 no glib-compile-resources/meson.build:8:0: ERROR: Could not execute glib-compile-resources."
}
]
}

7
test cases/failing/36 pkgconfig dependency impossible conditions/test.json Normal file
View File

@ -0,0 +1,7 @@
{
"stdout": [
{
"line": "test cases/failing/36 pkgconfig dependency impossible conditions/meson.build:7:0: ERROR: Dependency 'zlib' was already checked and was not found"
}
]
}

7
test cases/failing/67 subproj different versions/test.json Normal file
View File

@ -0,0 +1,7 @@
{
"stdout": [
{
"line": "test cases/failing/67 subproj different versions/subprojects/b/meson.build:3:0: ERROR: Dependency 'c' was already checked and was not found"
}
]
}

7
test cases/failing/84 gtest dependency with version/test.json Normal file
View File

@ -0,0 +1,7 @@
{
"stdout": [
{
"line": "test cases/failing/84 gtest dependency with version/meson.build:8:0: ERROR: Dependency 'gtest' was already checked and was not found"
}
]
}

7
test cases/failing/98 fallback consistency/test.json Normal file
View File

@ -0,0 +1,7 @@
{
"stdout": [
{
"line": "test cases/failing/98 fallback consistency/meson.build:4:0: ERROR: Inconsistency: Subproject has overridden the dependency with another variable than 'dep2'"
}
]
}