Virtualization
/
meson
mirror of https://gitlab.com/qemu-project/meson.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

mtest: tap: accept out-of-order or partly-numbered tests 

Resolves: #13802
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
This commit is contained in:
Paolo Bonzini 2024-12-09 17:39:12 +01:00 committed by Dylan Baker
parent 303916dd5c
commit 1c8b523c86
2 changed files with 74 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
mesonbuild/mtest.py
View File

@ -326,6 +326,8 @@ class TAPParser:
plan: T.Optional[Plan] = None
lineno = 0
num_tests = 0
last_test = 0
highest_test = 0
yaml_lineno: T.Optional[int] = None
yaml_indent = ''
state = _MAIN
@ -396,10 +398,11 @@ class TAPParser:
yield self.Error('unexpected test after late plan')
self.found_late_test = True
self.num_tests += 1
num = self.num_tests if m.group(2) is None else int(m.group(2))
if num != self.num_tests:
yield self.Error('out of order test numbers')
yield from self.parse_test(m.group(1) == 'ok', num,
self.last_test = self.last_test + 1 if m.group(2) is None else int(m.group(2))
self.highest_test = max(self.highest_test, self.last_test)
if self.plan and self.last_test > self.plan.num_tests:
yield self.Error('test number exceeds maximum specified in test plan')
yield from self.parse_test(m.group(1) == 'ok', self.last_test,
m.group(3), m.group(4), m.group(5))
self.state = self._AFTER_TEST
return
@ -449,11 +452,21 @@ class TAPParser:
if self.state == self._YAML:
yield self.Error(f'YAML block not terminated (started on line {self.yaml_lineno})')
if not self.bailed_out and self.plan and self.num_tests != self.plan.num_tests:
if self.bailed_out:
return
if self.plan and self.num_tests != self.plan.num_tests:
if self.num_tests < self.plan.num_tests:
yield self.Error(f'Too few tests run (expected {self.plan.num_tests}, got {self.num_tests})')
else:
yield self.Error(f'Too many tests run (expected {self.plan.num_tests}, got {self.num_tests})')
return
if self.highest_test != self.num_tests:
if self.highest_test < self.num_tests:
yield self.Error(f'Duplicate test numbers (expected {self.num_tests}, got test numbered {self.highest_test}')
else:
yield self.Error(f'Missing test numbers (expected {self.num_tests}, got test numbered {self.highest_test}')
class TestLogger:
def flush(self) -> None:

62
unittests/taptests.py
View File

@ -163,10 +163,57 @@ class TAPParserTests(unittest.TestCase):
self.assert_plan(events, num_tests=1, late=True)
self.assert_last(events)
def test_out_of_order(self):
events = self.parse_tap('ok 2')
self.assert_error(events)
def test_low_max_early_plan(self):
events = self.parse_tap('1..2\nok 1\nok 1')
self.assert_plan(events, num_tests=2, late=False)
self.assert_test(events, number=1, name='', result=TestResult.OK)
self.assert_test(events, number=1, name='', result=TestResult.OK)
self.assert_error(events) # incorrect high test number
self.assert_last(events)
def test_low_max_late_plan(self):
events = self.parse_tap('ok 1\nok 1\n1..2')
self.assert_test(events, number=1, name='', result=TestResult.OK)
self.assert_test(events, number=1, name='', result=TestResult.OK)
self.assert_plan(events, num_tests=2, late=True)
self.assert_error(events) # incorrect high test number
self.assert_last(events)
def test_high_max_early_plan(self):
events = self.parse_tap('1..2\nok 2\nok 3')
self.assert_plan(events, num_tests=2, late=False)
self.assert_test(events, number=2, name='', result=TestResult.OK)
self.assert_error(events) # high id
self.assert_test(events, number=3, name='', result=TestResult.OK)
self.assert_error(events) # incorrect high test number
self.assert_last(events)
def test_high_max_late_plan(self):
events = self.parse_tap('ok 2\nok 3\n1..2')
self.assert_test(events, number=2, name='', result=TestResult.OK)
self.assert_test(events, number=3, name='', result=TestResult.OK)
self.assert_plan(events, num_tests=2, late=True)
self.assert_error(events)
self.assert_last(events)
def test_out_of_order(self):
events = self.parse_tap('1..2\nok 2\nok 1')
self.assert_plan(events, num_tests=2, late=False)
self.assert_test(events, number=2, name='', result=TestResult.OK)
self.assert_test(events, number=1, name='', result=TestResult.OK)
self.assert_last(events)
def test_out_of_order_no_plan(self):
events = self.parse_tap('ok 2')
self.assert_test(events, number=2, name='', result=TestResult.OK)
self.assert_error(events)
def test_out_of_order_missing_numbers(self):
events = self.parse_tap('1..3\nok 2\nok\nok 1')
self.assert_plan(events, num_tests=3, late=False)
self.assert_test(events, number=2, name='', result=TestResult.OK)
self.assert_test(events, number=3, name='', result=TestResult.OK)
self.assert_test(events, number=1, name='', result=TestResult.OK)
self.assert_last(events)
def test_middle_plan(self):
@ -184,7 +231,7 @@ class TAPParserTests(unittest.TestCase):
self.assert_test(events, number=1, name='', result=TestResult.OK)
self.assert_last(events)
def test_too_many(self):
def test_too_many_late_plan(self):
events = self.parse_tap('ok 1\nnot ok 2\n1..1')
self.assert_test(events, number=1, name='', result=TestResult.OK)
self.assert_test(events, number=2, name='', result=TestResult.FAIL)
@ -192,14 +239,16 @@ class TAPParserTests(unittest.TestCase):
self.assert_error(events)
self.assert_last(events)
def test_too_many_early_plan(self):
events = self.parse_tap('1..1\nok 1\nnot ok 2')
self.assert_plan(events, num_tests=1, late=False)
self.assert_test(events, number=1, name='', result=TestResult.OK)
self.assert_error(events) # test number too high
self.assert_test(events, number=2, name='', result=TestResult.FAIL)
self.assert_error(events)
self.assert_error(events) # too many tests run
self.assert_last(events)
def test_too_few(self):
def test_too_few_late_plan(self):
events = self.parse_tap('ok 1\nnot ok 2\n1..3')
self.assert_test(events, number=1, name='', result=TestResult.OK)
self.assert_test(events, number=2, name='', result=TestResult.FAIL)
@ -207,6 +256,7 @@ class TAPParserTests(unittest.TestCase):
self.assert_error(events)
self.assert_last(events)
def test_too_few_early_plan(self):
events = self.parse_tap('1..3\nok 1\nnot ok 2')
self.assert_plan(events, num_tests=3, late=False)
self.assert_test(events, number=1, name='', result=TestResult.OK)