From ee5e7977e3e6a24806b467010deedc71898c0aa2 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 18 Nov 2020 12:31:46 +0100
Subject: [PATCH 01/20] mtest: allow parsing TAP line by line

This is the first step towards asynchronous parsing of the TAP output.
We will need to call the same code from both a "for" loop (for unit
tests) and an "async for" loop (for mtest itself).  Because the same
function cannot be both a generator and an asynchronous generator, we
need to build both on a common core.  This commit therefore introduces
a parse_line function that "parse" can call in a loop.  All the local
variables of TAPParser.parse move into "self".

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 131 +++++++++++++++++++++++---------------------
 1 file changed, 68 insertions(+), 63 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 9db271e6f..ea73a7cf8 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -206,6 +206,8 @@ class TestResult(enum.Enum):
         return decorator(result_str).get_text(colorize)
 
 
+TYPE_TAPResult = T.Union['TAPParser.Test', 'TAPParser.Error', 'TAPParser.Version', 'TAPParser.Plan', 'TAPParser.Bailout']
+
 class TAPParser:
     Plan = namedtuple('Plan', ['count', 'late', 'skipped', 'explanation'])
     Bailout = namedtuple('Bailout', ['message'])
@@ -225,6 +227,16 @@ class TAPParser:
     _RE_YAML_START = re.compile(r'(\s+)---.*')
     _RE_YAML_END = re.compile(r'\s+\.\.\.\s*')
 
+    found_late_test = False
+    bailed_out = False
+    plan: T.Optional[Plan] = None
+    lineno = 0
+    num_tests = 0
+    yaml_lineno: T.Optional[int] = None
+    yaml_indent = ''
+    state = _MAIN
+    version = 12
+
     def __init__(self, io: T.Iterator[str]):
         self.io = io
 
@@ -246,64 +258,57 @@ class TAPParser:
 
         yield self.Test(num, name, TestResult.OK if ok else TestResult.FAIL, explanation)
 
-    def parse(self) -> T.Generator[T.Union['TAPParser.Test', 'TAPParser.Error', 'TAPParser.Version', 'TAPParser.Plan', 'TAPParser.Bailout'], None, None]:
-        found_late_test = False
-        bailed_out = False
-        plan = None
-        lineno = 0
-        num_tests = 0
-        yaml_lineno = None
-        yaml_indent = ''
-        state = self._MAIN
-        version = 12
-        while True:
-            lineno += 1
-            try:
-                line = next(self.io).rstrip()
-            except StopIteration:
-                break
+    def parse(self) -> T.Iterator[TYPE_TAPResult]:
+        for line in self.io:
+            yield from self.parse_line(line)
+        yield from self.parse_line(None)
+
+    def parse_line(self, line: T.Optional[str]) -> T.Iterator[TYPE_TAPResult]:
+        if line is not None:
+            self.lineno += 1
+            line = line.rstrip()
 
             # YAML blocks are only accepted after a test
-            if state == self._AFTER_TEST:
-                if version >= 13:
+            if self.state == self._AFTER_TEST:
+                if self.version >= 13:
                     m = self._RE_YAML_START.match(line)
                     if m:
-                        state = self._YAML
-                        yaml_lineno = lineno
-                        yaml_indent = m.group(1)
-                        continue
-                state = self._MAIN
+                        self.state = self._YAML
+                        self.yaml_lineno = self.lineno
+                        self.yaml_indent = m.group(1)
+                        return
+                self.state = self._MAIN
 
-            elif state == self._YAML:
+            elif self.state == self._YAML:
                 if self._RE_YAML_END.match(line):
-                    state = self._MAIN
-                    continue
-                if line.startswith(yaml_indent):
-                    continue
-                yield self.Error('YAML block not terminated (started on line {})'.format(yaml_lineno))
-                state = self._MAIN
+                    self.state = self._MAIN
+                    return
+                if line.startswith(self.yaml_indent):
+                    return
+                yield self.Error('YAML block not terminated (started on line {})'.format(self.yaml_lineno))
+                self.state = self._MAIN
 
-            assert state == self._MAIN
+            assert self.state == self._MAIN
             if line.startswith('#'):
-                continue
+                return
 
             m = self._RE_TEST.match(line)
             if m:
-                if plan and plan.late and not found_late_test:
+                if self.plan and self.plan.late and not self.found_late_test:
                     yield self.Error('unexpected test after late plan')
-                    found_late_test = True
-                num_tests += 1
-                num = num_tests if m.group(2) is None else int(m.group(2))
-                if num != num_tests:
+                    self.found_late_test = True
+                self.num_tests += 1
+                num = self.num_tests if m.group(2) is None else int(m.group(2))
+                if num != self.num_tests:
                     yield self.Error('out of order test numbers')
                 yield from self.parse_test(m.group(1) == 'ok', num,
                                            m.group(3), m.group(4), m.group(5))
-                state = self._AFTER_TEST
-                continue
+                self.state = self._AFTER_TEST
+                return
 
             m = self._RE_PLAN.match(line)
             if m:
-                if plan:
+                if self.plan:
                     yield self.Error('more than one plan found')
                 else:
                     count = int(m.group(1))
@@ -315,44 +320,44 @@ class TAPParser:
                             skipped = True
                         else:
                             yield self.Error('invalid directive for plan')
-                    plan = self.Plan(count=count, late=(num_tests > 0),
-                                     skipped=skipped, explanation=m.group(3))
-                    yield plan
-                continue
+                    self.plan = self.Plan(count=count, late=(self.num_tests > 0),
+                                          skipped=skipped, explanation=m.group(3))
+                    yield self.plan
+                return
 
             m = self._RE_BAILOUT.match(line)
             if m:
                 yield self.Bailout(m.group(1))
-                bailed_out = True
-                continue
+                self.bailed_out = True
+                return
 
             m = self._RE_VERSION.match(line)
             if m:
                 # The TAP version is only accepted as the first line
-                if lineno != 1:
+                if self.lineno != 1:
                     yield self.Error('version number must be on the first line')
-                    continue
-                version = int(m.group(1))
-                if version < 13:
+                    return
+                self.version = int(m.group(1))
+                if self.version < 13:
                     yield self.Error('version number should be at least 13')
                 else:
-                    yield self.Version(version=version)
-                continue
+                    yield self.Version(version=self.version)
+                return
 
             if not line:
-                continue
+                return
 
-            yield self.Error('unexpected input at line {}'.format((lineno,)))
-
-        if state == self._YAML:
-            yield self.Error('YAML block not terminated (started on line {})'.format(yaml_lineno))
-
-        if not bailed_out and plan and num_tests != plan.count:
-            if num_tests < plan.count:
-                yield self.Error('Too few tests run (expected {}, got {})'.format(plan.count, num_tests))
-            else:
-                yield self.Error('Too many tests run (expected {}, got {})'.format(plan.count, num_tests))
+            yield self.Error('unexpected input at line {}'.format((self.lineno,)))
+        else:
+            # end of file
+            if self.state == self._YAML:
+                yield self.Error('YAML block not terminated (started on line {})'.format(self.yaml_lineno))
 
+            if not self.bailed_out and self.plan and self.num_tests != self.plan.count:
+                if self.num_tests < self.plan.count:
+                    yield self.Error('Too few tests run (expected {}, got {})'.format(self.plan.count, self.num_tests))
+                else:
+                    yield self.Error('Too many tests run (expected {}, got {})'.format(self.plan.count, self.num_tests))
 
 class TestLogger:
     def flush(self) -> None:

From 304abaf9ee3fd3c2cfd092811f26117d231a6a28 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 18 Nov 2020 13:41:12 +0100
Subject: [PATCH 02/20] mtest: remove argument to the TAPParser constructor

Pass the StringIO object to the parse method instead, because
there will be no T.Iterator[str] to use in the asynchronous
case.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 9 +++------
 run_unittests.py    | 4 ++--
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index ea73a7cf8..8af37f0cf 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -237,9 +237,6 @@ class TAPParser:
     state = _MAIN
     version = 12
 
-    def __init__(self, io: T.Iterator[str]):
-        self.io = io
-
     def parse_test(self, ok: bool, num: int, name: str, directive: T.Optional[str], explanation: T.Optional[str]) -> \
             T.Generator[T.Union['TAPParser.Test', 'TAPParser.Error'], None, None]:
         name = name.strip()
@@ -258,8 +255,8 @@ class TAPParser:
 
         yield self.Test(num, name, TestResult.OK if ok else TestResult.FAIL, explanation)
 
-    def parse(self) -> T.Iterator[TYPE_TAPResult]:
-        for line in self.io:
+    def parse(self, io: T.Iterator[str]) -> T.Iterator[TYPE_TAPResult]:
+        for line in io:
             yield from self.parse_line(line)
         yield from self.parse_line(None)
 
@@ -747,7 +744,7 @@ class TestRun:
         results = {}  # type: T.Dict[str, TestResult]
         failed = False
 
-        for n, i in enumerate(TAPParser(io.StringIO(stdo)).parse()):
+        for n, i in enumerate(TAPParser().parse(io.StringIO(stdo))):
             if isinstance(i, TAPParser.Bailout):
                 results[str(n)] = TestResult.ERROR
                 failed = True
diff --git a/run_unittests.py b/run_unittests.py
index 1ff54eb37..709f019ae 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -8875,8 +8875,8 @@ class TAPParserTests(unittest.TestCase):
             next(events)
 
     def parse_tap(self, s):
-        parser = TAPParser(io.StringIO(s))
-        return iter(parser.parse())
+        parser = TAPParser()
+        return iter(parser.parse(io.StringIO(s)))
 
     def parse_tap_v13(self, s):
         events = self.parse_tap('TAP version 13\n' + s)

From e50861e62f6cf695ba087ab161b3938be7c2b131 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 10:41:03 +0100
Subject: [PATCH 03/20] mtest/TAPParser: use typing.NamedTuple

It is cleaner than collections.namedtuple.  It also catches that "count()" is
a method on tuple, so rename the field to num_tests.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 43 +++++++++++++++++++++++++++++--------------
 run_unittests.py    | 42 +++++++++++++++++++++---------------------
 2 files changed, 50 insertions(+), 35 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 8af37f0cf..22ab8e56d 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -15,7 +15,7 @@
 # A tool to run tests in many different ways.
 
 from pathlib import Path
-from collections import deque, namedtuple
+from collections import deque
 from copy import deepcopy
 import argparse
 import asyncio
@@ -209,11 +209,26 @@ class TestResult(enum.Enum):
 TYPE_TAPResult = T.Union['TAPParser.Test', 'TAPParser.Error', 'TAPParser.Version', 'TAPParser.Plan', 'TAPParser.Bailout']
 
 class TAPParser:
-    Plan = namedtuple('Plan', ['count', 'late', 'skipped', 'explanation'])
-    Bailout = namedtuple('Bailout', ['message'])
-    Test = namedtuple('Test', ['number', 'name', 'result', 'explanation'])
-    Error = namedtuple('Error', ['message'])
-    Version = namedtuple('Version', ['version'])
+    class Plan(T.NamedTuple):
+        num_tests: int
+        late: bool
+        skipped: bool
+        explanation: T.Optional[str]
+
+    class Bailout(T.NamedTuple):
+        message: str
+
+    class Test(T.NamedTuple):
+        number: int
+        name: str
+        result: TestResult
+        explanation: T.Optional[str]
+
+    class Error(T.NamedTuple):
+        message: str
+
+    class Version(T.NamedTuple):
+        version: int
 
     _MAIN = 1
     _AFTER_TEST = 2
@@ -308,16 +323,16 @@ class TAPParser:
                 if self.plan:
                     yield self.Error('more than one plan found')
                 else:
-                    count = int(m.group(1))
-                    skipped = (count == 0)
+                    num_tests = int(m.group(1))
+                    skipped = (num_tests == 0)
                     if m.group(2):
                         if m.group(2).upper().startswith('SKIP'):
-                            if count > 0:
+                            if num_tests > 0:
                                 yield self.Error('invalid SKIP directive for plan')
                             skipped = True
                         else:
                             yield self.Error('invalid directive for plan')
-                    self.plan = self.Plan(count=count, late=(self.num_tests > 0),
+                    self.plan = self.Plan(num_tests=num_tests, late=(self.num_tests > 0),
                                           skipped=skipped, explanation=m.group(3))
                     yield self.plan
                 return
@@ -350,11 +365,11 @@ class TAPParser:
             if self.state == self._YAML:
                 yield self.Error('YAML block not terminated (started on line {})'.format(self.yaml_lineno))
 
-            if not self.bailed_out and self.plan and self.num_tests != self.plan.count:
-                if self.num_tests < self.plan.count:
-                    yield self.Error('Too few tests run (expected {}, got {})'.format(self.plan.count, self.num_tests))
+            if not self.bailed_out and self.plan and self.num_tests != self.plan.num_tests:
+                if self.num_tests < self.plan.num_tests:
+                    yield self.Error('Too few tests run (expected {}, got {})'.format(self.plan.num_tests, self.num_tests))
                 else:
-                    yield self.Error('Too many tests run (expected {}, got {})'.format(self.plan.count, self.num_tests))
+                    yield self.Error('Too many tests run (expected {}, got {})'.format(self.plan.num_tests, self.num_tests))
 
 class TestLogger:
     def flush(self) -> None:
diff --git a/run_unittests.py b/run_unittests.py
index 709f019ae..9569e145e 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -8889,25 +8889,25 @@ class TAPParserTests(unittest.TestCase):
 
     def test_empty_plan(self):
         events = self.parse_tap('1..0')
-        self.assert_plan(events, count=0, late=False, skipped=True)
+        self.assert_plan(events, num_tests=0, late=False, skipped=True)
         self.assert_last(events)
 
     def test_plan_directive(self):
         events = self.parse_tap('1..0 # skipped for some reason')
-        self.assert_plan(events, count=0, late=False, skipped=True,
+        self.assert_plan(events, num_tests=0, late=False, skipped=True,
                          explanation='for some reason')
         self.assert_last(events)
 
         events = self.parse_tap('1..1 # skipped for some reason\nok 1')
         self.assert_error(events)
-        self.assert_plan(events, count=1, late=False, skipped=True,
+        self.assert_plan(events, num_tests=1, late=False, skipped=True,
                          explanation='for some reason')
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)
 
         events = self.parse_tap('1..1 # todo not supported here\nok 1')
         self.assert_error(events)
-        self.assert_plan(events, count=1, late=False, skipped=False,
+        self.assert_plan(events, num_tests=1, late=False, skipped=False,
                          explanation='not supported here')
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)
@@ -8953,7 +8953,7 @@ class TAPParserTests(unittest.TestCase):
 
     def test_many_early_plan(self):
         events = self.parse_tap('1..4\nok 1\nnot ok 2\nok 3\nnot ok 4')
-        self.assert_plan(events, count=4, late=False)
+        self.assert_plan(events, num_tests=4, late=False)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_test(events, number=2, name='', result=TestResult.FAIL)
         self.assert_test(events, number=3, name='', result=TestResult.OK)
@@ -8966,7 +8966,7 @@ class TAPParserTests(unittest.TestCase):
         self.assert_test(events, number=2, name='', result=TestResult.FAIL)
         self.assert_test(events, number=3, name='', result=TestResult.OK)
         self.assert_test(events, number=4, name='', result=TestResult.FAIL)
-        self.assert_plan(events, count=4, late=True)
+        self.assert_plan(events, num_tests=4, late=True)
         self.assert_last(events)
 
     def test_directive_case(self):
@@ -8991,14 +8991,14 @@ class TAPParserTests(unittest.TestCase):
 
     def test_one_test_early_plan(self):
         events = self.parse_tap('1..1\nok')
-        self.assert_plan(events, count=1, late=False)
+        self.assert_plan(events, num_tests=1, late=False)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)
 
     def test_one_test_late_plan(self):
         events = self.parse_tap('ok\n1..1')
         self.assert_test(events, number=1, name='', result=TestResult.OK)
-        self.assert_plan(events, count=1, late=True)
+        self.assert_plan(events, num_tests=1, late=True)
         self.assert_last(events)
 
     def test_out_of_order(self):
@@ -9010,14 +9010,14 @@ class TAPParserTests(unittest.TestCase):
     def test_middle_plan(self):
         events = self.parse_tap('ok 1\n1..2\nok 2')
         self.assert_test(events, number=1, name='', result=TestResult.OK)
-        self.assert_plan(events, count=2, late=True)
+        self.assert_plan(events, num_tests=2, late=True)
         self.assert_error(events)
         self.assert_test(events, number=2, name='', result=TestResult.OK)
         self.assert_last(events)
 
     def test_too_many_plans(self):
         events = self.parse_tap('1..1\n1..2\nok 1')
-        self.assert_plan(events, count=1, late=False)
+        self.assert_plan(events, num_tests=1, late=False)
         self.assert_error(events)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)
@@ -9026,12 +9026,12 @@ class TAPParserTests(unittest.TestCase):
         events = self.parse_tap('ok 1\nnot ok 2\n1..1')
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_test(events, number=2, name='', result=TestResult.FAIL)
-        self.assert_plan(events, count=1, late=True)
+        self.assert_plan(events, num_tests=1, late=True)
         self.assert_error(events)
         self.assert_last(events)
 
         events = self.parse_tap('1..1\nok 1\nnot ok 2')
-        self.assert_plan(events, count=1, late=False)
+        self.assert_plan(events, num_tests=1, late=False)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_test(events, number=2, name='', result=TestResult.FAIL)
         self.assert_error(events)
@@ -9041,12 +9041,12 @@ class TAPParserTests(unittest.TestCase):
         events = self.parse_tap('ok 1\nnot ok 2\n1..3')
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_test(events, number=2, name='', result=TestResult.FAIL)
-        self.assert_plan(events, count=3, late=True)
+        self.assert_plan(events, num_tests=3, late=True)
         self.assert_error(events)
         self.assert_last(events)
 
         events = self.parse_tap('1..3\nok 1\nnot ok 2')
-        self.assert_plan(events, count=3, late=False)
+        self.assert_plan(events, num_tests=3, late=False)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_test(events, number=2, name='', result=TestResult.FAIL)
         self.assert_error(events)
@@ -9054,7 +9054,7 @@ class TAPParserTests(unittest.TestCase):
 
     def test_too_few_bailout(self):
         events = self.parse_tap('1..3\nok 1\nnot ok 2\nBail out! no third test')
-        self.assert_plan(events, count=3, late=False)
+        self.assert_plan(events, num_tests=3, late=False)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_test(events, number=2, name='', result=TestResult.FAIL)
         self.assert_bailout(events, message='no third test')
@@ -9062,29 +9062,29 @@ class TAPParserTests(unittest.TestCase):
 
     def test_diagnostics(self):
         events = self.parse_tap('1..1\n# ignored\nok 1')
-        self.assert_plan(events, count=1, late=False)
+        self.assert_plan(events, num_tests=1, late=False)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)
 
         events = self.parse_tap('# ignored\n1..1\nok 1\n# ignored too')
-        self.assert_plan(events, count=1, late=False)
+        self.assert_plan(events, num_tests=1, late=False)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)
 
         events = self.parse_tap('# ignored\nok 1\n1..1\n# ignored too')
         self.assert_test(events, number=1, name='', result=TestResult.OK)
-        self.assert_plan(events, count=1, late=True)
+        self.assert_plan(events, num_tests=1, late=True)
         self.assert_last(events)
 
     def test_empty_line(self):
         events = self.parse_tap('1..1\n\nok 1')
-        self.assert_plan(events, count=1, late=False)
+        self.assert_plan(events, num_tests=1, late=False)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)
 
     def test_unexpected(self):
         events = self.parse_tap('1..1\ninvalid\nok 1')
-        self.assert_plan(events, count=1, late=False)
+        self.assert_plan(events, num_tests=1, late=False)
         self.assert_error(events)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)
@@ -9099,7 +9099,7 @@ class TAPParserTests(unittest.TestCase):
         self.assert_last(events)
 
         events = self.parse_tap('1..0\nTAP version 13\n')
-        self.assert_plan(events, count=0, late=False, skipped=True)
+        self.assert_plan(events, num_tests=0, late=False, skipped=True)
         self.assert_error(events)
         self.assert_last(events)
 

From 723c4c9fefade3a806e8ece73126c8c0e21477b2 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Thu, 19 Nov 2020 09:59:06 +0100
Subject: [PATCH 04/20] mtest: extract TAP parsing out of TestRun.make_tap

For now this is just a refactoring that simplifies the next patch.  However,
it will also come in handy when we will make the parsing asynchronous, because
it will make it possible to access subtest results while the test runs.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 97 +++++++++++++++++++++++++--------------------
 1 file changed, 53 insertions(+), 44 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 22ab8e56d..fe9104048 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -195,6 +195,9 @@ class TestResult(enum.Enum):
         return self in {TestResult.FAIL, TestResult.TIMEOUT, TestResult.INTERRUPT,
                         TestResult.UNEXPECTEDPASS, TestResult.ERROR}
 
+    def was_killed(self) -> bool:
+        return self in (TestResult.TIMEOUT, TestResult.INTERRUPT)
+
     def get_text(self, colorize: bool) -> str:
         result_str = '{res:{reslen}}'.format(res=self.value, reslen=self.maxlen())
         if self.is_bad():
@@ -752,48 +755,46 @@ class TestRun:
             res = TestResult.EXPECTEDFAIL if bool(returncode) else TestResult.UNEXPECTEDPASS
         else:
             res = TestResult.FAIL if bool(returncode) else TestResult.OK
-        self.complete(res, {}, returncode, stdo, stde, cmd, **kwargs)
+        self.complete(res, returncode, stdo, stde, cmd, **kwargs)
 
-    def complete_tap(self, returncode: int, stdo: str, stde: str, cmd: T.List[str]) -> None:
+    def parse_tap(self, lines: T.Iterator[str]) -> T.Tuple[TestResult, str]:
         res = None    # type: T.Optional[TestResult]
-        results = {}  # type: T.Dict[str, TestResult]
-        failed = False
+        error = ''
 
-        for n, i in enumerate(TAPParser().parse(io.StringIO(stdo))):
+        for n, i in enumerate(TAPParser().parse(lines)):
             if isinstance(i, TAPParser.Bailout):
-                results[str(n)] = TestResult.ERROR
-                failed = True
+                self.results[str(n)] = i.result
+                res = TestResult.ERROR
             elif isinstance(i, TAPParser.Test):
-                results[str(n)] = i.result
+                self.results[str(n)] = i.result
                 if i.result not in {TestResult.OK, TestResult.EXPECTEDFAIL, TestResult.SKIP}:
-                    failed = True
+                    res = TestResult.FAIL
             elif isinstance(i, TAPParser.Error):
-                results[str(n)] = TestResult.ERROR
-                stde += '\nTAP parsing error: ' + i.message
-                failed = True
+                self.results[str(n)] = TestResult.ERROR
+                error += '\nTAP parsing error: ' + i.message
+                res = TestResult.ERROR
 
-        if returncode != 0:
+        if all(t is TestResult.SKIP for t in self.results):
+            # This includes the case where self.results is empty
+            res = TestResult.SKIP
+        return res or TestResult.OK, error
+
+    def complete_tap(self, returncode: int, res: TestResult,
+                     stdo: str, stde: str, cmd: T.List[str]) -> None:
+        if self.should_fail and res in (TestResult.OK, TestResult.FAIL):
+            res = TestResult.UNEXPECTEDPASS if res.is_ok() else TestResult.EXPECTEDFAIL
+        if returncode != 0 and not res.was_killed():
             res = TestResult.ERROR
             stde += '\n(test program exited with status code {})'.format(returncode,)
 
-        if res is None:
-            # Now determine the overall result of the test based on the outcome of the subcases
-            if all(t is TestResult.SKIP for t in results.values()):
-                # This includes the case where num_tests is zero
-                res = TestResult.SKIP
-            elif self.should_fail:
-                res = TestResult.EXPECTEDFAIL if failed else TestResult.UNEXPECTEDPASS
-            else:
-                res = TestResult.FAIL if failed else TestResult.OK
-
-        self.complete(res, results, returncode, stdo, stde, cmd)
+        self.complete(res, returncode, stdo, stde, cmd)
 
     def complete_rust(self, returncode: int, stdo: str, stde: str, cmd: T.List[str]) -> None:
-        results = parse_rust_test(stdo)
+        self.results = parse_rust_test(stdo)
 
-        failed = TestResult.FAIL in results.values()
+        failed = TestResult.FAIL in self.results.values()
         # Now determine the overall result of the test based on the outcome of the subcases
-        if all(t is TestResult.SKIP for t in results.values()):
+        if all(t is TestResult.SKIP for t in self.results.values()):
             # This includes the case where num_tests is zero
             res = TestResult.SKIP
         elif self.should_fail:
@@ -801,7 +802,7 @@ class TestRun:
         else:
             res = TestResult.FAIL if failed else TestResult.OK
 
-        self.complete(res, results, returncode, stdo, stde, cmd)
+        self.complete(res, returncode, stdo, stde, cmd)
 
     @property
     def num(self) -> int:
@@ -810,13 +811,11 @@ class TestRun:
             self._num = TestRun.TEST_NUM
         return self._num
 
-    def complete(self, res: TestResult, results: T.Dict[str, TestResult],
-                 returncode: int,
+    def complete(self, res: TestResult, returncode: int,
                  stdo: T.Optional[str], stde: T.Optional[str],
                  cmd: T.List[str], *, junit: T.Optional[et.ElementTree] = None) -> None:
         assert isinstance(res, TestResult)
         self.res = res
-        self.results = results  # May be empty
         self.returncode = returncode
         self.duration = time.time() - self.starttime
         self.stdo = stdo
@@ -960,7 +959,7 @@ class SingleTestRunner:
         self.runobj.start()
         if cmd is None:
             skip_stdout = 'Not run because can not execute cross compiled binaries.'
-            self.runobj.complete(TestResult.SKIP, {}, GNU_SKIP_RETURNCODE, skip_stdout, None, None)
+            self.runobj.complete(TestResult.SKIP, GNU_SKIP_RETURNCODE, skip_stdout, None, None)
         else:
             wrap = TestHarness.get_wrapper(self.options)
             if self.options.gdb:
@@ -1116,19 +1115,29 @@ class SingleTestRunner:
         else:
             stdo = ""
             stde = additional_error
-        if result:
-            self.runobj.complete(result, {}, returncode, stdo, stde, cmd)
-        else:
-            if self.test.protocol is TestProtocol.EXITCODE:
-                self.runobj.complete_exitcode(returncode, stdo, stde, cmd)
-            elif self.test.protocol is TestProtocol.GTEST:
-                self.runobj.complete_gtest(returncode, stdo, stde, cmd)
-            elif self.test.protocol is TestProtocol.RUST:
-                return self.runobj.complete_rust(returncode, stdo, stde, cmd)
-            else:
+
+        # Print lines along the way if requested
+        def lines() -> T.Iterator[str]:
+            for line in io.StringIO(stdo):
                 if self.options.verbose:
-                    print(stdo, end='')
-                self.runobj.complete_tap(returncode, stdo, stde, cmd)
+                    print(line, end='')
+                yield line
+
+        if self.test.protocol is TestProtocol.TAP:
+            res, error = self.runobj.parse_tap(lines())
+            if error:
+                stde += '\n' + error
+            self.runobj.complete_tap(returncode, result or res, stdo, stde, cmd)
+
+        elif result:
+            self.runobj.complete(result, returncode, stdo, stde, cmd)
+        elif self.test.protocol is TestProtocol.EXITCODE:
+            self.runobj.complete_exitcode(returncode, stdo, stde, cmd)
+        elif self.test.protocol is TestProtocol.RUST:
+            return self.runobj.complete_rust(returncode, stdo, stde, cmd)
+        elif self.test.protocol is TestProtocol.GTEST:
+            self.runobj.complete_gtest(returncode, stdo, stde, cmd)
+
 
 class TestHarness:
     def __init__(self, options: argparse.Namespace):

From 35d3baaa2fab0feb9e66a7603bb1dfa2178f2de4 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 11:50:22 +0100
Subject: [PATCH 05/20] mtest: improvements to JUnit XML generation

Omit the classname attribute, as it is optional, and add the duration.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index fe9104048..d59bb3f1e 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -620,12 +620,13 @@ class JunitBuilder(TestLogger):
                 failures=str(sum(1 for r in test.results.values() if r in
                                  {TestResult.FAIL, TestResult.UNEXPECTEDPASS, TestResult.TIMEOUT})),
                 skipped=str(sum(1 for r in test.results.values() if r is TestResult.SKIP)),
+                time=str(test.duration),
             )
 
-            for i, result in test.results.items():
-                # Both name and classname are required. Set them both to the
-                # number of the test in a TAP test, as TAP doesn't give names.
-                testcase = et.SubElement(suite, 'testcase', name=i, classname=i)
+            for i, result in enumerate(test.results):
+                # Set the name to the number of the test in a TAP test, as we cannot
+                # access the name yet.
+                testcase = et.SubElement(suite, 'testcase', name=str(i))
                 if result is TestResult.SKIP:
                     et.SubElement(testcase, 'skipped')
                 elif result is TestResult.ERROR:
@@ -651,12 +652,13 @@ class JunitBuilder(TestLogger):
             if test.project not in self.suites:
                 suite = self.suites[test.project] = et.Element(
                     'testsuite', name=test.project, tests='1', errors='0',
-                    failures='0', skipped='0')
+                    failures='0', skipped='0', time=str(test.duration))
             else:
                 suite = self.suites[test.project]
                 suite.attrib['tests'] = str(int(suite.attrib['tests']) + 1)
 
-            testcase = et.SubElement(suite, 'testcase', name=test.name, classname=test.name)
+            testcase = et.SubElement(suite, 'testcase', name=test.name,
+                                     time=str(test.duration))
             if test.res is TestResult.SKIP:
                 et.SubElement(testcase, 'skipped')
                 suite.attrib['skipped'] = str(int(suite.attrib['skipped']) + 1)

From c48713e58c5a56d23d09afa3c5b7c2fc5ba7effa Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 11:50:22 +0100
Subject: [PATCH 06/20] mtest: improve JUnit XML generation for TAP testsuites

Include the names from the TAP output and the SKIP/TODO explanations
if present.  Omit the classname attribute, it is optional.

In order to enable this, TestRun.results becomes a list of TAPParser.Test
objects.  If in the future there are other kinds of subtest results a
new class can be introduced, but for now it is enough.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 94 ++++++++++++++++++++++++---------------------
 1 file changed, 50 insertions(+), 44 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index d59bb3f1e..3bedb95d2 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -227,6 +227,9 @@ class TAPParser:
         result: TestResult
         explanation: T.Optional[str]
 
+        def __str__(self) -> str:
+            return '{} {}'.format(self.number, self.name).strip()
+
     class Error(T.NamedTuple):
         message: str
 
@@ -615,33 +618,33 @@ class JunitBuilder(TestLogger):
                 'testsuite',
                 name=suitename,
                 tests=str(len(test.results)),
-                errors=str(sum(1 for r in test.results.values() if r in
+                errors=str(sum(1 for r in test.results if r.result in
                                {TestResult.INTERRUPT, TestResult.ERROR})),
-                failures=str(sum(1 for r in test.results.values() if r in
+                failures=str(sum(1 for r in test.results if r.result in
                                  {TestResult.FAIL, TestResult.UNEXPECTEDPASS, TestResult.TIMEOUT})),
-                skipped=str(sum(1 for r in test.results.values() if r is TestResult.SKIP)),
+                skipped=str(sum(1 for r in test.results if r.result is TestResult.SKIP)),
                 time=str(test.duration),
             )
 
-            for i, result in enumerate(test.results):
-                # Set the name to the number of the test in a TAP test, as we cannot
-                # access the name yet.
-                testcase = et.SubElement(suite, 'testcase', name=str(i))
-                if result is TestResult.SKIP:
+            for subtest in test.results:
+                testcase = et.SubElement(suite, 'testcase', name=str(subtest))
+                if subtest.result is TestResult.SKIP:
                     et.SubElement(testcase, 'skipped')
-                elif result is TestResult.ERROR:
+                elif subtest.result is TestResult.ERROR:
                     et.SubElement(testcase, 'error')
-                elif result is TestResult.FAIL:
+                elif subtest.result is TestResult.FAIL:
                     et.SubElement(testcase, 'failure')
-                elif result is TestResult.UNEXPECTEDPASS:
+                elif subtest.result is TestResult.UNEXPECTEDPASS:
                     fail = et.SubElement(testcase, 'failure')
                     fail.text = 'Test unexpected passed.'
-                elif result is TestResult.INTERRUPT:
-                    fail = et.SubElement(testcase, 'failure')
+                elif subtest.result is TestResult.INTERRUPT:
+                    fail = et.SubElement(testcase, 'error')
                     fail.text = 'Test was interrupted by user.'
-                elif result is TestResult.TIMEOUT:
-                    fail = et.SubElement(testcase, 'failure')
+                elif subtest.result is TestResult.TIMEOUT:
+                    fail = et.SubElement(testcase, 'error')
                     fail.text = 'Test did not finish before configured timeout.'
+                if subtest.explanation:
+                    et.SubElement(testcase, 'system-out').text = subtest.explanation
             if test.stdo:
                 out = et.SubElement(suite, 'system-out')
                 out.text = test.stdo.rstrip()
@@ -688,26 +691,36 @@ class JunitBuilder(TestLogger):
             tree.write(f, encoding='utf-8', xml_declaration=True)
 
 
-def parse_rust_test(stdout: str) -> T.Dict[str, TestResult]:
+def parse_rust_test(stdout: str) -> T.Tuple[T.List[TAPParser.Test], TestResult]:
     """Parse the output of rust tests."""
-    res = {}  # type; T.Dict[str, TestResult]
+    results = []          # type: T.List[TAPParser.Test]
 
-    def parse_res(res: str) -> TestResult:
-        if res == 'ok':
-            return TestResult.OK
-        elif res == 'ignored':
-            return TestResult.SKIP
-        elif res == 'FAILED':
-            return TestResult.FAIL
-        raise MesonException('Unsupported output from rust test: {}'.format(res))
+    def parse_res(n: int, name: str, result: str) -> TAPParser.Test:
+        if result == 'ok':
+            return TAPParser.Test(n, name, TestResult.OK, None)
+        elif result == 'ignored':
+            return TAPParser.Test(n, name, TestResult.SKIP, None)
+        elif result == 'FAILED':
+            return TAPParser.Test(n, name, TestResult.FAIL, None)
+        return TAPParser.Test(n, name, TestResult.ERROR,
+                              'Unsupported output from rust test: {}'.format(result))
 
+    n = 1
     for line in stdout.splitlines():
         if line.startswith('test ') and not line.startswith('test result'):
             _, name, _, result = line.split(' ')
             name = name.replace('::', '.')
-            res[name] = parse_res(result)
+            results.append(parse_res(n, name, result))
+            n += 1
 
-    return res
+    if all(t.result is TestResult.SKIP for t in results):
+        # This includes the case where results is empty
+        return results, TestResult.SKIP
+    elif any(t.result is TestResult.ERROR for t in results):
+        return results, TestResult.ERROR
+    elif any(t.result is TestResult.FAIL for t in results):
+        return results, TestResult.FAIL
+    return results, TestResult.OK
 
 
 class TestRun:
@@ -719,7 +732,7 @@ class TestRun:
         self.test = test
         self._num = None       # type: T.Optional[int]
         self.name = name
-        self.results: T.Dict[str, TestResult] = {}
+        self.results = list()  # type: T.List[TAPParser.Test]
         self.returncode = 0
         self.starttime = None  # type: T.Optional[float]
         self.duration = None   # type: T.Optional[float]
@@ -763,20 +776,18 @@ class TestRun:
         res = None    # type: T.Optional[TestResult]
         error = ''
 
-        for n, i in enumerate(TAPParser().parse(lines)):
+        for i in TAPParser().parse(lines):
             if isinstance(i, TAPParser.Bailout):
-                self.results[str(n)] = i.result
                 res = TestResult.ERROR
             elif isinstance(i, TAPParser.Test):
-                self.results[str(n)] = i.result
-                if i.result not in {TestResult.OK, TestResult.EXPECTEDFAIL, TestResult.SKIP}:
+                self.results.append(i)
+                if i.result.is_bad():
                     res = TestResult.FAIL
             elif isinstance(i, TAPParser.Error):
-                self.results[str(n)] = TestResult.ERROR
-                error += '\nTAP parsing error: ' + i.message
+                error = '\nTAP parsing error: ' + i.message
                 res = TestResult.ERROR
 
-        if all(t is TestResult.SKIP for t in self.results):
+        if all(t.result is TestResult.SKIP for t in self.results):
             # This includes the case where self.results is empty
             res = TestResult.SKIP
         return res or TestResult.OK, error
@@ -792,17 +803,12 @@ class TestRun:
         self.complete(res, returncode, stdo, stde, cmd)
 
     def complete_rust(self, returncode: int, stdo: str, stde: str, cmd: T.List[str]) -> None:
-        self.results = parse_rust_test(stdo)
+        self.results, result = parse_rust_test(stdo)
 
-        failed = TestResult.FAIL in self.results.values()
-        # Now determine the overall result of the test based on the outcome of the subcases
-        if all(t is TestResult.SKIP for t in self.results.values()):
-            # This includes the case where num_tests is zero
-            res = TestResult.SKIP
-        elif self.should_fail:
-            res = TestResult.EXPECTEDFAIL if failed else TestResult.UNEXPECTEDPASS
+        if self.should_fail:
+            res = TestResult.EXPECTEDFAIL if result is TestResult.FAIL else TestResult.UNEXPECTEDPASS
         else:
-            res = TestResult.FAIL if failed else TestResult.OK
+            res = result
 
         self.complete(res, returncode, stdo, stde, cmd)
 

From d2da7565fd9454bb71077975c7578e875d2f1c56 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 6 Jan 2021 09:07:58 +0100
Subject: [PATCH 07/20] mtest: reorder arguments to "complete"

Put them in the same order as complete_*.
---
 mesonbuild/mtest.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 3bedb95d2..f3879e26a 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -770,7 +770,7 @@ class TestRun:
             res = TestResult.EXPECTEDFAIL if bool(returncode) else TestResult.UNEXPECTEDPASS
         else:
             res = TestResult.FAIL if bool(returncode) else TestResult.OK
-        self.complete(res, returncode, stdo, stde, cmd, **kwargs)
+        self.complete(returncode, res, stdo, stde, cmd, **kwargs)
 
     def parse_tap(self, lines: T.Iterator[str]) -> T.Tuple[TestResult, str]:
         res = None    # type: T.Optional[TestResult]
@@ -800,7 +800,7 @@ class TestRun:
             res = TestResult.ERROR
             stde += '\n(test program exited with status code {})'.format(returncode,)
 
-        self.complete(res, returncode, stdo, stde, cmd)
+        self.complete(returncode, res, stdo, stde, cmd)
 
     def complete_rust(self, returncode: int, stdo: str, stde: str, cmd: T.List[str]) -> None:
         self.results, result = parse_rust_test(stdo)
@@ -810,7 +810,7 @@ class TestRun:
         else:
             res = result
 
-        self.complete(res, returncode, stdo, stde, cmd)
+        self.complete(returncode, res, stdo, stde, cmd)
 
     @property
     def num(self) -> int:
@@ -819,7 +819,7 @@ class TestRun:
             self._num = TestRun.TEST_NUM
         return self._num
 
-    def complete(self, res: TestResult, returncode: int,
+    def complete(self, returncode: int, res: TestResult,
                  stdo: T.Optional[str], stde: T.Optional[str],
                  cmd: T.List[str], *, junit: T.Optional[et.ElementTree] = None) -> None:
         assert isinstance(res, TestResult)
@@ -967,7 +967,7 @@ class SingleTestRunner:
         self.runobj.start()
         if cmd is None:
             skip_stdout = 'Not run because can not execute cross compiled binaries.'
-            self.runobj.complete(TestResult.SKIP, GNU_SKIP_RETURNCODE, skip_stdout, None, None)
+            self.runobj.complete(GNU_SKIP_RETURNCODE, TestResult.SKIP, skip_stdout, None, None)
         else:
             wrap = TestHarness.get_wrapper(self.options)
             if self.options.gdb:
@@ -1138,7 +1138,7 @@ class SingleTestRunner:
             self.runobj.complete_tap(returncode, result or res, stdo, stde, cmd)
 
         elif result:
-            self.runobj.complete(result, returncode, stdo, stde, cmd)
+            self.runobj.complete(returncode, result, stdo, stde, cmd)
         elif self.test.protocol is TestProtocol.EXITCODE:
             self.runobj.complete_exitcode(returncode, stdo, stde, cmd)
         elif self.test.protocol is TestProtocol.RUST:

From 63e26ba05fe9f367b8e83800ebd22ccb06204eda Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 6 Jan 2021 09:11:12 +0100
Subject: [PATCH 08/20] mtest: handle should_fail in common code

This is common to all protocols, place the code in a single
place.
---
 mesonbuild/mtest.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index f3879e26a..454646d8a 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -766,8 +766,6 @@ class TestRun:
             res = TestResult.SKIP
         elif returncode == GNU_ERROR_RETURNCODE:
             res = TestResult.ERROR
-        elif self.should_fail:
-            res = TestResult.EXPECTEDFAIL if bool(returncode) else TestResult.UNEXPECTEDPASS
         else:
             res = TestResult.FAIL if bool(returncode) else TestResult.OK
         self.complete(returncode, res, stdo, stde, cmd, **kwargs)
@@ -794,8 +792,6 @@ class TestRun:
 
     def complete_tap(self, returncode: int, res: TestResult,
                      stdo: str, stde: str, cmd: T.List[str]) -> None:
-        if self.should_fail and res in (TestResult.OK, TestResult.FAIL):
-            res = TestResult.UNEXPECTEDPASS if res.is_ok() else TestResult.EXPECTEDFAIL
         if returncode != 0 and not res.was_killed():
             res = TestResult.ERROR
             stde += '\n(test program exited with status code {})'.format(returncode,)
@@ -803,12 +799,7 @@ class TestRun:
         self.complete(returncode, res, stdo, stde, cmd)
 
     def complete_rust(self, returncode: int, stdo: str, stde: str, cmd: T.List[str]) -> None:
-        self.results, result = parse_rust_test(stdo)
-
-        if self.should_fail:
-            res = TestResult.EXPECTEDFAIL if result is TestResult.FAIL else TestResult.UNEXPECTEDPASS
-        else:
-            res = result
+        self.results, res = parse_rust_test(stdo)
 
         self.complete(returncode, res, stdo, stde, cmd)
 
@@ -823,6 +814,9 @@ class TestRun:
                  stdo: T.Optional[str], stde: T.Optional[str],
                  cmd: T.List[str], *, junit: T.Optional[et.ElementTree] = None) -> None:
         assert isinstance(res, TestResult)
+        if self.should_fail and res in (TestResult.OK, TestResult.FAIL):
+            res = TestResult.UNEXPECTEDPASS if res.is_ok() else TestResult.EXPECTEDFAIL
+
         self.res = res
         self.returncode = returncode
         self.duration = time.time() - self.starttime

From 0ccc70ae1bb85095c5d7313d68bbda5ddb0d1530 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 13:19:55 +0100
Subject: [PATCH 09/20] mtest: do not wait inside _run_subprocess

We would like SingleTestRunner to run code before waiting on the process,
for example starting tasks to read stdout and stderr.

Return a new object that is able to complete _run_subprocess's task.
In the next patch, SingleTestRunner will also use the object to get hold
of the stdout and stderr StreamReaders.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 147 ++++++++++++++++++++++++--------------------
 1 file changed, 81 insertions(+), 66 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 454646d8a..e2c2c565d 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -924,6 +924,69 @@ async def complete_all(futures: T.Iterable[asyncio.Future]) -> None:
             if not f.cancelled():
                 f.result()
 
+class TestSubprocess:
+    def __init__(self, p: asyncio.subprocess.Process, postwait_fn: T.Callable[[], None] = None):
+        self._process = p
+        self.postwait_fn = postwait_fn   # type: T.Callable[[], None]
+
+    async def _kill(self) -> T.Optional[str]:
+        # Python does not provide multiplatform support for
+        # killing a process and all its children so we need
+        # to roll our own.
+        p = self._process
+        try:
+            if is_windows():
+                subprocess.run(['taskkill', '/F', '/T', '/PID', str(p.pid)])
+            else:
+                # Send a termination signal to the process group that setsid()
+                # created - giving it a chance to perform any cleanup.
+                os.killpg(p.pid, signal.SIGTERM)
+
+                # Make sure the termination signal actually kills the process
+                # group, otherwise retry with a SIGKILL.
+                await try_wait_one(p.wait(), timeout=0.5)
+                if p.returncode is not None:
+                    return None
+
+                os.killpg(p.pid, signal.SIGKILL)
+
+            await try_wait_one(p.wait(), timeout=1)
+            if p.returncode is not None:
+                return None
+
+            # An earlier kill attempt has not worked for whatever reason.
+            # Try to kill it one last time with a direct call.
+            # If the process has spawned children, they will remain around.
+            p.kill()
+            await try_wait_one(p.wait(), timeout=1)
+            if p.returncode is not None:
+                return None
+            return 'Test process could not be killed.'
+        except ProcessLookupError:
+            # Sometimes (e.g. with Wine) this happens.  There's nothing
+            # we can do, probably the process already died so just wait
+            # for the event loop to pick that up.
+            await p.wait()
+            return None
+
+    async def wait(self, timeout: T.Optional[int]) -> T.Tuple[int, TestResult, T.Optional[str]]:
+        p = self._process
+        result = None
+        additional_error = None
+        try:
+            await try_wait_one(p.wait(), timeout=timeout)
+            if p.returncode is None:
+                additional_error = await self._kill()
+                result = TestResult.TIMEOUT
+        except asyncio.CancelledError:
+            # The main loop must have seen Ctrl-C.
+            additional_error = await self._kill()
+            result = TestResult.INTERRUPT
+        finally:
+            if self.postwait_fn:
+                self.postwait_fn()
+
+        return p.returncode or 0, result, additional_error
 
 class SingleTestRunner:
 
@@ -969,48 +1032,9 @@ class SingleTestRunner:
             await self._run_cmd(wrap + cmd + self.test.cmd_args + self.options.test_args)
         return self.runobj
 
-    async def _run_subprocess(self, args: T.List[str], *, timeout: T.Optional[int],
+    async def _run_subprocess(self, args: T.List[str], *,
                               stdout: T.IO, stderr: T.IO,
-                              env: T.Dict[str, str], cwd: T.Optional[str]) -> T.Tuple[int, TestResult, T.Optional[str]]:
-        async def kill_process(p: asyncio.subprocess.Process) -> T.Optional[str]:
-            # Python does not provide multiplatform support for
-            # killing a process and all its children so we need
-            # to roll our own.
-            try:
-                if is_windows():
-                    subprocess.run(['taskkill', '/F', '/T', '/PID', str(p.pid)])
-                else:
-                    # Send a termination signal to the process group that setsid()
-                    # created - giving it a chance to perform any cleanup.
-                    os.killpg(p.pid, signal.SIGTERM)
-
-                    # Make sure the termination signal actually kills the process
-                    # group, otherwise retry with a SIGKILL.
-                    await try_wait_one(p.wait(), timeout=0.5)
-                    if p.returncode is not None:
-                        return None
-
-                    os.killpg(p.pid, signal.SIGKILL)
-
-                await try_wait_one(p.wait(), timeout=1)
-                if p.returncode is not None:
-                    return None
-
-                # An earlier kill attempt has not worked for whatever reason.
-                # Try to kill it one last time with a direct call.
-                # If the process has spawned children, they will remain around.
-                p.kill()
-                await try_wait_one(p.wait(), timeout=1)
-                if p.returncode is not None:
-                    return None
-                return 'Test process could not be killed.'
-            except ProcessLookupError:
-                # Sometimes (e.g. with Wine) this happens.  There's nothing
-                # we can do, probably the process already died so just wait
-                # for the event loop to pick that up.
-                await p.wait()
-                return None
-
+                              env: T.Dict[str, str], cwd: T.Optional[str]) -> TestSubprocess:
         # Let gdb handle ^C instead of us
         if self.options.gdb:
             previous_sigint_handler = signal.getsignal(signal.SIGINT)
@@ -1028,31 +1052,18 @@ class SingleTestRunner:
                 # errors avoid not being able to use the terminal.
                 os.setsid()
 
+        def postwait_fn() -> None:
+            if self.options.gdb:
+                # Let us accept ^C again
+                signal.signal(signal.SIGINT, previous_sigint_handler)
+
         p = await asyncio.create_subprocess_exec(*args,
                                                  stdout=stdout,
                                                  stderr=stderr,
                                                  env=env,
                                                  cwd=cwd,
                                                  preexec_fn=preexec_fn if not is_windows() else None)
-        result = None
-        additional_error = None
-        try:
-            await try_wait_one(p.wait(), timeout=timeout)
-            if p.returncode is None:
-                if self.options.verbose:
-                    print('{} time out (After {} seconds)'.format(self.test.name, timeout))
-                additional_error = await kill_process(p)
-                result = TestResult.TIMEOUT
-        except asyncio.CancelledError:
-            # The main loop must have seen Ctrl-C.
-            additional_error = await kill_process(p)
-            result = TestResult.INTERRUPT
-        finally:
-            if self.options.gdb:
-                # Let us accept ^C again
-                signal.signal(signal.SIGINT, previous_sigint_handler)
-
-        return p.returncode or 0, result, additional_error
+        return TestSubprocess(p, postwait_fn=postwait_fn if not is_windows() else None)
 
     async def _run_cmd(self, cmd: T.List[str]) -> None:
         if self.test.extra_paths:
@@ -1097,12 +1108,16 @@ class SingleTestRunner:
         else:
             timeout = self.test.timeout
 
-        returncode, result, additional_error = await self._run_subprocess(cmd + extra_cmd,
-                                                                          timeout=timeout,
-                                                                          stdout=stdout,
-                                                                          stderr=stderr,
-                                                                          env=self.env,
-                                                                          cwd=self.test.workdir)
+        p = await self._run_subprocess(cmd + extra_cmd,
+                                       stdout=stdout,
+                                       stderr=stderr,
+                                       env=self.env,
+                                       cwd=self.test.workdir)
+
+        returncode, result, additional_error = await p.wait(timeout)
+        if result is TestResult.TIMEOUT and self.options.verbose:
+            print('{} time out (After {} seconds)'.format(self.test.name, timeout))
+
         if additional_error is None:
             if stdout is None:
                 stdo = ''

From 755412b5261c448025d40214e3f2c0197e2e27bc Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 14:47:44 +0100
Subject: [PATCH 10/20] mtest: read test stdout/stderr via asyncio pipes

Instead of creating temporary files, get the StreamReaders from
_run_subprocess's returned object. Through asyncio magic, their
contents will be read as it becomes ready and then returned when
the StreamReader.read future is awaited.

Because of this change, the stdout and stderr can be easily
preserved when TestSubprocess returns an additional_error.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 50 ++++++++++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 21 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index e2c2c565d..e1782b35a 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -32,7 +32,6 @@ import re
 import signal
 import subprocess
 import sys
-import tempfile
 import textwrap
 import time
 import typing as T
@@ -929,6 +928,14 @@ class TestSubprocess:
         self._process = p
         self.postwait_fn = postwait_fn   # type: T.Callable[[], None]
 
+    @property
+    def stdout(self) -> T.Optional[asyncio.StreamReader]:
+        return self._process.stdout
+
+    @property
+    def stderr(self) -> T.Optional[asyncio.StreamReader]:
+        return self._process.stderr
+
     async def _kill(self) -> T.Optional[str]:
         # Python does not provide multiplatform support for
         # killing a process and all its children so we need
@@ -1033,7 +1040,7 @@ class SingleTestRunner:
         return self.runobj
 
     async def _run_subprocess(self, args: T.List[str], *,
-                              stdout: T.IO, stderr: T.IO,
+                              stdout: int, stderr: int,
                               env: T.Dict[str, str], cwd: T.Optional[str]) -> TestSubprocess:
         # Let gdb handle ^C instead of us
         if self.options.gdb:
@@ -1088,11 +1095,12 @@ class SingleTestRunner:
 
         stdout = None
         stderr = None
-        if not self.options.verbose:
-            stdout = tempfile.TemporaryFile("wb+")
-            stderr = tempfile.TemporaryFile("wb+") if self.options.split else stdout
-        if self.test.protocol is TestProtocol.TAP and stderr is stdout:
-            stdout = tempfile.TemporaryFile("wb+")
+        if self.test.protocol is TestProtocol.TAP:
+            stdout = asyncio.subprocess.PIPE
+            stderr = None if self.options.verbose else asyncio.subprocess.PIPE
+        elif not self.options.verbose:
+            stdout = asyncio.subprocess.PIPE
+            stderr = asyncio.subprocess.PIPE if self.options.split else asyncio.subprocess.STDOUT
 
         extra_cmd = []  # type: T.List[str]
         if self.test.protocol is TestProtocol.GTEST:
@@ -1114,24 +1122,24 @@ class SingleTestRunner:
                                        env=self.env,
                                        cwd=self.test.workdir)
 
+        stdo = stde = ''
+        stdo_task = stde_task = None
+        if stdout is not None:
+            stdo_task = p.stdout.read(-1)
+        if stderr is not None and stderr != asyncio.subprocess.STDOUT:
+            stde_task = p.stderr.read(-1)
+
         returncode, result, additional_error = await p.wait(timeout)
         if result is TestResult.TIMEOUT and self.options.verbose:
             print('{} time out (After {} seconds)'.format(self.test.name, timeout))
 
-        if additional_error is None:
-            if stdout is None:
-                stdo = ''
-            else:
-                stdout.seek(0)
-                stdo = decode(stdout.read())
-            if stderr is None or stderr is stdout:
-                stde = ''
-            else:
-                stderr.seek(0)
-                stde = decode(stderr.read())
-        else:
-            stdo = ""
-            stde = additional_error
+        if stdo_task is not None:
+            stdo = decode(await stdo_task)
+        if stde_task is not None:
+            stde = decode(await stde_task)
+
+        if additional_error is not None:
+            stde += '\n' + additional_error
 
         # Print lines along the way if requested
         def lines() -> T.Iterator[str]:

From f1938349c74f4fee2fbec718227985e259f02e47 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 6 Jan 2021 10:11:25 +0100
Subject: [PATCH 11/20] mtest: move Rust parsing inside TestRun

Make the code look like the TAP parser.  This simplifies the introduction of
asynchronous parsing.
---
 mesonbuild/mtest.py | 81 ++++++++++++++++++++++-----------------------
 1 file changed, 40 insertions(+), 41 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index e1782b35a..2efa9994d 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -690,38 +690,6 @@ class JunitBuilder(TestLogger):
             tree.write(f, encoding='utf-8', xml_declaration=True)
 
 
-def parse_rust_test(stdout: str) -> T.Tuple[T.List[TAPParser.Test], TestResult]:
-    """Parse the output of rust tests."""
-    results = []          # type: T.List[TAPParser.Test]
-
-    def parse_res(n: int, name: str, result: str) -> TAPParser.Test:
-        if result == 'ok':
-            return TAPParser.Test(n, name, TestResult.OK, None)
-        elif result == 'ignored':
-            return TAPParser.Test(n, name, TestResult.SKIP, None)
-        elif result == 'FAILED':
-            return TAPParser.Test(n, name, TestResult.FAIL, None)
-        return TAPParser.Test(n, name, TestResult.ERROR,
-                              'Unsupported output from rust test: {}'.format(result))
-
-    n = 1
-    for line in stdout.splitlines():
-        if line.startswith('test ') and not line.startswith('test result'):
-            _, name, _, result = line.split(' ')
-            name = name.replace('::', '.')
-            results.append(parse_res(n, name, result))
-            n += 1
-
-    if all(t.result is TestResult.SKIP for t in results):
-        # This includes the case where results is empty
-        return results, TestResult.SKIP
-    elif any(t.result is TestResult.ERROR for t in results):
-        return results, TestResult.ERROR
-    elif any(t.result is TestResult.FAIL for t in results):
-        return results, TestResult.FAIL
-    return results, TestResult.OK
-
-
 class TestRun:
     TEST_NUM = 0
 
@@ -797,10 +765,33 @@ class TestRun:
 
         self.complete(returncode, res, stdo, stde, cmd)
 
-    def complete_rust(self, returncode: int, stdo: str, stde: str, cmd: T.List[str]) -> None:
-        self.results, res = parse_rust_test(stdo)
+    def parse_rust(self, lines: T.Iterator[str]) -> T.Tuple[TestResult, str]:
+        def parse_res(n: int, name: str, result: str) -> TAPParser.Test:
+            if result == 'ok':
+                return TAPParser.Test(n, name, TestResult.OK, None)
+            elif result == 'ignored':
+                return TAPParser.Test(n, name, TestResult.SKIP, None)
+            elif result == 'FAILED':
+                return TAPParser.Test(n, name, TestResult.FAIL, None)
+            return TAPParser.Test(n, name, TestResult.ERROR,
+                                  'Unsupported output from rust test: {}'.format(result))
 
-        self.complete(returncode, res, stdo, stde, cmd)
+        n = 1
+        for line in lines:
+            if line.startswith('test ') and not line.startswith('test result'):
+                _, name, _, result = line.rstrip().split(' ')
+                name = name.replace('::', '.')
+                self.results.append(parse_res(n, name, result))
+                n += 1
+
+        if all(t.result is TestResult.SKIP for t in self.results):
+            # This includes the case where self.results is empty
+            return TestResult.SKIP, ''
+        elif any(t.result is TestResult.ERROR for t in self.results):
+            return TestResult.ERROR, ''
+        elif any(t.result is TestResult.FAIL for t in self.results):
+            return TestResult.FAIL, ''
+        return TestResult.OK, ''
 
     @property
     def num(self) -> int:
@@ -1124,6 +1115,13 @@ class SingleTestRunner:
 
         stdo = stde = ''
         stdo_task = stde_task = None
+
+        parser = None
+        if self.test.protocol is TestProtocol.TAP:
+            parser = self.runobj.parse_tap
+        elif self.test.protocol is TestProtocol.RUST:
+            parser = self.runobj.parse_rust
+
         if stdout is not None:
             stdo_task = p.stdout.read(-1)
         if stderr is not None and stderr != asyncio.subprocess.STDOUT:
@@ -1148,18 +1146,19 @@ class SingleTestRunner:
                     print(line, end='')
                 yield line
 
-        if self.test.protocol is TestProtocol.TAP:
-            res, error = self.runobj.parse_tap(lines())
+        if parser is not None:
+            res, error = parser(lines())
             if error:
                 stde += '\n' + error
-            self.runobj.complete_tap(returncode, result or res, stdo, stde, cmd)
+            result = result or res
+            if self.test.protocol is TestProtocol.TAP:
+                self.runobj.complete_tap(returncode, result, stdo, stde, cmd)
+                return
 
-        elif result:
+        if result:
             self.runobj.complete(returncode, result, stdo, stde, cmd)
         elif self.test.protocol is TestProtocol.EXITCODE:
             self.runobj.complete_exitcode(returncode, stdo, stde, cmd)
-        elif self.test.protocol is TestProtocol.RUST:
-            return self.runobj.complete_rust(returncode, stdo, stde, cmd)
         elif self.test.protocol is TestProtocol.GTEST:
             self.runobj.complete_gtest(returncode, stdo, stde, cmd)
 

From fa4fb3e350c39103aa47b19b321eaf8586059fd0 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 18 Nov 2020 13:43:01 +0100
Subject: [PATCH 12/20] mtest: make test output parsing asynchronous

Instead of slurping in the entire stream, build the TestResult along
the way.  This allows reporting the results of TAP and Rust subtests as
they come in, either as part of the progress report or (in the future)
as individual lines of the output.
---
 mesonbuild/mtest.py | 56 +++++++++++++++++++++++++++------------------
 1 file changed, 34 insertions(+), 22 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 2efa9994d..0790a0453 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -21,7 +21,6 @@ import argparse
 import asyncio
 import datetime
 import enum
-import io
 import json
 import multiprocessing
 import os
@@ -275,6 +274,13 @@ class TAPParser:
 
         yield self.Test(num, name, TestResult.OK if ok else TestResult.FAIL, explanation)
 
+    async def parse_async(self, lines: T.AsyncIterator[str]) -> T.AsyncIterator[TYPE_TAPResult]:
+        async for line in lines:
+            for event in self.parse_line(line):
+                yield event
+        for event in self.parse_line(None):
+            yield event
+
     def parse(self, io: T.Iterator[str]) -> T.Iterator[TYPE_TAPResult]:
         for line in io:
             yield from self.parse_line(line)
@@ -737,11 +743,11 @@ class TestRun:
             res = TestResult.FAIL if bool(returncode) else TestResult.OK
         self.complete(returncode, res, stdo, stde, cmd, **kwargs)
 
-    def parse_tap(self, lines: T.Iterator[str]) -> T.Tuple[TestResult, str]:
-        res = None    # type: T.Optional[TestResult]
+    async def parse_tap(self, lines: T.AsyncIterator[str]) -> T.Tuple[TestResult, str]:
+        res = TestResult.OK
         error = ''
 
-        for i in TAPParser().parse(lines):
+        async for i in TAPParser().parse_async(lines):
             if isinstance(i, TAPParser.Bailout):
                 res = TestResult.ERROR
             elif isinstance(i, TAPParser.Test):
@@ -755,7 +761,7 @@ class TestRun:
         if all(t.result is TestResult.SKIP for t in self.results):
             # This includes the case where self.results is empty
             res = TestResult.SKIP
-        return res or TestResult.OK, error
+        return res, error
 
     def complete_tap(self, returncode: int, res: TestResult,
                      stdo: str, stde: str, cmd: T.List[str]) -> None:
@@ -765,7 +771,7 @@ class TestRun:
 
         self.complete(returncode, res, stdo, stde, cmd)
 
-    def parse_rust(self, lines: T.Iterator[str]) -> T.Tuple[TestResult, str]:
+    async def parse_rust(self, lines: T.AsyncIterator[str]) -> T.Tuple[TestResult, str]:
         def parse_res(n: int, name: str, result: str) -> TAPParser.Test:
             if result == 'ok':
                 return TAPParser.Test(n, name, TestResult.OK, None)
@@ -777,7 +783,7 @@ class TestRun:
                                   'Unsupported output from rust test: {}'.format(result))
 
         n = 1
-        for line in lines:
+        async for line in lines:
             if line.startswith('test ') and not line.startswith('test result'):
                 _, name, _, result = line.rstrip().split(' ')
                 name = name.replace('::', '.')
@@ -1114,15 +1120,28 @@ class SingleTestRunner:
                                        cwd=self.test.workdir)
 
         stdo = stde = ''
-        stdo_task = stde_task = None
+        stdo_task = stde_task = parse_task = None
+
+        # Extract lines out of the StreamReader and print them
+        # along the way if requested
+        async def lines() -> T.AsyncIterator[str]:
+            stdo_lines = []
+            reader = p.stdout
+            while not reader.at_eof():
+                line = decode(await reader.readline())
+                stdo_lines.append(line)
+                if self.options.verbose:
+                    print(line, end='')
+                yield line
+
+            nonlocal stdo
+            stdo = ''.join(stdo_lines)
 
-        parser = None
         if self.test.protocol is TestProtocol.TAP:
-            parser = self.runobj.parse_tap
+            parse_task = self.runobj.parse_tap(lines())
         elif self.test.protocol is TestProtocol.RUST:
-            parser = self.runobj.parse_rust
-
-        if stdout is not None:
+            parse_task = self.runobj.parse_rust(lines())
+        elif stdout is not None:
             stdo_task = p.stdout.read(-1)
         if stderr is not None and stderr != asyncio.subprocess.STDOUT:
             stde_task = p.stderr.read(-1)
@@ -1139,15 +1158,8 @@ class SingleTestRunner:
         if additional_error is not None:
             stde += '\n' + additional_error
 
-        # Print lines along the way if requested
-        def lines() -> T.Iterator[str]:
-            for line in io.StringIO(stdo):
-                if self.options.verbose:
-                    print(line, end='')
-                yield line
-
-        if parser is not None:
-            res, error = parser(lines())
+        if parse_task is not None:
+            res, error = await parse_task
             if error:
                 stde += '\n' + error
             result = result or res

From f97521a1ff61f0738ebb242921ed241454d18976 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 13:04:45 +0100
Subject: [PATCH 13/20] mtest: align correctly tests with wide Unicode
 characters

This correctly formats tests with CJK names or, well, emoji.  It is not perfect
(for example it does not correctly format emoji that are variations of 1-wide
characters), but it is as good as most terminal emulators.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 0790a0453..5054e13dd 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -34,6 +34,7 @@ import sys
 import textwrap
 import time
 import typing as T
+import unicodedata
 import xml.etree.ElementTree as et
 
 from . import build
@@ -61,6 +62,14 @@ def is_windows() -> bool:
 def is_cygwin() -> bool:
     return sys.platform == 'cygwin'
 
+UNIWIDTH_MAPPING = {'F': 2, 'H': 1, 'W': 2, 'Na': 1, 'N': 1, 'A': 1}
+def uniwidth(s: str) -> int:
+    result = 0
+    for c in s:
+        w = unicodedata.east_asian_width(c)
+        result += UNIWIDTH_MAPPING[w]
+    return result
+
 def determine_worker_count() -> int:
     varname = 'MESON_TESTTHREADS'
     if varname in os.environ:
@@ -1272,12 +1281,13 @@ class TestHarness:
             l.log(self, result)
 
     def format(self, result: TestRun, colorize: bool) -> str:
-        result_str = '{num:{numlen}}/{testcount} {name:{name_max_len}} {res} {dur:.2f}s'.format(
+        extra_name_width = self.name_max_len + 1 - uniwidth(result.name)
+        result_str = '{num:{numlen}}/{testcount} {name}{extra_name_padding}{res} {dur:.2f}s'.format(
             numlen=len(str(self.test_count)),
             num=result.num,
             testcount=self.test_count,
-            name_max_len=self.name_max_len,
             name=result.name,
+            extra_name_padding=' ' * max(1, extra_name_width),
             res=result.res.get_text(colorize),
             dur=result.duration)
         if result.res is TestResult.FAIL:
@@ -1315,7 +1325,7 @@ class TestHarness:
             sys.exit(125)
 
         self.test_count = len(tests)
-        self.name_max_len = max([len(self.get_pretty_suite(test)) for test in tests])
+        self.name_max_len = max([uniwidth(self.get_pretty_suite(test)) for test in tests])
         self.run_tests(tests)
         return self.total_failure_count()
 

From 4b8364b1e2a5ecbef07676c2e00ddab743694469 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 12:07:06 +0100
Subject: [PATCH 14/20] mtest: add more formatting options to
 TestHarness.format

Allow leaving extra space in the left column, as well as customizing parts of
the printed line.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 49 +++++++++++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 5054e13dd..faacadbeb 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -157,10 +157,10 @@ def returncode_to_status(retcode: int) -> str:
             signame = signal.Signals(signum).name
         except ValueError:
             signame = 'SIGinvalid'
-        return '(killed by signal {} {})'.format(signum, signame)
+        return 'killed by signal {} {}'.format(signum, signame)
 
     if retcode <= 128:
-        return '(exit status {})'.format(retcode)
+        return 'exit status {}'.format(retcode)
 
     signum = retcode - 128
     try:
@@ -815,6 +815,12 @@ class TestRun:
             self._num = TestRun.TEST_NUM
         return self._num
 
+    @property
+    def detail(self) -> str:
+        if self.res is TestResult.FAIL:
+            return returncode_to_status(self.returncode)
+        return ''
+
     def complete(self, returncode: int, res: TestResult,
                  stdo: T.Optional[str], stde: T.Optional[str],
                  cmd: T.List[str], *, junit: T.Optional[et.ElementTree] = None) -> None:
@@ -1280,19 +1286,32 @@ class TestHarness:
         for l in self.loggers:
             l.log(self, result)
 
-    def format(self, result: TestRun, colorize: bool) -> str:
-        extra_name_width = self.name_max_len + 1 - uniwidth(result.name)
-        result_str = '{num:{numlen}}/{testcount} {name}{extra_name_padding}{res} {dur:.2f}s'.format(
-            numlen=len(str(self.test_count)),
-            num=result.num,
-            testcount=self.test_count,
-            name=result.name,
-            extra_name_padding=' ' * max(1, extra_name_width),
-            res=result.res.get_text(colorize),
-            dur=result.duration)
-        if result.res is TestResult.FAIL:
-            result_str += ' ' + returncode_to_status(result.returncode)
-        return result_str
+    def format(self, result: TestRun, colorize: bool,
+               max_left_width: int = 0,
+               left: T.Optional[str] = None,
+               right: T.Optional[str] = None) -> str:
+        numlen = len(str(self.test_count))
+
+        if left is None:
+            left = '{num:{numlen}}/{testcount} '.format(
+                numlen=numlen,
+                num=result.num,
+                testcount=self.test_count)
+
+        # A non-default max_left_width lets the logger print more stuff before the
+        # name, while ensuring that the rightmost columns remain aligned.
+        max_left_width = max(max_left_width, 2 * numlen + 2)
+        extra_name_width = max_left_width + self.name_max_len + 1 - uniwidth(result.name) - uniwidth(left)
+        middle = result.name + (' ' * max(1, extra_name_width))
+
+        if right is None:
+            right = '{res} {dur:.2f}s'.format(
+                res=result.res.get_text(colorize),
+                dur=result.duration)
+            detail = result.detail
+            if detail:
+                right += ' (' + detail + ')'
+        return left + middle + right
 
     def summary(self) -> str:
         return textwrap.dedent('''

From d1ee242e0d0e8642756202e9992a0973aca2c0fa Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 12:33:04 +0100
Subject: [PATCH 15/20] mtest: store timeout in TestRun

This will be useful when printing the progress report.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index faacadbeb..98c7d5bfa 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -709,11 +709,12 @@ class TestRun:
     TEST_NUM = 0
 
     def __init__(self, test: TestSerialisation, test_env: T.Dict[str, str],
-                 name: str):
+                 name: str, timeout: T.Optional[int]):
         self.res = TestResult.PENDING
         self.test = test
         self._num = None       # type: T.Optional[int]
         self.name = name
+        self.timeout = timeout
         self.results = list()  # type: T.List[TAPParser.Test]
         self.returncode = 0
         self.starttime = None  # type: T.Optional[float]
@@ -1016,7 +1017,15 @@ class SingleTestRunner:
         self.test_env = test_env
         self.env = env
         self.options = options
-        self.runobj = TestRun(test, test_env, name)
+
+        if self.options.gdb or self.test.timeout is None:
+            timeout = None
+        elif self.options.timeout_multiplier is not None:
+            timeout = self.test.timeout * self.options.timeout_multiplier
+        else:
+            timeout = self.test.timeout
+
+        self.runobj = TestRun(test, test_env, name, timeout)
 
     def _get_cmd(self) -> T.Optional[T.List[str]]:
         if self.test.fname[0].endswith('.jar'):
@@ -1046,8 +1055,6 @@ class SingleTestRunner:
             self.runobj.complete(GNU_SKIP_RETURNCODE, TestResult.SKIP, skip_stdout, None, None)
         else:
             wrap = TestHarness.get_wrapper(self.options)
-            if self.options.gdb:
-                self.test.timeout = None
             await self._run_cmd(wrap + cmd + self.test.cmd_args + self.options.test_args)
         return self.runobj
 
@@ -1121,13 +1128,6 @@ class SingleTestRunner:
                 gtestname = os.path.join(self.test.workdir, self.test.name)
             extra_cmd.append('--gtest_output=xml:{}.xml'.format(gtestname))
 
-        if self.test.timeout is None:
-            timeout = None
-        elif self.options.timeout_multiplier is not None:
-            timeout = self.test.timeout * self.options.timeout_multiplier
-        else:
-            timeout = self.test.timeout
-
         p = await self._run_subprocess(cmd + extra_cmd,
                                        stdout=stdout,
                                        stderr=stderr,
@@ -1161,9 +1161,9 @@ class SingleTestRunner:
         if stderr is not None and stderr != asyncio.subprocess.STDOUT:
             stde_task = p.stderr.read(-1)
 
-        returncode, result, additional_error = await p.wait(timeout)
+        returncode, result, additional_error = await p.wait(self.runobj.timeout)
         if result is TestResult.TIMEOUT and self.options.verbose:
-            print('{} time out (After {} seconds)'.format(self.test.name, timeout))
+            print('{} time out (After {} seconds)'.format(self.test.name, self.runobj.timeout))
 
         if stdo_task is not None:
             stdo = decode(await stdo_task)

From d470a915dae07d50f4f417eb188b2c19f1313e07 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 15:24:38 +0100
Subject: [PATCH 16/20] mtest: move timeout message to ConsoleLogger

This adds a point where to call the progress report flush() method.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 98c7d5bfa..5cb4d7c02 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -518,6 +518,10 @@ class ConsoleLogger(TestLogger):
 
     def log(self, harness: 'TestHarness', result: 'TestRun') -> None:
         self.running_tests.remove(result)
+        if result.res is TestResult.TIMEOUT and harness.options.verbose:
+            self.flush()
+            print('{} time out (After {} seconds)'.format(result.name, result.timeout))
+
         if not harness.options.quiet or not result.res.is_ok():
             self.flush()
             print(harness.format(result, mlog.colorize_console()), flush=True)

From 72e96a470a13928526a292cc8066e230df612d19 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 6 Jan 2021 16:10:50 +0100
Subject: [PATCH 17/20] mtest: create runners in advance

Compute all options in advance so that we can compute the maximum
timeout.
---
 mesonbuild/mtest.py | 41 ++++++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 5cb4d7c02..0b87c0325 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -1051,6 +1051,10 @@ class SingleTestRunner:
                 return self.test.exe_runner.get_command() + self.test.fname
         return self.test.fname
 
+    @property
+    def visible_name(self) -> str:
+        return self.runobj.name
+
     async def run(self) -> TestRun:
         cmd = self._get_cmd()
         self.runobj.start()
@@ -1254,7 +1258,8 @@ class TestHarness:
             options.wrapper = current.exe_wrapper
         return current.env.get_env(os.environ.copy())
 
-    def get_test_runner(self, test: TestSerialisation, name: str) -> SingleTestRunner:
+    def get_test_runner(self, test: TestSerialisation) -> SingleTestRunner:
+        name = self.get_pretty_suite(test)
         options = deepcopy(self.options)
         if not options.setup:
             options.setup = self.build_data.test_setup_default_name
@@ -1349,7 +1354,15 @@ class TestHarness:
 
         self.test_count = len(tests)
         self.name_max_len = max([uniwidth(self.get_pretty_suite(test)) for test in tests])
-        self.run_tests(tests)
+        startdir = os.getcwd()
+        try:
+            if self.options.wd:
+                os.chdir(self.options.wd)
+            self.build_data = build.load(os.getcwd())
+            runners = [self.get_test_runner(test) for test in tests]
+            self.run_tests(runners)
+        finally:
+            os.chdir(startdir)
         return self.total_failure_count()
 
     @staticmethod
@@ -1488,23 +1501,19 @@ class TestHarness:
         else:
             return test.name
 
-    def run_tests(self, tests: T.List[TestSerialisation]) -> None:
+    def run_tests(self, runners: T.List[SingleTestRunner]) -> None:
         try:
             self.open_logfiles()
             # Replace with asyncio.run once we can require Python 3.7
             loop = asyncio.get_event_loop()
-            loop.run_until_complete(self._run_tests(tests))
+            loop.run_until_complete(self._run_tests(runners))
         finally:
             self.close_logfiles()
 
-    async def _run_tests(self, tests: T.List[TestSerialisation]) -> None:
+    async def _run_tests(self, runners: T.List[SingleTestRunner]) -> None:
         semaphore = asyncio.Semaphore(self.options.num_processes)
         futures = deque()  # type: T.Deque[asyncio.Future]
         running_tests = dict() # type: T.Dict[asyncio.Future, str]
-        startdir = os.getcwd()
-        if self.options.wd:
-            os.chdir(self.options.wd)
-        self.build_data = build.load(os.getcwd())
         interrupted = False
 
         async def run_test(test: SingleTestRunner) -> None:
@@ -1565,17 +1574,16 @@ class TestHarness:
             asyncio.get_event_loop().add_signal_handler(signal.SIGTERM, sigterm_handler)
         try:
             for _ in range(self.options.repeat):
-                for test in tests:
-                    visible_name = self.get_pretty_suite(test)
-                    single_test = self.get_test_runner(test, visible_name)
+                for runner in runners:
+                    test = runner.test
 
-                    if not test.is_parallel or single_test.options.gdb:
+                    if not test.is_parallel or runner.options.gdb:
                         await complete_all(futures)
-                    future = asyncio.ensure_future(run_test(single_test))
+                    future = asyncio.ensure_future(run_test(runner))
                     futures.append(future)
-                    running_tests[future] = visible_name
+                    running_tests[future] = runner.visible_name
                     future.add_done_callback(test_done)
-                    if not test.is_parallel or single_test.options.gdb:
+                    if not test.is_parallel or runner.options.gdb:
                         await complete(future)
                 if self.options.repeat > 1 and self.fail_count:
                     break
@@ -1587,7 +1595,6 @@ class TestHarness:
                 asyncio.get_event_loop().remove_signal_handler(signal.SIGTERM)
             for l in self.loggers:
                 await l.finish(self)
-            os.chdir(startdir)
 
 def list_tests(th: TestHarness) -> bool:
     tests = th.get_tests()

From 4371756182e9adbd80dd00c10fdce17bbec8ffda Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 6 Jan 2021 16:31:19 +0100
Subject: [PATCH 18/20] mtest: align decimal point of test durations

---
 mesonbuild/mtest.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 0b87c0325..c8dc02321 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -1055,6 +1055,10 @@ class SingleTestRunner:
     def visible_name(self) -> str:
         return self.runobj.name
 
+    @property
+    def timeout(self) -> T.Optional[int]:
+        return self.runobj.timeout
+
     async def run(self) -> TestRun:
         cmd = self._get_cmd()
         self.runobj.start()
@@ -1314,9 +1318,10 @@ class TestHarness:
         middle = result.name + (' ' * max(1, extra_name_width))
 
         if right is None:
-            right = '{res} {dur:.2f}s'.format(
+            right = '{res} {dur:{durlen}.2f}s'.format(
                 res=result.res.get_text(colorize),
-                dur=result.duration)
+                dur=result.duration,
+                durlen=self.duration_max_len + 3)
             detail = result.detail
             if detail:
                 right += ' (' + detail + ')'
@@ -1360,6 +1365,8 @@ class TestHarness:
                 os.chdir(self.options.wd)
             self.build_data = build.load(os.getcwd())
             runners = [self.get_test_runner(test) for test in tests]
+            self.duration_max_len = max([len(str(int(runner.timeout or 99)))
+                                         for runner in runners])
             self.run_tests(runners)
         finally:
             os.chdir(startdir)

From b05aee5c94938fdad51e44141db0552fb453ca25 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 12:27:32 +0100
Subject: [PATCH 19/20] mtest: print time that the test has been running

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 mesonbuild/mtest.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index c8dc02321..6df23d6ce 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -431,6 +431,7 @@ class ConsoleLogger(TestLogger):
         self.running_tests = OrderedSet()  # type: OrderedSet['TestRun']
         self.progress_test = None          # type: T.Optional['TestRun']
         self.progress_task = None          # type: T.Optional[asyncio.Future]
+        self.max_left_width = 0            # type: int
         self.stop = False
         self.update = asyncio.Event()
         self.should_erase_line = ''
@@ -450,7 +451,7 @@ class ConsoleLogger(TestLogger):
     def request_update(self) -> None:
         self.update.set()
 
-    def emit_progress(self) -> None:
+    def emit_progress(self, harness: 'TestHarness') -> None:
         if self.progress_test is None:
             self.flush()
             return
@@ -461,8 +462,17 @@ class ConsoleLogger(TestLogger):
             count = '{}-{}/{}'.format(self.started_tests - len(self.running_tests) + 1,
                                       self.started_tests, self.test_count)
 
-        line = '[{}] {} {}'.format(count, self.SPINNER[self.spinner_index], self.progress_test.name)
+        left = '[{}] {} '.format(count, self.SPINNER[self.spinner_index])
         self.spinner_index = (self.spinner_index + 1) % len(self.SPINNER)
+
+        right = '{spaces} {dur:{durlen}}/{timeout:{durlen}}s'.format(
+            spaces=' ' * TestResult.maxlen(),
+            dur=int(time.time() - self.progress_test.starttime),
+            durlen=harness.duration_max_len,
+            timeout=int(self.progress_test.timeout))
+        line = harness.format(self.progress_test, colorize=True,
+                              max_left_width=self.max_left_width,
+                              left=left, right=right)
         self.print_progress(line)
 
     @staticmethod
@@ -501,13 +511,16 @@ class ConsoleLogger(TestLogger):
                     self.progress_test = self.running_tests.pop(last=False)
                     self.running_tests.add(self.progress_test)
 
-                self.emit_progress()
+                self.emit_progress(harness)
             self.flush()
 
         self.test_count = harness.test_count
+
         # In verbose mode, the progress report gets in the way of the tests'
         # stdout and stderr.
         if self.is_tty() and not harness.options.verbose:
+            # Account for "[aa-bb/cc] OO " in the progress report
+            self.max_left_width = 3 * len(str(self.test_count)) + 8
             self.progress_task = asyncio.ensure_future(report_progress())
 
     def start_test(self, test: 'TestRun') -> None:
@@ -524,7 +537,8 @@ class ConsoleLogger(TestLogger):
 
         if not harness.options.quiet or not result.res.is_ok():
             self.flush()
-            print(harness.format(result, mlog.colorize_console()), flush=True)
+            print(harness.format(result, mlog.colorize_console(), max_left_width=self.max_left_width),
+                  flush=True)
         self.request_update()
 
     async def finish(self, harness: 'TestHarness') -> None:

From f13dde1f0851b3bb3d3b1406adaad5a48662cb02 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Wed, 25 Nov 2020 12:56:02 +0100
Subject: [PATCH 20/20] mtest: print TAP subtest count

The parentheses look ugly in the progress report.  To keep it aligned
with the test outcomes, remove them from the outcomes as well.
---
 mesonbuild/mtest.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
index 6df23d6ce..59d238910 100644
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -470,6 +470,9 @@ class ConsoleLogger(TestLogger):
             dur=int(time.time() - self.progress_test.starttime),
             durlen=harness.duration_max_len,
             timeout=int(self.progress_test.timeout))
+        detail = self.progress_test.detail
+        if detail:
+            right += '   ' + detail
         line = harness.format(self.progress_test, colorize=True,
                               max_left_width=self.max_left_width,
                               left=left, right=right)
@@ -836,8 +839,18 @@ class TestRun:
 
     @property
     def detail(self) -> str:
-        if self.res is TestResult.FAIL:
+        if self.res is TestResult.PENDING:
+            return ''
+        if self.returncode:
             return returncode_to_status(self.returncode)
+        if self.results:
+            # running or succeeded
+            passed = sum((x.result.is_ok() for x in self.results))
+            ran = sum((x.result is not TestResult.SKIP for x in self.results))
+            if passed == ran:
+                return '{} subtests passed'.format(passed)
+            else:
+                return '{}/{} subtests passed'.format(passed, ran)
         return ''
 
     def complete(self, returncode: int, res: TestResult,
@@ -1338,7 +1351,7 @@ class TestHarness:
                 durlen=self.duration_max_len + 3)
             detail = result.detail
             if detail:
-                right += ' (' + detail + ')'
+                right += '   ' + detail
         return left + middle + right
 
     def summary(self) -> str: