[test] Add ability to expect timeouts

Summary:
This adds ability to mark test that do not complete due to hangs, crashes, etc., as "expected",
to avoid flagging the build red for a known problem. Functionally, this extends the scope of the
existing expectedFailureXXX decorators to cover these states as well. Once this is in, I will
start replacing the magic list of failing tests in dosep.py with our regular annotations which
should hopefully make code simpler.

Reviewers: tfiala

Subscribers: lldb-commits

Differential Revision: http://reviews.llvm.org/D15530

llvm-svn: 255763

lldb/packages/Python/lldbsuite/test/dosep.py

@@ -1143,7 +1143,6 @@ def getExpectedTimeouts(platform_name):
         target = m.group(1)
 
     expected_timeout = set()
-    expected_timeout.add("TestExpectedTimeout.py")
 
     if target.startswith("linux"):
         expected_timeout |= {

lldb/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park

@@ -10,6 +10,7 @@ class ExpectedTimeoutTestCase(lldbtest.TestBase):
     """Forces test timeout."""
     mydir = lldbtest.TestBase.compute_mydir(__file__)
 
+    @lldbtest.expectedFailureAll()
     def test_buildbot_sees_expected_timeout(self):
         """Tests that expected timeout logic kicks in and is picked up."""
         while True:

lldb/packages/Python/lldbsuite/test/lldbtest.py

@@ -598,6 +598,10 @@ def expectedFailure(expected_fn, bugnumber=None):
             from unittest2 import case
             self = args[0]
             if expected_fn(self):
+                if configuration.results_formatter_object is not None:
+                    # Mark this test as expected to fail.
+                    configuration.results_formatter_object.handle_event(
+                        EventBuilder.event_for_mark_test_expected_failure(self))
                 xfail_func = unittest2.expectedFailure(func)
                 xfail_func(*args, **kwargs)
             else:

lldb/packages/Python/lldbsuite/test/result_formatter.py

@@ -165,6 +165,7 @@ class EventBuilder(object):
     TYPE_TEST_RESULT = "test_result"
     TYPE_TEST_START = "test_start"
     TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun"
+    TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure"
     TYPE_SESSION_TERMINATE = "terminate"
 
     RESULT_TYPES = set([
@@ -527,6 +528,20 @@ class EventBuilder(object):
             EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE)
         return event
 
+    @staticmethod
+    def event_for_mark_test_expected_failure(test):
+        """Creates an event that indicates the specified test is expected
+        to fail.
+
+        @param test the TestCase instance to which this pertains.
+
+        @return an event that specifies the given test is expected to fail.
+        """
+        event = EventBuilder._event_dictionary_common(
+            test,
+            EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE)
+        return event
+
     @staticmethod
     def add_entries_to_all_events(entries_dict):
         """Specifies a dictionary of entries to add to all test events.
@@ -681,6 +696,11 @@ class ResultsFormatter(object):
         # timeout test status for this.
         self.expected_timeouts_by_basename = set()
 
+        # Tests which have reported that they are expecting to fail. These will
+        # be marked as expected failures even if they return a failing status,
+        # probably because they crashed or deadlocked.
+        self.expected_failures = set()
+
         # Keep track of rerun-eligible tests.
         # This is a set that contains tests saved as:
         # {test_filename}:{test_class}:{test_name}
@@ -721,6 +741,15 @@ class ResultsFormatter(object):
             component_count += 1
         return key
 
+    def _mark_test_as_expected_failure(self, test_result_event):
+        key = self._make_key(test_result_event)
+        if key is not None:
+            self.expected_failures.add(key)
+        else:
+            sys.stderr.write(
+                "\nerror: test marked as expected failure but "
+                "failed to create key.\n")
+
     def _mark_test_for_rerun_eligibility(self, test_result_event):
         key = self._make_key(test_result_event)
         if key is not None:
@@ -796,6 +825,20 @@ class ResultsFormatter(object):
             # Convert to an expected timeout.
             event["status"] = EventBuilder.STATUS_EXPECTED_TIMEOUT
 
+    def _maybe_remap_expected_failure(self, event):
+        if event is None:
+            return
+
+        key = self._make_key(event)
+        if key not in self.expected_failures:
+            return
+
+        status = event.get("status", None)
+        if status in EventBuilder.TESTRUN_ERROR_STATUS_VALUES:
+            event["status"] = EventBuilder.STATUS_EXPECTED_FAILURE
+        elif status == EventBuilder.STATUS_SUCCESS:
+            event["status"] = EventBuilder.STATUS_UNEXPECTED_SUCCESS
+
     def handle_event(self, test_event):
         """Handles the test event for collection into the formatter output.
 
@@ -824,6 +867,7 @@ class ResultsFormatter(object):
             # Remap timeouts to expected timeouts.
             if event_type in EventBuilder.RESULT_TYPES:
                 self._maybe_remap_expected_timeout(test_event)
+                self._maybe_remap_expected_failure(test_event)
                 event_type = test_event.get("event", "")
 
             if event_type == "terminate":
@@ -887,6 +931,8 @@ class ResultsFormatter(object):
 
             elif event_type == EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE:
                 self._mark_test_for_rerun_eligibility(test_event)
+            elif event_type == EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE:
+                self._mark_test_as_expected_failure(test_event)
 
     def set_expected_timeouts_by_basename(self, basenames):
         """Specifies a list of test file basenames that are allowed to timeout