framework: if a test with subtests crashes mark the offending subtest

This relies on the fact that subtests are guaranteed to be ordered to
mark the crashing subtest as such. This ensures that the correct status
will be propagated up the totals tree.

Signed-off-by: Fabian Bieler <fabianbieler@fastmail.fm>
Reviewed-by: Dylan Baker <dylan@pnwbakers.com>

2 files changed, 44 insertions, 0 deletions

diff --git a/framework/test/base.py b/framework/test/base.py
index 2d761efa9..134b87245 100644
--- a/framework/test/base.py
+++ b/framework/test/base.py
@@ -236,6 +236,14 @@ class Test(object):
         """
         if is_crash_returncode(self.result.returncode):
             self.result.result = status.CRASH
+            if self.result.subtests:
+                # We know because subtests are ordered that the first test with
+                # a status of NOTRUN is the subtest that crashed, mark that
+                # test and move on.
+                for k, v in six.iteritems(self.result.subtests):
+                    if v == status.NOTRUN:
+                        self.result.subtests[k] = status.CRASH
+                        break
         elif self.result.returncode != 0:
             if self.result.result == status.PASS:
                 self.result.result = status.WARN
diff --git a/unittests/framework/test/test_base.py b/unittests/framework/test/test_base.py
index 656d839f7..ad355b1d8 100644
--- a/unittests/framework/test/test_base.py
+++ b/unittests/framework/test/test_base.py
@@ -33,6 +33,8 @@ except ImportError:
 import pytest
 import six
 
+from six.moves import range
+
 from framework import dmesg
 from framework import log
 from framework import monitoring
@@ -283,6 +285,40 @@ class TestTest(object):
 
             assert test.result.result is status.FAIL
 
+        def test_crash_subtest_before_start(self):
+            """A test for a test with a subtest, that crashes at the start
+            of the run.
+            """
+            test = _Test(['foobar'])
+            test.result.returncode = -1
+            for x in (str(y) for y in range(5)):
+                test.result.subtests[x] = status.NOTRUN
+            test.interpret_result()
+
+            assert test.result.result is status.CRASH
+            assert test.result.subtests['0'] is status.CRASH
+            for x in (str(y) for y in range(1, 5)):
+                assert test.result.subtests[x] is status.NOTRUN
+
+        def test_crash_subtest_mid(self):
+            """A test for a test with a subtest, that crashes in the middle
+            of the run.
+            """
+            test = _Test(['foobar'])
+            test.result.returncode = -1
+            for x in (str(y) for y in range(2)):
+                test.result.subtests[x] = status.PASS
+            for x in (str(y) for y in range(2, 5)):
+                test.result.subtests[x] = status.NOTRUN
+            test.interpret_result()
+
+            assert test.result.result is status.CRASH
+            for x in (str(y) for y in range(2)):
+                assert test.result.subtests[x] is status.PASS
+            assert test.result.subtests['2'] is status.CRASH
+            for x in (str(y) for y in range(3, 5)):
+                assert test.result.subtests[x] is status.NOTRUN
+
 
 class TestWindowResizeMixin(object):
     """Tests for the WindowResizeMixin class."""