runner_tests: Test that dynamic subtest failure is handled correctly

See also: commit 0e6457f1bfe2 ("lib: Don't dump log buffer when
dynamic subtest failure is inherited")

This is quite an explicit top-to-bottom test that we don't get an
incorrect warn result for an innocent dynamic subtest. It is tested
here in runner_test instead of testing in lib/tests that the extra
lines are not printed, because not printing the extra lines is an
implementation detail that might change later. The issue is after all
about test results parsed by igt_runner.

v2: Squash adding the new mockup test binary to this commit

Signed-off-by: Petri Latvala <petri.latvala@intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>

4 files changed, 67 insertions, 3 deletions

diff --git a/runner/meson.build b/runner/meson.build
index 404b8e04..7c2e8e0d 100644
--- a/runner/meson.build
+++ b/runner/meson.build
@@ -59,7 +59,7 @@ if jsonc.found()
 				 c_args : '-DTESTDATA_DIRECTORY="@0@"'.format(testdata_dir),
 				 link_with : runnerlib,
 				 install : false,
-				 dependencies : igt_deps)
+				 dependencies : [igt_deps, jsonc])
 	test('runner', runner_test, timeout : 300)
 
 	runner_json_test = executable('runner_json_test', runner_json_test_sources,
diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index 9bbc4252..6b0e6cc4 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -4,11 +4,14 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <json.h>
+
 #include "igt.h"
 
 #include "settings.h"
 #include "job_list.h"
 #include "executor.h"
+#include "resultgen.h"
 
 /*
  * NOTE: this test is using a lot of variables that are changed in igt_fixture,
@@ -25,9 +28,9 @@ static const char testdatadir[] = TESTDATA_DIRECTORY;
  * that test binaries without subtests should still be counted as one
  * for this macro.
  */
-#define NUM_TESTDATA_SUBTESTS 5
+#define NUM_TESTDATA_SUBTESTS 6
 /* The total number of test binaries in runner/testdata/ */
-#define NUM_TESTDATA_BINARIES 3
+#define NUM_TESTDATA_BINARIES 4
 
 static void igt_assert_eqstr(const char *one, const char *two)
 {
@@ -1348,6 +1351,54 @@ igt_main
 			free(list);
 	}
 
+	igt_subtest_group {
+		struct job_list *list = malloc(sizeof(*list));
+		volatile int dirfd = -1;
+		char dirname[] = "tmpdirXXXXXX";
+
+		igt_fixture {
+			igt_require(mkdtemp(dirname) != NULL);
+			rmdir(dirname);
+
+			init_job_list(list);
+		}
+
+		igt_subtest("dynamic-subtest-failure-should-not-cause-warn") {
+			struct execute_state state;
+			struct json_object *results, *obj;
+			const char *argv[] = { "runner",
+					       "-t", "dynamic",
+					       testdatadir,
+					       dirname,
+			};
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+			igt_assert(create_job_list(list, settings));
+			igt_assert(initialize_execute_state(&state, settings, list));
+			igt_assert(execute(&state, settings, list));
+
+			igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
+				     "Execute didn't create the results directory\n");
+			igt_assert_f((results = generate_results_json(dirfd)) != NULL,
+				     "Results parsing failed\n");
+
+			obj = results;
+			igt_assert(json_object_object_get_ex(obj, "tests", &obj));
+			igt_assert(json_object_object_get_ex(obj, "igt@dynamic@dynamic-subtest@passing", &obj));
+			igt_assert(json_object_object_get_ex(obj, "result", &obj));
+
+			igt_assert_eqstr(json_object_get_string(obj), "pass");
+
+			igt_assert_eq(json_object_put(results), 1);
+		}
+
+		igt_fixture {
+			close(dirfd);
+			clear_directory(dirname);
+			free_job_list(list);
+		}
+	}
+
 	igt_subtest("file-descriptor-leakage") {
 		int i;
 
diff --git a/runner/testdata/dynamic.c b/runner/testdata/dynamic.c
new file mode 100644
index 00000000..8e5de7d9
--- /dev/null
+++ b/runner/testdata/dynamic.c
@@ -0,0 +1,12 @@
+#include "igt.h"
+
+igt_main
+{
+	igt_subtest_with_dynamic("dynamic-subtest") {
+		igt_dynamic("failing")
+			igt_assert(false);
+
+		igt_dynamic("passing")
+			;
+	}
+}
diff --git a/runner/testdata/meson.build b/runner/testdata/meson.build
index 2456f82a..631ba5b9 100644
--- a/runner/testdata/meson.build
+++ b/runner/testdata/meson.build
@@ -2,6 +2,7 @@
 testdata_progs = [ 'successtest',
 		   'no-subtests',
 		   'skippers',
+		   'dynamic',
 		 ]
 
 testdata_executables = []