Update test harness to handle test return codes; fix comment format in harness; update Main test suite to handle globally disabled features

3 files changed, 102 insertions, 57 deletions

diff --git a/include/SDL_test_harness.h b/include/SDL_test_harness.h
index e5fe673afc..f2c553cb23 100644
--- a/include/SDL_test_harness.h
+++ b/include/SDL_test_harness.h
@@ -51,8 +51,9 @@ extern "C" {
 
 //! Definition of all the possible test return values of the test case method
 #define TEST_ABORTED		-1
-#define TEST_COMPLETED		 0
-#define TEST_SKIPPED		 1
+#define TEST_STARTED		 0
+#define TEST_COMPLETED		 1
+#define TEST_SKIPPED		 2
 
 //! Definition of all the possible test results for the harness
 #define TEST_RESULT_PASSED				0
@@ -65,7 +66,7 @@ extern "C" {
 typedef void (*SDLTest_TestCaseSetUpFp)(void *arg);
 
 //!< Function pointer to a test case function
-typedef void (*SDLTest_TestCaseFp)(void *arg);
+typedef int (*SDLTest_TestCaseFp)(void *arg);
 
 //!< Function pointer to a test case teardown function (run after every test)
 typedef void  (*SDLTest_TestCaseTearDownFp)(void *arg);
diff --git a/src/test/SDL_test_harness.c b/src/test/SDL_test_harness.c
index fcd2e4f5c7..d3c0266cf9 100644
--- a/src/test/SDL_test_harness.c
+++ b/src/test/SDL_test_harness.c
@@ -57,20 +57,20 @@ SDLTest_GenerateRunSeed(const int length)
 	SDLTest_RandomContext randomContext;
 	int counter;
 
-	// Sanity check input
+	/* Sanity check input */
 	if (length <= 0) {
 		SDLTest_LogError("The length of the harness seed must be >0.");
 		return NULL;
 	}
 
-	// Allocate output buffer
+	/* Allocate output buffer */
 	seed = (char *)SDL_malloc((length + 1) * sizeof(char));
 	if (seed == NULL) {
 		SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
 		return NULL;
 	}
 
-	// Generate a random string of alphanumeric characters
+	/* Generate a random string of alphanumeric characters */
 	SDLTest_RandomInitTime(&randomContext);
 	for (counter = 0; counter < length - 1; ++counter) {
 		unsigned int number = SDLTest_Random(&randomContext);
@@ -129,11 +129,11 @@ SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iter
 		return -1;
 	}
 
-	// Convert iteration number into a string
+	/* Convert iteration number into a string */
 	SDL_memset(iterationString, 0, sizeof(iterationString));
 	SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
 
-	// Combine the parameters into single string
+	/* Combine the parameters into single string */
 	runSeedLength = SDL_strlen(runSeed);
 	suiteNameLength = SDL_strlen(suiteName);
 	testNameLength = SDL_strlen(testName);
@@ -146,7 +146,7 @@ SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iter
 	}
 	SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
 
-	// Hash string and use half of the digest as 64bit exec key
+	/* Hash string and use half of the digest as 64bit exec key */
 	SDLTest_Md5Init(&md5Context);
 	SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
 	SDLTest_Md5Final(&md5Context);
@@ -208,7 +208,7 @@ void
 	SDLTest_BailOut()
 {
 	SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
-	exit(TEST_ABORTED); // bail out from the test
+	exit(TEST_ABORTED); /* bail out from the test */
 }
 
 /**
@@ -224,6 +224,7 @@ int
 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey)
 {
 	SDL_TimerID timer = 0;
+	int testCaseResult = 0;
 	int testResult = 0;
 	int fuzzerCount;
 
@@ -240,16 +241,16 @@ SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference
 	}
 
 
-	// Initialize fuzzer
+	/* Initialize fuzzer */
 	SDLTest_FuzzerInit(execKey);
 
-	// Reset assert tracker
+	/* Reset assert tracker */
 	SDLTest_ResetAssertSummary();
 
-	// Set timeout timer
+	/* Set timeout timer */
 	timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
 
-	// Maybe run suite initalizer function
+	/* Maybe run suite initalizer function */
 	if (testSuite->testSetUp) {
 		testSuite->testSetUp(0x0);
 		if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
@@ -258,26 +259,53 @@ SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference
 		}
 	}
 
-	// Run test case function
-	testCase->testCase(0x0);
-	testResult = SDLTest_AssertSummaryToTestResult();
+	/* Run test case function */
+	testCaseResult = testCase->testCase(0x0);
+	
+	/* Convert test execution result into harness result */
+	if (testCaseResult == TEST_SKIPPED) {
+		/* Test was programatically skipped */
+		testResult = TEST_RESULT_SKIPPED;
+	} else if (testCaseResult == TEST_STARTED) {
+		/* Test did not return a TEST_COMPLETED value; assume it failed */
+		testResult = TEST_RESULT_FAILED;
+	} else if (testCaseResult == TEST_ABORTED) {
+		/* Test was aborted early; assume it failed */
+		testResult = TEST_RESULT_FAILED;
+	} else {
+		/* Perform failure analysis based on asserts */
+		testResult = SDLTest_AssertSummaryToTestResult();
+	}
 
-	// Maybe run suite cleanup function (ignore failed asserts)
+	/* Maybe run suite cleanup function (ignore failed asserts) */
 	if (testSuite->testTearDown) {
 		testSuite->testTearDown(0x0);
 	}
 
-	// Cancel timeout timer
+	/* Cancel timeout timer */
 	if (timer) {
 		SDL_RemoveTimer(timer);
 	}
 
-	// Report on asserts and fuzzer usage
+	/* Report on asserts and fuzzer usage */
 	fuzzerCount = SDLTest_GetFuzzerInvocationCount();
 	if (fuzzerCount > 0) {
 		SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
 	}
-	SDLTest_LogAssertSummary();
+
+	/* Final log based on test execution result */
+	if (testCaseResult == TEST_SKIPPED) {
+		/* Test was programatically skipped */
+		SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Skipped (Programmatically)");
+	} else if (testCaseResult == TEST_STARTED) {
+		/* Test did not return a TEST_COMPLETED value; assume it failed */
+		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
+	} else if (testCaseResult == TEST_ABORTED) {
+		/* Test was aborted early; assume it failed */
+		SDLTest_LogError((char *)SDLTest_FinalResultFormat, "Test", testCase->name, "Failed (Aborted)");
+	} else {
+		SDLTest_LogAssertSummary();
+	}
 
 	return testResult;
 }
@@ -290,7 +318,7 @@ void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
 	SDLTest_TestSuiteReference *testSuite;
 	SDLTest_TestCaseReference *testCase;
 
-	// Loop over all suites
+	/* Loop over all suites */
 	suiteCounter = 0;
 	while(&testSuites[suiteCounter]) {
 		testSuite=&testSuites[suiteCounter];
@@ -298,7 +326,7 @@ void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
 		SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 
 			(testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
 
-		// Loop over all test cases
+		/* Loop over all test cases */
 		testCounter = 0;
 		while(testSuite->testCases[testCounter])
 		{
@@ -365,12 +393,12 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
 	Uint32 countSum = 0;
 	char *logFormat = (char *)SDLTest_LogSummaryFormat;
 
-	// Sanitize test iterations
+	/* Sanitize test iterations */
 	if (testIterations < 1) {
 		testIterations = 1;
 	}
 
-	// Generate run see if we don't have one already
+	/* Generate run see if we don't have one already */
 	if (userRunSeed == NULL || SDL_strlen(userRunSeed) == 0) {
 		runSeed = SDLTest_GenerateRunSeed(16);
 		if (runSeed == NULL) {
@@ -382,18 +410,18 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
 	}
 
 
-	// Reset per-run counters
+	/* Reset per-run counters */
 	totalTestFailedCount = 0;
 	totalTestPassedCount = 0;
 	totalTestSkippedCount = 0;
 
-	// Take time - run start
+	/* Take time - run start */
 	runStartSeconds = GetClock();
 
-	// Log run with fuzzer parameters
+	/* Log run with fuzzer parameters */
 	SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
 
-	// Initialize filtering
+	/* Initialize filtering */
 	if (filter != NULL && SDL_strlen(filter) > 0) {
 		/* Loop over all suites to check if we have a filter match */
 		suiteCounter = 0;
@@ -433,36 +461,36 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
 		}		
 	}
 
-	// Loop over all suites
+	/* Loop over all suites */
 	suiteCounter = 0;
 	while(testSuites[suiteCounter]) {
 		testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
 		currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTest_InvalidNameFormat);
 		suiteCounter++;
 
-		// Filter suite if flag set and we have a name
+		/* Filter suite if flag set and we have a name */
 		if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
 			SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
-				// Skip suite
+				/* Skip suite */
 				SDLTest_Log("===== Test Suite %i: '%s' skipped\n", 
 					suiteCounter, 
 					currentSuiteName);
 		} else {
 
-			// Reset per-suite counters
+			/* Reset per-suite counters */
 			testFailedCount = 0;
 			testPassedCount = 0;
 			testSkippedCount = 0;
 
-			// Take time - suite start
+			/* Take time - suite start */
 			suiteStartSeconds = GetClock();
 
-			// Log suite started
+			/* Log suite started */
 			SDLTest_Log("===== Test Suite %i: '%s' started\n", 
 				suiteCounter, 
 				currentSuiteName);
 
-			// Loop over all test cases
+			/* Loop over all test cases */
 			testCounter = 0;
 			while(testSuite->testCases[testCounter])
 			{
@@ -470,25 +498,25 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
 				currentTestName = (char *)((testCase->name) ? testCase->name : SDLTest_InvalidNameFormat);
 				testCounter++;
 
-				// Filter tests if flag set and we have a name
+				/* Filter tests if flag set and we have a name */
 				if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
 					SDL_strcmp(testFilterName, testCase->name) != 0) {
-						// Skip test
+						/* Skip test */
 						SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n", 
 							suiteCounter,
 							testCounter,
 							currentTestName);
 				} else {
-					// Override 'disabled' flag if we specified a test filter (i.e. force run for debugging)
+					/* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
 					if (testFilter == 1 && !testCase->enabled) {
 						SDLTest_Log("Force run of disabled test since test filter was set");
 						testCase->enabled = 1;
 					}
 
-					// Take time - test start
+					/* Take time - test start */
 					testStartSeconds = GetClock();
 
-					// Log test started
+					/* Log test started */
 					SDLTest_Log("----- Test Case %i.%i: '%s' started",
 						suiteCounter,
 						testCounter, 
@@ -498,7 +526,7 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
 							(testCase->description) ? testCase->description : SDLTest_InvalidNameFormat);
 					}
 
-					// Loop over all iterations
+					/* Loop over all iterations */
 					iterationCounter = 0;
 					while(iterationCounter < testIterations)
 					{
@@ -525,21 +553,21 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
 						}
 					}
 
-					// Take time - test end
+					/* Take time - test end */
 					testEndSeconds = GetClock();
 					runtime = testEndSeconds - testStartSeconds;
 					if (runtime < 0.0f) runtime = 0.0f;
 
 					if (testIterations > 1) {
-						// Log test runtime
+						/* Log test runtime */
 						SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
 						SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
 					} else {
-						// Log test runtime
+						/* Log test runtime */
 						SDLTest_Log("Total Test runtime: %.1f sec", runtime);
 					}
 
-					// Log final test result
+					/* Log final test result */
 					switch (testResult) {
 					case TEST_RESULT_PASSED:
 						SDLTest_Log((char *)SDLTest_FinalResultFormat, "Test", currentTestName, "Passed");
@@ -555,15 +583,15 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
 				}
 			}
 
-			// Take time - suite end
+			/* Take time - suite end */
 			suiteEndSeconds = GetClock();
 			runtime = suiteEndSeconds - suiteStartSeconds;
 			if (runtime < 0.0f) runtime = 0.0f;
 
-			// Log suite runtime
+			/* Log suite runtime */
 			SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
 
-			// Log summary and final Suite result
+			/* Log summary and final Suite result */
 			countSum = testPassedCount + testFailedCount + testSkippedCount;
 			if (testFailedCount == 0)
 			{
@@ -579,15 +607,15 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
 		}
 	}
 
-	// Take time - run end
+	/* Take time - run end */
 	runEndSeconds = GetClock();
 	runtime = runEndSeconds - runStartSeconds;
 	if (runtime < 0.0f) runtime = 0.0f;
 
-	// Log total runtime
+	/* Log total runtime */
 	SDLTest_Log("Total Run runtime: %.1f sec", runtime);
 
-	// Log summary and final run result
+	/* Log summary and final run result */
 	countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
 	if (totalTestFailedCount == 0)
 	{
diff --git a/test/testautomation_main.c b/test/testautomation_main.c
index 9873c7e41c..a39bf3ca1d 100644
--- a/test/testautomation_main.c
+++ b/test/testautomation_main.c
@@ -11,13 +11,16 @@
 
 
 /*!
- * \brief Tests SDL_Init() and SDL_Quit()
+ * \brief Tests SDL_Init() and SDL_Quit() of Joystick and Haptic subsystems
  * \sa
  * http://wiki.libsdl.org/moin.cgi/SDL_Init
  * http://wiki.libsdl.org/moin.cgi/SDL_Quit
  */
-static int main_testInitQuit (void *arg)
+static int main_testInitQuitJoystickHaptic (void *arg)
 {
+#if defined SDL_JOYSTICK_DISABLED || defined SDL_HAPTIC_DISABLED
+    return TEST_SKIPPED;
+#else
     int enabled_subsystems;
     int initialized_subsystems = SDL_INIT_JOYSTICK | SDL_INIT_HAPTIC;
 
@@ -32,6 +35,7 @@ static int main_testInitQuit (void *arg)
     SDLTest_AssertCheck( enabled_subsystems == 0, "SDL_Quit should shut down everything (%i)", enabled_subsystems );
 
     return TEST_COMPLETED;
+#endif
 }
 
 /*!
@@ -42,6 +46,9 @@ static int main_testInitQuit (void *arg)
  */
 static int main_testInitQuitSubSystem (void *arg)
 {
+#if defined SDL_JOYSTICK_DISABLED || defined SDL_HAPTIC_DISABLED || defined SDL_GAMECONTROLLER_DISABLED
+    return TEST_SKIPPED;
+#else
     int i;
     int subsystems[] = { SDL_INIT_JOYSTICK, SDL_INIT_HAPTIC, SDL_INIT_GAMECONTROLLER };
 
@@ -61,11 +68,15 @@ static int main_testInitQuitSubSystem (void *arg)
     }
 
     return TEST_COMPLETED;
+#endif
 }
 
 const int joy_and_controller = SDL_INIT_JOYSTICK | SDL_INIT_GAMECONTROLLER;
 static int main_testImpliedJoystickInit (void *arg)
 {
+#if defined SDL_JOYSTICK_DISABLED || defined SDL_GAMECONTROLLER_DISABLED
+    return TEST_SKIPPED;
+#else
     int initialized_system;
 
     // First initialize the controller
@@ -82,11 +93,15 @@ static int main_testImpliedJoystickInit (void *arg)
     initialized_system = SDL_WasInit(joy_and_controller);
     SDLTest_AssertCheck( (initialized_system & joy_and_controller) == 0, "SDL_WasInit() should be false for joystick & controller (%x)", initialized_system );
 
-	return TEST_COMPLETED;
+    return TEST_COMPLETED;
+#endif
 }
 
 static int main_testImpliedJoystickQuit (void *arg)
 {
+#if defined SDL_JOYSTICK_DISABLED || defined SDL_GAMECONTROLLER_DISABLED
+    return TEST_SKIPPED;
+#else
     int initialized_system;
 
     // First initialize the controller and the joystick (explicitly)
@@ -106,11 +121,12 @@ static int main_testImpliedJoystickQuit (void *arg)
 
     SDL_QuitSubSystem(SDL_INIT_JOYSTICK);
 
-	return TEST_COMPLETED;
+    return TEST_COMPLETED;
+#endif
 }
 
 static const SDLTest_TestCaseReference mainTest1 =
-		{ (SDLTest_TestCaseFp)main_testInitQuit, "main_testInitQuit", "Tests SDL_Init/Quit", TEST_ENABLED};
+		{ (SDLTest_TestCaseFp)main_testInitQuitJoystickHaptic, "main_testInitQuitJoystickHaptic", "Tests SDL_Init/Quit of Joystick and Haptic subsystem", TEST_ENABLED};
 
 static const SDLTest_TestCaseReference mainTest2 =
 		{ (SDLTest_TestCaseFp)main_testInitQuitSubSystem, "main_testInitQuitSubSystem", "Tests SDL_InitSubSystem/QuitSubSystem", TEST_ENABLED};