test-dbus: Produce machine-readable TAP output

See http://testanything.org/ for more information on TAP.

Reviewed-by: Philip Withnall <withnall@endlessm.com>
Signed-off-by: Simon McVittie <smcv@collabora.com>
Bug: https://bugs.freedesktop.org/show_bug.cgi?id=103601

6 files changed, 190 insertions, 45 deletions

diff --git a/dbus/dbus-test-main.c b/dbus/dbus-test-main.c
index f8088f823..4624fd5c9 100644
--- a/dbus/dbus-test-main.c
+++ b/dbus/dbus-test-main.c
@@ -25,6 +25,7 @@
 #include <config.h>
 #include "dbus-types.h"
 #include "dbus-test.h"
+#include "dbus-test-tap.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -61,8 +62,7 @@ main (int    argc,
     specific_test = argv[2];
   else
     specific_test = NULL;
-  
-  dbus_internal_do_not_use_run_tests (test_data_dir, specific_test);
-  
-  return 0;
+
+  _dbus_run_tests (test_data_dir, specific_test);
+  return _dbus_test_done_testing ();
 }
diff --git a/dbus/dbus-test-tap.c b/dbus/dbus-test-tap.c
index a6f99b547..710e5eb87 100644
--- a/dbus/dbus-test-tap.c
+++ b/dbus/dbus-test-tap.c
@@ -40,6 +40,9 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+static unsigned int failures = 0;
+static unsigned int tap_test_counter = 0;
+
 /*
  * Output TAP indicating a fatal error, and exit unsuccessfully.
  */
@@ -74,4 +77,129 @@ _dbus_test_diag (const char *format,
   printf ("\n");
 }
 
+/*
+ * Output TAP indicating that all tests have been skipped, and exit
+ * successfully.
+ *
+ * This is only valid if _dbus_test_ok(), _dbus_test_not_ok(),
+ * etc. have not yet been called.
+ */
+void
+_dbus_test_skip_all (const char *format,
+    ...)
+{
+  va_list ap;
+
+  _dbus_assert (tap_test_counter == 0);
+
+  printf ("1..0 # SKIP - ");
+  va_start (ap, format);
+  vprintf (format, ap);
+  va_end (ap);
+  printf ("\n");
+  fflush (stdout);
+  exit (0);
+}
+
+/*
+ * Output TAP indicating that a test has passed, and increment the
+ * test counter.
+ */
+void
+_dbus_test_ok (const char *format,
+    ...)
+{
+  va_list ap;
+
+  printf ("ok %u - ", ++tap_test_counter);
+  va_start (ap, format);
+  vprintf (format, ap);
+  va_end (ap);
+  printf ("\n");
+  fflush (stdout);
+}
+
+/*
+ * Output TAP indicating that a test has failed (in a way that is not
+ * fatal to the test executable), and increment the test counter.
+ */
+void
+_dbus_test_not_ok (const char *format,
+    ...)
+{
+  va_list ap;
+
+  printf ("not ok %u - ", ++tap_test_counter);
+  va_start (ap, format);
+  vprintf (format, ap);
+  va_end (ap);
+  printf ("\n");
+  failures++;
+  fflush (stdout);
+}
+
+/*
+ * Output TAP indicating that a test has been skipped (in a way that is
+ * not fatal to the test executable), and increment the test counter.
+ */
+void
+_dbus_test_skip (const char *format,
+    ...)
+{
+  va_list ap;
+
+  printf ("ok %u # SKIP ", ++tap_test_counter);
+  va_start (ap, format);
+  vprintf (format, ap);
+  va_end (ap);
+  printf ("\n");
+  fflush (stdout);
+}
+
+/*
+ * Shut down libdbus, check that exactly previously_allocated memory
+ * blocks are allocated, and output TAP indicating a test pass or failure.
+ *
+ * Return TRUE if no leaks were detected.
+ */
+void
+_dbus_test_check_memleaks (const char *test_name)
+{
+  dbus_shutdown ();
+
+  if (_dbus_get_malloc_blocks_outstanding () == 0)
+    {
+      printf ("ok %u - %s did not leak memory\n", ++tap_test_counter,
+          test_name);
+    }
+  else
+    {
+      printf ("not ok %u - %s leaked %d blocks\n",
+          ++tap_test_counter, test_name,
+          _dbus_get_malloc_blocks_outstanding ());
+      failures++;
+    }
+}
+
+/*
+ * Output TAP indicating that testing has finished and no more tests
+ * are going to be run. Return the Unix-style exit code.
+ */
+int
+_dbus_test_done_testing (void)
+{
+  if (failures == 0)
+    _dbus_test_diag ("%u tests passed", tap_test_counter);
+  else
+    _dbus_test_diag ("%u/%u tests failed", failures, tap_test_counter);
+
+  printf ("1..%u\n", tap_test_counter);
+  fflush (stdout);
+
+  if (failures == 0)
+    return 0;
+
+  return 1;
+}
+
 #endif
diff --git a/dbus/dbus-test-tap.h b/dbus/dbus-test-tap.h
index 706475bdd..ea1169212 100644
--- a/dbus/dbus-test-tap.h
+++ b/dbus/dbus-test-tap.h
@@ -39,6 +39,26 @@ DBUS_PRIVATE_EXPORT
 void _dbus_test_diag (const char *format,
     ...) _DBUS_GNUC_PRINTF (1, 2);
 
+DBUS_PRIVATE_EXPORT
+void _dbus_test_skip_all (const char *format,
+    ...) _DBUS_GNUC_NORETURN _DBUS_GNUC_PRINTF (1, 2);
+
+DBUS_PRIVATE_EXPORT
+void _dbus_test_ok (const char *format,
+    ...) _DBUS_GNUC_PRINTF (1, 2);
+DBUS_PRIVATE_EXPORT
+void _dbus_test_not_ok (const char *format,
+    ...) _DBUS_GNUC_PRINTF (1, 2);
+DBUS_PRIVATE_EXPORT
+void _dbus_test_skip (const char *format,
+    ...) _DBUS_GNUC_PRINTF (1, 2);
+
+DBUS_PRIVATE_EXPORT
+void _dbus_test_check_memleaks (const char *test_name);
+
+DBUS_PRIVATE_EXPORT
+int _dbus_test_done_testing (void);
+
 #endif
 
 #endif
diff --git a/dbus/dbus-test.c b/dbus/dbus-test.c
index 8ae91d657..ae0b791c4 100644
--- a/dbus/dbus-test.c
+++ b/dbus/dbus-test.c
@@ -30,49 +30,50 @@
 #include <stdlib.h>
 
 #ifdef DBUS_ENABLE_EMBEDDED_TESTS
-static void
-check_memleaks (void)
-{
-  dbus_shutdown ();
-
-  _dbus_test_diag ("%s: checking for memleaks", "test-dbus");
-  if (_dbus_get_malloc_blocks_outstanding () != 0)
-    _dbus_test_fatal ("%d dbus_malloc blocks were not freed",
-                      _dbus_get_malloc_blocks_outstanding ());
-}
-
 typedef dbus_bool_t (*TestFunc)(void);
 typedef dbus_bool_t (*TestDataFunc)(const char *data);
 
 static void
 run_test (const char             *test_name,
-	  const char             *specific_test,
-	  TestFunc                test)
+          const char             *specific_test,
+          TestFunc                test)
 {
-  if (!specific_test || strcmp (specific_test, test_name) == 0)
+  if (specific_test != NULL && strcmp (specific_test, test_name) != 0)
     {
-      _dbus_test_diag ("%s: running %s tests", "test-dbus", test_name);
-      if (!test ())
-        _dbus_test_fatal ("%s test failed", test_name);
-
-      check_memleaks ();
+      _dbus_test_skip ("%s - Only intending to run %s", test_name, specific_test);
+      return;
     }
+
+  _dbus_test_diag ("%s: running %s tests", "test-dbus", test_name);
+
+  if (test ())
+    _dbus_test_ok ("%s", test_name);
+  else
+    _dbus_test_not_ok ("%s", test_name);
+
+  _dbus_test_check_memleaks (test_name);
 }
 
 static void
 run_data_test (const char             *test_name,
-	       const char             *specific_test,
-	       TestDataFunc            test,
-	       const char             *test_data_dir)
+               const char             *specific_test,
+               TestDataFunc            test,
+               const char             *test_data_dir)
 {
-  if (!specific_test || strcmp (specific_test, test_name) == 0)
+  if (specific_test != NULL && strcmp (specific_test, test_name) != 0)
     {
-      _dbus_test_diag ("%s: running %s tests", "test-dbus", test_name);
-      if (!test (test_data_dir))
-        _dbus_test_fatal ("%s test failed", test_name);
-
-      check_memleaks ();
+      _dbus_test_skip ("%s - Only intending to run %s", test_name, specific_test);
+      return;
     }
+
+  _dbus_test_diag ("%s: running %s tests", "test-dbus", test_name);
+
+  if (test (test_data_dir))
+    _dbus_test_ok ("%s", test_name);
+  else
+    _dbus_test_not_ok ("%s", test_name);
+
+  _dbus_test_check_memleaks (test_name);
 }
 
 /**
@@ -86,7 +87,8 @@ run_data_test (const char             *test_name,
  * @param specific_test run specific test or #NULL to run all tests
  */
 void
-dbus_internal_do_not_use_run_tests (const char *test_data_dir, const char *specific_test)
+_dbus_run_tests (const char   *test_data_dir,
+                 const char   *specific_test)
 {
   if (!_dbus_threads_init_debug ())
     _dbus_test_fatal ("debug threads init failed");
@@ -152,8 +154,6 @@ dbus_internal_do_not_use_run_tests (const char *test_data_dir, const char *speci
   run_data_test ("sha", specific_test, _dbus_sha_test, test_data_dir);
   
   run_data_test ("auth", specific_test, _dbus_auth_test, test_data_dir);
-
-  _dbus_test_diag ("%s: completed successfully", "test-dbus");
 }
 
 #endif /* DBUS_ENABLE_EMBEDDED_TESTS */
diff --git a/dbus/dbus-test.h b/dbus/dbus-test.h
index 71876a102..113aded42 100644
--- a/dbus/dbus-test.h
+++ b/dbus/dbus-test.h
@@ -96,8 +96,9 @@ dbus_bool_t _dbus_object_tree_test       (void);
 
 dbus_bool_t _dbus_credentials_test       (const char *test_data_dir);
 
-void        dbus_internal_do_not_use_run_tests         (const char          *test_data_dir,
-							const char          *specific_test);
+void        _dbus_run_tests                            (const char          *test_data_dir,
+                                                        const char          *specific_test);
+
 dbus_bool_t dbus_internal_do_not_use_try_message_file  (const DBusString    *filename,
                                                         DBusValidity         expected_validity);
 dbus_bool_t dbus_internal_do_not_use_try_message_data  (const DBusString    *data,
diff --git a/test/Makefile.am b/test/Makefile.am
index bc5bc1852..2721a4a90 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -70,28 +70,24 @@ TEST_BINARIES = \
 
 ## These are conceptually part of directories that come earlier in SUBDIRS
 ## order, but we don't want to run them til we arrive in this directory,
-## since they depend on stuff from this directory. We wrap them in a
+## since they depend on stuff from this directory. We wrap some of them in a
 ## simple shell script to get TAP output.
 
 wrap_bus_tests = test-bus.sh
-wrap_dbus_tests = test-dbus.sh
 
 if DBUS_UNIX
 wrap_bus_tests += test-bus-launch-helper.sh
 wrap_bus_tests += test-bus-system.sh
 endif
 
-TESTS += $(wrap_bus_tests) $(wrap_dbus_tests)
-CLEANFILES += $(wrap_bus_tests) $(wrap_dbus_tests)
+TESTS += $(wrap_bus_tests)
+TESTS += ../dbus/test-dbus$(EXEEXT)
+CLEANFILES += $(wrap_bus_tests)
 
 $(wrap_bus_tests): test-bus%.sh: ../bus/test-bus%$(EXEEXT) tap-test.sh.in Makefile
 	sed -e 's![@]RUN[@]!$<!' \
 		< $(srcdir)/tap-test.sh.in > $@
 
-$(wrap_dbus_tests): test-dbus%.sh: ../dbus/test-dbus%$(EXEEXT) tap-test.sh.in Makefile
-	sed -e 's![@]RUN[@]!$<!' \
-		< $(srcdir)/tap-test.sh.in > $@
-
 else !DBUS_ENABLE_EMBEDDED_TESTS
 
 TEST_BINARIES=