diff options
| -rw-r--r-- | Makefile.am | 9 | ||||
| -rw-r--r-- | configure.ac | 25 | ||||
| -rw-r--r-- | tools/zunitc/src/zuc_junit_reporter.c | 470 | ||||
| -rw-r--r-- | tools/zunitc/src/zuc_junit_reporter.h | 38 | ||||
| -rw-r--r-- | tools/zunitc/src/zunitc_impl.c | 17 |
5 files changed, 559 insertions, 0 deletions
diff --git a/Makefile.am b/Makefile.am index d7b45a84..8f2cda3e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -984,6 +984,8 @@ libzunitc_la_SOURCES = \ tools/zunitc/src/zuc_context.h \ tools/zunitc/src/zuc_event.h \ tools/zunitc/src/zuc_event_listener.h \ + tools/zunitc/src/zuc_junit_reporter.c \ + tools/zunitc/src/zuc_junit_reporter.h \ tools/zunitc/src/zuc_types.h \ tools/zunitc/src/zunitc_impl.c \ shared/helpers.h @@ -995,6 +997,13 @@ libzunitc_la_CFLAGS = \ libzunitc_la_LIBADD = \ libshared.la +if ENABLE_JUNIT_XML +libzunitc_la_CFLAGS += \ + $(LIBXML2_CFLAGS) +libzunitc_la_LIBADD += \ + $(LIBXML2_LIBS) +endif + libzunitcmain_la_SOURCES = \ tools/zunitc/src/main.c diff --git a/configure.ac b/configure.ac index 404418eb..d24fb0b4 100644 --- a/configure.ac +++ b/configure.ac @@ -426,6 +426,30 @@ if test "x$enable_dbus" != "xno"; then fi AM_CONDITIONAL(ENABLE_DBUS, test "x$enable_dbus" = "xyes") +# Note that other features might want libxml2, or this feature might use +# alternative xml libraries at some point. Therefore the feature and +# pre-requisite concepts are split. +AC_ARG_ENABLE(junit_xml, + AS_HELP_STRING([--disable-junit-xml], + [do not build with JUnit XML output]),, + enable_junit_xml=auto) +if test "x$enable_junit_xml" != "xno"; then + PKG_CHECK_MODULES(LIBXML2, + [libxml-2.0 >= 2.6], + have_libxml2=yes, + have_libxml2=no) + if test "x$have_libxml2" = "xno" -a "x$enable_junit_xml" = "xyes"; then + AC_MSG_ERROR([JUnit XML support explicitly requested, but libxml2 couldn't be found]) + fi + if test "x$have_libxml2" = "xyes"; then + enable_junit_xml=yes + AC_DEFINE(ENABLE_JUNIT_XML, [1], [Build Weston with JUnit output support]) + else + enable_junit_xml=no + fi +fi +AM_CONDITIONAL(ENABLE_JUNIT_XML, test "x$enable_junit_xml" = "xyes") + # ivi-shell support AC_ARG_ENABLE(ivi-shell, AS_HELP_STRING([--disable-ivi-shell], @@ -537,6 +561,7 @@ AC_MSG_RESULT([ FBDEV Compositor ${enable_fbdev_compositor} RDP Compositor ${enable_rdp_compositor} Screen Sharing ${enable_screen_sharing} + JUnit XML output ${enable_junit_xml} Raspberry Pi BCM headers ${have_bcm_host} diff --git a/tools/zunitc/src/zuc_junit_reporter.c b/tools/zunitc/src/zuc_junit_reporter.c new file mode 100644 index 00000000..5c30eaab --- /dev/null +++ b/tools/zunitc/src/zuc_junit_reporter.c @@ -0,0 +1,470 @@ +/* + * Copyright © 2015 Samsung Electronics Co., Ltd + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "config.h" + +#include "zuc_junit_reporter.h" + +#if ENABLE_JUNIT_XML + +#include <fcntl.h> +#include <libxml/parser.h> +#include <memory.h> +#include <stdio.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <time.h> +#include <unistd.h> + +#include "zuc_event_listener.h" +#include "zuc_types.h" + +#include "shared/zalloc.h" + +/** + * Hardcoded output name. + * @todo follow-up with refactoring to avoid filename hardcoding. + * Will allow for better testing in parallel etc. in general. + */ +#define XML_FNAME "test_detail.xml" + +#define ISO_8601_FORMAT "%Y-%m-%dT%H:%M:%SZ" + +/** + * Internal data. + */ +struct junit_data +{ + int fd; + time_t begin; +}; + +#define MAX_64BIT_STRLEN 20 + +static void +set_attribute(xmlNodePtr node, const char *name, int value) +{ + xmlChar scratch[MAX_64BIT_STRLEN + 1] = {}; + xmlStrPrintf(scratch, sizeof(scratch), BAD_CAST "%d", value); + xmlSetProp(node, BAD_CAST name, scratch); +} + +/** + * Output the given event. + * + * @param parent the parent node to add new content to. + * @param event the event to write out. + */ +static void +emit_event(xmlNodePtr parent, struct zuc_event *event) +{ + char *msg = NULL; + + switch (event->op) { + case ZUC_OP_TRUE: + if (asprintf(&msg, "%s:%d: error: Value of: %s\n" + " Actual: false\n" + "Expected: true\n", event->file, event->line, + event->expr1) < 0) { + msg = NULL; + } + break; + case ZUC_OP_FALSE: + if (asprintf(&msg, "%s:%d: error: Value of: %s\n" + " Actual: true\n" + "Expected: false\n", event->file, event->line, + event->expr1) < 0) { + msg = NULL; + } + break; + case ZUC_OP_NULL: + if (asprintf(&msg, "%s:%d: error: Value of: %s\n" + " Actual: %p\n" + "Expected: %p\n", event->file, event->line, + event->expr1, (void *)event->val1, NULL) < 0) { + msg = NULL; + } + break; + case ZUC_OP_NOT_NULL: + if (asprintf(&msg, "%s:%d: error: Value of: %s\n" + " Actual: %p\n" + "Expected: not %p\n", event->file, event->line, + event->expr1, (void *)event->val1, NULL) < 0) { + msg = NULL; + } + break; + case ZUC_OP_EQ: + if (event->valtype == ZUC_VAL_CSTR) { + if (asprintf(&msg, "%s:%d: error: Value of: %s\n" + " Actual: %s\n" + "Expected: %s\n" + "Which is: %s\n", + event->file, event->line, event->expr2, + (char *)event->val2, event->expr1, + (char *)event->val1) < 0) { + msg = NULL; + } + } else { + if (asprintf(&msg, "%s:%d: error: Value of: %s\n" + " Actual: %ld\n" + "Expected: %s\n" + "Which is: %ld\n", + event->file, event->line, event->expr2, + event->val2, event->expr1, + event->val1) < 0) { + msg = NULL; + } + } + break; + case ZUC_OP_NE: + if (event->valtype == ZUC_VAL_CSTR) { + if (asprintf(&msg, "%s:%d: error: " + "Expected: (%s) %s (%s)," + " actual: %s == %s\n", + event->file, event->line, + event->expr1, zuc_get_opstr(event->op), + event->expr2, (char *)event->val1, + (char *)event->val2) < 0) { + msg = NULL; + } + } else { + if (asprintf(&msg, "%s:%d: error: " + "Expected: (%s) %s (%s)," + " actual: %ld vs %ld\n", + event->file, event->line, + event->expr1, zuc_get_opstr(event->op), + event->expr2, event->val1, + event->val2) < 0) { + msg = NULL; + } + } + break; + case ZUC_OP_TERMINATE: + { + char const *level = (event->val1 == 0) ? "error" + : (event->val1 == 1) ? "warning" + : "note"; + if (asprintf(&msg, "%s:%d: %s: %s\n", + event->file, event->line, level, + event->expr1) < 0) { + msg = NULL; + } + break; + } + case ZUC_OP_TRACEPOINT: + if (asprintf(&msg, "%s:%d: note: %s\n", + event->file, event->line, event->expr1) < 0) { + msg = NULL; + } + break; + default: + if (asprintf(&msg, "%s:%d: error: " + "Expected: (%s) %s (%s), actual: %ld vs %ld\n", + event->file, event->line, + event->expr1, zuc_get_opstr(event->op), + event->expr2, event->val1, event->val2) < 0) { + msg = NULL; + } + } + + if ((event->op == ZUC_OP_TERMINATE) && (event->val1 > 1)) { + xmlNewChild(parent, NULL, BAD_CAST "skipped", NULL); + } else { + xmlNodePtr node = xmlNewChild(parent, NULL, + BAD_CAST "failure", NULL); + + if (msg) { + xmlSetProp(node, BAD_CAST "message", BAD_CAST msg); + } + xmlSetProp(node, BAD_CAST "type", BAD_CAST ""); + if (msg) { + xmlNodePtr cdata = xmlNewCDataBlock(node->doc, + BAD_CAST msg, + strlen(msg)); + xmlAddChild(node, cdata); + } + } + + free(msg); +} + +/** + * Formats a time in milliseconds to the normal JUnit elapsed form, or + * NULL if there is a problem. + * The caller should release this with free() + * + * @return the formatted time string upon success, NULL otherwise. + */ +static char * +as_duration(long ms) { + char *str = NULL; + + if (asprintf(&str, "%1.3f", ms / 1000.0) < 0) { + str = NULL; + } else { + /* + * Special case to match behavior of standard JUnit output + * writers. Asumption is certain readers might have + * limitations, etc. so it is best to keep 100% identical + * output. + */ + if (!strcmp("0.000", str)) { + free(str); + str = strdup("0"); + } + } + return str; +} + +/** + * Returns the status string for the tests (run/notrun). + * + * @param test the test to check status of. + * @return the status string. + */ +static char const * +get_test_status(struct zuc_test *test) +{ + if (test->disabled || test->skipped) + return "notrun"; + else + return "run"; +} + +/** + * Output the given test. + * + * @param parent the parent node to add new content to. + * @param test the test to write out. + */ +static void +emit_test(xmlNodePtr parent, struct zuc_test *test) +{ + char *time_str = as_duration(test->elapsed); + xmlNodePtr node = xmlNewChild(parent, NULL, BAD_CAST "testcase", NULL); + + xmlSetProp(node, BAD_CAST "name", BAD_CAST test->name); + xmlSetProp(node, BAD_CAST "status", BAD_CAST get_test_status(test)); + + if (time_str) { + xmlSetProp(node, BAD_CAST "time", BAD_CAST time_str); + + free(time_str); + time_str = NULL; + } + + xmlSetProp(node, BAD_CAST "classname", BAD_CAST test->test_case->name); + + if ((test->failed || test->fatal || test->skipped) && test->events) { + struct zuc_event *evt; + for (evt = test->events; evt; evt = evt->next) + emit_event(node, evt); + } +} + +/** + * Output the given test case. + * + * @param parent the parent node to add new content to. + * @param test_case the test case to write out. + */ +static void +emit_case(xmlNodePtr parent, struct zuc_case *test_case) +{ + int i; + int skipped = 0; + int disabled = 0; + int failures = 0; + xmlNodePtr node = NULL; + char *time_str = as_duration(test_case->elapsed); + + for (i = 0; i < test_case->test_count; ++i) { + if (test_case->tests[i]->disabled ) + disabled++; + if (test_case->tests[i]->skipped ) + skipped++; + if (test_case->tests[i]->failed + || test_case->tests[i]->fatal ) + failures++; + } + + node = xmlNewChild(parent, NULL, BAD_CAST "testsuite", NULL); + xmlSetProp(node, BAD_CAST "name", BAD_CAST test_case->name); + + set_attribute(node, "tests", test_case->test_count); + set_attribute(node, "failures", failures); + set_attribute(node, "disabled", disabled); + set_attribute(node, "skipped", skipped); + + if (time_str) { + xmlSetProp(node, BAD_CAST "time", BAD_CAST time_str); + free(time_str); + time_str = NULL; + } + + for (i = 0; i < test_case->test_count; ++i) + emit_test(node, test_case->tests[i]); +} + +/** + * Formats a time in milliseconds to the full ISO-8601 date/time string + * format, or NULL if there is a problem. + * The caller should release this with free() + * + * @return the formatted time string upon success, NULL otherwise. + */ +static char * +as_iso_8601(time_t const *t) +{ + char *result = NULL; + char buf[32] = {}; + struct tm when; + + if (gmtime_r(t, &when) != NULL) + if (strftime(buf, sizeof(buf), ISO_8601_FORMAT, &when)) + result = strdup(buf); + + return result; +} + + +static void +run_started(void *data, int live_case_count, int live_test_count, + int disabled_count) +{ + struct junit_data *jdata = data; + + jdata->begin = time(NULL); + jdata->fd = open(XML_FNAME, O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH); +} + +static void +run_ended(void *data, int case_count, struct zuc_case **cases, + int live_case_count, int live_test_count, int total_passed, + int total_failed, int total_disabled, long total_elapsed) +{ + int i; + long time = 0; + char *time_str = NULL; + char *timestamp = NULL; + xmlNodePtr root = NULL; + xmlDocPtr doc = NULL; + xmlChar *xmlchars = NULL; + int xmlsize = 0; + struct junit_data *jdata = data; + + for (i = 0; i < case_count; ++i) + time += cases[i]->elapsed; + + time_str = as_duration(time); + timestamp = as_iso_8601(&jdata->begin); + + /* here would be where to add errors? */ + + doc = xmlNewDoc(BAD_CAST "1.0"); + root = xmlNewNode(NULL, BAD_CAST "testsuites"); + xmlDocSetRootElement(doc, root); + + set_attribute(root, "tests", live_test_count); + set_attribute(root, "failures", total_failed); + set_attribute(root, "disabled", total_disabled); + + if (timestamp) { + xmlSetProp(root, BAD_CAST "timestamp", BAD_CAST timestamp); + free(timestamp); + timestamp = NULL; + } + + if (time_str) { + xmlSetProp(root, BAD_CAST "time", BAD_CAST time_str); + free(time_str); + time_str = NULL; + } + + xmlSetProp(root, BAD_CAST "name", BAD_CAST "AllTests"); + + for (i = 0; i < case_count; ++i) { + emit_case(root, cases[i]); + } + + xmlDocDumpFormatMemoryEnc(doc, &xmlchars, &xmlsize, "UTF-8", 1); + dprintf(jdata->fd, "%s", (char *) xmlchars); + xmlFree(xmlchars); + xmlchars = NULL; + xmlFreeDoc(doc); + + if ((jdata->fd != fileno(stdout)) + && (jdata->fd != fileno(stderr)) + && (jdata->fd != -1)) { + close(jdata->fd); + jdata->fd = -1; + } +} + +static void +destroy(void *data) +{ + xmlCleanupParser(); + + free(data); +} + +struct zuc_event_listener * +zuc_junit_reporter_create(void) +{ + struct zuc_event_listener *listener = + zalloc(sizeof(struct zuc_event_listener)); + + struct junit_data *data = zalloc(sizeof(struct junit_data)); + data->fd = -1; + + listener->data = data; + listener->destroy = destroy; + listener->run_started = run_started; + listener->run_ended = run_ended; + + return listener; +} + +#else /* ENABLE_JUNIT_XML */ + +#include "shared/zalloc.h" +#include "zuc_event_listener.h" + +/* + * Simple stub version if junit output (including libxml2 support) has + * been disabled. + * Will return NULL to cause failures as calling this when the #define + * has not been enabled is an invalid scenario. + */ + +struct zuc_event_listener * +zuc_junit_reporter_create(void) +{ + return NULL; +} + +#endif /* ENABLE_JUNIT_XML */ diff --git a/tools/zunitc/src/zuc_junit_reporter.h b/tools/zunitc/src/zuc_junit_reporter.h new file mode 100644 index 00000000..66f3c7b3 --- /dev/null +++ b/tools/zunitc/src/zuc_junit_reporter.h @@ -0,0 +1,38 @@ +/* + * Copyright © 2015 Samsung Electronics Co., Ltd + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ZUC_JUNIT_REPORTER_H +#define ZUC_JUNIT_REPORTER_H + +struct zuc_event_listener; + +/** + * Creates an instance of a reporter that will write data in the JUnit + * XML format. + */ +struct zuc_event_listener * +zuc_junit_reporter_create(void); + +#endif /* ZUC_JUNIT_REPORTER_H */ diff --git a/tools/zunitc/src/zunitc_impl.c b/tools/zunitc/src/zunitc_impl.c index b8ab0b2c..6f591f08 100644 --- a/tools/zunitc/src/zunitc_impl.c +++ b/tools/zunitc/src/zunitc_impl.c @@ -44,6 +44,7 @@ #include "zuc_collector.h" #include "zuc_context.h" #include "zuc_event_listener.h" +#include "zuc_junit_reporter.h" #include "shared/config-parser.h" #include "shared/helpers.h" @@ -152,6 +153,12 @@ zuc_set_break_on_failure(bool break_on_failure) g_ctx.break_on_failure = break_on_failure; } +void +zuc_set_output_junit(bool enable) +{ + g_ctx.output_junit = enable; +} + const char * zuc_get_program_name(void) { @@ -523,6 +530,7 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged) int opt_repeat = 0; int opt_random = 0; int opt_break_on_failure = 0; + int opt_junit = 0; char *opt_filter = NULL; char *help_param = NULL; @@ -535,6 +543,9 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged) { WESTON_OPTION_INTEGER, "zuc-random", 0, &opt_random }, { WESTON_OPTION_BOOLEAN, "zuc-break-on-failure", 0, &opt_break_on_failure }, +#if ENABLE_JUNIT_XML + { WESTON_OPTION_BOOLEAN, "zuc-output-xml", 0, &opt_junit }, +#endif { WESTON_OPTION_STRING, "zuc-filter", 0, &opt_filter }, }; @@ -623,6 +634,9 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged) " --zuc-filter=FILTER\n" " --zuc-list-tests\n" " --zuc-nofork\n" +#if ENABLE_JUNIT_XML + " --zuc-output-xml\n" +#endif " --zuc-random=N [0|1|<seed number>]\n" " --zuc-repeat=N\n" " --help\n", @@ -638,6 +652,7 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged) zuc_set_random(opt_random); zuc_set_spawn(!opt_nofork); zuc_set_break_on_failure(opt_break_on_failure); + zuc_set_output_junit(opt_junit); rc = EXIT_SUCCESS; } @@ -1301,6 +1316,8 @@ zucimpl_run_tests(void) if (g_ctx.listeners == NULL) { zuc_add_event_listener(zuc_collector_create(&(g_ctx.fds[1]))); zuc_add_event_listener(zuc_base_logger_create()); + if (g_ctx.output_junit) + zuc_add_event_listener(zuc_junit_reporter_create()); } if (g_ctx.case_count < 1) { |