tests: optionally write results as JUnit-style .xml
authorJohannes Schindelin <johannes.schindelin@gmx.de>
Tue, 29 Jan 2019 14:19:27 +0000 (06:19 -0800)
committerJunio C Hamano <gitster@pobox.com>
Tue, 29 Jan 2019 17:26:46 +0000 (09:26 -0800)
This will come in handy when publishing the results of Git's test suite
during an automated Azure DevOps run.

Note: we need to make extra sure that invalid UTF-8 encoding is turned
into valid UTF-8 (using the Replacement Character, \uFFFD) because
t9902's trace contains such invalid byte sequences, and the task in the
Azure Pipeline that uploads the test results would refuse to do anything
if it was asked to parse an .xml file with invalid UTF-8 in it.

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
Makefile
t/.gitignore
t/helper/test-tool.c
t/helper/test-tool.h
t/helper/test-xml-encode.c [new file with mode: 0644]
t/test-lib.sh
index 1a44c811aa56330327172cf693c61f9a221e4e16..044b4f77bd7be3b4cab1ae78b23afad448c2ce2a 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -754,6 +754,7 @@ TEST_BUILTINS_OBJS += test-submodule-config.o
 TEST_BUILTINS_OBJS += test-submodule-nested-repo-config.o
 TEST_BUILTINS_OBJS += test-subprocess.o
 TEST_BUILTINS_OBJS += test-urlmatch-normalization.o
+TEST_BUILTINS_OBJS += test-xml-encode.o
 TEST_BUILTINS_OBJS += test-wildmatch.o
 TEST_BUILTINS_OBJS += test-windows-named-pipe.o
 TEST_BUILTINS_OBJS += test-write-cache.o
index 348715f0e4bcfebf9df680fda5b9c4be4f3e7527..91cf5772fe5643dbe075da98ed5166e1899b9a54 100644 (file)
@@ -2,3 +2,4 @@
 /test-results
 /.prove
 /chainlinttmp
+/out/
index bfb195b1a828a34988912e66f9e07f493588d5b4..4b4b397d9365e3cc594702a5ccc4f28e8c5f8b6d 100644 (file)
@@ -49,6 +49,7 @@ static struct test_cmd cmds[] = {
        { "submodule-nested-repo-config", cmd__submodule_nested_repo_config },
        { "subprocess", cmd__subprocess },
        { "urlmatch-normalization", cmd__urlmatch_normalization },
+       { "xml-encode", cmd__xml_encode },
        { "wildmatch", cmd__wildmatch },
 #ifdef GIT_WINDOWS_NATIVE
        { "windows-named-pipe", cmd__windows_named_pipe },
index 042f12464b2a17afeb37e01d90bf49ba33ce879a..c0ab65e370eb6eec7e783013bdfe541ea4c29ab5 100644 (file)
@@ -45,6 +45,7 @@ int cmd__submodule_config(int argc, const char **argv);
 int cmd__submodule_nested_repo_config(int argc, const char **argv);
 int cmd__subprocess(int argc, const char **argv);
 int cmd__urlmatch_normalization(int argc, const char **argv);
+int cmd__xml_encode(int argc, const char **argv);
 int cmd__wildmatch(int argc, const char **argv);
 #ifdef GIT_WINDOWS_NATIVE
 int cmd__windows_named_pipe(int argc, const char **argv);
diff --git a/t/helper/test-xml-encode.c b/t/helper/test-xml-encode.c
new file mode 100644 (file)
index 0000000..a648bbd
--- /dev/null
@@ -0,0 +1,80 @@
+#include "test-tool.h"
+
+static const char *utf8_replace_character = "&#xfffd;";
+
+/*
+ * Encodes (possibly incorrect) UTF-8 on <stdin> to <stdout>, to be embedded
+ * in an XML file.
+ */
+int cmd__xml_encode(int argc, const char **argv)
+{
+       unsigned char buf[1024], tmp[4], *tmp2 = NULL;
+       ssize_t cur = 0, len = 1, remaining = 0;
+       unsigned char ch;
+
+       for (;;) {
+               if (++cur == len) {
+                       len = xread(0, buf, sizeof(buf));
+                       if (!len)
+                               return 0;
+                       if (len < 0)
+                               die_errno("Could not read <stdin>");
+                       cur = 0;
+               }
+               ch = buf[cur];
+
+               if (tmp2) {
+                       if ((ch & 0xc0) != 0x80) {
+                               fputs(utf8_replace_character, stdout);
+                               tmp2 = NULL;
+                               cur--;
+                               continue;
+                       }
+                       *tmp2 = ch;
+                       tmp2++;
+                       if (--remaining == 0) {
+                               fwrite(tmp, tmp2 - tmp, 1, stdout);
+                               tmp2 = NULL;
+                       }
+                       continue;
+               }
+
+               if (!(ch & 0x80)) {
+                       /* 0xxxxxxx */
+                       if (ch == '&')
+                               fputs("&amp;", stdout);
+                       else if (ch == '\'')
+                               fputs("&apos;", stdout);
+                       else if (ch == '"')
+                               fputs("&quot;", stdout);
+                       else if (ch == '<')
+                               fputs("&lt;", stdout);
+                       else if (ch == '>')
+                               fputs("&gt;", stdout);
+                       else if (ch >= 0x20)
+                               fputc(ch, stdout);
+                       else if (ch == 0x09 || ch == 0x0a || ch == 0x0d)
+                               fprintf(stdout, "&#x%02x;", ch);
+                       else
+                               fputs(utf8_replace_character, stdout);
+               } else if ((ch & 0xe0) == 0xc0) {
+                       /* 110XXXXx 10xxxxxx */
+                       tmp[0] = ch;
+                       remaining = 1;
+                       tmp2 = tmp + 1;
+               } else if ((ch & 0xf0) == 0xe0) {
+                       /* 1110XXXX 10Xxxxxx 10xxxxxx */
+                       tmp[0] = ch;
+                       remaining = 2;
+                       tmp2 = tmp + 1;
+               } else if ((ch & 0xf8) == 0xf0) {
+                       /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */
+                       tmp[0] = ch;
+                       remaining = 3;
+                       tmp2 = tmp + 1;
+               } else
+                       fputs(utf8_replace_character, stdout);
+       }
+
+       return 0;
+}
index a1abb1177a15c85ef7c75f0b9b056a59ff9843e9..a3b2166cb5908152df5a2b0d73fa128720b7a095 100644 (file)
@@ -139,6 +139,9 @@ do
                verbose_log=t
                tee=t
                ;;
+       --write-junit-xml)
+               write_junit_xml=t
+               ;;
        --stress)
                stress=t ;;
        --stress=*)
@@ -622,11 +625,24 @@ trap 'exit $?' INT TERM HUP
 # the test_expect_* functions instead.
 
 test_ok_ () {
+       if test -n "$write_junit_xml"
+       then
+               write_junit_xml_testcase "$*"
+       fi
        test_success=$(($test_success + 1))
        say_color "" "ok $test_count - $@"
 }
 
 test_failure_ () {
+       if test -n "$write_junit_xml"
+       then
+               junit_insert="<failure message=\"not ok $test_count -"
+               junit_insert="$junit_insert $(xml_attr_encode "$1")\">"
+               junit_insert="$junit_insert $(xml_attr_encode \
+                       "$(printf '%s\n' "$@" | sed 1d)")"
+               junit_insert="$junit_insert</failure>"
+               write_junit_xml_testcase "$1" "      $junit_insert"
+       fi
        test_failure=$(($test_failure + 1))
        say_color error "not ok $test_count - $1"
        shift
@@ -635,11 +651,19 @@ test_failure_ () {
 }
 
 test_known_broken_ok_ () {
+       if test -n "$write_junit_xml"
+       then
+               write_junit_xml_testcase "$* (breakage fixed)"
+       fi
        test_fixed=$(($test_fixed+1))
        say_color error "ok $test_count - $@ # TODO known breakage vanished"
 }
 
 test_known_broken_failure_ () {
+       if test -n "$write_junit_xml"
+       then
+               write_junit_xml_testcase "$* (known breakage)"
+       fi
        test_broken=$(($test_broken+1))
        say_color warn "not ok $test_count - $@ # TODO known breakage"
 }
@@ -897,6 +921,10 @@ test_start_ () {
        test_count=$(($test_count+1))
        maybe_setup_verbose
        maybe_setup_valgrind
+       if test -n "$write_junit_xml"
+       then
+               junit_start=$(test-tool date getnanos)
+       fi
 }
 
 test_finish_ () {
@@ -934,6 +962,13 @@ test_skip () {
 
        case "$to_skip" in
        t)
+               if test -n "$write_junit_xml"
+               then
+                       message="$(xml_attr_encode "$skipped_reason")"
+                       write_junit_xml_testcase "$1" \
+                               "      <skipped message=\"$message\" />"
+               fi
+
                say_color skip >&3 "skipping test: $@"
                say_color skip "ok $test_count # skip $1 ($skipped_reason)"
                : true
@@ -949,9 +984,51 @@ test_at_end_hook_ () {
        :
 }
 
+write_junit_xml () {
+       case "$1" in
+       --truncate)
+               >"$junit_xml_path"
+               junit_have_testcase=
+               shift
+               ;;
+       esac
+       printf '%s\n' "$@" >>"$junit_xml_path"
+}
+
+xml_attr_encode () {
+       printf '%s\n' "$@" | test-tool xml-encode
+}
+
+write_junit_xml_testcase () {
+       junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\""
+       shift
+       junit_attrs="$junit_attrs classname=\"$this_test\""
+       junit_attrs="$junit_attrs time=\"$(test-tool \
+               date getnanos $junit_start)\""
+       write_junit_xml "$(printf '%s\n' \
+               "    <testcase $junit_attrs>" "$@" "    </testcase>")"
+       junit_have_testcase=t
+}
+
 test_done () {
        GIT_EXIT_OK=t
 
+       if test -n "$write_junit_xml" && test -n "$junit_xml_path"
+       then
+               test -n "$junit_have_testcase" || {
+                       junit_start=$(test-tool date getnanos)
+                       write_junit_xml_testcase "all tests skipped"
+               }
+
+               # adjust the overall time
+               junit_time=$(test-tool date getnanos $junit_suite_start)
+               sed "s/<testsuite [^>]*/& time=\"$junit_time\"/" \
+                       <"$junit_xml_path" >"$junit_xml_path.new"
+               mv "$junit_xml_path.new" "$junit_xml_path"
+
+               write_junit_xml "  </testsuite>" "</testsuites>"
+       fi
+
        if test -z "$HARNESS_ACTIVE"
        then
                mkdir -p "$TEST_RESULTS_DIR"
@@ -1178,6 +1255,7 @@ then
 else
        mkdir -p "$TRASH_DIRECTORY"
 fi
+
 # Use -P to resolve symlinks in our working directory so that the cwd
 # in subprocesses like git equals our $PWD (for pathname comparisons).
 cd -P "$TRASH_DIRECTORY" || exit 1
@@ -1191,6 +1269,19 @@ then
        test_done
 fi
 
+if test -n "$write_junit_xml"
+then
+       junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out"
+       mkdir -p "$junit_xml_dir"
+       junit_xml_base=${0##*/}
+       junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml"
+       junit_attrs="name=\"${junit_xml_base%.sh}\""
+       junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \
+               date +%Y-%m-%dT%H:%M:%S)\""
+       write_junit_xml --truncate "<testsuites>" "  <testsuite $junit_attrs>"
+       junit_suite_start=$(test-tool date getnanos)
+fi
+
 # Provide an implementation of the 'yes' utility
 yes () {
        if test $# = 0