tests: include detailed trace logs with --write-junit-xml upon failure

The JUnit XML format lends itself to be presented in a powerful UI,
where you can drill down to the information you are interested in very
quickly.

For test failures, this usually means that you want to see the detailed
trace of the failing tests.

With Travis CI, we passed the `--verbose-log` option to get those
traces. However, that seems excessive, as we do not need/use the logs in
almost all of those cases: only when a test fails do we have a way to
include the trace.

So let's do something different when using Azure DevOps: let's run all
the tests with `--quiet` first, and only if a failure is encountered,
try to trace the commands as they are executed.

Of course, we cannot turn on `--verbose-log` after the fact. So let's
just re-run the test with all the same options, adding `--verbose-log`.
And then munging the output file into the JUnit XML on the fly.

Note: there is an off chance that re-running the test in verbose mode
"fixes" the failures (and this does happen from time to time!). That is
a possibility we should be able to live with. Ideally, we would label
this as "Passed upon rerun", and Azure Pipelines even know about that
outcome, but it is not available when using the JUnit XML format for
now:
https://github.com/Microsoft/azure-pipelines-agent/blob/master/src/Agent.Worker/TestResults/JunitResultReader.cs

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/t/helper/test-path-utils.c b/t/helper/test-path-utils.c
index 30211d6d641538453dd7a36c5627ef11a26e2dde..6efde6f5ba5b7203a73335c51114c2d47156231d 100644 (file)
--- a/t/helper/test-path-utils.c
+++ b/t/helper/test-path-utils.c
@@ -303,6 +303,27 @@ int cmd__path_utils(int argc, const char **argv)
                return !!res;
        }
 
+       if (argc == 4 && !strcmp(argv[1], "skip-n-bytes")) {
+               int fd = open(argv[2], O_RDONLY), offset = atoi(argv[3]);
+               char buffer[65536];
+
+               if (fd < 0)
+                       die_errno("could not open '%s'", argv[2]);
+               if (lseek(fd, offset, SEEK_SET) < 0)
+                       die_errno("could not skip %d bytes", offset);
+               for (;;) {
+                       ssize_t count = read(fd, buffer, sizeof(buffer));
+                       if (count < 0)
+                               die_errno("could not read '%s'", argv[2]);
+                       if (!count)
+                               break;
+                       if (write(1, buffer, count) < 0)
+                               die_errno("could not write to stdout");
+               }
+               close(fd);
+               return 0;
+       }
+
        fprintf(stderr, "%s: unknown function name: %s\n", argv[0],
                argv[1] ? argv[1] : "(there was none)");
        return 1;

diff --git a/t/test-lib.sh b/t/test-lib.sh
index a3b2166cb5908152df5a2b0d73fa128720b7a095..f31a1c8f796adbb37d04768923d60b2c39a74f87 100644 (file)
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -639,8 +639,19 @@ test_failure_ () {
                junit_insert="<failure message=\"not ok $test_count -"
                junit_insert="$junit_insert $(xml_attr_encode "$1")\">"
                junit_insert="$junit_insert $(xml_attr_encode \
-                       "$(printf '%s\n' "$@" | sed 1d)")"
+                       "$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
+                          then
+                               test-tool path-utils skip-n-bytes \
+                                       "$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET
+                          else
+                               printf '%s\n' "$@" | sed 1d
+                          fi)")"
                junit_insert="$junit_insert</failure>"
+               if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
+               then
+                       junit_insert="$junit_insert<system-err>$(xml_attr_encode \
+                               "$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")</system-err>"
+               fi
                write_junit_xml_testcase "$1" "      $junit_insert"
        fi
        test_failure=$(($test_failure + 1))
@@ -931,6 +942,11 @@ test_finish_ () {
        echo >&3 ""
        maybe_teardown_valgrind
        maybe_teardown_verbose
+       if test -n "$GIT_TEST_TEE_OFFSET"
+       then
+               GIT_TEST_TEE_OFFSET=$(test-tool path-utils file-size \
+                       "$GIT_TEST_TEE_OUTPUT_FILE")
+       fi
 }
 
 test_skip () {
@@ -1280,6 +1296,10 @@ then
                date +%Y-%m-%dT%H:%M:%S)\""
        write_junit_xml --truncate "<testsuites>" "  <testsuite $junit_attrs>"
        junit_suite_start=$(test-tool date getnanos)
+       if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
+       then
+               GIT_TEST_TEE_OFFSET=0
+       fi
 fi
 
 # Provide an implementation of the 'yes' utility