Merge branch 'sg/test-x' into next
authorJunio C Hamano <gitster@pobox.com>
Tue, 6 Mar 2018 23:11:48 +0000 (15:11 -0800)
committerJunio C Hamano <gitster@pobox.com>
Tue, 6 Mar 2018 23:11:48 +0000 (15:11 -0800)
Running test scripts under -x option of the shell is often not a
useful way to debug them, because the error messages from the
commands tests try to capture and inspect are contaminated by the
tracing output by the shell. An earlier work done to make it more
pleasant to run tests under -x with recent versions of bash is
extended to cover posix shells that do not support BASH_XTRACEFD.

* sg/test-x:
travis-ci: run tests with '-x' tracing
t/README: add a note about don't saving stderr of compound commands
t1510-repo-setup: mark as untraceable with '-x'
t9903-bash-prompt: don't check the stderr of __git_ps1()
t5570-git-daemon: don't check the stderr of a subshell
t5526: use $TRASH_DIRECTORY to specify the path of GIT_TRACE log file
t5500-fetch-pack: don't check the stderr of a subshell
t3030-merge-recursive: don't check the stderr of a subshell
t1507-rev-parse-upstream: don't check the stderr of a shell function
t: add means to disable '-x' tracing for individual test scripts
t: prevent '-x' tracing from interfering with test helpers' stderr

12 files changed:
ci/lib-travisci.sh
t/README
t/lib-terminal.sh
t/t1507-rev-parse-upstream.sh
t/t1510-repo-setup.sh
t/t3030-merge-recursive.sh
t/t5500-fetch-pack.sh
t/t5526-fetch-submodules.sh
t/t5570-git-daemon.sh
t/t9903-bash-prompt.sh
t/test-lib-functions.sh
t/test-lib.sh
index 1efee55ef79cdcf2ca07135eed76073b91d91ff9..109ef280da9884027f17fe7f233b6b818bd71393 100755 (executable)
@@ -97,7 +97,7 @@ fi
 export DEVELOPER=1
 export DEFAULT_TEST_TARGET=prove
 export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save"
-export GIT_TEST_OPTS="--verbose-log"
+export GIT_TEST_OPTS="--verbose-log -x"
 export GIT_TEST_CLONE_2GB=YesPlease
 
 case "$jobname" in
index 1a1361a8063b8e247b6a92b83b6e8876953f95bd..24ddebfabf97be1251452fe5b6bba847597431fd 100644 (file)
--- a/t/README
+++ b/t/README
@@ -84,9 +84,10 @@ appropriately before running "make".
 
 -x::
        Turn on shell tracing (i.e., `set -x`) during the tests
-       themselves. Implies `--verbose`. Note that in non-bash shells,
-       this can cause failures in some tests which redirect and test
-       the output of shell functions. Use with caution.
+       themselves. Implies `--verbose`.
+       Ignored in test scripts that set the variable 'test_untraceable'
+       to a non-empty value, unless it's run with a Bash version
+       supporting BASH_XTRACEFD, i.e. v4.1 or later.
 
 -d::
 --debug::
@@ -452,6 +453,22 @@ Don't:
    causing the next test to start in an unexpected directory.  Do so
    inside a subshell if necessary.
 
+ - save and verify the standard error of compound commands, i.e. group
+   commands, subshells, and shell functions (except test helper
+   functions like 'test_must_fail') like this:
+
+     ( cd dir && git cmd ) 2>error &&
+     test_cmp expect error
+
+   When running the test with '-x' tracing, then the trace of commands
+   executed in the compound command will be included in standard error
+   as well, quite possibly throwing off the subsequent checks examining
+   the output.  Instead, save only the relevant git command's standard
+   error:
+
+     ( cd dir && git cmd 2>../error ) &&
+     test_cmp expect error
+
  - Break the TAP output
 
    The raw output from your test may be interpreted by a TAP harness. TAP
index cd220e378e201fc5bfbaf9c78b0a3183ebf12d11..e3809dcead1818400829b27721cf8197de71432b 100644 (file)
@@ -9,8 +9,8 @@ test_terminal () {
                echo >&4 "test_terminal: need to declare TTY prerequisite"
                return 127
        fi
-       perl "$TEST_DIRECTORY"/test-terminal.perl "$@"
-}
+       perl "$TEST_DIRECTORY"/test-terminal.perl "$@" 2>&7
+} 7>&2 2>&4
 
 test_lazy_prereq TTY '
        test_have_prereq PERL &&
index b23c4e3fab604f957ec6359eae75400dd1e53174..2ce68cc277a1ed742ce7b644de31a4a847e3529b 100755 (executable)
@@ -42,7 +42,7 @@ commit_subject () {
 
 error_message () {
        (cd clone &&
-        test_must_fail git rev-parse --verify "$@")
+        test_must_fail git rev-parse --verify "$@" 2>../error)
 }
 
 test_expect_success '@{upstream} resolves to correct full name' '
@@ -159,8 +159,8 @@ test_expect_success 'branch@{u} error message when no upstream' '
        cat >expect <<-EOF &&
        fatal: no upstream configured for branch ${sq}non-tracking${sq}
        EOF
-       error_message non-tracking@{u} 2>actual &&
-       test_i18ncmp expect actual
+       error_message non-tracking@{u} &&
+       test_i18ncmp expect error
 '
 
 test_expect_success '@{u} error message when no upstream' '
@@ -175,8 +175,8 @@ test_expect_success 'branch@{u} error message with misspelt branch' '
        cat >expect <<-EOF &&
        fatal: no such branch: ${sq}no-such-branch${sq}
        EOF
-       error_message no-such-branch@{u} 2>actual &&
-       test_i18ncmp expect actual
+       error_message no-such-branch@{u} &&
+       test_i18ncmp expect error
 '
 
 test_expect_success '@{u} error message when not on a branch' '
@@ -192,8 +192,8 @@ test_expect_success 'branch@{u} error message if upstream branch not fetched' '
        cat >expect <<-EOF &&
        fatal: upstream branch ${sq}refs/heads/side${sq} not stored as a remote-tracking branch
        EOF
-       error_message bad-upstream@{u} 2>actual &&
-       test_i18ncmp expect actual
+       error_message bad-upstream@{u} &&
+       test_i18ncmp expect error
 '
 
 test_expect_success 'pull works when tracking a local branch' '
index 13ae12dfa7d494f50fc2e728d3e7ee36ff5534bc..e6854b828e2e68ad217721eb6139970b7c5958c0 100755 (executable)
@@ -39,6 +39,10 @@ A few rules for repo setup:
 11. When user's cwd is outside worktree, cwd remains unchanged,
     prefix is NULL.
 "
+
+# This test heavily relies on the standard error of nested function calls.
+test_untraceable=UnfortunatelyYes
+
 . ./test-lib.sh
 
 here=$(pwd)
index cdc38fe5d1a0d171aa3cdf9ac8ee73dd2cd3b18d..3563e77b374c69a138bdfa3b274b75b6094c55a7 100755 (executable)
@@ -525,20 +525,22 @@ test_expect_success 'merge-recursive w/ empty work tree - ours has rename' '
                GIT_INDEX_FILE="$PWD/ours-has-rename-index" &&
                export GIT_INDEX_FILE &&
                mkdir "$GIT_WORK_TREE" &&
-               git read-tree -i -m $c7 &&
-               git update-index --ignore-missing --refresh &&
-               git merge-recursive $c0 -- $c7 $c3 &&
-               git ls-files -s >actual-files
-       ) 2>actual-err &&
-       >expected-err &&
+               git read-tree -i -m $c7 2>actual-err &&
+               test_must_be_empty actual-err &&
+               git update-index --ignore-missing --refresh 2>actual-err &&
+               test_must_be_empty actual-err &&
+               git merge-recursive $c0 -- $c7 $c3 2>actual-err &&
+               test_must_be_empty actual-err &&
+               git ls-files -s >actual-files 2>actual-err &&
+               test_must_be_empty actual-err
+       ) &&
        cat >expected-files <<-EOF &&
        100644 $o3 0    b/c
        100644 $o0 0    c
        100644 $o0 0    d/e
        100644 $o0 0    e
        EOF
-       test_cmp expected-files actual-files &&
-       test_cmp expected-err actual-err
+       test_cmp expected-files actual-files
 '
 
 test_expect_success 'merge-recursive w/ empty work tree - theirs has rename' '
@@ -548,20 +550,22 @@ test_expect_success 'merge-recursive w/ empty work tree - theirs has rename' '
                GIT_INDEX_FILE="$PWD/theirs-has-rename-index" &&
                export GIT_INDEX_FILE &&
                mkdir "$GIT_WORK_TREE" &&
-               git read-tree -i -m $c3 &&
-               git update-index --ignore-missing --refresh &&
-               git merge-recursive $c0 -- $c3 $c7 &&
-               git ls-files -s >actual-files
-       ) 2>actual-err &&
-       >expected-err &&
+               git read-tree -i -m $c3 2>actual-err &&
+               test_must_be_empty actual-err &&
+               git update-index --ignore-missing --refresh 2>actual-err &&
+               test_must_be_empty actual-err &&
+               git merge-recursive $c0 -- $c3 $c7 2>actual-err &&
+               test_must_be_empty actual-err &&
+               git ls-files -s >actual-files 2>actual-err &&
+               test_must_be_empty actual-err
+       ) &&
        cat >expected-files <<-EOF &&
        100644 $o3 0    b/c
        100644 $o0 0    c
        100644 $o0 0    d/e
        100644 $o0 0    e
        EOF
-       test_cmp expected-files actual-files &&
-       test_cmp expected-err actual-err
+       test_cmp expected-files actual-files
 '
 
 test_expect_success 'merge removes empty directories' '
index ec9ba9bf6e7c709a0a1cc25fcfdec92c4c5c9b55..0680dec808574fb334612163ebd43d31418a37d5 100755 (executable)
@@ -482,24 +482,24 @@ test_expect_success 'set up tests of missing reference' '
 test_expect_success 'test lonely missing ref' '
        (
                cd client &&
-               test_must_fail git fetch-pack --no-progress .. refs/heads/xyzzy
-       ) >/dev/null 2>error-m &&
+               test_must_fail git fetch-pack --no-progress .. refs/heads/xyzzy 2>../error-m
+       ) &&
        test_i18ncmp expect-error error-m
 '
 
 test_expect_success 'test missing ref after existing' '
        (
                cd client &&
-               test_must_fail git fetch-pack --no-progress .. refs/heads/A refs/heads/xyzzy
-       ) >/dev/null 2>error-em &&
+               test_must_fail git fetch-pack --no-progress .. refs/heads/A refs/heads/xyzzy 2>../error-em
+       ) &&
        test_i18ncmp expect-error error-em
 '
 
 test_expect_success 'test missing ref before existing' '
        (
                cd client &&
-               test_must_fail git fetch-pack --no-progress .. refs/heads/xyzzy refs/heads/A
-       ) >/dev/null 2>error-me &&
+               test_must_fail git fetch-pack --no-progress .. refs/heads/xyzzy refs/heads/A 2>../error-me
+       ) &&
        test_i18ncmp expect-error error-me
 '
 
index 74486c73b0bcc20ee69e49d5c1cc072994e4d7b0..9cc4b569c0566da2f5c2778f404e67e20b29ed06 100755 (executable)
@@ -85,7 +85,7 @@ test_expect_success "fetch --recurse-submodules -j2 has the same output behaviou
        add_upstream_commit &&
        (
                cd downstream &&
-               GIT_TRACE=$(pwd)/../trace.out git fetch --recurse-submodules -j2 2>../actual.err
+               GIT_TRACE="$TRASH_DIRECTORY/trace.out" git fetch --recurse-submodules -j2 2>../actual.err
        ) &&
        test_must_be_empty actual.out &&
        test_i18ncmp expect.err actual.err &&
index 755b05a8ae9de940be8d9ad4c89473d836dcdf20..0d4c52016b2b3651fa638fcc416f22d236c87fc2 100755 (executable)
@@ -50,7 +50,7 @@ test_expect_success 'no-op fetch -v stderr is as expected' '
 '
 
 test_expect_success 'no-op fetch without "-v" is quiet' '
-       (cd clone && git fetch) 2>stderr &&
+       (cd clone && git fetch 2>../stderr) &&
        ! test -s stderr
 '
 
index 97c9b32c2ecfa608f1e9c37c1c6402a1772e4187..8f5c811dd7d627ea5a0611be5161cea98deedb95 100755 (executable)
@@ -735,22 +735,12 @@ test_expect_success 'prompt - hide if pwd ignored - env var set, config unset, p
        test_cmp expected "$actual"
 '
 
-test_expect_success 'prompt - hide if pwd ignored - inside gitdir (stdout)' '
+test_expect_success 'prompt - hide if pwd ignored - inside gitdir' '
        printf " (GIT_DIR!)" >expected &&
        (
                GIT_PS1_HIDE_IF_PWD_IGNORED=y &&
                cd .git &&
-               __git_ps1 >"$actual" 2>/dev/null
-       ) &&
-       test_cmp expected "$actual"
-'
-
-test_expect_success 'prompt - hide if pwd ignored - inside gitdir (stderr)' '
-       printf "" >expected &&
-       (
-               GIT_PS1_HIDE_IF_PWD_IGNORED=y &&
-               cd .git &&
-               __git_ps1 >/dev/null 2>"$actual"
+               __git_ps1 >"$actual"
        ) &&
        test_cmp expected "$actual"
 '
index 6e342804f96fb62ff0b3cfaf667e7efce79dc6af..b895366feef6027ac00bf47271b2530e8b7e7162 100644 (file)
@@ -629,7 +629,7 @@ test_must_fail () {
                _test_ok=
                ;;
        esac
-       "$@"
+       "$@" 2>&7
        exit_code=$?
        if test $exit_code -eq 0 && ! list_contains "$_test_ok" success
        then
@@ -652,7 +652,7 @@ test_must_fail () {
                return 1
        fi
        return 0
-}
+} 7>&2 2>&4
 
 # Similar to test_must_fail, but tolerates success, too.  This is
 # meant to be used in contexts like:
@@ -668,8 +668,8 @@ test_must_fail () {
 # Accepts the same options as test_must_fail.
 
 test_might_fail () {
-       test_must_fail ok=success "$@"
-}
+       test_must_fail ok=success "$@" 2>&7
+} 7>&2 2>&4
 
 # Similar to test_must_fail and test_might_fail, but check that a
 # given command exited with a given exit code. Meant to be used as:
@@ -681,7 +681,7 @@ test_might_fail () {
 test_expect_code () {
        want_code=$1
        shift
-       "$@"
+       "$@" 2>&7
        exit_code=$?
        if test $exit_code = $want_code
        then
@@ -690,7 +690,7 @@ test_expect_code () {
 
        echo >&4 "test_expect_code: command exited with $exit_code, we wanted $want_code $*"
        return 1
-}
+} 7>&2 2>&4
 
 # test_cmp is a helper function to compare actual and expected output.
 # You can use it like:
@@ -896,8 +896,8 @@ test_write_lines () {
 }
 
 perl () {
-       command "$PERL_PATH" "$@"
-}
+       command "$PERL_PATH" "$@" 2>&7
+} 7>&2 2>&4
 
 # Is the value one of the various ways to spell a boolean true/false?
 test_normalize_bool () {
@@ -1037,13 +1037,13 @@ test_env () {
                                shift
                                ;;
                        *)
-                               "$@"
+                               "$@" 2>&7
                                exit
                                ;;
                        esac
                done
        )
-}
+} 7>&2 2>&4
 
 # Returns true if the numeric exit code in "$2" represents the expected signal
 # in "$1". Signals should be given numerically.
@@ -1085,9 +1085,9 @@ nongit () {
                GIT_CEILING_DIRECTORIES=$(pwd) &&
                export GIT_CEILING_DIRECTORIES &&
                cd non-repo &&
-               "$@"
+               "$@" 2>&7
        )
-}
+} 7>&2 2>&4
 
 # convert stdin to pktline representation; note that empty input becomes an
 # empty packet, not a flush packet (for that you can just print 0000 yourself).
index 9535d2e0a936eb1b7a84686a9b702d3ca9fd680d..7740d511d289f44bb1313308fe49d5894f64b3c2 100644 (file)
@@ -264,7 +264,24 @@ do
                GIT_TEST_CHAIN_LINT=0
                shift ;;
        -x)
-               trace=t
+               # Some test scripts can't be reliably traced  with '-x',
+               # unless the test is run with a Bash version supporting
+               # BASH_XTRACEFD (introduced in Bash v4.1).  Check whether
+               # this test is marked as such, and ignore '-x' if it
+               # isn't executed with a suitable Bash version.
+               if test -z "$test_untraceable" || {
+                    test -n "$BASH_VERSION" && {
+                      test ${BASH_VERSINFO[0]} -gt 4 || {
+                        test ${BASH_VERSINFO[0]} -eq 4 &&
+                        test ${BASH_VERSINFO[1]} -ge 1
+                      }
+                    }
+                  }
+               then
+                       trace=t
+               else
+                       echo >&2 "warning: ignoring -x; '$0' is untraceable without BASH_XTRACEFD"
+               fi
                shift ;;
        --verbose-log)
                verbose_log=t