Merge branch 'sg/stress-test'
authorJunio C Hamano <gitster@pobox.com>
Thu, 14 Feb 2019 02:18:42 +0000 (18:18 -0800)
committerJunio C Hamano <gitster@pobox.com>
Thu, 14 Feb 2019 02:18:42 +0000 (18:18 -0800)
Test improvement.

* sg/stress-test:
test-lib: fix non-portable pattern bracket expressions
test-lib: make '--stress' more bisect-friendly

1  2 
t/README
t/test-lib.sh
diff --combined t/README
index 1326fd7505fde3a9e34fef1e197623c8d409d6ff,3aed32124819fe9b5ce7bfec3b330f2ef7a15d3b..886bbec5bc8e40995817c042874be4860a4b58b8
+++ b/t/README
@@@ -170,15 -170,6 +170,15 @@@ appropriately before running "make"
        implied by other options like --valgrind and
        GIT_TEST_INSTALLED.
  
 +--no-bin-wrappers::
 +      By default, the test suite uses the wrappers in
 +      `../bin-wrappers/` to execute `git` and friends. With this option,
 +      `../git` and friends are run directly. This is not recommended
 +      in general, as the wrappers contain safeguards to ensure that no
 +      files from an installed Git are used, but can speed up test runs
 +      especially on platforms where running shell scripts is expensive
 +      (most notably, Windows).
 +
  --root=<directory>::
        Create "trash" directories used to store all temporary data during
        testing under <directory>, instead of the t/ directory.
        '.stress-<nr>' suffix, and the trash directory of the failed
        test job is renamed to end with a '.stress-failed' suffix.
  
+ --stress-limit=<N>::
+       When combined with --stress run the test script repeatedly
+       this many times in each of the parallel jobs or until one of
+       them fails, whichever comes first.
  You can also set the GIT_TEST_INSTALLED environment variable to
  the bindir of an existing git installation to test that installation.
  You still need to have built this git sandbox, from which various
@@@ -367,10 -363,6 +372,10 @@@ GIT_TEST_INDEX_VERSION=<n> exercises th
  for the index version specified.  Can be set to any valid version
  (currently 2, 3, or 4).
  
 +GIT_TEST_PACK_SPARSE=<boolean> if enabled will default the pack-objects
 +builtin to use the sparse object walk. This can still be overridden by
 +the --no-sparse command-line argument.
 +
  GIT_TEST_PRELOAD_INDEX=<boolean> exercises the preload-index code path
  by overriding the minimum number of cache entries required per thread.
  
@@@ -387,11 -379,6 +392,11 @@@ GIT_TEST_MULTI_PACK_INDEX=<boolean>, wh
  index to be written after every 'git repack' command, and overrides the
  'core.multiPackIndex' setting to true.
  
 +GIT_TEST_SIDEBAND_ALL=<boolean>, when true, overrides the
 +'uploadpack.allowSidebandAll' setting to true, and when false, forces
 +fetch-pack to not request sideband-all (even if the server advertises
 +sideband-all).
 +
  Naming Tests
  ------------
  
diff --combined t/test-lib.sh
index 42b1a0aa7f0b8f5b06735293089f640b7f228f62,4e7cb52b57a59b76404f701a542fc58ce06ec685..8665b0a9b6186a7ad17889213f5225265c9f27d1
@@@ -111,8 -111,6 +111,8 @@@ d
                test -z "$HARNESS_ACTIVE" && quiet=t ;;
        --with-dashes)
                with_dashes=t ;;
 +      --no-bin-wrappers)
 +              no_bin_wrappers=t ;;
        --no-color)
                color= ;;
        --va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind)
                verbose_log=t
                tee=t
                ;;
 +      --write-junit-xml)
 +              write_junit_xml=t
 +              ;;
        --stress)
                stress=t ;;
        --stress=*)
                stress=${opt#--*=}
                case "$stress" in
-               *[^0-9]*|0*|"")
+               *[!0-9]*|0*|"")
                        echo "error: --stress=<N> requires the number of jobs to run" >&2
                        exit 1
                        ;;
                        ;;
                esac
                ;;
+       --stress-limit=*)
+               stress_limit=${opt#--*=}
+               case "$stress_limit" in
+               *[!0-9]*|0*|"")
+                       echo "error: --stress-limit=<N> requires the number of repetitions" >&2
+                       exit 1
+                       ;;
+               *)      # Good.
+                       ;;
+               esac
+               ;;
        *)
                echo "error: unknown test option '$opt'" >&2; exit 1 ;;
        esac
@@@ -242,8 -248,10 +253,10 @@@ the
                                exit 1
                        ' TERM INT
  
-                       cnt=0
-                       while ! test -e "$stressfail"
+                       cnt=1
+                       while ! test -e "$stressfail" &&
+                             { test -z "$stress_limit" ||
+                               test $cnt -le $stress_limit ; }
                        do
                                $TEST_SHELL_PATH "$0" "$@" >"$TEST_RESULTS_BASE.stress-$job_nr.out" 2>&1 &
                                test_pid=$!
  
        if test -f "$stressfail"
        then
+               stress_exit=1
                echo "Log(s) of failed test run(s):"
                for failed_job_nr in $(sort -n "$stressfail")
                do
@@@ -627,35 -636,11 +641,35 @@@ trap 'exit $?' INT TERM HU
  # the test_expect_* functions instead.
  
  test_ok_ () {
 +      if test -n "$write_junit_xml"
 +      then
 +              write_junit_xml_testcase "$*"
 +      fi
        test_success=$(($test_success + 1))
        say_color "" "ok $test_count - $@"
  }
  
  test_failure_ () {
 +      if test -n "$write_junit_xml"
 +      then
 +              junit_insert="<failure message=\"not ok $test_count -"
 +              junit_insert="$junit_insert $(xml_attr_encode "$1")\">"
 +              junit_insert="$junit_insert $(xml_attr_encode \
 +                      "$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
 +                         then
 +                              test-tool path-utils skip-n-bytes \
 +                                      "$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET
 +                         else
 +                              printf '%s\n' "$@" | sed 1d
 +                         fi)")"
 +              junit_insert="$junit_insert</failure>"
 +              if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
 +              then
 +                      junit_insert="$junit_insert<system-err>$(xml_attr_encode \
 +                              "$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")</system-err>"
 +              fi
 +              write_junit_xml_testcase "$1" "      $junit_insert"
 +      fi
        test_failure=$(($test_failure + 1))
        say_color error "not ok $test_count - $1"
        shift
  }
  
  test_known_broken_ok_ () {
 +      if test -n "$write_junit_xml"
 +      then
 +              write_junit_xml_testcase "$* (breakage fixed)"
 +      fi
        test_fixed=$(($test_fixed+1))
        say_color error "ok $test_count - $@ # TODO known breakage vanished"
  }
  
  test_known_broken_failure_ () {
 +      if test -n "$write_junit_xml"
 +      then
 +              write_junit_xml_testcase "$* (known breakage)"
 +      fi
        test_broken=$(($test_broken+1))
        say_color warn "not ok $test_count - $@ # TODO known breakage"
  }
@@@ -934,21 -911,12 +948,21 @@@ test_start_ () 
        test_count=$(($test_count+1))
        maybe_setup_verbose
        maybe_setup_valgrind
 +      if test -n "$write_junit_xml"
 +      then
 +              junit_start=$(test-tool date getnanos)
 +      fi
  }
  
  test_finish_ () {
        echo >&3 ""
        maybe_teardown_valgrind
        maybe_teardown_verbose
 +      if test -n "$GIT_TEST_TEE_OFFSET"
 +      then
 +              GIT_TEST_TEE_OFFSET=$(test-tool path-utils file-size \
 +                      "$GIT_TEST_TEE_OUTPUT_FILE")
 +      fi
  }
  
  test_skip () {
  
        case "$to_skip" in
        t)
 +              if test -n "$write_junit_xml"
 +              then
 +                      message="$(xml_attr_encode "$skipped_reason")"
 +                      write_junit_xml_testcase "$1" \
 +                              "      <skipped message=\"$message\" />"
 +              fi
 +
                say_color skip >&3 "skipping test: $@"
                say_color skip "ok $test_count # skip $1 ($skipped_reason)"
                : true
@@@ -1002,51 -963,9 +1016,51 @@@ test_at_end_hook_ () 
        :
  }
  
 +write_junit_xml () {
 +      case "$1" in
 +      --truncate)
 +              >"$junit_xml_path"
 +              junit_have_testcase=
 +              shift
 +              ;;
 +      esac
 +      printf '%s\n' "$@" >>"$junit_xml_path"
 +}
 +
 +xml_attr_encode () {
 +      printf '%s\n' "$@" | test-tool xml-encode
 +}
 +
 +write_junit_xml_testcase () {
 +      junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\""
 +      shift
 +      junit_attrs="$junit_attrs classname=\"$this_test\""
 +      junit_attrs="$junit_attrs time=\"$(test-tool \
 +              date getnanos $junit_start)\""
 +      write_junit_xml "$(printf '%s\n' \
 +              "    <testcase $junit_attrs>" "$@" "    </testcase>")"
 +      junit_have_testcase=t
 +}
 +
  test_done () {
        GIT_EXIT_OK=t
  
 +      if test -n "$write_junit_xml" && test -n "$junit_xml_path"
 +      then
 +              test -n "$junit_have_testcase" || {
 +                      junit_start=$(test-tool date getnanos)
 +                      write_junit_xml_testcase "all tests skipped"
 +              }
 +
 +              # adjust the overall time
 +              junit_time=$(test-tool date getnanos $junit_suite_start)
 +              sed "s/<testsuite [^>]*/& time=\"$junit_time\"/" \
 +                      <"$junit_xml_path" >"$junit_xml_path.new"
 +              mv "$junit_xml_path.new" "$junit_xml_path"
 +
 +              write_junit_xml "  </testsuite>" "</testsuites>"
 +      fi
 +
        if test -z "$HARNESS_ACTIVE"
        then
                mkdir -p "$TEST_RESULTS_DIR"
                        error "Tests passed but trash directory already removed before test cleanup; aborting"
  
                        cd "$TRASH_DIRECTORY/.." &&
 -                      rm -fr "$TRASH_DIRECTORY" ||
 +                      rm -fr "$TRASH_DIRECTORY" || {
 +                              # try again in a bit
 +                              sleep 5;
 +                              rm -fr "$TRASH_DIRECTORY"
 +                      } ||
                        error "Tests passed but test cleanup failed; aborting"
                fi
                test_at_end_hook_
        PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$PATH
        GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH}
  else # normal case, use ../bin-wrappers only unless $with_dashes:
 -      git_bin_dir="$GIT_BUILD_DIR/bin-wrappers"
 -      if ! test -x "$git_bin_dir/git"
 +      if test -n "$no_bin_wrappers"
        then
 -              if test -z "$with_dashes"
 +              with_dashes=t
 +      else
 +              git_bin_dir="$GIT_BUILD_DIR/bin-wrappers"
 +              if ! test -x "$git_bin_dir/git"
                then
 -                      say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH"
 +                      if test -z "$with_dashes"
 +                      then
 +                              say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH"
 +                      fi
 +                      with_dashes=t
                fi
 -              with_dashes=t
 +              PATH="$git_bin_dir:$PATH"
        fi
 -      PATH="$git_bin_dir:$PATH"
        GIT_EXEC_PATH=$GIT_BUILD_DIR
        if test -n "$with_dashes"
        then
 -              PATH="$GIT_BUILD_DIR:$PATH"
 +              PATH="$GIT_BUILD_DIR:$GIT_BUILD_DIR/t/helper:$PATH"
        fi
  fi
  GIT_TEMPLATE_DIR="$GIT_BUILD_DIR"/templates/blt
@@@ -1258,7 -1168,7 +1272,7 @@@ test -d "$GIT_BUILD_DIR"/templates/blt 
        error "You haven't built things yet, have you?"
  }
  
 -if ! test -x "$GIT_BUILD_DIR"/t/helper/test-tool
 +if ! test -x "$GIT_BUILD_DIR"/t/helper/test-tool$X
  then
        echo >&2 'You need to build test-tool:'
        echo >&2 'Run "make t/helper/test-tool" in the source (toplevel) directory'
@@@ -1282,7 -1192,6 +1296,7 @@@ the
  else
        mkdir -p "$TRASH_DIRECTORY"
  fi
 +
  # Use -P to resolve symlinks in our working directory so that the cwd
  # in subprocesses like git equals our $PWD (for pathname comparisons).
  cd -P "$TRASH_DIRECTORY" || exit 1
        test_done
  fi
  
 +if test -n "$write_junit_xml"
 +then
 +      junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out"
 +      mkdir -p "$junit_xml_dir"
 +      junit_xml_base=${0##*/}
 +      junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml"
 +      junit_attrs="name=\"${junit_xml_base%.sh}\""
 +      junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \
 +              date +%Y-%m-%dT%H:%M:%S)\""
 +      write_junit_xml --truncate "<testsuites>" "  <testsuite $junit_attrs>"
 +      junit_suite_start=$(test-tool date getnanos)
 +      if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
 +      then
 +              GIT_TEST_TEE_OFFSET=0
 +      fi
 +fi
 +
  # Provide an implementation of the 'yes' utility
  yes () {
        if test $# = 0