From: Junio C Hamano Date: Thu, 14 Feb 2019 02:18:42 +0000 (-0800) Subject: Merge branch 'sg/stress-test' X-Git-Tag: v2.21.0-rc1~5 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/feb9a9b202c12289150de45473846f4272aac1bd?ds=inline;hp=-c Merge branch 'sg/stress-test' Test improvement. * sg/stress-test: test-lib: fix non-portable pattern bracket expressions test-lib: make '--stress' more bisect-friendly --- feb9a9b202c12289150de45473846f4272aac1bd diff --combined t/README index 1326fd7505,3aed321248..886bbec5bc --- a/t/README +++ b/t/README @@@ -170,15 -170,6 +170,15 @@@ appropriately before running "make" implied by other options like --valgrind and GIT_TEST_INSTALLED. +--no-bin-wrappers:: + By default, the test suite uses the wrappers in + `../bin-wrappers/` to execute `git` and friends. With this option, + `../git` and friends are run directly. This is not recommended + in general, as the wrappers contain safeguards to ensure that no + files from an installed Git are used, but can speed up test runs + especially on platforms where running shell scripts is expensive + (most notably, Windows). + --root=:: Create "trash" directories used to store all temporary data during testing under , instead of the t/ directory. @@@ -211,6 -202,11 +211,11 @@@ '.stress-' suffix, and the trash directory of the failed test job is renamed to end with a '.stress-failed' suffix. + --stress-limit=:: + When combined with --stress run the test script repeatedly + this many times in each of the parallel jobs or until one of + them fails, whichever comes first. + You can also set the GIT_TEST_INSTALLED environment variable to the bindir of an existing git installation to test that installation. You still need to have built this git sandbox, from which various @@@ -367,10 -363,6 +372,10 @@@ GIT_TEST_INDEX_VERSION= exercises th for the index version specified. Can be set to any valid version (currently 2, 3, or 4). +GIT_TEST_PACK_SPARSE= if enabled will default the pack-objects +builtin to use the sparse object walk. This can still be overridden by +the --no-sparse command-line argument. + GIT_TEST_PRELOAD_INDEX= exercises the preload-index code path by overriding the minimum number of cache entries required per thread. @@@ -387,11 -379,6 +392,11 @@@ GIT_TEST_MULTI_PACK_INDEX=, wh index to be written after every 'git repack' command, and overrides the 'core.multiPackIndex' setting to true. +GIT_TEST_SIDEBAND_ALL=, when true, overrides the +'uploadpack.allowSidebandAll' setting to true, and when false, forces +fetch-pack to not request sideband-all (even if the server advertises +sideband-all). + Naming Tests ------------ diff --combined t/test-lib.sh index 42b1a0aa7f,4e7cb52b57..8665b0a9b6 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@@ -111,8 -111,6 +111,8 @@@ d test -z "$HARNESS_ACTIVE" && quiet=t ;; --with-dashes) with_dashes=t ;; + --no-bin-wrappers) + no_bin_wrappers=t ;; --no-color) color= ;; --va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind) @@@ -141,15 -139,12 +141,15 @@@ verbose_log=t tee=t ;; + --write-junit-xml) + write_junit_xml=t + ;; --stress) stress=t ;; --stress=*) stress=${opt#--*=} case "$stress" in - *[^0-9]*|0*|"") + *[!0-9]*|0*|"") echo "error: --stress= requires the number of jobs to run" >&2 exit 1 ;; @@@ -157,6 -152,17 +157,17 @@@ ;; esac ;; + --stress-limit=*) + stress_limit=${opt#--*=} + case "$stress_limit" in + *[!0-9]*|0*|"") + echo "error: --stress-limit= requires the number of repetitions" >&2 + exit 1 + ;; + *) # Good. + ;; + esac + ;; *) echo "error: unknown test option '$opt'" >&2; exit 1 ;; esac @@@ -242,8 -248,10 +253,10 @@@ the exit 1 ' TERM INT - cnt=0 - while ! test -e "$stressfail" + cnt=1 + while ! test -e "$stressfail" && + { test -z "$stress_limit" || + test $cnt -le $stress_limit ; } do $TEST_SHELL_PATH "$0" "$@" >"$TEST_RESULTS_BASE.stress-$job_nr.out" 2>&1 & test_pid=$! @@@ -266,6 -274,7 +279,7 @@@ if test -f "$stressfail" then + stress_exit=1 echo "Log(s) of failed test run(s):" for failed_job_nr in $(sort -n "$stressfail") do @@@ -627,35 -636,11 +641,35 @@@ trap 'exit $?' INT TERM HU # the test_expect_* functions instead. test_ok_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$*" + fi test_success=$(($test_success + 1)) say_color "" "ok $test_count - $@" } test_failure_ () { + if test -n "$write_junit_xml" + then + junit_insert="" + junit_insert="$junit_insert $(xml_attr_encode \ + "$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + test-tool path-utils skip-n-bytes \ + "$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET + else + printf '%s\n' "$@" | sed 1d + fi)")" + junit_insert="$junit_insert" + if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + junit_insert="$junit_insert$(xml_attr_encode \ + "$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")" + fi + write_junit_xml_testcase "$1" " $junit_insert" + fi test_failure=$(($test_failure + 1)) say_color error "not ok $test_count - $1" shift @@@ -664,19 -649,11 +678,19 @@@ } test_known_broken_ok_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$* (breakage fixed)" + fi test_fixed=$(($test_fixed+1)) say_color error "ok $test_count - $@ # TODO known breakage vanished" } test_known_broken_failure_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$* (known breakage)" + fi test_broken=$(($test_broken+1)) say_color warn "not ok $test_count - $@ # TODO known breakage" } @@@ -934,21 -911,12 +948,21 @@@ test_start_ () test_count=$(($test_count+1)) maybe_setup_verbose maybe_setup_valgrind + if test -n "$write_junit_xml" + then + junit_start=$(test-tool date getnanos) + fi } test_finish_ () { echo >&3 "" maybe_teardown_valgrind maybe_teardown_verbose + if test -n "$GIT_TEST_TEE_OFFSET" + then + GIT_TEST_TEE_OFFSET=$(test-tool path-utils file-size \ + "$GIT_TEST_TEE_OUTPUT_FILE") + fi } test_skip () { @@@ -980,13 -948,6 +994,13 @@@ case "$to_skip" in t) + if test -n "$write_junit_xml" + then + message="$(xml_attr_encode "$skipped_reason")" + write_junit_xml_testcase "$1" \ + " " + fi + say_color skip >&3 "skipping test: $@" say_color skip "ok $test_count # skip $1 ($skipped_reason)" : true @@@ -1002,51 -963,9 +1016,51 @@@ test_at_end_hook_ () : } +write_junit_xml () { + case "$1" in + --truncate) + >"$junit_xml_path" + junit_have_testcase= + shift + ;; + esac + printf '%s\n' "$@" >>"$junit_xml_path" +} + +xml_attr_encode () { + printf '%s\n' "$@" | test-tool xml-encode +} + +write_junit_xml_testcase () { + junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\"" + shift + junit_attrs="$junit_attrs classname=\"$this_test\"" + junit_attrs="$junit_attrs time=\"$(test-tool \ + date getnanos $junit_start)\"" + write_junit_xml "$(printf '%s\n' \ + " " "$@" " ")" + junit_have_testcase=t +} + test_done () { GIT_EXIT_OK=t + if test -n "$write_junit_xml" && test -n "$junit_xml_path" + then + test -n "$junit_have_testcase" || { + junit_start=$(test-tool date getnanos) + write_junit_xml_testcase "all tests skipped" + } + + # adjust the overall time + junit_time=$(test-tool date getnanos $junit_suite_start) + sed "s/]*/& time=\"$junit_time\"/" \ + <"$junit_xml_path" >"$junit_xml_path.new" + mv "$junit_xml_path.new" "$junit_xml_path" + + write_junit_xml " " "" + fi + if test -z "$HARNESS_ACTIVE" then mkdir -p "$TEST_RESULTS_DIR" @@@ -1106,11 -1025,7 +1120,11 @@@ error "Tests passed but trash directory already removed before test cleanup; aborting" cd "$TRASH_DIRECTORY/.." && - rm -fr "$TRASH_DIRECTORY" || + rm -fr "$TRASH_DIRECTORY" || { + # try again in a bit + sleep 5; + rm -fr "$TRASH_DIRECTORY" + } || error "Tests passed but test cleanup failed; aborting" fi test_at_end_hook_ @@@ -1216,25 -1131,20 +1230,25 @@@ the PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$PATH GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH} else # normal case, use ../bin-wrappers only unless $with_dashes: - git_bin_dir="$GIT_BUILD_DIR/bin-wrappers" - if ! test -x "$git_bin_dir/git" + if test -n "$no_bin_wrappers" then - if test -z "$with_dashes" + with_dashes=t + else + git_bin_dir="$GIT_BUILD_DIR/bin-wrappers" + if ! test -x "$git_bin_dir/git" then - say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH" + if test -z "$with_dashes" + then + say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH" + fi + with_dashes=t fi - with_dashes=t + PATH="$git_bin_dir:$PATH" fi - PATH="$git_bin_dir:$PATH" GIT_EXEC_PATH=$GIT_BUILD_DIR if test -n "$with_dashes" then - PATH="$GIT_BUILD_DIR:$PATH" + PATH="$GIT_BUILD_DIR:$GIT_BUILD_DIR/t/helper:$PATH" fi fi GIT_TEMPLATE_DIR="$GIT_BUILD_DIR"/templates/blt @@@ -1258,7 -1168,7 +1272,7 @@@ test -d "$GIT_BUILD_DIR"/templates/blt error "You haven't built things yet, have you?" } -if ! test -x "$GIT_BUILD_DIR"/t/helper/test-tool +if ! test -x "$GIT_BUILD_DIR"/t/helper/test-tool$X then echo >&2 'You need to build test-tool:' echo >&2 'Run "make t/helper/test-tool" in the source (toplevel) directory' @@@ -1282,7 -1192,6 +1296,7 @@@ the else mkdir -p "$TRASH_DIRECTORY" fi + # Use -P to resolve symlinks in our working directory so that the cwd # in subprocesses like git equals our $PWD (for pathname comparisons). cd -P "$TRASH_DIRECTORY" || exit 1 @@@ -1296,23 -1205,6 +1310,23 @@@ the test_done fi +if test -n "$write_junit_xml" +then + junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out" + mkdir -p "$junit_xml_dir" + junit_xml_base=${0##*/} + junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml" + junit_attrs="name=\"${junit_xml_base%.sh}\"" + junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \ + date +%Y-%m-%dT%H:%M:%S)\"" + write_junit_xml --truncate "" " " + junit_suite_start=$(test-tool date getnanos) + if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + GIT_TEST_TEE_OFFSET=0 + fi +fi + # Provide an implementation of the 'yes' utility yes () { if test $# = 0