From: Junio C Hamano Date: Mon, 16 Jun 2014 19:18:56 +0000 (-0700) Subject: Merge branch 'ib/test-selectively-run' X-Git-Tag: v2.1.0-rc0~100 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/7e1a5381b0048572e72971af41e13a85804d48d9?ds=inline;hp=-c Merge branch 'ib/test-selectively-run' Allow specifying only certain individual test pieces to be run using a range notation (e.g. "t1234-test.sh --run='1-4 6 8 9-'"). * ib/test-selectively-run: t0000-*.sh: fix the GIT_SKIP_TESTS sub-tests test-lib: '--run' to run only specific tests test-lib: tests skipped by GIT_SKIP_TESTS say so test-lib: document short options in t/README --- 7e1a5381b0048572e72971af41e13a85804d48d9 diff --combined t/README index 8a9d4995f5,cd99d212e6..52c77ae936 --- a/t/README +++ b/t/README @@@ -71,6 -71,7 +71,7 @@@ You can pass --verbose (or -v), --debu (or -i) command line argument to the test, or by setting GIT_TEST_OPTS appropriately before running "make". + -v:: --verbose:: This makes the test more verbose. Specifically, the command being run and their output if any are also @@@ -81,6 -82,7 +82,7 @@@ numbers matching . The number matched against is simply the running count of the test within the file. + -d:: --debug:: This may help the person who is developing a new test. It causes the command defined with test_debug to run. @@@ -89,6 -91,7 +91,7 @@@ failed tests so that you can inspect its contents after the test finished. + -i:: --immediate:: This causes the test to immediately exit upon the first failed test. Cleanup commands requested with @@@ -96,10 -99,17 +99,17 @@@ in order to keep the state for inspection by the tester to diagnose the bug. + -l:: --long-tests:: This causes additional long-running tests to be run (where available), for more exhaustive testing. + -r:: + --run=:: + Run only the subset of tests indicated by + . See section "Skipping Tests" below for + syntax. + --valgrind=:: Execute all Git binaries under valgrind tool and exit with status 126 on errors (just like regular tests, this will @@@ -187,10 -197,77 +197,77 @@@ and either can match the "t[0-9]{4}" pa test, or t[0-9]{4} followed by ".$number" to say which particular test to skip. - Note that some tests in the existing test suite rely on previous - test item, so you cannot arbitrarily disable one and expect the - remainder of test to check what the test originally was intended - to check. + For an individual test suite --run could be used to specify that + only some tests should be run or that some tests should be + excluded from a run. + + The argument for --run is a list of individual test numbers or + ranges with an optional negation prefix that define what tests in + a test suite to include in the run. A range is two numbers + separated with a dash and matches a range of tests with both ends + been included. You may omit the first or the second number to + mean "from the first test" or "up to the very last test" + respectively. + + Optional prefix of '!' means that the test or a range of tests + should be excluded from the run. + + If --run starts with an unprefixed number or range the initial + set of tests to run is empty. If the first item starts with '!' + all the tests are added to the initial set. After initial set is + determined every test number or range is added or excluded from + the set one by one, from left to right. + + Individual numbers or ranges could be separated either by a space + or a comma. + + For example, to run only tests up to a specific test (21), one + could do this: + + $ sh ./t9200-git-cvsexport-commit.sh --run='1-21' + + or this: + + $ sh ./t9200-git-cvsexport-commit.sh --run='-21' + + Common case is to run several setup tests (1, 2, 3) and then a + specific test (21) that relies on that setup: + + $ sh ./t9200-git-cvsexport-commit.sh --run='1 2 3 21' + + or: + + $ sh ./t9200-git-cvsexport-commit.sh --run=1,2,3,21 + + or: + + $ sh ./t9200-git-cvsexport-commit.sh --run='-3 21' + + As noted above, the test set is built going though items left to + right, so this: + + $ sh ./t9200-git-cvsexport-commit.sh --run='1-4 !3' + + will run tests 1, 2, and 4. Items that comes later have higher + precendence. It means that this: + + $ sh ./t9200-git-cvsexport-commit.sh --run='!3 1-4' + + would just run tests from 1 to 4, including 3. + + You may use negation with ranges. The following will run all + test in the test suite except from 7 up to 11: + + $ sh ./t9200-git-cvsexport-commit.sh --run='!7-11' + + Some tests in a test suite rely on the previous tests performing + certain actions, specifically some tests are designated as + "setup" test, so you cannot _arbitrarily_ disable one test and + expect the rest to function correctly. + + --run is mostly useful when you want to focus on a specific test + and know what setup is needed for it. Or when you want to run + everything up to a certain test. Naming Tests @@@ -596,27 -673,6 +673,27 @@@ library for your script to use ... ' + - test_write_lines + + Write on standard output, one line per argument. + Useful to prepare multi-line files in a compact form. + + Example: + + test_write_lines a b c d e f g >foo + + Is a more compact equivalent of: + cat >foo <<-EOF + a + b + c + d + e + f + g + EOF + + - test_pause This command is useful for writing and debugging tests and must be diff --combined t/test-lib.sh index f7e55b1001,91000fef2e..81394c8c7c --- a/t/test-lib.sh +++ b/t/test-lib.sh @@@ -91,7 -91,6 +91,7 @@@ unset VISUAL EMAIL LANGUAGE COLUMNS $(" VALGRIND UNZIP PERF_ + CURL_VERBOSE )); my @vars = grep(/^GIT_/ && !/^GIT_($ok)/o, @env); print join("\n", @vars); @@@ -192,6 -191,14 +192,14 @@@ d immediate=t; shift ;; -l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests) GIT_TEST_LONG=t; export GIT_TEST_LONG; shift ;; + -r) + shift; test "$#" -ne 0 || { + echo 'error: -r requires an argument' >&2; + exit 1; + } + run_list=$1; shift ;; + --run=*) + run_list=$(expr "z$1" : 'z[^=]*=\(.*\)'); shift ;; -h|--h|--he|--hel|--help) help=t; shift ;; -v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose) @@@ -367,6 -374,99 +375,99 @@@ match_pattern_list () return 1 } + match_test_selector_list () { + title="$1" + shift + arg="$1" + shift + test -z "$1" && return 0 + + # Both commas and whitespace are accepted as separators. + OLDIFS=$IFS + IFS=' ,' + set -- $1 + IFS=$OLDIFS + + # If the first selector is negative we include by default. + include= + case "$1" in + !*) include=t ;; + esac + + for selector + do + orig_selector=$selector + + positive=t + case "$selector" in + !*) + positive= + selector=${selector##?} + ;; + esac + + test -z "$selector" && continue + + case "$selector" in + *-*) + if expr "z${selector%%-*}" : "z[0-9]*[^0-9]" >/dev/null + then + echo "error: $title: invalid non-numeric in range" \ + "start: '$orig_selector'" >&2 + exit 1 + fi + if expr "z${selector#*-}" : "z[0-9]*[^0-9]" >/dev/null + then + echo "error: $title: invalid non-numeric in range" \ + "end: '$orig_selector'" >&2 + exit 1 + fi + ;; + *) + if expr "z$selector" : "z[0-9]*[^0-9]" >/dev/null + then + echo "error: $title: invalid non-numeric in test" \ + "selector: '$orig_selector'" >&2 + exit 1 + fi + esac + + # Short cut for "obvious" cases + test -z "$include" && test -z "$positive" && continue + test -n "$include" && test -n "$positive" && continue + + case "$selector" in + -*) + if test $arg -le ${selector#-} + then + include=$positive + fi + ;; + *-) + if test $arg -ge ${selector%-} + then + include=$positive + fi + ;; + *-*) + if test ${selector%%-*} -le $arg \ + && test $arg -le ${selector#*-} + then + include=$positive + fi + ;; + *) + if test $arg -eq $selector + then + include=$positive + fi + ;; + esac + done + + test -n "$include" + } + maybe_teardown_verbose () { test -z "$verbose_only" && return exec 4>/dev/null 3>/dev/null @@@ -453,25 -553,35 +554,35 @@@ test_finish_ () test_skip () { to_skip= + skipped_reason= if match_pattern_list $this_test.$test_count $GIT_SKIP_TESTS then to_skip=t + skipped_reason="GIT_SKIP_TESTS" fi if test -z "$to_skip" && test -n "$test_prereq" && ! test_have_prereq "$test_prereq" then to_skip=t - fi - case "$to_skip" in - t) + of_prereq= if test "$missing_prereq" != "$test_prereq" then of_prereq=" of $test_prereq" fi + skipped_reason="missing $missing_prereq${of_prereq}" + fi + if test -z "$to_skip" && test -n "$run_list" && + ! match_test_selector_list '--run' $test_count "$run_list" + then + to_skip=t + skipped_reason="--run" + fi + case "$to_skip" in + t) say_color skip >&3 "skipping test: $@" - say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})" + say_color skip "ok $test_count # skip $1 ($skipped_reason)" : true ;; *)