Merge branch 'cc/perf-bisect'
authorJunio C Hamano <gitster@pobox.com>
Wed, 25 Apr 2018 04:29:04 +0000 (13:29 +0900)
committerJunio C Hamano <gitster@pobox.com>
Wed, 25 Apr 2018 04:29:04 +0000 (13:29 +0900)
Performance measuring framework in t/perf learned to help bisecting
performance regressions.

* cc/perf-bisect:
t/perf: add scripts to bisect performance regressions
perf/run: add --subsection option

t/perf/bisect_regression [new file with mode: 0755]
t/perf/bisect_run_script [new file with mode: 0755]
t/perf/run
diff --git a/t/perf/bisect_regression b/t/perf/bisect_regression
new file mode 100755 (executable)
index 0000000..a94d995
--- /dev/null
@@ -0,0 +1,73 @@
+#!/bin/sh
+
+# Read a line coming from `./aggregate.perl --sort-by regression ...`
+# and automatically bisect to find the commit responsible for the
+# performance regression.
+#
+# Lines from `./aggregate.perl --sort-by regression ...` look like:
+#
+# +100.0% p7821-grep-engines-fixed.1 0.04(0.10+0.03) 0.08(0.11+0.08) v2.14.3 v2.15.1
+# +33.3% p7820-grep-engines.1 0.03(0.08+0.02) 0.04(0.08+0.02) v2.14.3 v2.15.1
+#
+
+die () {
+       echo >&2 "error: $*"
+       exit 1
+}
+
+while [ $# -gt 0 ]; do
+       arg="$1"
+       case "$arg" in
+       --help)
+               echo "usage: $0 [--config file] [--subsection subsection]"
+               exit 0
+               ;;
+       --config)
+               shift
+               GIT_PERF_CONFIG_FILE=$(cd "$(dirname "$1")"; pwd)/$(basename "$1")
+               export GIT_PERF_CONFIG_FILE
+               shift ;;
+       --subsection)
+               shift
+               GIT_PERF_SUBSECTION="$1"
+               export GIT_PERF_SUBSECTION
+               shift ;;
+       --*)
+               die "unrecognised option: '$arg'" ;;
+       *)
+               die "unknown argument '$arg'"
+               ;;
+       esac
+done
+
+read -r regression subtest oldtime newtime oldrev newrev
+
+test_script=$(echo "$subtest" | sed -e 's/\(.*\)\.[0-9]*$/\1.sh/')
+test_number=$(echo "$subtest" | sed -e 's/.*\.\([0-9]*\)$/\1/')
+
+# oldtime and newtime are decimal number, not integers
+
+oldtime=$(echo "$oldtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/')
+newtime=$(echo "$newtime" | sed -e 's/^\([0-9]\+\.[0-9]\+\).*$/\1/')
+
+test $(echo "$newtime" "$oldtime" | awk '{ print ($1 > $2) }') = 1 ||
+       die "New time '$newtime' shoud be greater than old time '$oldtime'"
+
+tmpdir=$(mktemp -d -t bisect_regression_XXXXXX) || die "Failed to create temp directory"
+echo "$oldtime" >"$tmpdir/oldtime" || die "Failed to write to '$tmpdir/oldtime'"
+echo "$newtime" >"$tmpdir/newtime" || die "Failed to write to '$tmpdir/newtime'"
+
+# Bisecting must be performed from the top level directory (even with --no-checkout)
+(
+       toplevel_dir=$(git rev-parse --show-toplevel) || die "Failed to find top level directory"
+       cd "$toplevel_dir" || die "Failed to cd into top level directory '$toplevel_dir'"
+
+       git bisect start --no-checkout "$newrev" "$oldrev" || die "Failed to start bisecting"
+
+       git bisect run t/perf/bisect_run_script "$test_script" "$test_number" "$tmpdir"
+       res="$?"
+
+       git bisect reset
+
+       exit "$res"
+)
diff --git a/t/perf/bisect_run_script b/t/perf/bisect_run_script
new file mode 100755 (executable)
index 0000000..038255d
--- /dev/null
@@ -0,0 +1,47 @@
+#!/bin/sh
+
+script="$1"
+test_number="$2"
+info_dir="$3"
+
+# This aborts the bisection immediately
+die () {
+       echo >&2 "error: $*"
+       exit 255
+}
+
+bisect_head=$(git rev-parse --verify BISECT_HEAD) || die "Failed to find BISECT_HEAD ref"
+
+script_number=$(echo "$script" | sed -e "s/^p\([0-9]*\).*\$/\1/") || die "Failed to get script number for '$script'"
+
+oldtime=$(cat "$info_dir/oldtime") || die "Failed to access '$info_dir/oldtime'"
+newtime=$(cat "$info_dir/newtime") || die "Failed to access '$info_dir/newtime'"
+
+cd t/perf || die "Failed to cd into 't/perf'"
+
+result_file="$info_dir/perf_${script_number}_${bisect_head}_results.txt"
+
+GIT_PERF_DIRS_OR_REVS="$bisect_head"
+export GIT_PERF_DIRS_OR_REVS
+
+./run "$script" >"$result_file" 2>&1 || die "Failed to run perf test '$script'"
+
+rtime=$(sed -n "s/^$script_number\.$test_number:.*\([0-9]\+\.[0-9]\+\)(.*).*\$/\1/p" "$result_file")
+
+echo "newtime: $newtime"
+echo "rtime: $rtime"
+echo "oldtime: $oldtime"
+
+# Compare ($newtime - $rtime) with ($rtime - $oldtime)
+# Times are decimal number, not integers
+
+if test $(echo "$newtime" "$rtime" "$oldtime" | awk '{ print ($1 - $2 > $2 - $3) }') = 1
+then
+       # Current commit is considered "good/old"
+       echo "$rtime" >"$info_dir/oldtime"
+       exit 0
+else
+       # Current commit is considered "bad/new"
+       echo "$rtime" >"$info_dir/newtime"
+       exit 1
+fi
index 213da5d6b9437b7db7f3e5b824afea1df465d934..9aaa733c770fb97157e273b6c6d1efd38f9a630a 100755 (executable)
@@ -1,21 +1,34 @@
 #!/bin/sh
 
-case "$1" in
+die () {
+       echo >&2 "error: $*"
+       exit 1
+}
+
+while [ $# -gt 0 ]; do
+       arg="$1"
+       case "$arg" in
+       --)
+               break ;;
        --help)
-               echo "usage: $0 [--config file] [other_git_tree...] [--] [test_scripts]"
-               exit 0
-               ;;
+               echo "usage: $0 [--config file] [--subsection subsec] [other_git_tree...] [--] [test_scripts]"
+               exit 0 ;;
        --config)
                shift
                GIT_PERF_CONFIG_FILE=$(cd "$(dirname "$1")"; pwd)/$(basename "$1")
                export GIT_PERF_CONFIG_FILE
                shift ;;
-esac
-
-die () {
-       echo >&2 "error: $*"
-       exit 1
-}
+       --subsection)
+               shift
+               GIT_PERF_SUBSECTION="$1"
+               export GIT_PERF_SUBSECTION
+               shift ;;
+       --*)
+               die "unrecognised option: '$arg'" ;;
+       *)
+               break ;;
+       esac
+done
 
 run_one_dir () {
        if test $# -eq 0; then
@@ -172,9 +185,32 @@ get_subsections "perf" >test-results/run_subsections.names
 
 if test $(wc -l <test-results/run_subsections.names) -eq 0
 then
+       if test -n "$GIT_PERF_SUBSECTION"
+       then
+               if test -n "$GIT_PERF_CONFIG_FILE"
+               then
+                       die "no subsections are defined in config file '$GIT_PERF_CONFIG_FILE'"
+               else
+                       die "subsection '$GIT_PERF_SUBSECTION' defined without a config file"
+               fi
+       fi
        (
                run_subsection "$@"
        )
+elif test -n "$GIT_PERF_SUBSECTION"
+then
+       egrep "^$GIT_PERF_SUBSECTION\$" test-results/run_subsections.names >/dev/null ||
+               die "subsection '$GIT_PERF_SUBSECTION' not found in '$GIT_PERF_CONFIG_FILE'"
+
+       egrep "^$GIT_PERF_SUBSECTION\$" test-results/run_subsections.names | while read -r subsec
+       do
+               (
+                       GIT_PERF_SUBSECTION="$subsec"
+                       export GIT_PERF_SUBSECTION
+                       echo "======== Run for subsection '$GIT_PERF_SUBSECTION' ========"
+                       run_subsection "$@"
+               )
+       done
 else
        while read -r subsec
        do