tests: paint unexpectedly fixed known breakages in bold red

Change color of unexpectedly fixed known breakages to bold red.  An
unexpectedly passing test indicates that the test code is somehow
broken or out of sync with the code it is testing.  Either way this is
an error which is potentially as bad as a failing test, and as such is
no longer portrayed as a pass in the output.

Signed-off-by: Adam Spiers <git@adamspiers.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh
index 384b0aee47ba1606435be6e48f8a49c4db881598..8973d2ccc8406bb55ae771a2a1a2b136f7159891 100755 (executable)
--- a/t/t0000-basic.sh
+++ b/t/t0000-basic.sh
@@ -145,13 +145,31 @@ test_expect_success 'pretend we have fixed a known breakage' "
        test_done
        EOF
        check_sub_test_lib_test passing-todo <<-\\EOF
        test_done
        EOF
        check_sub_test_lib_test passing-todo <<-\\EOF

-       > ok 1 - pretend we have fixed a known breakage # TODO known breakage
-       > # fixed 1 known breakage(s)
-       > # passed all 1 test(s)
+       > ok 1 - pretend we have fixed a known breakage # TODO known breakage vanished
+       > # 1 known breakage(s) vanished; please update test(s)

        > 1..1
        EOF
 "
 
        > 1..1
        EOF
 "
 

+test_expect_success 'pretend we have fixed one of two known breakages (run in sub test-lib)' "
+       run_sub_test_lib_test partially-passing-todos \
+               '2 TODO tests, one passing' <<-\\EOF &&
+       test_expect_failure 'pretend we have a known breakage' 'false'
+       test_expect_success 'pretend we have a passing test' 'true'
+       test_expect_failure 'pretend we have fixed another known breakage' 'true'
+       test_done
+       EOF
+       check_sub_test_lib_test partially-passing-todos <<-\\EOF
+       > not ok 1 - pretend we have a known breakage # TODO known breakage
+       > ok 2 - pretend we have a passing test
+       > ok 3 - pretend we have fixed another known breakage # TODO known breakage vanished
+       > # 1 known breakage(s) vanished; please update test(s)
+       > # still have 1 known breakage(s)
+       > # passed all remaining 1 test(s)
+       > 1..3
+       EOF
+"
+

 test_expect_success 'pretend we have a pass, fail, and known breakage' "
        test_must_fail run_sub_test_lib_test \
                mixed-results1 'mixed results #1' <<-\\EOF &&
 test_expect_success 'pretend we have a pass, fail, and known breakage' "
        test_must_fail run_sub_test_lib_test \
                mixed-results1 'mixed results #1' <<-\\EOF &&


@@ -199,10 +217,10 @@ test_expect_success 'pretend we have a mix of all possible results' "
        > #     false
        > not ok 8 - pretend we have a known breakage # TODO known breakage
        > not ok 9 - pretend we have a known breakage # TODO known breakage
        > #     false
        > not ok 8 - pretend we have a known breakage # TODO known breakage
        > not ok 9 - pretend we have a known breakage # TODO known breakage

-       > ok 10 - pretend we have fixed a known breakage # TODO known breakage
-       > # fixed 1 known breakage(s)
+       > ok 10 - pretend we have fixed a known breakage # TODO known breakage vanished
+       > # 1 known breakage(s) vanished; please update test(s)

        > # still have 2 known breakage(s)
        > # still have 2 known breakage(s)

-       > # failed 3 among remaining 8 test(s)
+       > # failed 3 among remaining 7 test(s)

        > 1..10
        EOF
 "
        > 1..10
        EOF
 "

diff --git a/t/test-lib.sh b/t/test-lib.sh
index 8b75c9a2750449dcb8f3d04756415b03b66a6477..30a0937dd726c8bf9667e2f9e8145d21584d271d 100644 (file)
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -278,7 +278,7 @@ test_failure_ () {
 
 test_known_broken_ok_ () {
        test_fixed=$(($test_fixed+1))
 
 test_known_broken_ok_ () {
        test_fixed=$(($test_fixed+1))

-       say_color "" "ok $test_count - $@ # TODO known breakage"
+       say_color error "ok $test_count - $@ # TODO known breakage vanished"

 }
 
 test_known_broken_failure_ () {
 }
 
 test_known_broken_failure_ () {


@@ -373,13 +373,18 @@ test_done () {
 
        if test "$test_fixed" != 0
        then
 
        if test "$test_fixed" != 0
        then

-               say_color pass "# fixed $test_fixed known breakage(s)"
+               say_color error "# $test_fixed known breakage(s) vanished; please update test(s)"

        fi
        if test "$test_broken" != 0
        then
                say_color warn "# still have $test_broken known breakage(s)"
        fi
        if test "$test_broken" != 0
        then
                say_color warn "# still have $test_broken known breakage(s)"

-               msg="remaining $(($test_count-$test_broken)) test(s)"
+       fi
+       if test "$test_broken" != 0 || test "$test_fixed" != 0
+       then
+               test_remaining=$(( $test_count - $test_broken - $test_fixed ))
+               msg="remaining $test_remaining test(s)"

        else
        else

+               test_remaining=$test_count

                msg="$test_count test(s)"
        fi
        case "$test_failure" in
                msg="$test_count test(s)"
        fi
        case "$test_failure" in


@@ -393,7 +398,7 @@ test_done () {
 
                if test $test_external_has_tap -eq 0
                then
 
                if test $test_external_has_tap -eq 0
                then

-                       if test $test_count -gt 0
+                       if test $test_remaining -gt 0

                        then
                                say_color pass "# passed all $msg"
                        fi
                        then
                                say_color pass "# passed all $msg"
                        fi