Win32: reliably detect console pipe handles

As of "Win32: Thread-safe windows console output", child processes may
print to the console even if stdout has been redirected to a file. E.g.:

git config tar.cat.command "cat"
git archive -o test.cat HEAD

Detecting whether stdout / stderr point to our console pipe is currently
based on the assumption that OS HANDLE values are never reused. This is
apparently not true if stdout / stderr is replaced via dup2() (as in
builtin/archive.c:17).

Instead of comparing handle values, check if the file descriptor isatty()
backed by a pipe OS handle. This is only possible by swapping the handles
in MSVCRT's internal data structures, as we do in winansi_init().

Reported-by: Johannes Sixt <j6t@kdbg.org>
Signed-off-by: Karsten Blees <blees@dcon.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/compat/winansi.c b/compat/winansi.c
index f96d5c28fc0d634139e0c9667142294fc1b9afc1..efc5bb3a4b63166eccb33f6ec6a8ad57e4c9ac36 100644 (file)
--- a/compat/winansi.c
+++ b/compat/winansi.c
@@ -20,7 +20,6 @@ static WORD attr;
 static int negative;
 static int non_ascii_used = 0;
 static HANDLE hthread, hread, hwrite;
-static HANDLE hwrite1 = INVALID_HANDLE_VALUE, hwrite2 = INVALID_HANDLE_VALUE;
 static HANDLE hconsole1, hconsole2;
 
 #ifdef __MINGW32__
@@ -435,10 +434,6 @@ static void winansi_exit(void)
        WaitForSingleObject(hthread, INFINITE);
 
        /* cleanup handles... */
-       if (hwrite1 != INVALID_HANDLE_VALUE)
-               CloseHandle(hwrite1);
-       if (hwrite2 != INVALID_HANDLE_VALUE)
-               CloseHandle(hwrite2);
        CloseHandle(hwrite);
        CloseHandle(hthread);
 }
@@ -565,14 +560,9 @@ void winansi_init(void)
 
        /* redirect stdout / stderr to the pipe */
        if (con1)
-               hconsole1 = swap_osfhnd(1, hwrite1 = duplicate_handle(hwrite));
+               hconsole1 = swap_osfhnd(1, duplicate_handle(hwrite));
        if (con2)
-               hconsole2 = swap_osfhnd(2, hwrite2 = duplicate_handle(hwrite));
-}
-
-static int is_same_handle(HANDLE hnd, int fd)
-{
-       return hnd != INVALID_HANDLE_VALUE && hnd == (HANDLE) _get_osfhandle(fd);
+               hconsole2 = swap_osfhnd(2, duplicate_handle(hwrite));
 }
 
 /*
@@ -581,10 +571,9 @@ static int is_same_handle(HANDLE hnd, int fd)
  */
 HANDLE winansi_get_osfhandle(int fd)
 {
-       if (fd == 1 && is_same_handle(hwrite1, 1))
-               return hconsole1;
-       else if (fd == 2 && is_same_handle(hwrite2, 2))
-               return hconsole2;
-       else
-               return (HANDLE) _get_osfhandle(fd);
+       HANDLE hnd = (HANDLE) _get_osfhandle(fd);
+       if ((fd == 1 || fd == 2) && isatty(fd)
+           && GetFileType(hnd) == FILE_TYPE_PIPE)
+               return (fd == 1) ? hconsole1 : hconsole2;
+       return hnd;
 }