Win32: detect console streams more reliably

GetStdHandle(STD_OUTPUT_HANDLE) doesn't work for stderr if stdout is
redirected. Use _get_osfhandle of the FILE* instead.

_isatty() is true for all character devices (including parallel and serial
ports). Check return value of GetConsoleScreenBufferInfo instead to
reliably detect console handles (also don't initialize internal state from
an uninitialized CONSOLE_SCREEN_BUFFER_INFO structure if the function
fails).

Signed-off-by: Karsten Blees <blees@dcon.de>
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Stepan Kasal <kasal@ucw.cz>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/compat/winansi.c b/compat/winansi.c
index abe0feaa2c207fd8ea0393e08998916ca198c293..c4be401a6eaffe6180ffb1c4ef7d44a24f61d374 100644 (file)
--- a/compat/winansi.c
+++ b/compat/winansi.c
@@ -25,27 +25,39 @@ static HANDLE console;
 static WORD plain_attr;
 static WORD attr;
 static int negative;
+static FILE *last_stream = NULL;
 
-static void init(void)
+static int is_console(FILE *stream)
 {
        CONSOLE_SCREEN_BUFFER_INFO sbi;
+       HANDLE hcon;
 
        static int initialized = 0;
-       if (initialized)
-               return;
 
-       console = GetStdHandle(STD_OUTPUT_HANDLE);
-       if (console == INVALID_HANDLE_VALUE)
-               console = NULL;
+       /* use cached value if stream hasn't changed */
+       if (stream == last_stream)
+               return console != NULL;
 
-       if (!console)
-               return;
+       last_stream = stream;
+       console = NULL;
 
-       GetConsoleScreenBufferInfo(console, &sbi);
-       attr = plain_attr = sbi.wAttributes;
-       negative = 0;
+       /* get OS handle of the stream */
+       hcon = (HANDLE) _get_osfhandle(_fileno(stream));
+       if (hcon == INVALID_HANDLE_VALUE)
+               return 0;
+
+       /* check if its a handle to a console output screen buffer */
+       if (!GetConsoleScreenBufferInfo(hcon, &sbi))
+               return 0;
+
+       if (!initialized) {
+               attr = plain_attr = sbi.wAttributes;
+               negative = 0;
+               initialized = 1;
+       }
 
-       initialized = 1;
+       console = hcon;
+       return 1;
 }
 
 static int write_console(const char *str, size_t len)
@@ -292,12 +304,7 @@ int winansi_fputs(const char *str, FILE *stream)
 {
        int rv;
 
-       if (!isatty(fileno(stream)))
-               return fputs(str, stream);
-
-       init();
-
-       if (!console)
+       if (!is_console(stream))
                return fputs(str, stream);
 
        rv = ansi_emulate(str, stream);
@@ -315,12 +322,7 @@ int winansi_vfprintf(FILE *stream, const char *format, va_list list)
        char *buf = small_buf;
        va_list cp;
 
-       if (!isatty(fileno(stream)))
-               goto abort;
-
-       init();
-
-       if (!console)
+       if (!is_console(stream))
                goto abort;
 
        va_copy(cp, list);