Merge branch 'sp/clip-read-write-to-8mb'

Send a large request to read(2)/write(2) as a smaller but still
reasonably large chunks, which would improve the latency when the
operation needs to be killed and incidentally works around broken
64-bit systems that cannot take a 2GB write or read in one go.

* sp/clip-read-write-to-8mb:
Revert "compat/clipped-write.c: large write(2) fails on Mac OS X/XNU"
xread, xwrite: limit size of IO to 8MB

diff --git a/Makefile b/Makefile
index 9135cc9358fb3cb8305eebe8ef209652f758a395..e2abb7b4ea3d3d02dc6e5a8f67516ad1069e31a4 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -69,9 +69,6 @@ all::
 # Define NO_MSGFMT_EXTENDED_OPTIONS if your implementation of msgfmt
 # doesn't support GNU extensions like --check and --statistics
 #
-# Define NEEDS_CLIPPED_WRITE if your write(2) cannot write more than
-# INT_MAX bytes at once (e.g. MacOS X).
-#
 # Define HAVE_PATHS_H if you have paths.h and want to use the default PATH
 # it specifies.
 #
@@ -1499,11 +1496,6 @@ ifndef NO_MSGFMT_EXTENDED_OPTIONS
        MSGFMT += --check --statistics
 endif
 
-ifdef NEEDS_CLIPPED_WRITE
-       BASIC_CFLAGS += -DNEEDS_CLIPPED_WRITE
-       COMPAT_OBJS += compat/clipped-write.o
-endif
-
 ifneq (,$(XDL_FAST_HASH))
        BASIC_CFLAGS += -DXDL_FAST_HASH
 endif

diff --git a/compat/clipped-write.c b/compat/clipped-write.c
deleted file mode 100644 (file)
index b8f98ff..0000000
--- a/compat/clipped-write.c
+++ /dev/null
@@ -1,13 +0,0 @@
-#include "../git-compat-util.h"
-#undef write
-
-/*
- * Version of write that will write at most INT_MAX bytes.
- * Workaround a xnu bug on Mac OS X
- */
-ssize_t clipped_write(int fildes, const void *buf, size_t nbyte)
-{
-       if (nbyte > INT_MAX)
-               nbyte = INT_MAX;
-       return write(fildes, buf, nbyte);
-}

diff --git a/config.mak.uname b/config.mak.uname
index b27f51d4862c67f2dcc61c93f9e2db46f8e44bf6..7d615314f41f33df245efcecbe49f2aee3c90ad7 100644 (file)
--- a/config.mak.uname
+++ b/config.mak.uname
@@ -95,7 +95,6 @@ ifeq ($(uname_S),Darwin)
        NO_MEMMEM = YesPlease
        USE_ST_TIMESPEC = YesPlease
        HAVE_DEV_TTY = YesPlease
-       NEEDS_CLIPPED_WRITE = YesPlease
        COMPAT_OBJS += compat/precompose_utf8.o
        BASIC_CFLAGS += -DPRECOMPOSE_UNICODE
 endif

diff --git a/git-compat-util.h b/git-compat-util.h
index 8752317fe9a52e0e3ba941c63484b00968b2b695..9549de6318147839c73b60b1fc72ceb44d9adb47 100644 (file)
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@ -185,11 +185,6 @@ typedef unsigned long uintptr_t;
 #define probe_utf8_pathname_composition(a,b)
 #endif
 
-#ifdef NEEDS_CLIPPED_WRITE
-ssize_t clipped_write(int fildes, const void *buf, size_t nbyte);
-#define write(x,y,z) clipped_write((x),(y),(z))
-#endif
-
 #ifdef MKDIR_WO_TRAILING_SLASH
 #define mkdir(a,b) compat_mkdir_wo_trailing_slash((a),(b))
 extern int compat_mkdir_wo_trailing_slash(const char*, mode_t);

diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh
index e50f0f742fdc4dca766e3f236cd32e388f0c89aa..b92e6cb0469ebb2e328c490cba9b8aaa7c1dbb84 100755 (executable)
--- a/t/t0021-conversion.sh
+++ b/t/t0021-conversion.sh
@@ -190,4 +190,18 @@ test_expect_success 'required filter clean failure' '
        test_must_fail git add test.fc
 '
 
+test -n "$GIT_TEST_LONG" && test_set_prereq EXPENSIVE
+
+test_expect_success EXPENSIVE 'filter large file' '
+       git config filter.largefile.smudge cat &&
+       git config filter.largefile.clean cat &&
+       for i in $(test_seq 1 2048); do printf "%1048576d" 1; done >2GB &&
+       echo "2GB filter=largefile" >.gitattributes &&
+       git add 2GB 2>err &&
+       ! test -s err &&
+       rm -f 2GB &&
+       git checkout -- 2GB 2>err &&
+       ! test -s err
+'
+
 test_done

diff --git a/wrapper.c b/wrapper.c
index 6a015de5f0564e1ccc9c8cffca891f4ddf4a3ac3..f92b14759833b1ae77070b4188d8ad8ea8d3799e 100644 (file)
--- a/wrapper.c
+++ b/wrapper.c
@@ -130,6 +130,14 @@ void *xcalloc(size_t nmemb, size_t size)
        return ret;
 }
 
+/*
+ * Limit size of IO chunks, because huge chunks only cause pain.  OS X
+ * 64-bit is buggy, returning EINVAL if len >= INT_MAX; and even in
+ * the absense of bugs, large chunks can result in bad latencies when
+ * you decide to kill the process.
+ */
+#define MAX_IO_SIZE (8*1024*1024)
+
 /*
  * xread() is the same a read(), but it automatically restarts read()
  * operations with a recoverable error (EAGAIN and EINTR). xread()
@@ -138,6 +146,8 @@ void *xcalloc(size_t nmemb, size_t size)
 ssize_t xread(int fd, void *buf, size_t len)
 {
        ssize_t nr;
+       if (len > MAX_IO_SIZE)
+           len = MAX_IO_SIZE;
        while (1) {
                nr = read(fd, buf, len);
                if ((nr < 0) && (errno == EAGAIN || errno == EINTR))
@@ -154,6 +164,8 @@ ssize_t xread(int fd, void *buf, size_t len)
 ssize_t xwrite(int fd, const void *buf, size_t len)
 {
        ssize_t nr;
+       if (len > MAX_IO_SIZE)
+           len = MAX_IO_SIZE;
        while (1) {
                nr = write(fd, buf, len);
                if ((nr < 0) && (errno == EAGAIN || errno == EINTR))