Avoid accessing a slow working copy during diffcore operations.

The Cygwin folks have done a fine job at creating a POSIX layer
on Windows That Just Works(tm).  However it comes with a penalty;
accessing files in the working tree by way of stat/open/mmap can
be slower for diffcore than inflating the data from a blob which
is stored in a packfile.

This performance problem is especially an issue in merge-recursive
when dealing with nearly 7000 added files, as we are loading
each file's content from the working directory to perform rename
detection.  I have literally seen (and sadly watched) paint dry in
less time than it takes for merge-recursive to finish such a merge.
On the other hand this very same merge runs very fast on Solaris.

If Git is compiled with NO_FAST_WORKING_DIRECTORY set then we will
avoid looking at the working directory when the blob in question
is available within a packfile and the caller doesn't need the data
unpacked into a temporary file.

We don't use loose objects as they have the same open/mmap/close
costs as the working directory file access, but have the additional
CPU overhead of needing to inflate the content before use.  So it
is still faster to use the working tree file over the loose object.

If the caller needs the file data unpacked into a temporary file
its likely because they are going to call an external diff program,
passing the file as a parameter.  In this case reusing the working
tree file will be faster as we don't need to inflate the data and
write it out to a temporary file.

The NO_FAST_WORKING_DIRECTORY feature is enabled by default on
Cygwin, as that is the platform which currently appears to benefit
the most from this option.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/Makefile b/Makefile
index 2d17fa702795ca08a8ffc3aeaf6393b300401e33..676d426a9e707361ac3922e77cbf08d59728b607 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -69,6 +69,9 @@ all:
 #
 # Define NO_MMAP if you want to avoid mmap.
 #
 #
 # Define NO_MMAP if you want to avoid mmap.
 #

+# Define NO_FAST_WORKING_DIRECTORY if accessing objects in pack files is
+# generally faster on your platform than accessing the working directory.
+#

 # Define NO_IPV6 if you lack IPv6 support and getaddrinfo().
 #
 # Define NO_SOCKADDR_STORAGE if your platform does not have struct
 # Define NO_IPV6 if you lack IPv6 support and getaddrinfo().
 #
 # Define NO_SOCKADDR_STORAGE if your platform does not have struct


@@ -355,6 +358,7 @@ ifeq ($(uname_O),Cygwin)
        NO_SYMLINK_HEAD = YesPlease
        NEEDS_LIBICONV = YesPlease
        NO_C99_FORMAT = YesPlease
        NO_SYMLINK_HEAD = YesPlease
        NEEDS_LIBICONV = YesPlease
        NO_C99_FORMAT = YesPlease

+       NO_FAST_WORKING_DIRECTORY = UnfortunatelyYes

        # There are conflicting reports about this.
        # On some boxes NO_MMAP is needed, and not so elsewhere.
        # Try uncommenting this if you see things break -- YMMV.
        # There are conflicting reports about this.
        # On some boxes NO_MMAP is needed, and not so elsewhere.
        # Try uncommenting this if you see things break -- YMMV.


@@ -506,6 +510,9 @@ ifdef NO_MMAP
        COMPAT_CFLAGS += -DNO_MMAP
        COMPAT_OBJS += compat/mmap.o
 endif
        COMPAT_CFLAGS += -DNO_MMAP
        COMPAT_OBJS += compat/mmap.o
 endif

+ifdef NO_FAST_WORKING_DIRECTORY
+       BASIC_CFLAGS += -DNO_FAST_WORKING_DIRECTORY
+endif

 ifdef NO_IPV6
        BASIC_CFLAGS += -DNO_IPV6
 endif
 ifdef NO_IPV6
        BASIC_CFLAGS += -DNO_IPV6
 endif

diff --git a/diff.c b/diff.c
index 2df14b2469362bfd8a30a45457953d993baae66d..77ba641aa48974a7601875bd0a6a351baa30e726 100644 (file)
--- a/diff.c
+++ b/diff.c
@@ -12,6 +12,12 @@
 #include "xdiff-interface.h"
 #include "color.h"
 
 #include "xdiff-interface.h"
 #include "color.h"
 

+#ifdef NO_FAST_WORKING_DIRECTORY
+#define FAST_WORKING_DIRECTORY 0
+#else
+#define FAST_WORKING_DIRECTORY 1
+#endif
+

 static int use_size_cache;
 
 static int diff_detect_rename_default;
 static int use_size_cache;
 
 static int diff_detect_rename_default;


@@ -1158,7 +1164,7 @@ void fill_filespec(struct diff_filespec *spec, const unsigned char *sha1,
  * the work tree has that object contents, return true, so that
  * prepare_temp_file() does not have to inflate and extract.
  */
  * the work tree has that object contents, return true, so that
  * prepare_temp_file() does not have to inflate and extract.
  */

-static int work_tree_matches(const char *name, const unsigned char *sha1)
+static int reuse_worktree_file(const char *name, const unsigned char *sha1, int want_file)

 {
        struct cache_entry *ce;
        struct stat st;
 {
        struct cache_entry *ce;
        struct stat st;


@@ -1179,6 +1185,18 @@ static int work_tree_matches(const char *name, const unsigned char *sha1)
        if (!active_cache)
                return 0;
 
        if (!active_cache)
                return 0;
 

+       /* We want to avoid the working directory if our caller
+        * doesn't need the data in a normal file, this system
+        * is rather slow with its stat/open/mmap/close syscalls,
+        * and the object is contained in a pack file.  The pack
+        * is probably already open and will be faster to obtain
+        * the data through than the working directory.  Loose
+        * objects however would tend to be slower as they need
+        * to be individually opened and inflated.
+        */
+       if (FAST_WORKING_DIRECTORY && !want_file && has_sha1_pack(sha1, NULL))
+               return 0;
+

        len = strlen(name);
        pos = cache_name_pos(name, len);
        if (pos < 0)
        len = strlen(name);
        pos = cache_name_pos(name, len);
        if (pos < 0)


@@ -1265,7 +1283,7 @@ int diff_populate_filespec(struct diff_filespec *s, int size_only)
        if (s->data)
                return err;
        if (!s->sha1_valid ||
        if (s->data)
                return err;
        if (!s->sha1_valid ||

-           work_tree_matches(s->path, s->sha1)) {
+           reuse_worktree_file(s->path, s->sha1, 0)) {

                struct stat st;
                int fd;
                if (lstat(s->path, &st) < 0) {
                struct stat st;
                int fd;
                if (lstat(s->path, &st) < 0) {


@@ -1372,7 +1390,7 @@ static void prepare_temp_file(const char *name,
        }
 
        if (!one->sha1_valid ||
        }
 
        if (!one->sha1_valid ||

-           work_tree_matches(name, one->sha1)) {
+           reuse_worktree_file(name, one->sha1, 1)) {

                struct stat st;
                if (lstat(name, &st) < 0) {
                        if (errno == ENOENT)
                struct stat st;
                if (lstat(name, &st) < 0) {
                        if (errno == ENOENT)