Merge branch 'lt/preload-lstat'

* lt/preload-lstat:
Fix index preloading for racy dirty case
Add cache preload facility

diff --combined Documentation/config.txt
index 8fee2531147357da9df3a77184717af1f0177e5b,0d96e763ac1dad30ccd42290d1163bd746a07501..b233fe53520ab755902882906f8e76957957b658
--- 1/Documentation/config.txt
--- 2/Documentation/config.txt
+++ b/Documentation/config.txt
@@@ -413,6 -413,15 +413,15 @@@ data writes properly, but can be usefu
  journalling (traditional UNIX filesystems) or that only journal metadata
  and not file contents (OS X's HFS+, or Linux ext3 with "data=writeback").
  
+ core.preloadindex::
+       Enable parallel index preload for operations like 'git diff'
+ +
+ This can speed up operations like 'git diff' and 'git status' especially
+ on filesystems like NFS that have weak caching semantics and thus
+ relatively high IO latencies.  With this set to 'true', git will do the
+ index comparison to the filesystem data in parallel, allowing
+ overlapping IO's.
+ 
  alias.*::
        Command aliases for the linkgit:git[1] command wrapper - e.g.
        after defining "alias.last = cat-file commit HEAD", the invocation
@@@ -572,6 -581,9 +581,6 @@@ color.status.<slot>:
        to red). The values of these variables may be specified as in
        color.branch.<slot>.
  
 -commit.template::
 -      Specify a file to use as the template for new commit messages.
 -
  color.ui::
        When set to `always`, always use colors in all git commands which
        are capable of colored output. When false (or `never`), never. When
@@@ -579,9 -591,6 +588,9 @@@
        terminal. When more specific variables of color.* are set, they always
        take precedence over this setting. Defaults to false.
  
 +commit.template::
 +      Specify a file to use as the template for new commit messages.
 +
  diff.autorefreshindex::
        When using 'git-diff' to compare with work tree
        files, do not consider stat-only change as changed.
@@@ -714,6 -723,18 +723,6 @@@ gc.rerereunresolved:
        kept for this many days when 'git-rerere gc' is run.
        The default is 15 days.  See linkgit:git-rerere[1].
  
 -rerere.autoupdate::
 -      When set to true, `git-rerere` updates the index with the
 -      resulting contents after it cleanly resolves conflicts using
 -      previously recorded resolution.  Defaults to false.
 -
 -rerere.enabled::
 -      Activate recording of resolved conflicts, so that identical
 -      conflict hunks can be resolved automatically, should they
 -      be encountered again.  linkgit:git-rerere[1] command is by
 -      default enabled if you create `rr-cache` directory under
 -      `$GIT_DIR`, but can be disabled by setting this option to false.
 -
  gitcvs.enabled::
        Whether the CVS server interface is enabled for this repository.
        See linkgit:git-cvsserver[1].
@@@ -784,14 -805,6 +793,14 @@@ gui.diffcontext:
        Specifies how many context lines should be used in calls to diff
        made by the linkgit:git-gui[1]. The default is "5".
  
 +gui.encoding::
 +      Specifies the default encoding to use for displaying of
 +      file contents in linkgit:git-gui[1] and linkgit:gitk[1].
 +      It can be overridden by setting the 'encoding' attribute
 +      for relevant files (see linkgit:gitattributes[5]).
 +      If this option is not set, the tools default to the
 +      locale encoding.
 +
  gui.matchtrackingbranch::
        Determines if new branches created with linkgit:git-gui[1] should
        default to tracking remote branches with matching names or
@@@ -814,22 -827,6 +823,22 @@@ gui.spellingdictionary:
        the linkgit:git-gui[1]. When set to "none" spell checking is turned
        off.
  
 +gui.fastcopyblame::
 +      If true, 'git gui blame' uses '-C' instead of '-C -C' for original
 +      location detection. It makes blame significantly faster on huge
 +      repositories at the expense of less thorough copy detection.
 +
 +gui.copyblamethreshold::
 +      Specifies the threshold to use in 'git gui blame' original location
 +      detection, measured in alphanumeric characters. See the
 +      linkgit:git-blame[1] manual for more information on copy detection.
 +
 +gui.blamehistoryctx::
 +      Specifies the radius of history context in days to show in
 +      linkgit:gitk[1] for the selected commit, when the `Show History
 +      Context` menu item is invoked from 'git gui blame'. If this
 +      variable is set to zero, the whole history is shown.
 +
  help.browser::
        Specify the browser that will be used to display help in the
        'web' format. See linkgit:git-help[1].
@@@ -905,10 -902,6 +914,10 @@@ i18n.logOutputEncoding:
        Character encoding the commit messages are converted to when
        running 'git-log' and friends.
  
 +imap::
 +      The configuration variables in the 'imap' section are described
 +      in linkgit:git-imap-send[1].
 +
  instaweb.browser::
        Specify the program that will be used to browse your working
        repository in gitweb. See linkgit:git-instaweb[1].
@@@ -944,6 -937,8 +953,6 @@@ man.viewer:
        Specify the programs that may be used to display help in the
        'man' format. See linkgit:git-help[1].
  
 -include::merge-config.txt[]
 -
  man.<tool>.cmd::
        Specify the command to invoke the specified man viewer. The
        specified command is evaluated in shell with the man page
@@@ -953,7 -948,13 +962,7 @@@ man.<tool>.path:
        Override the path for the given tool that may be used to
        display help in the 'man' format. See linkgit:git-help[1].
  
 -merge.conflictstyle::
 -      Specify the style in which conflicted hunks are written out to
 -      working tree files upon merge.  The default is "merge", which
 -      shows `<<<<<<<` conflict marker, change made by one side,
 -      `=======` marker, change made by the other side, and then
 -      `>>>>>>>` marker.  An alternate style, "diff3", adds `|||||||`
 -      marker and the original text before `=======` marker.
 +include::merge-config.txt[]
  
  mergetool.<tool>.path::
        Override the path for the given tool.  This is useful in case
@@@ -1063,41 -1064,6 +1072,41 @@@ pull.octopus:
  pull.twohead::
        The default merge strategy to use when pulling a single branch.
  
 +receive.fsckObjects::
 +      If it is set to true, git-receive-pack will check all received
 +      objects. It will abort in the case of a malformed object or a
 +      broken link. The result of an abort are only dangling objects.
 +      Defaults to false.
 +
 +receive.unpackLimit::
 +      If the number of objects received in a push is below this
 +      limit then the objects will be unpacked into loose object
 +      files. However if the number of received objects equals or
 +      exceeds this limit then the received pack will be stored as
 +      a pack, after adding any missing delta bases.  Storing the
 +      pack from a push can make the push operation complete faster,
 +      especially on slow filesystems.  If not set, the value of
 +      `transfer.unpackLimit` is used instead.
 +
 +receive.denyDeletes::
 +      If set to true, git-receive-pack will deny a ref update that deletes
 +      the ref. Use this to prevent such a ref deletion via a push.
 +
 +receive.denyCurrentBranch::
 +      If set to true or "refuse", receive-pack will deny a ref update
 +      to the currently checked out branch of a non-bare repository.
 +      Such a push is potentially dangerous because it brings the HEAD
 +      out of sync with the index and working tree. If set to "warn",
 +      print a warning of such a push to stderr, but allow the push to
 +      proceed. If set to false or "ignore", allow such pushes with no
 +      message. Defaults to "warn".
 +
 +receive.denyNonFastForwards::
 +      If set to true, git-receive-pack will deny a ref update which is
 +      not a fast forward. Use this to prevent such an update via a push,
 +      even if that push is forced. This configuration variable is
 +      set when initializing a shared repository.
 +
  remote.<name>.url::
        The URL of a remote repository.  See linkgit:git-fetch[1] or
        linkgit:git-push[1].
@@@ -1147,18 -1113,6 +1156,18 @@@ repack.usedeltabaseoffset:
        "false" and repack. Access from old git versions over the
        native protocol are unaffected by this option.
  
 +rerere.autoupdate::
 +      When set to true, `git-rerere` updates the index with the
 +      resulting contents after it cleanly resolves conflicts using
 +      previously recorded resolution.  Defaults to false.
 +
 +rerere.enabled::
 +      Activate recording of resolved conflicts, so that identical
 +      conflict hunks can be resolved automatically, should they
 +      be encountered again.  linkgit:git-rerere[1] command is by
 +      default enabled if you create `rr-cache` directory under
 +      `$GIT_DIR`, but can be disabled by setting this option to false.
 +
  showbranch.default::
        The default set of branches for linkgit:git-show-branch[1].
        See linkgit:git-show-branch[1].
@@@ -1195,11 -1149,6 +1204,11 @@@ tar.umask:
        archiving user's umask will be used instead.  See umask(2) and
        linkgit:git-archive[1].
  
 +transfer.unpackLimit::
 +      When `fetch.unpackLimit` or `receive.unpackLimit` are
 +      not set, the value of this variable is used instead.
 +      The default value is 100.
 +
  url.<base>.insteadOf::
        Any URL that starts with this value will be rewritten to
        start, instead, with <base>. In cases where some site serves a
@@@ -1228,6 -1177,50 +1237,6 @@@ user.signingkey:
        unchanged to gpg's --local-user parameter, so you may specify a key
        using any method that gpg supports.
  
 -imap::
 -      The configuration variables in the 'imap' section are described
 -      in linkgit:git-imap-send[1].
 -
 -receive.fsckObjects::
 -      If it is set to true, git-receive-pack will check all received
 -      objects. It will abort in the case of a malformed object or a
 -      broken link. The result of an abort are only dangling objects.
 -      Defaults to false.
 -
 -receive.unpackLimit::
 -      If the number of objects received in a push is below this
 -      limit then the objects will be unpacked into loose object
 -      files. However if the number of received objects equals or
 -      exceeds this limit then the received pack will be stored as
 -      a pack, after adding any missing delta bases.  Storing the
 -      pack from a push can make the push operation complete faster,
 -      especially on slow filesystems.  If not set, the value of
 -      `transfer.unpackLimit` is used instead.
 -
 -receive.denyDeletes::
 -      If set to true, git-receive-pack will deny a ref update that deletes
 -      the ref. Use this to prevent such a ref deletion via a push.
 -
 -receive.denyNonFastForwards::
 -      If set to true, git-receive-pack will deny a ref update which is
 -      not a fast forward. Use this to prevent such an update via a push,
 -      even if that push is forced. This configuration variable is
 -      set when initializing a shared repository.
 -
 -receive.denyCurrentBranch::
 -      If set to true or "refuse", receive-pack will deny a ref update
 -      to the currently checked out branch of a non-bare repository.
 -      Such a push is potentially dangerous because it brings the HEAD
 -      out of sync with the index and working tree. If set to "warn",
 -      print a warning of such a push to stderr, but allow the push to
 -      proceed. If set to false or "ignore", allow such pushes with no
 -      message. Defaults to "warn".
 -
 -transfer.unpackLimit::
 -      When `fetch.unpackLimit` or `receive.unpackLimit` are
 -      not set, the value of this variable is used instead.
 -      The default value is 100.
 -
  web.browser::
        Specify a web browser that may be used by some commands.
        Currently only linkgit:git-instaweb[1] and linkgit:git-help[1]

diff --combined builtin-diff.c
index b90d8bcdb6e732729432d8764b429304474bb3e8,1447c1d6baf99c405676a54f0527b254a44ef22e..dddcf697d7447b11d618ac5983516fd1789b0d39
--- 1/builtin-diff.c
--- 2/builtin-diff.c
+++ b/builtin-diff.c
@@@ -134,8 -134,8 +134,8 @@@ static int builtin_diff_index(struct re
            revs->max_count != -1 || revs->min_age != -1 ||
            revs->max_age != -1)
                usage(builtin_diff_usage);
-       if (read_cache() < 0) {
-               perror("read_cache");
+       if (read_cache_preload(revs->diffopt.paths) < 0) {
+               perror("read_cache_preload");
                return -1;
        }
        return run_diff_index(revs, cached);
@@@ -234,8 -234,8 +234,8 @@@ static int builtin_diff_files(struct re
                revs->combine_merges = revs->dense_combined_merges = 1;
  
        setup_work_tree();
-       if (read_cache() < 0) {
-               perror("read_cache");
+       if (read_cache_preload(revs->diffopt.paths) < 0) {
+               perror("read_cache_preload");
                return -1;
        }
        result = run_diff_files(revs, options);
@@@ -290,9 -290,6 +290,9 @@@ int cmd_diff(int argc, const char **arg
        /* Otherwise, we are doing the usual "git" diff */
        rev.diffopt.skip_stat_unmatch = !!diff_auto_refresh_index;
  
 +      /* Default to let external be used */
 +      DIFF_OPT_SET(&rev.diffopt, ALLOW_EXTERNAL);
 +
        if (nongit)
                die("Not a git repository");
        argc = setup_revisions(argc, argv, &rev, NULL);
@@@ -301,7 -298,7 +301,7 @@@
                if (diff_setup_done(&rev.diffopt) < 0)
                        die("diff_setup_done failed");
        }
 -      DIFF_OPT_SET(&rev.diffopt, ALLOW_EXTERNAL);
 +
        DIFF_OPT_SET(&rev.diffopt, RECURSIVE);
        DIFF_OPT_SET(&rev.diffopt, ALLOW_TEXTCONV);
  

diff --combined cache.h
index 7efba7756d3658996f2cda4f79f1c5d2072ed51c,685a8666b64967e4c76514dc3bcf391fd168681e..487e5e1b1bf66007a2d2d753daaf97ddb879b85a
--- 1/cache.h
--- 2/cache.h
+++ b/cache.h
@@@ -262,6 -262,7 +262,7 @@@ static inline void remove_name_hash(str
  
  #define read_cache() read_index(&the_index)
  #define read_cache_from(path) read_index_from(&the_index, (path))
+ #define read_cache_preload(pathspec) read_index_preload(&the_index, (pathspec))
  #define is_cache_unborn() is_index_unborn(&the_index)
  #define read_cache_unmerged() read_index_unmerged(&the_index)
  #define write_cache(newfd, cache, entries) write_index(&the_index, (newfd))
@@@ -368,6 -369,7 +369,7 @@@ extern int init_db(const char *template
  
  /* Initialize and use the cache information */
  extern int read_index(struct index_state *);
+ extern int read_index_preload(struct index_state *, const char **pathspec);
  extern int read_index_from(struct index_state *, const char *path);
  extern int is_index_unborn(struct index_state *);
  extern int read_index_unmerged(struct index_state *);
@@@ -458,6 -460,7 +460,7 @@@ extern size_t packed_git_limit
  extern size_t delta_base_cache_limit;
  extern int auto_crlf;
  extern int fsync_object_files;
+ extern int core_preload_index;
  
  enum safe_crlf {
        SAFE_CRLF_FALSE = 0,
@@@ -530,12 -533,6 +533,12 @@@ static inline void hashclr(unsigned cha
  }
  extern int is_empty_blob_sha1(const unsigned char *sha1);
  
 +#define EMPTY_TREE_SHA1_HEX \
 +      "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
 +#define EMPTY_TREE_SHA1_BIN \
 +       "\x4b\x82\x5d\xc6\x42\xcb\x6e\xb9\xa0\x60" \
 +       "\xe5\x4b\xf8\xd6\x92\x88\xfb\xee\x49\x04"
 +
  int git_mkstemp(char *path, size_t n, const char *template);
  
  /*