From: Junio C Hamano <gitster@pobox.com>
Date: Tue, 21 Nov 2017 05:07:50 +0000 (+0900)
Subject: Merge branch 'bp/fsmonitor'
X-Git-Tag: v2.16.0-rc0~112
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/e05336bddacb90cf243aacc0f7b7f34f900453d7?ds=inline;hp=-c

Merge branch 'bp/fsmonitor'

We learned to talk to watchman to speed up "git status" and other
operations that need to see which paths have been modified.

* bp/fsmonitor:
fsmonitor: preserve utf8 filenames in fsmonitor-watchman log
fsmonitor: read entirety of watchman output
fsmonitor: MINGW support for watchman integration
fsmonitor: add a performance test
fsmonitor: add a sample integration script for Watchman
fsmonitor: add test cases for fsmonitor extension
split-index: disable the fsmonitor extension when running the split index test
fsmonitor: add a test tool to dump the index extension
update-index: add fsmonitor support to update-index
ls-files: Add support in ls-files to display the fsmonitor valid bit
fsmonitor: add documentation for the fsmonitor extension.
fsmonitor: teach git to optionally utilize a file system monitor to speed up detecting new or changed files.
update-index: add a new --force-write-index option
preload-index: add override to enable testing preload-index
bswap: add 64 bit endianness helper get_be64
---

e05336bddacb90cf243aacc0f7b7f34f900453d7
diff --combined Documentation/config.txt
index 671fcbaa0f,db52645cb4..5f65fa9234
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@@ -351,9 -351,6 +351,9 @@@ advice.*:
  	addEmbeddedRepo::
  		Advice on what to do when you've accidentally added one
  		git repo inside of another.
 +	ignoredHook::
 +		Advice shown if an hook is ignored because the hook is not
 +		set as executable.
  --
  
  core.fileMode::
@@@ -416,6 -413,13 +416,13 @@@ core.protectNTFS:
  	8.3 "short" names.
  	Defaults to `true` on Windows, and `false` elsewhere.
  
+ core.fsmonitor::
+ 	If set, the value of this variable is used as a command which
+ 	will identify all files that may have changed since the
+ 	requested date/time. This information is used to speed up git by
+ 	avoiding unnecessary processing of files that have not changed.
+ 	See the "fsmonitor-watchman" section of linkgit:githooks[5].
+ 
  core.trustctime::
  	If false, the ctime differences between the index and the
  	working tree are ignored; useful when the inode change time
@@@ -952,23 -956,6 +959,23 @@@ apply.whitespace:
  	Tells 'git apply' how to handle whitespaces, in the same way
  	as the `--whitespace` option. See linkgit:git-apply[1].
  
 +blame.showRoot::
 +	Do not treat root commits as boundaries in linkgit:git-blame[1].
 +	This option defaults to false.
 +
 +blame.blankBoundary::
 +	Show blank commit object name for boundary commits in
 +	linkgit:git-blame[1]. This option defaults to false.
 +
 +blame.showEmail::
 +	Show the author email instead of author name in linkgit:git-blame[1].
 +	This option defaults to false.
 +
 +blame.date::
 +	Specifies the format used to output dates in linkgit:git-blame[1].
 +	If unset the iso format is used. For supported values,
 +	see the discussion of the `--date` option at linkgit:git-log[1].
 +
  branch.autoSetupMerge::
  	Tells 'git branch' and 'git checkout' to set up new branches
  	so that linkgit:git-pull[1] will appropriately merge from the
@@@ -2641,35 -2628,6 +2648,35 @@@ push.gpgSign:
  	override a value from a lower-priority config file. An explicit
  	command-line flag always overrides this config option.
  
 +push.pushOption::
 +	When no `--push-option=<option>` argument is given from the
 +	command line, `git push` behaves as if each <value> of
 +	this variable is given as `--push-option=<value>`.
 ++
 +This is a multi-valued variable, and an empty value can be used in a
 +higher priority configuration file (e.g. `.git/config` in a
 +repository) to clear the values inherited from a lower priority
 +configuration files (e.g. `$HOME/.gitconfig`).
 ++
 +--
 +
 +Example:
 +
 +/etc/gitconfig
 +  push.pushoption = a
 +  push.pushoption = b
 +
 +~/.gitconfig
 +  push.pushoption = c
 +
 +repo/.git/config
 +  push.pushoption =
 +  push.pushoption = b
 +
 +This will result in only b (a and c are cleared).
 +
 +--
 +
  push.recurseSubmodules::
  	Make sure all submodule commits used by the revisions to be pushed
  	are available on a remote-tracking branch. If the value is 'check'
@@@ -3134,14 -3092,10 +3141,14 @@@ submodule.<name>.url:
  	See linkgit:git-submodule[1] and linkgit:gitmodules[5] for details.
  
  submodule.<name>.update::
 -	The default update procedure for a submodule. This variable
 -	is populated by `git submodule init` from the
 -	linkgit:gitmodules[5] file. See description of 'update'
 -	command in linkgit:git-submodule[1].
 +	The method by which a submodule is updated by 'git submodule update',
 +	which is the only affected command, others such as
 +	'git checkout --recurse-submodules' are unaffected. It exists for
 +	historical reasons, when 'git submodule' was the only command to
 +	interact with submodules; settings like `submodule.active`
 +	and `pull.rebase` are more specific. It is populated by
 +	`git submodule init` from the linkgit:gitmodules[5] file.
 +	See description of 'update' command in linkgit:git-submodule[1].
  
  submodule.<name>.branch::
  	The remote branch name for a submodule, used by `git submodule
diff --combined Documentation/git-update-index.txt
index 75c7dd9dea,7c2f880a22..bdb0342593
--- a/Documentation/git-update-index.txt
+++ b/Documentation/git-update-index.txt
@@@ -16,9 -16,11 +16,11 @@@ SYNOPSI
  	     [--chmod=(+|-)x]
  	     [--[no-]assume-unchanged]
  	     [--[no-]skip-worktree]
+ 	     [--[no-]fsmonitor-valid]
  	     [--ignore-submodules]
  	     [--[no-]split-index]
  	     [--[no-|test-|force-]untracked-cache]
+ 	     [--[no-]fsmonitor]
  	     [--really-refresh] [--unresolve] [--again | -g]
  	     [--info-only] [--index-info]
  	     [-z] [--stdin] [--index-version <n>]
@@@ -111,6 -113,12 +113,12 @@@ you will need to handle the situation m
  	set and unset the "skip-worktree" bit for the paths. See
  	section "Skip-worktree bit" below for more information.
  
+ --[no-]fsmonitor-valid::
+ 	When one of these flags is specified, the object name recorded
+ 	for the paths are not updated. Instead, these options
+ 	set and unset the "fsmonitor valid" bit for the paths. See
+ 	section "File System Monitor" below for more information.
+ 
  -g::
  --again::
  	Runs 'git update-index' itself on the paths whose index
@@@ -153,7 -161,7 +161,7 @@@
  +
  Version 4 performs a simple pathname compression that reduces index
  size by 30%-50% on large repositories, which results in faster load
 -time. Version 4 is relatively young (first released in in 1.8.0 in
 +time. Version 4 is relatively young (first released in 1.8.0 in
  October 2012). Other Git implementations such as JGit and libgit2
  may not support it yet.
  
@@@ -201,6 -209,15 +209,15 @@@ will remove the intended effect of the 
  	`--untracked-cache` used to imply `--test-untracked-cache` but
  	this option would enable the extension unconditionally.
  
+ --fsmonitor::
+ --no-fsmonitor::
+ 	Enable or disable files system monitor feature. These options
+ 	take effect whatever the value of the `core.fsmonitor`
+ 	configuration variable (see linkgit:git-config[1]). But a warning
+ 	is emitted when the change goes against the configured value, as
+ 	the configured value will take effect next time the index is
+ 	read and this will remove the intended effect of the option.
+ 
  \--::
  	Do not interpret any more arguments as options.
  
@@@ -447,6 -464,34 +464,34 @@@ command reads the index; while when `--
  are used, the untracked cache is immediately added to or removed from
  the index.
  
+ File System Monitor
+ -------------------
+ 
+ This feature is intended to speed up git operations for repos that have
+ large working directories.
+ 
+ It enables git to work together with a file system monitor (see the
+ "fsmonitor-watchman" section of linkgit:githooks[5]) that can
+ inform it as to what files have been modified. This enables git to avoid
+ having to lstat() every file to find modified files.
+ 
+ When used in conjunction with the untracked cache, it can further improve
+ performance by avoiding the cost of scanning the entire working directory
+ looking for new files.
+ 
+ If you want to enable (or disable) this feature, it is easier to use
+ the `core.fsmonitor` configuration variable (see
+ linkgit:git-config[1]) than using the `--fsmonitor` option to
+ `git update-index` in each repository, especially if you want to do so
+ across all repositories you use, because you can set the configuration
+ variable to `true` (or `false`) in your `$HOME/.gitconfig` just once
+ and have it affect all repositories you touch.
+ 
+ When the `core.fsmonitor` configuration variable is changed, the
+ file system monitor is added to or removed from the index the next time
+ a command reads the index. When `--[no-]fsmonitor` are used, the file
+ system monitor is immediately added to or removed from the index.
+ 
  Configuration
  -------------
  
diff --combined Documentation/githooks.txt
index 5d3f45560e,ae60559cd9..0bb0042d8c
--- a/Documentation/githooks.txt
+++ b/Documentation/githooks.txt
@@@ -127,10 -127,11 +127,10 @@@ help message found in the commented por
  commit-msg
  ~~~~~~~~~~
  
 -This hook is invoked by 'git commit', and can be bypassed
 -with the `--no-verify` option.  It takes a single parameter, the
 -name of the file that holds the proposed commit log message.
 -Exiting with a non-zero status causes the 'git commit' to
 -abort.
 +This hook is invoked by 'git commit' and 'git merge', and can be
 +bypassed with the `--no-verify` option.  It takes a single parameter,
 +the name of the file that holds the proposed commit log message.
 +Exiting with a non-zero status causes the command to abort.
  
  The hook is allowed to edit the message file in place, and can be used
  to normalize the message into some project standard format. It
@@@ -454,6 -455,34 +454,34 @@@ the name of the file that holds the e-m
  non-zero status causes 'git send-email' to abort before sending any
  e-mails.
  
+ fsmonitor-watchman
+ ~~~~~~~~~~~~~~~~~~
+ 
+ This hook is invoked when the configuration option core.fsmonitor is
+ set to .git/hooks/fsmonitor-watchman.  It takes two arguments, a version
+ (currently 1) and the time in elapsed nanoseconds since midnight,
+ January 1, 1970.
+ 
+ The hook should output to stdout the list of all files in the working
+ directory that may have changed since the requested time.  The logic
+ should be inclusive so that it does not miss any potential changes.
+ The paths should be relative to the root of the working directory
+ and be separated by a single NUL.
+ 
+ It is OK to include files which have not actually changed.  All changes
+ including newly-created and deleted files should be included. When
+ files are renamed, both the old and the new name should be included.
+ 
+ Git will limit what files it checks for changes as well as which
+ directories are checked for untracked files based on the path names
+ given.
+ 
+ An optimized way to tell git "all files have changed" is to return
+ the filename '/'.
+ 
+ The exit status determines whether git will use the data from the
+ hook to limit its search.  On error, it will fall back to verifying
+ all files and folders.
  
  GIT
  ---
diff --combined Makefile
index ee9d5eb11e,b2653ee64f..e53750ca01
--- a/Makefile
+++ b/Makefile
@@@ -162,11 -162,6 +162,11 @@@ all:
  # algorithm. This is slower, but may detect attempted collision attacks.
  # Takes priority over other *_SHA1 knobs.
  #
 +# Define DC_SHA1_EXTERNAL in addition to DC_SHA1 if you want to build / link
 +# git with the external SHA1 collision-detect library.
 +# Without this option, i.e. the default behavior is to build git with its
 +# own built-in code (or submodule).
 +#
  # Define DC_SHA1_SUBMODULE in addition to DC_SHA1 to use the
  # sha1collisiondetection shipped as a submodule instead of the
  # non-submodule copy in sha1dc/. This is an experimental option used
@@@ -205,9 -200,6 +205,9 @@@
  #
  # Define NO_MMAP if you want to avoid mmap.
  #
 +# Define MMAP_PREVENTS_DELETE if a file that is currently mmapped cannot be
 +# deleted or cannot be replaced using rename().
 +#
  # Define NO_SYS_POLL_H if you don't have sys/poll.h.
  #
  # Define NO_POLL if you do not have or don't want to use poll().
@@@ -646,7 -638,9 +646,9 @@@ TEST_PROGRAMS_NEED_X += test-ctyp
  TEST_PROGRAMS_NEED_X += test-config
  TEST_PROGRAMS_NEED_X += test-date
  TEST_PROGRAMS_NEED_X += test-delta
+ TEST_PROGRAMS_NEED_X += test-drop-caches
  TEST_PROGRAMS_NEED_X += test-dump-cache-tree
+ TEST_PROGRAMS_NEED_X += test-dump-fsmonitor
  TEST_PROGRAMS_NEED_X += test-dump-split-index
  TEST_PROGRAMS_NEED_X += test-dump-untracked-cache
  TEST_PROGRAMS_NEED_X += test-fake-ssh
@@@ -794,6 -788,7 +796,7 @@@ LIB_OBJS += ewah/ewah_rlw.
  LIB_OBJS += exec_cmd.o
  LIB_OBJS += fetch-pack.o
  LIB_OBJS += fsck.o
+ LIB_OBJS += fsmonitor.o
  LIB_OBJS += gettext.o
  LIB_OBJS += gpg-interface.o
  LIB_OBJS += graph.o
@@@ -824,7 -819,6 +827,7 @@@ LIB_OBJS += notes-cache.
  LIB_OBJS += notes-merge.o
  LIB_OBJS += notes-utils.o
  LIB_OBJS += object.o
 +LIB_OBJS += oidmap.o
  LIB_OBJS += oidset.o
  LIB_OBJS += packfile.o
  LIB_OBJS += pack-bitmap.o
@@@ -1045,9 -1039,6 +1048,9 @@@ BASIC_CFLAGS += -fno-omit-frame-pointe
  ifneq ($(filter undefined,$(SANITIZERS)),)
  BASIC_CFLAGS += -DNO_UNALIGNED_LOADS
  endif
 +ifneq ($(filter leak,$(SANITIZERS)),)
 +BASIC_CFLAGS += -DSUPPRESS_ANNOTATED_LEAKS
 +endif
  endif
  
  ifndef sysconfdir
@@@ -1395,9 -1386,6 +1398,9 @@@ els
  		COMPAT_OBJS += compat/win32mmap.o
  	endif
  endif
 +ifdef MMAP_PREVENTS_DELETE
 +	BASIC_CFLAGS += -DMMAP_PREVENTS_DELETE
 +endif
  ifdef OBJECT_CREATION_USES_RENAMES
  	COMPAT_CFLAGS += -DOBJECT_CREATION_MODE=1
  endif
@@@ -1490,15 -1478,6 +1493,15 @@@ ifdef APPLE_COMMON_CRYPT
  	BASIC_CFLAGS += -DSHA1_APPLE
  else
  	DC_SHA1 := YesPlease
 +	BASIC_CFLAGS += -DSHA1_DC
 +	LIB_OBJS += sha1dc_git.o
 +ifdef DC_SHA1_EXTERNAL
 +	ifdef DC_SHA1_SUBMODULE
 +$(error Only set DC_SHA1_EXTERNAL or DC_SHA1_SUBMODULE, not both)
 +	endif
 +	BASIC_CFLAGS += -DDC_SHA1_EXTERNAL
 +	EXTLIBS += -lsha1detectcoll
 +else
  ifdef DC_SHA1_SUBMODULE
  	LIB_OBJS += sha1collisiondetection/lib/sha1.o
  	LIB_OBJS += sha1collisiondetection/lib/ubc_check.o
@@@ -1508,15 -1487,17 +1511,15 @@@ els
  	LIB_OBJS += sha1dc/ubc_check.o
  endif
  	BASIC_CFLAGS += \
 -		-DSHA1_DC \
  		-DSHA1DC_NO_STANDARD_INCLUDES \
  		-DSHA1DC_INIT_SAFE_HASH_DEFAULT=0 \
  		-DSHA1DC_CUSTOM_INCLUDE_SHA1_C="\"cache.h\"" \
 -		-DSHA1DC_CUSTOM_TRAILING_INCLUDE_SHA1_C="\"sha1dc_git.c\"" \
 -		-DSHA1DC_CUSTOM_TRAILING_INCLUDE_SHA1_H="\"sha1dc_git.h\"" \
  		-DSHA1DC_CUSTOM_INCLUDE_UBC_CHECK_C="\"git-compat-util.h\""
  endif
  endif
  endif
  endif
 +endif
  
  ifdef SHA1_MAX_BLOCK_SIZE
  	LIB_OBJS += compat/sha1-chunked.o
@@@ -1940,8 -1921,7 +1943,8 @@@ $(SCRIPT_LIB) : % : %.sh GIT-SCRIPT-DEF
  
  git.res: git.rc GIT-VERSION-FILE
  	$(QUIET_RC)$(RC) \
 -	  $(join -DMAJOR= -DMINOR=, $(wordlist 1,2,$(subst -, ,$(subst ., ,$(GIT_VERSION))))) \
 +	  $(join -DMAJOR= -DMINOR= -DMICRO= -DPATCHLEVEL=, $(wordlist 1, 4, \
 +	    $(shell echo $(GIT_VERSION) 0 0 0 0 | tr '.a-zA-Z-' ' '))) \
  	  -DGIT_VERSION="\\\"$(GIT_VERSION)\\\"" -i $< -o $@
  
  # This makes sure we depend on the NO_PERL setting itself.
@@@ -2465,10 -2445,6 +2468,10 @@@ $(SP_OBJ): %.sp: %.c GIT-CFLAGS FORC
  .PHONY: sparse $(SP_OBJ)
  sparse: $(SP_OBJ)
  
 +.PHONY: style
 +style:
 +	git clang-format --style file --diff --extensions c,h
 +
  check: common-cmds.h
  	@if sparse; \
  	then \
diff --combined builtin/ls-files.c
index 8c713c47ac,313962a0c1..2fc836e330
--- a/builtin/ls-files.c
+++ b/builtin/ls-files.c
@@@ -31,6 -31,7 +31,7 @@@ static int show_resolve_undo
  static int show_modified;
  static int show_killed;
  static int show_valid_bit;
+ static int show_fsmonitor_bit;
  static int line_terminator = '\n';
  static int debug_mode;
  static int show_eol;
@@@ -86,7 -87,8 +87,8 @@@ static const char *get_tag(const struc
  {
  	static char alttag[4];
  
- 	if (tag && *tag && show_valid_bit && (ce->ce_flags & CE_VALID)) {
+ 	if (tag && *tag && ((show_valid_bit && (ce->ce_flags & CE_VALID)) ||
+ 		(show_fsmonitor_bit && (ce->ce_flags & CE_FSMONITOR_VALID)))) {
  		memcpy(alttag, tag, 3);
  
  		if (isalpha(tag[0])) {
@@@ -515,6 -517,8 +517,8 @@@ int cmd_ls_files(int argc, const char *
  			N_("identify the file status with tags")),
  		OPT_BOOL('v', NULL, &show_valid_bit,
  			N_("use lowercase letters for 'assume unchanged' files")),
+ 		OPT_BOOL('f', NULL, &show_fsmonitor_bit,
+ 			N_("use lowercase letters for 'fsmonitor clean' files")),
  		OPT_BOOL('c', "cached", &show_cached,
  			N_("show cached files in the output (default)")),
  		OPT_BOOL('d', "deleted", &show_deleted,
@@@ -584,7 -588,7 +588,7 @@@
  	for (i = 0; i < exclude_list.nr; i++) {
  		add_exclude(exclude_list.items[i].string, "", 0, el, --exclude_args);
  	}
- 	if (show_tag || show_valid_bit) {
+ 	if (show_tag || show_valid_bit || show_fsmonitor_bit) {
  		tag_cached = "H ";
  		tag_unmerged = "M ";
  		tag_removed = "R ";
@@@ -673,6 -677,5 +677,6 @@@
  		return bad ? 1 : 0;
  	}
  
 +	UNLEAK(dir);
  	return 0;
  }
diff --combined builtin/update-index.c
index fefbe60167,41618db098..58d1c2d282
--- a/builtin/update-index.c
+++ b/builtin/update-index.c
@@@ -16,6 -16,7 +16,7 @@@
  #include "pathspec.h"
  #include "dir.h"
  #include "split-index.h"
+ #include "fsmonitor.h"
  
  /*
   * Default to not allowing changes to the list of files. The
@@@ -32,6 -33,7 +33,7 @@@ static int force_remove
  static int verbose;
  static int mark_valid_only;
  static int mark_skip_worktree_only;
+ static int mark_fsmonitor_only;
  #define MARK_FLAG 1
  #define UNMARK_FLAG 2
  static struct strbuf mtime_dir = STRBUF_INIT;
@@@ -228,6 -230,7 +230,7 @@@ static int mark_ce_flags(const char *pa
  	int namelen = strlen(path);
  	int pos = cache_name_pos(path, namelen);
  	if (0 <= pos) {
+ 		mark_fsmonitor_invalid(&the_index, active_cache[pos]);
  		if (mark)
  			active_cache[pos]->ce_flags |= flag;
  		else
@@@ -287,10 -290,8 +290,10 @@@ static int add_one_path(const struct ca
  	}
  	option = allow_add ? ADD_CACHE_OK_TO_ADD : 0;
  	option |= allow_replace ? ADD_CACHE_OK_TO_REPLACE : 0;
 -	if (add_cache_entry(ce, option))
 +	if (add_cache_entry(ce, option)) {
 +		free(ce);
  		return error("%s: cannot add to the index - missing --add option?", path);
 +	}
  	return 0;
  }
  
@@@ -328,7 -329,7 +331,7 @@@ static int process_directory(const cha
  		if (S_ISGITLINK(ce->ce_mode)) {
  
  			/* Do nothing to the index if there is no HEAD! */
 -			if (resolve_gitlink_ref(path, "HEAD", oid.hash) < 0)
 +			if (resolve_gitlink_ref(path, "HEAD", &oid) < 0)
  				return 0;
  
  			return add_one_path(ce, path, len, st);
@@@ -354,7 -355,7 +357,7 @@@
  	}
  
  	/* No match - should we add it as a gitlink? */
 -	if (!resolve_gitlink_ref(path, "HEAD", oid.hash))
 +	if (!resolve_gitlink_ref(path, "HEAD", &oid))
  		return add_one_path(NULL, path, len, st);
  
  	/* Error out. */
@@@ -460,6 -461,11 +463,11 @@@ static void update_one(const char *path
  			die("Unable to mark file %s", path);
  		return;
  	}
+ 	if (mark_fsmonitor_only) {
+ 		if (mark_ce_flags(path, CE_FSMONITOR_VALID, mark_fsmonitor_only == MARK_FLAG))
+ 			die("Unable to mark file %s", path);
+ 		return;
+ 	}
  
  	if (force_remove) {
  		if (remove_file_from_cache(path))
@@@ -679,9 -685,9 +687,9 @@@ static int unresolve_one(const char *pa
  
  static void read_head_pointers(void)
  {
 -	if (read_ref("HEAD", head_oid.hash))
 +	if (read_ref("HEAD", &head_oid))
  		die("No HEAD -- no initial commit yet?");
 -	if (read_ref("MERGE_HEAD", merge_head_oid.hash)) {
 +	if (read_ref("MERGE_HEAD", &merge_head_oid)) {
  		fprintf(stderr, "Not in the middle of a merge.\n");
  		exit(0);
  	}
@@@ -721,7 -727,7 +729,7 @@@ static int do_reupdate(int ac, const ch
  		       PATHSPEC_PREFER_CWD,
  		       prefix, av + 1);
  
 -	if (read_ref("HEAD", head_oid.hash))
 +	if (read_ref("HEAD", &head_oid))
  		/* If there is no HEAD, that means it is an initial
  		 * commit.  Update everything in the index.
  		 */
@@@ -917,7 -923,9 +925,9 @@@ int cmd_update_index(int argc, const ch
  	struct refresh_params refresh_args = {0, &has_errors};
  	int lock_error = 0;
  	int split_index = -1;
+ 	int force_write = 0;
+ 	int fsmonitor = -1;
 -	struct lock_file *lock_file;
 +	struct lock_file lock_file = LOCK_INIT;
  	struct parse_opt_ctx_t ctx;
  	strbuf_getline_fn getline_fn;
  	int parseopt_state = PARSE_OPT_UNKNOWN;
@@@ -1008,6 -1016,16 +1018,16 @@@
  			    N_("test if the filesystem supports untracked cache"), UC_TEST),
  		OPT_SET_INT(0, "force-untracked-cache", &untracked_cache,
  			    N_("enable untracked cache without testing the filesystem"), UC_FORCE),
+ 		OPT_SET_INT(0, "force-write-index", &force_write,
+ 			N_("write out the index even if is not flagged as changed"), 1),
+ 		OPT_BOOL(0, "fsmonitor", &fsmonitor,
+ 			N_("enable or disable file system monitor")),
+ 		{OPTION_SET_INT, 0, "fsmonitor-valid", &mark_fsmonitor_only, NULL,
+ 			N_("mark files as fsmonitor valid"),
+ 			PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, MARK_FLAG},
+ 		{OPTION_SET_INT, 0, "no-fsmonitor-valid", &mark_fsmonitor_only, NULL,
+ 			N_("clear fsmonitor valid bit"),
+ 			PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, UNMARK_FLAG},
  		OPT_END()
  	};
  
@@@ -1016,8 -1034,11 +1036,8 @@@
  
  	git_config(git_default_config, NULL);
  
 -	/* We can't free this memory, it becomes part of a linked list parsed atexit() */
 -	lock_file = xcalloc(1, sizeof(struct lock_file));
 -
  	/* we will diagnose later if it turns out that we need to update it */
 -	newfd = hold_locked_index(lock_file, 0);
 +	newfd = hold_locked_index(&lock_file, 0);
  	if (newfd < 0)
  		lock_error = errno;
  
@@@ -1146,17 -1167,33 +1166,33 @@@
  		die("BUG: bad untracked_cache value: %d", untracked_cache);
  	}
  
- 	if (active_cache_changed) {
+ 	if (fsmonitor > 0) {
+ 		if (git_config_get_fsmonitor() == 0)
+ 			warning(_("core.fsmonitor is unset; "
+ 				"set it if you really want to "
+ 				"enable fsmonitor"));
+ 		add_fsmonitor(&the_index);
+ 		report(_("fsmonitor enabled"));
+ 	} else if (!fsmonitor) {
+ 		if (git_config_get_fsmonitor() == 1)
+ 			warning(_("core.fsmonitor is set; "
+ 				"remove it if you really want to "
+ 				"disable fsmonitor"));
+ 		remove_fsmonitor(&the_index);
+ 		report(_("fsmonitor disabled"));
+ 	}
+ 
+ 	if (active_cache_changed || force_write) {
  		if (newfd < 0) {
  			if (refresh_args.flags & REFRESH_QUIET)
  				exit(128);
  			unable_to_lock_die(get_index_file(), lock_error);
  		}
 -		if (write_locked_index(&the_index, lock_file, COMMIT_LOCK))
 +		if (write_locked_index(&the_index, &lock_file, COMMIT_LOCK))
  			die("Unable to write new index file");
  	}
  
 -	rollback_lock_file(lock_file);
 +	rollback_lock_file(&lock_file);
  
  	return has_errors ? 1 : 0;
  }
diff --combined cache.h
index 89f5d24579,f1c903e1b6..7c71681236
--- a/cache.h
+++ b/cache.h
@@@ -4,7 -4,6 +4,7 @@@
  #include "git-compat-util.h"
  #include "strbuf.h"
  #include "hashmap.h"
 +#include "mru.h"
  #include "advice.h"
  #include "gettext.h"
  #include "convert.h"
@@@ -204,6 -203,7 +204,7 @@@ struct cache_entry 
  #define CE_ADDED             (1 << 19)
  
  #define CE_HASHED            (1 << 20)
+ #define CE_FSMONITOR_VALID   (1 << 21)
  #define CE_WT_REMOVE         (1 << 22) /* remove in work directory */
  #define CE_CONFLICTED        (1 << 23)
  
@@@ -327,6 -327,7 +328,7 @@@ static inline unsigned int canon_mode(u
  #define CACHE_TREE_CHANGED	(1 << 5)
  #define SPLIT_INDEX_ORDERED	(1 << 6)
  #define UNTRACKED_CHANGED	(1 << 7)
+ #define FSMONITOR_CHANGED	(1 << 8)
  
  struct split_index;
  struct untracked_cache;
@@@ -345,6 -346,7 +347,7 @@@ struct index_state 
  	struct hashmap dir_hash;
  	unsigned char sha1[20];
  	struct untracked_cache *untracked;
+ 	uint64_t fsmonitor_last_update;
  };
  
  extern struct index_state the_index;
@@@ -444,7 -446,6 +447,7 @@@ static inline enum object_type object_t
  #define GIT_NOGLOB_PATHSPECS_ENVIRONMENT "GIT_NOGLOB_PATHSPECS"
  #define GIT_ICASE_PATHSPECS_ENVIRONMENT "GIT_ICASE_PATHSPECS"
  #define GIT_QUARANTINE_ENVIRONMENT "GIT_QUARANTINE_PATH"
 +#define GIT_OPTIONAL_LOCKS_ENVIRONMENT "GIT_OPTIONAL_LOCKS"
  
  /*
   * This environment variable is expected to contain a boolean indicating
@@@ -602,28 -603,9 +605,28 @@@ extern int do_read_index(struct index_s
  extern int read_index_from(struct index_state *, const char *path);
  extern int is_index_unborn(struct index_state *);
  extern int read_index_unmerged(struct index_state *);
 +
 +/* For use with `write_locked_index()`. */
  #define COMMIT_LOCK		(1 << 0)
 -#define CLOSE_LOCK		(1 << 1)
 +
 +/*
 + * Write the index while holding an already-taken lock. Close the lock,
 + * and if `COMMIT_LOCK` is given, commit it.
 + *
 + * Unless a split index is in use, write the index into the lockfile.
 + *
 + * With a split index, write the shared index to a temporary file,
 + * adjust its permissions and rename it into place, then write the
 + * split index to the lockfile. If the temporary file for the shared
 + * index cannot be created, fall back to the behavior described in
 + * the previous paragraph.
 + *
 + * With `COMMIT_LOCK`, the lock is always committed or rolled back.
 + * Without it, the lock is closed, but neither committed nor rolled
 + * back.
 + */
  extern int write_locked_index(struct index_state *, struct lock_file *lock, unsigned flags);
 +
  extern int discard_index(struct index_state *);
  extern void move_index_extensions(struct index_state *dst, struct index_state *src);
  extern int unmerged_index(const struct index_state *);
@@@ -700,8 -682,10 +703,10 @@@ extern void *read_blob_data_from_index(
  #define CE_MATCH_IGNORE_MISSING		0x08
  /* enable stat refresh */
  #define CE_MATCH_REFRESH		0x10
- extern int ie_match_stat(const struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
- extern int ie_modified(const struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
+ /* don't refresh_fsmonitor state or do stat comparison even if CE_FSMONITOR_VALID is true */
+ #define CE_MATCH_IGNORE_FSMONITOR 0X20
+ extern int ie_match_stat(struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
+ extern int ie_modified(struct index_state *, const struct cache_entry *, struct stat *, unsigned int);
  
  #define HASH_WRITE_OBJECT 1
  #define HASH_FORMAT_CHECK 2
@@@ -735,17 -719,12 +740,17 @@@ extern void fill_stat_cache_info(struc
  extern int refresh_index(struct index_state *, unsigned int flags, const struct pathspec *pathspec, char *seen, const char *header_msg);
  extern struct cache_entry *refresh_cache_entry(struct cache_entry *, unsigned int);
  
 +/*
 + * Opportunistically update the index but do not complain if we can't.
 + * The lockfile is always committed or rolled back.
 + */
  extern void update_index_if_able(struct index_state *, struct lock_file *);
  
  extern int hold_locked_index(struct lock_file *, int);
  extern void set_alternate_index_output(const char *);
  
  extern int verify_index_checksum;
 +extern int verify_ce_order;
  
  /* Environment bits from configuration mechanism */
  extern int trust_executable_bit;
@@@ -799,6 -778,7 +804,7 @@@ extern int core_apply_sparse_checkout
  extern int precomposed_unicode;
  extern int protect_hfs;
  extern int protect_ntfs;
+ extern const char *core_fsmonitor;
  
  /*
   * Include broken refs in all ref iterations, which will
@@@ -808,11 -788,6 +814,11 @@@
   */
  extern int ref_paranoia;
  
 +/*
 + * Returns the boolean value of $GIT_OPTIONAL_LOCKS (or the default value).
 + */
 +int use_optional_locks(void);
 +
  /*
   * The character that begins a commented line in user-editable file
   * that is subject to stripspace.
@@@ -1274,8 -1249,8 +1280,8 @@@ static inline unsigned int hexval(unsig
   */
  static inline int hex2chr(const char *s)
  {
 -	int val = hexval(s[0]);
 -	return (val < 0) ? val : (val << 4) | hexval(s[1]);
 +	unsigned int val = hexval(s[0]);
 +	return (val & ~0xf) ? val : (val << 4) | hexval(s[1]);
  }
  
  /* Convert to/from hex/sha1 representation */
@@@ -1341,13 -1316,6 +1347,13 @@@ extern int set_disambiguate_hint_config
  extern int get_sha1_hex(const char *hex, unsigned char *sha1);
  extern int get_oid_hex(const char *hex, struct object_id *sha1);
  
 +/*
 + * Read `len` pairs of hexadecimal digits from `hex` and write the
 + * values to `binary` as `len` bytes. Return 0 on success, or -1 if
 + * the input does not consist of hex digits).
 + */
 +extern int hex_to_bytes(unsigned char *binary, const char *hex, size_t len);
 +
  /*
   * Convert a binary sha1 to its hex equivalent. The `_r` variant is reentrant,
   * and writes the NUL-terminated output to the buffer `out`, which must be at
@@@ -1627,7 -1595,8 +1633,7 @@@ extern struct packed_git 
   * A most-recently-used ordered version of the packed_git list, which can
   * be iterated instead of packed_git (and marked via mru_mark).
   */
 -struct mru;
 -extern struct mru *packed_git_mru;
 +extern struct mru packed_git_mru;
  
  struct pack_entry {
  	off_t offset;
diff --combined config.c
index 903abf9533,ddda96e584..03ab56d3d8
--- a/config.c
+++ b/config.c
@@@ -16,6 -16,7 +16,6 @@@
  #include "string-list.h"
  #include "utf8.h"
  #include "dir.h"
 -#include "color.h"
  
  struct config_source {
  	struct config_source *prev;
@@@ -955,6 -956,11 +955,6 @@@ int git_parse_maybe_bool(const char *va
  	return -1;
  }
  
 -int git_config_maybe_bool(const char *name, const char *value)
 -{
 -	return git_parse_maybe_bool(value);
 -}
 -
  int git_config_bool_or_int(const char *name, const char *value, int *is_bool)
  {
  	int v = git_parse_maybe_bool_text(value);
@@@ -1350,6 -1356,9 +1350,6 @@@ int git_default_config(const char *var
  	if (starts_with(var, "advice."))
  		return git_default_advice_config(var, value);
  
 -	if (git_color_config(var, value, dummy) < 0)
 -		return -1;
 -
  	if (!strcmp(var, "pager.color") || !strcmp(var, "color.pager")) {
  		pager_use_color = git_config_bool(var,value);
  		return 0;
@@@ -2156,6 -2165,20 +2156,20 @@@ int git_config_get_max_percent_split_ch
  	return -1; /* default value */
  }
  
+ int git_config_get_fsmonitor(void)
+ {
+ 	if (git_config_get_pathname("core.fsmonitor", &core_fsmonitor))
+ 		core_fsmonitor = getenv("GIT_FSMONITOR_TEST");
+ 
+ 	if (core_fsmonitor && !*core_fsmonitor)
+ 		core_fsmonitor = NULL;
+ 
+ 	if (core_fsmonitor)
+ 		return 1;
+ 
+ 	return 0;
+ }
+ 
  NORETURN
  void git_die_config_linenr(const char *key, const char *filename, int linenr)
  {
@@@ -2196,7 -2219,7 +2210,7 @@@ static struct 
  	size_t *offset;
  	unsigned int offset_alloc;
  	enum { START, SECTION_SEEN, SECTION_END_SEEN, KEY_SEEN } state;
 -	int seen;
 +	unsigned int seen;
  } store;
  
  static int matches(const char *key, const char *value)
@@@ -2288,10 -2311,10 +2302,10 @@@ static int write_error(const char *file
  	return 4;
  }
  
 -static int store_write_section(int fd, const char *key)
 +static struct strbuf store_create_section(const char *key)
  {
  	const char *dot;
 -	int i, success;
 +	int i;
  	struct strbuf sb = STRBUF_INIT;
  
  	dot = memchr(key, '.', store.baselen);
@@@ -2307,24 -2330,15 +2321,24 @@@
  		strbuf_addf(&sb, "[%.*s]\n", store.baselen, key);
  	}
  
 -	success = write_in_full(fd, sb.buf, sb.len) == sb.len;
 +	return sb;
 +}
 +
 +static ssize_t write_section(int fd, const char *key)
 +{
 +	struct strbuf sb = store_create_section(key);
 +	ssize_t ret;
 +
 +	ret = write_in_full(fd, sb.buf, sb.len) == sb.len;
  	strbuf_release(&sb);
  
 -	return success;
 +	return ret;
  }
  
 -static int store_write_pair(int fd, const char *key, const char *value)
 +static ssize_t write_pair(int fd, const char *key, const char *value)
  {
 -	int i, success;
 +	int i;
 +	ssize_t ret;
  	int length = strlen(key + store.baselen + 1);
  	const char *quote = "";
  	struct strbuf sb = STRBUF_INIT;
@@@ -2358,17 -2372,16 +2372,17 @@@
  		case '"':
  		case '\\':
  			strbuf_addch(&sb, '\\');
 +			/* fallthrough */
  		default:
  			strbuf_addch(&sb, value[i]);
  			break;
  		}
  	strbuf_addf(&sb, "%s\n", quote);
  
 -	success = write_in_full(fd, sb.buf, sb.len) == sb.len;
 +	ret = write_in_full(fd, sb.buf, sb.len);
  	strbuf_release(&sb);
  
 -	return success;
 +	return ret;
  }
  
  static ssize_t find_beginning_of_line(const char *contents, size_t size,
@@@ -2451,7 -2464,7 +2465,7 @@@ int git_config_set_multivar_in_file_gen
  {
  	int fd = -1, in_fd = -1;
  	int ret;
 -	struct lock_file *lock = NULL;
 +	struct lock_file lock = LOCK_INIT;
  	char *filename_buf = NULL;
  	char *contents = NULL;
  	size_t contents_sz;
@@@ -2470,7 -2483,8 +2484,7 @@@
  	 * The lock serves a purpose in addition to locking: the new
  	 * contents of .git/config will be written into it.
  	 */
 -	lock = xcalloc(1, sizeof(struct lock_file));
 -	fd = hold_lock_file_for_update(lock, config_filename, 0);
 +	fd = hold_lock_file_for_update(&lock, config_filename, 0);
  	if (fd < 0) {
  		error_errno("could not lock config file %s", config_filename);
  		free(store.key);
@@@ -2497,8 -2511,8 +2511,8 @@@
  		}
  
  		store.key = (char *)key;
 -		if (!store_write_section(fd, key) ||
 -		    !store_write_pair(fd, key, value))
 +		if (write_section(fd, key) < 0 ||
 +		    write_pair(fd, key, value) < 0)
  			goto write_err_out;
  	} else {
  		struct stat st;
@@@ -2583,8 -2597,8 +2597,8 @@@
  		close(in_fd);
  		in_fd = -1;
  
 -		if (chmod(get_lock_file_path(lock), st.st_mode & 07777) < 0) {
 -			error_errno("chmod on %s failed", get_lock_file_path(lock));
 +		if (chmod(get_lock_file_path(&lock), st.st_mode & 07777) < 0) {
 +			error_errno("chmod on %s failed", get_lock_file_path(&lock));
  			ret = CONFIG_NO_WRITE;
  			goto out_free;
  		}
@@@ -2608,10 -2622,11 +2622,10 @@@
  			/* write the first part of the config */
  			if (copy_end > copy_begin) {
  				if (write_in_full(fd, contents + copy_begin,
 -						  copy_end - copy_begin) <
 -				    copy_end - copy_begin)
 +						  copy_end - copy_begin) < 0)
  					goto write_err_out;
  				if (new_line &&
 -				    write_str_in_full(fd, "\n") != 1)
 +				    write_str_in_full(fd, "\n") < 0)
  					goto write_err_out;
  			}
  			copy_begin = store.offset[i];
@@@ -2620,36 -2635,46 +2634,36 @@@
  		/* write the pair (value == NULL means unset) */
  		if (value != NULL) {
  			if (store.state == START) {
 -				if (!store_write_section(fd, key))
 +				if (write_section(fd, key) < 0)
  					goto write_err_out;
  			}
 -			if (!store_write_pair(fd, key, value))
 +			if (write_pair(fd, key, value) < 0)
  				goto write_err_out;
  		}
  
  		/* write the rest of the config */
  		if (copy_begin < contents_sz)
  			if (write_in_full(fd, contents + copy_begin,
 -					  contents_sz - copy_begin) <
 -			    contents_sz - copy_begin)
 +					  contents_sz - copy_begin) < 0)
  				goto write_err_out;
  
  		munmap(contents, contents_sz);
  		contents = NULL;
  	}
  
 -	if (commit_lock_file(lock) < 0) {
 +	if (commit_lock_file(&lock) < 0) {
  		error_errno("could not write config file %s", config_filename);
  		ret = CONFIG_NO_WRITE;
 -		lock = NULL;
  		goto out_free;
  	}
  
 -	/*
 -	 * lock is committed, so don't try to roll it back below.
 -	 * NOTE: Since lockfile.c keeps a linked list of all created
 -	 * lock_file structures, it isn't safe to free(lock).  It's
 -	 * better to just leave it hanging around.
 -	 */
 -	lock = NULL;
  	ret = 0;
  
  	/* Invalidate the config cache */
  	git_config_clear();
  
  out_free:
 -	if (lock)
 -		rollback_lock_file(lock);
 +	rollback_lock_file(&lock);
  	free(filename_buf);
  	if (contents)
  		munmap(contents, contents_sz);
@@@ -2658,7 -2683,7 +2672,7 @@@
  	return ret;
  
  write_err_out:
 -	ret = write_error(get_lock_file_path(lock));
 +	ret = write_error(get_lock_file_path(&lock));
  	goto out_free;
  
  }
@@@ -2746,17 -2771,16 +2760,17 @@@ static int section_name_is_ok(const cha
  }
  
  /* if new_name == NULL, the section is removed instead */
 -int git_config_rename_section_in_file(const char *config_filename,
 -				      const char *old_name, const char *new_name)
 +static int git_config_copy_or_rename_section_in_file(const char *config_filename,
 +				      const char *old_name, const char *new_name, int copy)
  {
  	int ret = 0, remove = 0;
  	char *filename_buf = NULL;
 -	struct lock_file *lock;
 +	struct lock_file lock = LOCK_INIT;
  	int out_fd;
  	char buf[1024];
  	FILE *config_file = NULL;
  	struct stat st;
 +	struct strbuf copystr = STRBUF_INIT;
  
  	if (new_name && !section_name_is_ok(new_name)) {
  		ret = error("invalid section name: %s", new_name);
@@@ -2766,7 -2790,8 +2780,7 @@@
  	if (!config_filename)
  		config_filename = filename_buf = git_pathdup("config");
  
 -	lock = xcalloc(1, sizeof(struct lock_file));
 -	out_fd = hold_lock_file_for_update(lock, config_filename, 0);
 +	out_fd = hold_lock_file_for_update(&lock, config_filename, 0);
  	if (out_fd < 0) {
  		ret = error("could not lock config file %s", config_filename);
  		goto out;
@@@ -2785,39 -2810,21 +2799,39 @@@
  		goto out;
  	}
  
 -	if (chmod(get_lock_file_path(lock), st.st_mode & 07777) < 0) {
 +	if (chmod(get_lock_file_path(&lock), st.st_mode & 07777) < 0) {
  		ret = error_errno("chmod on %s failed",
 -				  get_lock_file_path(lock));
 +				  get_lock_file_path(&lock));
  		goto out;
  	}
  
  	while (fgets(buf, sizeof(buf), config_file)) {
  		int i;
  		int length;
 +		int is_section = 0;
  		char *output = buf;
  		for (i = 0; buf[i] && isspace(buf[i]); i++)
  			; /* do nothing */
  		if (buf[i] == '[') {
  			/* it's a section */
 -			int offset = section_name_match(&buf[i], old_name);
 +			int offset;
 +			is_section = 1;
 +
 +			/*
 +			 * When encountering a new section under -c we
 +			 * need to flush out any section we're already
 +			 * coping and begin anew. There might be
 +			 * multiple [branch "$name"] sections.
 +			 */
 +			if (copystr.len > 0) {
 +				if (write_in_full(out_fd, copystr.buf, copystr.len) != copystr.len) {
 +					ret = write_error(get_lock_file_path(&lock));
 +					goto out;
 +				}
 +				strbuf_reset(&copystr);
 +			}
 +
 +			offset = section_name_match(&buf[i], old_name);
  			if (offset > 0) {
  				ret++;
  				if (new_name == NULL) {
@@@ -2825,29 -2832,25 +2839,29 @@@
  					continue;
  				}
  				store.baselen = strlen(new_name);
 -				if (!store_write_section(out_fd, new_name)) {
 -					ret = write_error(get_lock_file_path(lock));
 -					goto out;
 -				}
 -				/*
 -				 * We wrote out the new section, with
 -				 * a newline, now skip the old
 -				 * section's length
 -				 */
 -				output += offset + i;
 -				if (strlen(output) > 0) {
 +				if (!copy) {
 +					if (write_section(out_fd, new_name) < 0) {
 +						ret = write_error(get_lock_file_path(&lock));
 +						goto out;
 +					}
  					/*
 -					 * More content means there's
 -					 * a declaration to put on the
 -					 * next line; indent with a
 -					 * tab
 +					 * We wrote out the new section, with
 +					 * a newline, now skip the old
 +					 * section's length
  					 */
 -					output -= 1;
 -					output[0] = '\t';
 +					output += offset + i;
 +					if (strlen(output) > 0) {
 +						/*
 +						 * More content means there's
 +						 * a declaration to put on the
 +						 * next line; indent with a
 +						 * tab
 +						 */
 +						output -= 1;
 +						output[0] = '\t';
 +					}
 +				} else {
 +					copystr = store_create_section(new_name);
  				}
  			}
  			remove = 0;
@@@ -2855,69 -2858,31 +2869,69 @@@
  		if (remove)
  			continue;
  		length = strlen(output);
 -		if (write_in_full(out_fd, output, length) != length) {
 -			ret = write_error(get_lock_file_path(lock));
 +
 +		if (!is_section && copystr.len > 0) {
 +			strbuf_add(&copystr, output, length);
 +		}
 +
 +		if (write_in_full(out_fd, output, length) < 0) {
 +			ret = write_error(get_lock_file_path(&lock));
 +			goto out;
 +		}
 +	}
 +
 +	/*
 +	 * Copy a trailing section at the end of the config, won't be
 +	 * flushed by the usual "flush because we have a new section
 +	 * logic in the loop above.
 +	 */
 +	if (copystr.len > 0) {
 +		if (write_in_full(out_fd, copystr.buf, copystr.len) != copystr.len) {
 +			ret = write_error(get_lock_file_path(&lock));
  			goto out;
  		}
 +		strbuf_reset(&copystr);
  	}
 +
  	fclose(config_file);
  	config_file = NULL;
  commit_and_out:
 -	if (commit_lock_file(lock) < 0)
 +	if (commit_lock_file(&lock) < 0)
  		ret = error_errno("could not write config file %s",
  				  config_filename);
  out:
  	if (config_file)
  		fclose(config_file);
 -	rollback_lock_file(lock);
 +	rollback_lock_file(&lock);
  out_no_rollback:
  	free(filename_buf);
  	return ret;
  }
  
 +int git_config_rename_section_in_file(const char *config_filename,
 +				      const char *old_name, const char *new_name)
 +{
 +	return git_config_copy_or_rename_section_in_file(config_filename,
 +					 old_name, new_name, 0);
 +}
 +
  int git_config_rename_section(const char *old_name, const char *new_name)
  {
  	return git_config_rename_section_in_file(NULL, old_name, new_name);
  }
  
 +int git_config_copy_section_in_file(const char *config_filename,
 +				      const char *old_name, const char *new_name)
 +{
 +	return git_config_copy_or_rename_section_in_file(config_filename,
 +					 old_name, new_name, 1);
 +}
 +
 +int git_config_copy_section(const char *old_name, const char *new_name)
 +{
 +	return git_config_copy_section_in_file(NULL, old_name, new_name);
 +}
 +
  /*
   * Call this to report error for your variable that should not
   * get a boolean value (i.e. "[my] var" means "true").
diff --combined config.h
index a49d264416,c9fcf691ba..524d411823
--- a/config.h
+++ b/config.h
@@@ -56,6 -56,7 +56,6 @@@ extern unsigned long git_config_ulong(c
  extern ssize_t git_config_ssize_t(const char *, const char *);
  extern int git_config_bool_or_int(const char *, const char *, int *);
  extern int git_config_bool(const char *, const char *);
 -extern int git_config_maybe_bool(const char *, const char *);
  extern int git_config_string(const char **, const char *, const char *);
  extern int git_config_pathname(const char **, const char *, const char *);
  extern int git_config_set_in_file_gently(const char *, const char *, const char *);
@@@ -70,8 -71,6 +70,8 @@@ extern int git_config_set_multivar_in_f
  extern void git_config_set_multivar_in_file(const char *, const char *, const char *, const char *, int);
  extern int git_config_rename_section(const char *, const char *);
  extern int git_config_rename_section_in_file(const char *, const char *, const char *);
 +extern int git_config_copy_section(const char *, const char *);
 +extern int git_config_copy_section_in_file(const char *, const char *, const char *);
  extern const char *git_etc_gitconfig(void);
  extern int git_env_bool(const char *, int);
  extern unsigned long git_env_ulong(const char *, unsigned long);
@@@ -212,6 -211,7 +212,7 @@@ extern int git_config_get_pathname(cons
  extern int git_config_get_untracked_cache(void);
  extern int git_config_get_split_index(void);
  extern int git_config_get_max_percent_split_change(void);
+ extern int git_config_get_fsmonitor(void);
  
  /* This dies if the configured or default date is in the future */
  extern int git_config_get_expiry(const char *key, const char **output);
diff --combined diff-lib.c
index 731f0886d6,23c6d03ca9..5173023cd3
--- a/diff-lib.c
+++ b/diff-lib.c
@@@ -12,6 -12,7 +12,7 @@@
  #include "refs.h"
  #include "submodule.h"
  #include "dir.h"
+ #include "fsmonitor.h"
  
  /*
   * diff-files
@@@ -36,7 -37,7 +37,7 @@@ static int check_removed(const struct c
  	if (has_symlink_leading_path(ce->name, ce_namelen(ce)))
  		return 1;
  	if (S_ISDIR(st->st_mode)) {
 -		unsigned char sub[20];
 +		struct object_id sub;
  
  		/*
  		 * If ce is already a gitlink, we can have a plain
@@@ -50,7 -51,7 +51,7 @@@
  		 * a directory --- the blob was removed!
  		 */
  		if (!S_ISGITLINK(ce->ce_mode) &&
 -		    resolve_gitlink_ref(ce->name, "HEAD", sub))
 +		    resolve_gitlink_ref(ce->name, "HEAD", &sub))
  			return 1;
  	}
  	return 0;
@@@ -71,15 -72,14 +72,15 @@@ static int match_stat_with_submodule(st
  {
  	int changed = ce_match_stat(ce, st, ce_option);
  	if (S_ISGITLINK(ce->ce_mode)) {
 -		unsigned orig_flags = diffopt->flags;
 -		if (!DIFF_OPT_TST(diffopt, OVERRIDE_SUBMODULE_CONFIG))
 +		struct diff_flags orig_flags = diffopt->flags;
 +		if (!diffopt->flags.override_submodule_config)
  			set_diffopt_flags_from_submodule_config(diffopt, ce->name);
 -		if (DIFF_OPT_TST(diffopt, IGNORE_SUBMODULES))
 +		if (diffopt->flags.ignore_submodules)
  			changed = 0;
 -		else if (!DIFF_OPT_TST(diffopt, IGNORE_DIRTY_SUBMODULES)
 -		    && (!changed || DIFF_OPT_TST(diffopt, DIRTY_SUBMODULES)))
 -			*dirty_submodule = is_submodule_modified(ce->name, DIFF_OPT_TST(diffopt, IGNORE_UNTRACKED_IN_SUBMODULES));
 +		else if (!diffopt->flags.ignore_dirty_submodules &&
 +			 (!changed || diffopt->flags.dirty_submodules))
 +			*dirty_submodule = is_submodule_modified(ce->name,
 +								 diffopt->flags.ignore_untracked_in_submodules);
  		diffopt->flags = orig_flags;
  	}
  	return changed;
@@@ -229,7 -229,8 +230,8 @@@ int run_diff_files(struct rev_info *rev
  
  		if (!changed && !dirty_submodule) {
  			ce_mark_uptodate(ce);
+ 			mark_fsmonitor_valid(ce);
 -			if (!DIFF_OPT_TST(&revs->diffopt, FIND_COPIES_HARDER))
 +			if (!revs->diffopt.flags.find_copies_harder)
  				continue;
  		}
  		oldmode = ce->ce_mode;
@@@ -363,7 -364,7 +365,7 @@@ static int show_modified(struct rev_inf
  
  	oldmode = old->ce_mode;
  	if (mode == oldmode && !oidcmp(oid, &old->oid) && !dirty_submodule &&
 -	    !DIFF_OPT_TST(&revs->diffopt, FIND_COPIES_HARDER))
 +	    !revs->diffopt.flags.find_copies_harder)
  		return 0;
  
  	diff_change(&revs->diffopt, oldmode, mode,
@@@ -494,7 -495,7 +496,7 @@@ static int diff_cache(struct rev_info *
  	opts.head_idx = 1;
  	opts.index_only = cached;
  	opts.diff_index_cached = (cached &&
 -				  !DIFF_OPT_TST(&revs->diffopt, FIND_COPIES_HARDER));
 +				  !revs->diffopt.flags.find_copies_harder);
  	opts.merge = 1;
  	opts.fn = oneway_diff;
  	opts.unpack_data = revs;
@@@ -535,7 -536,7 +537,7 @@@ int do_diff_cache(const struct object_i
  	return 0;
  }
  
 -int index_differs_from(const char *def, int diff_flags,
 +int index_differs_from(const char *def, const struct diff_flags *flags,
  		       int ita_invisible_in_index)
  {
  	struct rev_info rev;
@@@ -545,12 -546,12 +547,12 @@@
  	memset(&opt, 0, sizeof(opt));
  	opt.def = def;
  	setup_revisions(0, NULL, &rev, &opt);
 -	DIFF_OPT_SET(&rev.diffopt, QUICK);
 -	DIFF_OPT_SET(&rev.diffopt, EXIT_WITH_STATUS);
 -	rev.diffopt.flags |= diff_flags;
 +	rev.diffopt.flags.quick = 1;
 +	rev.diffopt.flags.exit_with_status = 1;
 +	if (flags)
 +		diff_flags_or(&rev.diffopt.flags, flags);
  	rev.diffopt.ita_invisible_in_index = ita_invisible_in_index;
  	run_diff_index(&rev, 1);
 -	if (rev.pending.alloc)
 -		free(rev.pending.objects);
 -	return (DIFF_OPT_TST(&rev.diffopt, HAS_CHANGES) != 0);
 +	object_array_clear(&rev.pending);
 +	return (rev.diffopt.flags.has_changes != 0);
  }
diff --combined dir.c
index 047a950f55,ac9833daec..3c54366a17
--- a/dir.c
+++ b/dir.c
@@@ -18,6 -18,7 +18,7 @@@
  #include "utf8.h"
  #include "varint.h"
  #include "ewah/ewok.h"
+ #include "fsmonitor.h"
  
  /*
   * Tells read_directory_recursive how a file or directory should be treated.
@@@ -49,7 -50,7 +50,7 @@@ struct cached_dir 
  static enum path_treatment read_directory_recursive(struct dir_struct *dir,
  	struct index_state *istate, const char *path, int len,
  	struct untracked_cache_dir *untracked,
 -	int check_only, const struct pathspec *pathspec);
 +	int check_only, int stop_at_first_file, const struct pathspec *pathspec);
  static int get_dtype(struct dirent *de, struct index_state *istate,
  		     const char *path, int len);
  
@@@ -1389,34 -1390,10 +1390,34 @@@ static enum path_treatment treat_direct
  	case index_nonexistent:
  		if (dir->flags & DIR_SHOW_OTHER_DIRECTORIES)
  			break;
 +		if (exclude &&
 +			(dir->flags & DIR_SHOW_IGNORED_TOO) &&
 +			(dir->flags & DIR_SHOW_IGNORED_TOO_MODE_MATCHING)) {
 +
 +			/*
 +			 * This is an excluded directory and we are
 +			 * showing ignored paths that match an exclude
 +			 * pattern.  (e.g. show directory as ignored
 +			 * only if it matches an exclude pattern).
 +			 * This path will either be 'path_excluded`
 +			 * (if we are showing empty directories or if
 +			 * the directory is not empty), or will be
 +			 * 'path_none' (empty directory, and we are
 +			 * not showing empty directories).
 +			 */
 +			if (!(dir->flags & DIR_HIDE_EMPTY_DIRECTORIES))
 +				return path_excluded;
 +
 +			if (read_directory_recursive(dir, istate, dirname, len,
 +						     untracked, 1, 1, pathspec) == path_excluded)
 +				return path_excluded;
 +
 +			return path_none;
 +		}
  		if (!(dir->flags & DIR_NO_GITLINKS)) {
 -			unsigned char sha1[20];
 -			if (resolve_gitlink_ref(dirname, "HEAD", sha1) == 0)
 -				return path_untracked;
 +			struct object_id oid;
 +			if (resolve_gitlink_ref(dirname, "HEAD", &oid) == 0)
 +				return exclude ? path_excluded : path_untracked;
  		}
  		return path_recurse;
  	}
@@@ -1428,13 -1405,8 +1429,13 @@@
  
  	untracked = lookup_untracked(dir->untracked, untracked,
  				     dirname + baselen, len - baselen);
 +
 +	/*
 +	 * If this is an excluded directory, then we only need to check if
 +	 * the directory contains any files.
 +	 */
  	return read_directory_recursive(dir, istate, dirname, len,
 -					untracked, 1, pathspec);
 +					untracked, 1, exclude, pathspec);
  }
  
  /*
@@@ -1585,7 -1557,6 +1586,7 @@@ static enum path_treatment treat_one_pa
  {
  	int exclude;
  	int has_path_in_index = !!index_file_exists(istate, path->buf, path->len, ignore_case);
 +	enum path_treatment path_treatment;
  
  	if (dtype == DT_UNKNOWN)
  		dtype = get_dtype(de, istate, path->buf, path->len);
@@@ -1632,23 -1603,8 +1633,23 @@@
  		return path_none;
  	case DT_DIR:
  		strbuf_addch(path, '/');
 -		return treat_directory(dir, istate, untracked, path->buf, path->len,
 -				       baselen, exclude, pathspec);
 +		path_treatment = treat_directory(dir, istate, untracked,
 +						 path->buf, path->len,
 +						 baselen, exclude, pathspec);
 +		/*
 +		 * If 1) we only want to return directories that
 +		 * match an exclude pattern and 2) this directory does
 +		 * not match an exclude pattern but all of its
 +		 * contents are excluded, then indicate that we should
 +		 * recurse into this directory (instead of marking the
 +		 * directory itself as an ignored path).
 +		 */
 +		if (!exclude &&
 +		    path_treatment == path_excluded &&
 +		    (dir->flags & DIR_SHOW_IGNORED_TOO) &&
 +		    (dir->flags & DIR_SHOW_IGNORED_TOO_MODE_MATCHING))
 +			return path_recurse;
 +		return path_treatment;
  	case DT_REG:
  	case DT_LNK:
  		return exclude ? path_excluded : path_untracked;
@@@ -1678,7 -1634,7 +1679,7 @@@ static enum path_treatment treat_path_f
  		 * with check_only set.
  		 */
  		return read_directory_recursive(dir, istate, path->buf, path->len,
 -						cdir->ucd, 1, pathspec);
 +						cdir->ucd, 1, 0, pathspec);
  	/*
  	 * We get path_recurse in the first run when
  	 * directory_exists_in_index() returns index_nonexistent. We
@@@ -1733,17 -1689,23 +1734,23 @@@ static int valid_cached_dir(struct dir_
  	if (!untracked)
  		return 0;
  
- 	if (stat(path->len ? path->buf : ".", &st)) {
- 		invalidate_directory(dir->untracked, untracked);
- 		memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
- 		return 0;
- 	}
- 	if (!untracked->valid ||
- 	    match_stat_data_racy(istate, &untracked->stat_data, &st)) {
- 		if (untracked->valid)
+ 	/*
+ 	 * With fsmonitor, we can trust the untracked cache's valid field.
+ 	 */
+ 	refresh_fsmonitor(istate);
+ 	if (!(dir->untracked->use_fsmonitor && untracked->valid)) {
+ 		if (stat(path->len ? path->buf : ".", &st)) {
  			invalidate_directory(dir->untracked, untracked);
- 		fill_stat_data(&untracked->stat_data, &st);
- 		return 0;
+ 			memset(&untracked->stat_data, 0, sizeof(untracked->stat_data));
+ 			return 0;
+ 		}
+ 		if (!untracked->valid ||
+ 			match_stat_data_racy(istate, &untracked->stat_data, &st)) {
+ 			if (untracked->valid)
+ 				invalidate_directory(dir->untracked, untracked);
+ 			fill_stat_data(&untracked->stat_data, &st);
+ 			return 0;
+ 		}
  	}
  
  	if (untracked->check_only != !!check_only) {
@@@ -1838,20 -1800,12 +1845,20 @@@ static void close_cached_dir(struct cac
   * Also, we ignore the name ".git" (even if it is not a directory).
   * That likely will not change.
   *
 + * If 'stop_at_first_file' is specified, 'path_excluded' is returned
 + * to signal that a file was found. This is the least significant value that
 + * indicates that a file was encountered that does not depend on the order of
 + * whether an untracked or exluded path was encountered first.
 + *
   * Returns the most significant path_treatment value encountered in the scan.
 + * If 'stop_at_first_file' is specified, `path_excluded` is the most
 + * significant path_treatment value that will be returned.
   */
 +
  static enum path_treatment read_directory_recursive(struct dir_struct *dir,
  	struct index_state *istate, const char *base, int baselen,
  	struct untracked_cache_dir *untracked, int check_only,
 -	const struct pathspec *pathspec)
 +	int stop_at_first_file, const struct pathspec *pathspec)
  {
  	struct cached_dir cdir;
  	enum path_treatment state, subdir_state, dir_state = path_none;
@@@ -1885,34 -1839,12 +1892,34 @@@
  			subdir_state =
  				read_directory_recursive(dir, istate, path.buf,
  							 path.len, ud,
 -							 check_only, pathspec);
 +							 check_only, stop_at_first_file, pathspec);
  			if (subdir_state > dir_state)
  				dir_state = subdir_state;
  		}
  
  		if (check_only) {
 +			if (stop_at_first_file) {
 +				/*
 +				 * If stopping at first file, then
 +				 * signal that a file was found by
 +				 * returning `path_excluded`. This is
 +				 * to return a consistent value
 +				 * regardless of whether an ignored or
 +				 * excluded file happened to be
 +				 * encountered 1st.
 +				 *
 +				 * In current usage, the
 +				 * `stop_at_first_file` is passed when
 +				 * an ancestor directory has matched
 +				 * an exclude pattern, so any found
 +				 * files will be excluded.
 +				 */
 +				if (dir_state >= path_excluded) {
 +					dir_state = path_excluded;
 +					break;
 +				}
 +			}
 +
  			/* abort early if maximum state has been reached */
  			if (dir_state == path_untracked) {
  				if (cdir.fdir)
@@@ -2183,7 -2115,7 +2190,7 @@@ int read_directory(struct dir_struct *d
  		 */
  		dir->untracked = NULL;
  	if (!len || treat_leading_path(dir, istate, path, len, pathspec))
 -		read_directory_recursive(dir, istate, path, len, untracked, 0, pathspec);
 +		read_directory_recursive(dir, istate, path, len, untracked, 0, 0, pathspec);
  	QSORT(dir->entries, dir->nr, cmp_dir_entry);
  	QSORT(dir->ignored, dir->ignored_nr, cmp_dir_entry);
  
@@@ -2319,10 -2251,10 +2326,10 @@@ static int remove_dir_recurse(struct st
  	int ret = 0, original_len = path->len, len, kept_down = 0;
  	int only_empty = (flag & REMOVE_DIR_EMPTY_ONLY);
  	int keep_toplevel = (flag & REMOVE_DIR_KEEP_TOPLEVEL);
 -	unsigned char submodule_head[20];
 +	struct object_id submodule_head;
  
  	if ((flag & REMOVE_DIR_KEEP_NESTED_GIT) &&
 -	    !resolve_gitlink_ref(path->buf, "HEAD", submodule_head)) {
 +	    !resolve_gitlink_ref(path->buf, "HEAD", &submodule_head)) {
  		/* Do not descend and nuke a nested git work tree. */
  		if (kept_up)
  			*kept_up = 1;
diff --combined dir.h
index 57b0943dae,fab8fc1561..233a2eb36b
--- a/dir.h
+++ b/dir.h
@@@ -139,6 -139,8 +139,8 @@@ struct untracked_cache 
  	int gitignore_invalidated;
  	int dir_invalidated;
  	int dir_opened;
+ 	/* fsmonitor invalidation data */
+ 	unsigned int use_fsmonitor : 1;
  };
  
  struct dir_struct {
@@@ -152,8 -154,7 +154,8 @@@
  		DIR_COLLECT_IGNORED = 1<<4,
  		DIR_SHOW_IGNORED_TOO = 1<<5,
  		DIR_COLLECT_KILLED_ONLY = 1<<6,
 -		DIR_KEEP_UNTRACKED_CONTENTS = 1<<7
 +		DIR_KEEP_UNTRACKED_CONTENTS = 1<<7,
 +		DIR_SHOW_IGNORED_TOO_MODE_MATCHING = 1<<8
  	} flags;
  	struct dir_entry **entries;
  	struct dir_entry **ignored;
diff --combined entry.c
index 944c183b0f,3a7b667373..30211447ac
--- a/entry.c
+++ b/entry.c
@@@ -4,6 -4,7 +4,7 @@@
  #include "streaming.h"
  #include "submodule.h"
  #include "progress.h"
+ #include "fsmonitor.h"
  
  static void create_directories(const char *path, int path_len,
  			       const struct checkout *state)
@@@ -253,13 -254,11 +254,13 @@@ static int write_entry(struct cache_ent
  		       char *path, const struct checkout *state, int to_tempfile)
  {
  	unsigned int ce_mode_s_ifmt = ce->ce_mode & S_IFMT;
 +	struct delayed_checkout *dco = state->delayed_checkout;
  	int fd, ret, fstat_done = 0;
  	char *new;
  	struct strbuf buf = STRBUF_INIT;
  	unsigned long size;
 -	size_t wrote, newsize = 0;
 +	ssize_t wrote;
 +	size_t newsize = 0;
  	struct stat st;
  	const struct submodule *sub;
  
@@@ -274,65 -273,55 +275,65 @@@
  	}
  
  	switch (ce_mode_s_ifmt) {
 -	case S_IFREG:
  	case S_IFLNK:
  		new = read_blob_entry(ce, &size);
  		if (!new)
  			return error("unable to read sha1 file of %s (%s)",
 -				path, oid_to_hex(&ce->oid));
 +				     path, oid_to_hex(&ce->oid));
  
 -		if (ce_mode_s_ifmt == S_IFLNK && has_symlinks && !to_tempfile) {
 -			ret = symlink(new, path);
 -			free(new);
 -			if (ret)
 -				return error_errno("unable to create symlink %s",
 -						   path);
 -			break;
 +		/*
 +		 * We can't make a real symlink; write out a regular file entry
 +		 * with the symlink destination as its contents.
 +		 */
 +		if (!has_symlinks || to_tempfile)
 +			goto write_file_entry;
 +
 +		ret = symlink(new, path);
 +		free(new);
 +		if (ret)
 +			return error_errno("unable to create symlink %s", path);
 +		break;
 +
 +	case S_IFREG:
 +		/*
 +		 * We do not send the blob in case of a retry, so do not
 +		 * bother reading it at all.
 +		 */
 +		if (dco && dco->state == CE_RETRY) {
 +			new = NULL;
 +			size = 0;
 +		} else {
 +			new = read_blob_entry(ce, &size);
 +			if (!new)
 +				return error("unable to read sha1 file of %s (%s)",
 +					     path, oid_to_hex(&ce->oid));
  		}
  
  		/*
  		 * Convert from git internal format to working tree format
  		 */
 -		if (ce_mode_s_ifmt == S_IFREG) {
 -			struct delayed_checkout *dco = state->delayed_checkout;
 -			if (dco && dco->state != CE_NO_DELAY) {
 -				/* Do not send the blob in case of a retry. */
 -				if (dco->state == CE_RETRY) {
 -					new = NULL;
 -					size = 0;
 -				}
 -				ret = async_convert_to_working_tree(
 -					ce->name, new, size, &buf, dco);
 -				if (ret && string_list_has_string(&dco->paths, ce->name)) {
 -					free(new);
 -					goto finish;
 -				}
 -			} else
 -				ret = convert_to_working_tree(
 -					ce->name, new, size, &buf);
 -
 -			if (ret) {
 +		if (dco && dco->state != CE_NO_DELAY) {
 +			ret = async_convert_to_working_tree(ce->name, new,
 +							    size, &buf, dco);
 +			if (ret && string_list_has_string(&dco->paths, ce->name)) {
  				free(new);
 -				new = strbuf_detach(&buf, &newsize);
 -				size = newsize;
 +				goto delayed;
  			}
 -			/*
 -			 * No "else" here as errors from convert are OK at this
 -			 * point. If the error would have been fatal (e.g.
 -			 * filter is required), then we would have died already.
 -			 */
 +		} else
 +			ret = convert_to_working_tree(ce->name, new, size, &buf);
 +
 +		if (ret) {
 +			free(new);
 +			new = strbuf_detach(&buf, &newsize);
 +			size = newsize;
  		}
 +		/*
 +		 * No "else" here as errors from convert are OK at this
 +		 * point. If the error would have been fatal (e.g.
 +		 * filter is required), then we would have died already.
 +		 */
  
 +	write_file_entry:
  		fd = open_output_fd(path, ce, to_tempfile);
  		if (fd < 0) {
  			free(new);
@@@ -344,10 -333,9 +345,10 @@@
  			fstat_done = fstat_output(fd, state, &st);
  		close(fd);
  		free(new);
 -		if (wrote != size)
 +		if (wrote < 0)
  			return error("unable to write file %s", path);
  		break;
 +
  	case S_IFGITLINK:
  		if (to_tempfile)
  			return error("cannot create temporary submodule %s", path);
@@@ -359,7 -347,6 +360,7 @@@
  				NULL, oid_to_hex(&ce->oid),
  				state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
  		break;
 +
  	default:
  		return error("unknown file mode for %s in index", path);
  	}
@@@ -368,14 -355,12 +369,15 @@@ finish
  	if (state->refresh_cache) {
  		assert(state->istate);
  		if (!fstat_done)
 -			lstat(ce->name, &st);
 +			if (lstat(ce->name, &st) < 0)
 +				return error_errno("unable to stat just-written file %s",
 +						   ce->name);
  		fill_stat_cache_info(ce, &st);
  		ce->ce_flags |= CE_UPDATE_IN_BASE;
+ 		mark_fsmonitor_invalid(state->istate, ce);
  		state->istate->cache_changed |= CE_ENTRY_CHANGED;
  	}
 +delayed:
  	return 0;
  }
  
diff --combined environment.c
index 8289c25b44,d0b9fc64d4..8fa032f307
--- a/environment.c
+++ b/environment.c
@@@ -76,6 -76,7 +76,7 @@@ int protect_hfs = PROTECT_HFS_DEFAULT
  #define PROTECT_NTFS_DEFAULT 0
  #endif
  int protect_ntfs = PROTECT_NTFS_DEFAULT;
+ const char *core_fsmonitor;
  
  /*
   * The character that begins a commented line in user-editable file
@@@ -97,7 -98,7 +98,7 @@@ int ignore_untracked_cache_config
  /* This is set by setup_git_dir_gently() and/or git_default_config() */
  char *git_work_tree_cfg;
  
 -static const char *namespace;
 +static char *namespace;
  
  static const char *super_prefix;
  
@@@ -152,10 -153,8 +153,10 @@@ void setup_git_env(void
  	if (getenv(NO_REPLACE_OBJECTS_ENVIRONMENT))
  		check_replace_refs = 0;
  	replace_ref_base = getenv(GIT_REPLACE_REF_BASE_ENVIRONMENT);
 +	free(git_replace_ref_base);
  	git_replace_ref_base = xstrdup(replace_ref_base ? replace_ref_base
  							  : "refs/replace/");
 +	free(namespace);
  	namespace = expand_namespace(getenv(GIT_NAMESPACE_ENVIRONMENT));
  	shallow_file = getenv(GIT_SHALLOW_FILE_ENVIRONMENT);
  	if (shallow_file)
@@@ -338,8 -337,3 +339,8 @@@ void reset_shared_repository(void
  {
  	need_shared_repository_from_config = 1;
  }
 +
 +int use_optional_locks(void)
 +{
 +	return git_env_bool(GIT_OPTIONAL_LOCKS_ENVIRONMENT, 1);
 +}
diff --combined read-cache.c
index b13a1cb8f2,05c0a33fdd..87e88b2642
--- a/read-cache.c
+++ b/read-cache.c
@@@ -19,6 -19,7 +19,7 @@@
  #include "varint.h"
  #include "split-index.h"
  #include "utf8.h"
+ #include "fsmonitor.h"
  
  /* Mask for the name length in ce_flags in the on-disk index */
  
@@@ -38,11 -39,12 +39,12 @@@
  #define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */
  #define CACHE_EXT_LINK 0x6c696e6b	  /* "link" */
  #define CACHE_EXT_UNTRACKED 0x554E5452	  /* "UNTR" */
+ #define CACHE_EXT_FSMONITOR 0x46534D4E	  /* "FSMN" */
  
  /* changes that can be kept in $GIT_DIR/index (basically all extensions) */
  #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \
  		 CE_ENTRY_ADDED | CE_ENTRY_REMOVED | CE_ENTRY_CHANGED | \
- 		 SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED)
+ 		 SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED | FSMONITOR_CHANGED)
  
  struct index_state the_index;
  static const char *alternate_index_output;
@@@ -62,6 -64,7 +64,7 @@@ static void replace_index_entry(struct 
  	free(old);
  	set_index_entry(istate, nr, ce);
  	ce->ce_flags |= CE_UPDATE_IN_BASE;
+ 	mark_fsmonitor_invalid(istate, ce);
  	istate->cache_changed |= CE_ENTRY_CHANGED;
  }
  
@@@ -150,8 -153,10 +153,10 @@@ void fill_stat_cache_info(struct cache_
  	if (assume_unchanged)
  		ce->ce_flags |= CE_VALID;
  
- 	if (S_ISREG(st->st_mode))
+ 	if (S_ISREG(st->st_mode)) {
  		ce_mark_uptodate(ce);
+ 		mark_fsmonitor_valid(ce);
+ 	}
  }
  
  static int ce_compare_data(const struct cache_entry *ce, struct stat *st)
@@@ -191,7 -196,7 +196,7 @@@ static int ce_compare_link(const struc
  
  static int ce_compare_gitlink(const struct cache_entry *ce)
  {
 -	unsigned char sha1[20];
 +	struct object_id oid;
  
  	/*
  	 * We don't actually require that the .git directory
@@@ -201,9 -206,9 +206,9 @@@
  	 *
  	 * If so, we consider it always to match.
  	 */
 -	if (resolve_gitlink_ref(ce->name, "HEAD", sha1) < 0)
 +	if (resolve_gitlink_ref(ce->name, "HEAD", &oid) < 0)
  		return 0;
 -	return hashcmp(sha1, ce->oid.hash);
 +	return oidcmp(&oid, &ce->oid);
  }
  
  static int ce_modified_check_fs(const struct cache_entry *ce, struct stat *st)
@@@ -220,7 -225,6 +225,7 @@@
  	case S_IFDIR:
  		if (S_ISGITLINK(ce->ce_mode))
  			return ce_compare_gitlink(ce) ? DATA_CHANGED : 0;
 +		/* else fallthrough */
  	default:
  		return TYPE_CHANGED;
  	}
@@@ -301,7 -305,7 +306,7 @@@ int match_stat_data_racy(const struct i
  	return match_stat_data(sd, st);
  }
  
- int ie_match_stat(const struct index_state *istate,
+ int ie_match_stat(struct index_state *istate,
  		  const struct cache_entry *ce, struct stat *st,
  		  unsigned int options)
  {
@@@ -309,7 -313,10 +314,10 @@@
  	int ignore_valid = options & CE_MATCH_IGNORE_VALID;
  	int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE;
  	int assume_racy_is_modified = options & CE_MATCH_RACY_IS_DIRTY;
+ 	int ignore_fsmonitor = options & CE_MATCH_IGNORE_FSMONITOR;
  
+ 	if (!ignore_fsmonitor)
+ 		refresh_fsmonitor(istate);
  	/*
  	 * If it's marked as always valid in the index, it's
  	 * valid whatever the checked-out copy says.
@@@ -320,6 -327,8 +328,8 @@@
  		return 0;
  	if (!ignore_valid && (ce->ce_flags & CE_VALID))
  		return 0;
+ 	if (!ignore_fsmonitor && (ce->ce_flags & CE_FSMONITOR_VALID))
+ 		return 0;
  
  	/*
  	 * Intent-to-add entries have not been added, so the index entry
@@@ -357,7 -366,7 +367,7 @@@
  	return changed;
  }
  
- int ie_modified(const struct index_state *istate,
+ int ie_modified(struct index_state *istate,
  		const struct cache_entry *ce,
  		struct stat *st, unsigned int options)
  {
@@@ -778,6 -787,7 +788,7 @@@ int chmod_index_entry(struct index_stat
  	}
  	cache_tree_invalidate_path(istate, ce->name);
  	ce->ce_flags |= CE_UPDATE_IN_BASE;
+ 	mark_fsmonitor_invalid(istate, ce);
  	istate->cache_changed |= CE_ENTRY_CHANGED;
  
  	return 0;
@@@ -1229,10 -1239,13 +1240,13 @@@ static struct cache_entry *refresh_cach
  	int ignore_valid = options & CE_MATCH_IGNORE_VALID;
  	int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE;
  	int ignore_missing = options & CE_MATCH_IGNORE_MISSING;
+ 	int ignore_fsmonitor = options & CE_MATCH_IGNORE_FSMONITOR;
  
  	if (!refresh || ce_uptodate(ce))
  		return ce;
  
+ 	if (!ignore_fsmonitor)
+ 		refresh_fsmonitor(istate);
  	/*
  	 * CE_VALID or CE_SKIP_WORKTREE means the user promised us
  	 * that the change to the work tree does not matter and told
@@@ -1246,6 -1259,10 +1260,10 @@@
  		ce_mark_uptodate(ce);
  		return ce;
  	}
+ 	if (!ignore_fsmonitor && (ce->ce_flags & CE_FSMONITOR_VALID)) {
+ 		ce_mark_uptodate(ce);
+ 		return ce;
+ 	}
  
  	if (has_symlink_leading_path(ce->name, ce_namelen(ce))) {
  		if (ignore_missing)
@@@ -1283,8 -1300,10 +1301,10 @@@
  			 * because CE_UPTODATE flag is in-core only;
  			 * we are not going to write this change out.
  			 */
- 			if (!S_ISGITLINK(ce->ce_mode))
+ 			if (!S_ISGITLINK(ce->ce_mode)) {
  				ce_mark_uptodate(ce);
+ 				mark_fsmonitor_valid(ce);
+ 			}
  			return ce;
  		}
  	}
@@@ -1392,6 -1411,7 +1412,7 @@@ int refresh_index(struct index_state *i
  				 */
  				ce->ce_flags &= ~CE_VALID;
  				ce->ce_flags |= CE_UPDATE_IN_BASE;
+ 				mark_fsmonitor_invalid(istate, ce);
  				istate->cache_changed |= CE_ENTRY_CHANGED;
  			}
  			if (quiet)
@@@ -1511,9 -1531,6 +1532,9 @@@ struct ondisk_cache_entry_extended 
  /* Allow fsck to force verification of the index checksum. */
  int verify_index_checksum;
  
 +/* Allow fsck to force verification of the cache entry order. */
 +int verify_ce_order;
 +
  static int verify_hdr(struct cache_header *hdr, unsigned long size)
  {
  	git_SHA_CTX c;
@@@ -1554,6 -1571,9 +1575,9 @@@ static int read_index_extension(struct 
  	case CACHE_EXT_UNTRACKED:
  		istate->untracked = read_untracked_extension(data, sz);
  		break;
+ 	case CACHE_EXT_FSMONITOR:
+ 		read_fsmonitor_extension(istate, data, sz);
+ 		break;
  	default:
  		if (*ext < 'A' || 'Z' < *ext)
  			return error("index uses %.4s extension, which we do not understand",
@@@ -1671,9 -1691,6 +1695,9 @@@ static void check_ce_order(struct index
  {
  	unsigned int i;
  
 +	if (!verify_ce_order)
 +		return;
 +
  	for (i = 1; i < istate->cache_nr; i++) {
  		struct cache_entry *ce = istate->cache[i - 1];
  		struct cache_entry *next_ce = istate->cache[i];
@@@ -1729,6 -1746,7 +1753,7 @@@ static void post_read_index_from(struc
  	check_ce_order(istate);
  	tweak_untracked_cache(istate);
  	tweak_split_index(istate);
+ 	tweak_fsmonitor(istate);
  }
  
  /* remember to discard_cache() before reading a different cache! */
@@@ -1929,7 -1947,7 +1954,7 @@@ static int ce_write_flush(git_SHA_CTX *
  	unsigned int buffered = write_buffer_len;
  	if (buffered) {
  		git_SHA1_Update(context, write_buffer, buffered);
 -		if (write_in_full(fd, write_buffer, buffered) != buffered)
 +		if (write_in_full(fd, write_buffer, buffered) < 0)
  			return -1;
  		write_buffer_len = 0;
  	}
@@@ -1978,7 -1996,7 +2003,7 @@@ static int ce_flush(git_SHA_CTX *contex
  
  	/* Flush first if not enough space for SHA1 signature */
  	if (left + 20 > WRITE_BUFFER_SIZE) {
 -		if (write_in_full(fd, write_buffer, left) != left)
 +		if (write_in_full(fd, write_buffer, left) < 0)
  			return -1;
  		left = 0;
  	}
@@@ -1987,7 -2005,7 +2012,7 @@@
  	git_SHA1_Final(write_buffer + left, context);
  	hashcpy(sha1, write_buffer + left);
  	left += 20;
 -	return (write_in_full(fd, write_buffer, left) != left) ? -1 : 0;
 +	return (write_in_full(fd, write_buffer, left) < 0) ? -1 : 0;
  }
  
  static void ce_smudge_racily_clean_entry(struct cache_entry *ce)
@@@ -2110,9 -2128,7 +2135,9 @@@ static int ce_write_entry(git_SHA_CTX *
  		if (!result)
  			result = ce_write(c, fd, to_remove_vi, prefix_size);
  		if (!result)
 -			result = ce_write(c, fd, ce->name + common, ce_namelen(ce) - common + 1);
 +			result = ce_write(c, fd, ce->name + common, ce_namelen(ce) - common);
 +		if (!result)
 +			result = ce_write(c, fd, padding, 1);
  
  		strbuf_splice(previous_name, common, to_remove,
  			      ce->name + common, ce_namelen(ce) - common);
@@@ -2182,22 -2198,17 +2207,22 @@@ static int has_racy_timestamp(struct in
  	return 0;
  }
  
 -/*
 - * Opportunistically update the index but do not complain if we can't
 - */
  void update_index_if_able(struct index_state *istate, struct lock_file *lockfile)
  {
  	if ((istate->cache_changed || has_racy_timestamp(istate)) &&
 -	    verify_index(istate) &&
 -	    write_locked_index(istate, lockfile, COMMIT_LOCK))
 +	    verify_index(istate))
 +		write_locked_index(istate, lockfile, COMMIT_LOCK);
 +	else
  		rollback_lock_file(lockfile);
  }
  
 +/*
 + * On success, `tempfile` is closed. If it is the temporary file
 + * of a `struct lock_file`, we will therefore effectively perform
 + * a 'close_lock_file_gently()`. Since that is an implementation
 + * detail of lockfiles, callers of `do_write_index()` should not
 + * rely on it.
 + */
  static int do_write_index(struct index_state *istate, struct tempfile *tempfile,
  			  int strip_extensions)
  {
@@@ -2320,13 -2331,21 +2345,23 @@@
  		if (err)
  			return -1;
  	}
+ 	if (!strip_extensions && istate->fsmonitor_last_update) {
+ 		struct strbuf sb = STRBUF_INIT;
+ 
+ 		write_fsmonitor_extension(&sb, istate);
+ 		err = write_index_ext_header(&c, newfd, CACHE_EXT_FSMONITOR, sb.len) < 0
+ 			|| ce_write(&c, newfd, sb.buf, sb.len) < 0;
+ 		strbuf_release(&sb);
+ 		if (err)
+ 			return -1;
+ 	}
  
  	if (ce_flush(&c, newfd, istate->sha1))
  		return -1;
 -	if (close_tempfile(tempfile))
 -		return error(_("could not close '%s'"), tempfile->filename.buf);
 +	if (close_tempfile_gently(tempfile)) {
 +		error(_("could not close '%s'"), tempfile->filename.buf);
 +		return -1;
 +	}
  	if (stat(tempfile->filename.buf, &st))
  		return -1;
  	istate->timestamp.sec = (unsigned int)st.st_mtime;
@@@ -2350,12 -2369,17 +2385,12 @@@ static int commit_locked_index(struct l
  static int do_write_locked_index(struct index_state *istate, struct lock_file *lock,
  				 unsigned flags)
  {
 -	int ret = do_write_index(istate, &lock->tempfile, 0);
 +	int ret = do_write_index(istate, lock->tempfile, 0);
  	if (ret)
  		return ret;
 -	assert((flags & (COMMIT_LOCK | CLOSE_LOCK)) !=
 -	       (COMMIT_LOCK | CLOSE_LOCK));
  	if (flags & COMMIT_LOCK)
  		return commit_locked_index(lock);
 -	else if (flags & CLOSE_LOCK)
 -		return close_lock_file(lock);
 -	else
 -		return ret;
 +	return close_lock_file_gently(lock);
  }
  
  static int write_split_index(struct index_state *istate,
@@@ -2428,33 -2452,34 +2463,33 @@@ static int clean_shared_index_files(con
  	return 0;
  }
  
 -static struct tempfile temporary_sharedindex;
 -
  static int write_shared_index(struct index_state *istate,
  			      struct lock_file *lock, unsigned flags)
  {
 +	struct tempfile *temp;
  	struct split_index *si = istate->split_index;
 -	int fd, ret;
 +	int ret;
  
 -	fd = mks_tempfile(&temporary_sharedindex, git_path("sharedindex_XXXXXX"));
 -	if (fd < 0) {
 +	temp = mks_tempfile(git_path("sharedindex_XXXXXX"));
 +	if (!temp) {
  		hashclr(si->base_sha1);
  		return do_write_locked_index(istate, lock, flags);
  	}
  	move_cache_to_base_index(istate);
 -	ret = do_write_index(si->base, &temporary_sharedindex, 1);
 +	ret = do_write_index(si->base, temp, 1);
  	if (ret) {
 -		delete_tempfile(&temporary_sharedindex);
 +		delete_tempfile(&temp);
  		return ret;
  	}
 -	ret = adjust_shared_perm(get_tempfile_path(&temporary_sharedindex));
 +	ret = adjust_shared_perm(get_tempfile_path(temp));
  	if (ret) {
  		int save_errno = errno;
 -		error("cannot fix permission bits on %s", get_tempfile_path(&temporary_sharedindex));
 -		delete_tempfile(&temporary_sharedindex);
 +		error("cannot fix permission bits on %s", get_tempfile_path(temp));
 +		delete_tempfile(&temp);
  		errno = save_errno;
  		return ret;
  	}
 -	ret = rename_tempfile(&temporary_sharedindex,
 +	ret = rename_tempfile(&temp,
  			      git_path("sharedindex.%s", sha1_to_hex(si->base->sha1)));
  	if (!ret) {
  		hashcpy(si->base_sha1, si->base->sha1);
@@@ -2504,8 -2529,7 +2539,8 @@@ int write_locked_index(struct index_sta
  	    (istate->cache_changed & ~EXTMASK)) {
  		if (si)
  			hashclr(si->base_sha1);
 -		return do_write_locked_index(istate, lock, flags);
 +		ret = do_write_locked_index(istate, lock, flags);
 +		goto out;
  	}
  
  	if (getenv("GIT_TEST_SPLIT_INDEX")) {
@@@ -2521,7 -2545,7 +2556,7 @@@
  	if (new_shared_index) {
  		ret = write_shared_index(istate, lock, flags);
  		if (ret)
 -			return ret;
 +			goto out;
  	}
  
  	ret = write_split_index(istate, lock, flags);
@@@ -2530,9 -2554,6 +2565,9 @@@
  	if (!ret && !new_shared_index)
  		freshen_shared_index(sha1_to_hex(si->base_sha1), 1);
  
 +out:
 +	if (flags & COMMIT_LOCK)
 +		rollback_lock_file(lock);
  	return ret;
  }
  
diff --combined submodule.c
index 3ee4a0caa7,8a931a1aaa..bb531e0e59
--- a/submodule.c
+++ b/submodule.c
@@@ -21,7 -21,7 +21,7 @@@
  #include "parse-options.h"
  
  static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF;
 -static struct string_list changed_submodule_paths = STRING_LIST_INIT_DUP;
 +static struct string_list changed_submodule_names = STRING_LIST_INIT_DUP;
  static int initialized_fetch_ref_tips;
  static struct oid_array ref_tips_before_fetch;
  static struct oid_array ref_tips_after_fetch;
@@@ -62,20 -62,13 +62,20 @@@ int is_staging_gitmodules_ok(const stru
  	if ((pos >= 0) && (pos < istate->cache_nr)) {
  		struct stat st;
  		if (lstat(GITMODULES_FILE, &st) == 0 &&
- 		    ce_match_stat(istate->cache[pos], &st, 0) & DATA_CHANGED)
+ 		    ce_match_stat(istate->cache[pos], &st, CE_MATCH_IGNORE_FSMONITOR) & DATA_CHANGED)
  			return 0;
  	}
  
  	return 1;
  }
  
 +static int for_each_remote_ref_submodule(const char *submodule,
 +					 each_ref_fn fn, void *cb_data)
 +{
 +	return refs_for_each_remote_ref(get_submodule_ref_store(submodule),
 +					fn, cb_data);
 +}
 +
  /*
   * Try to update the "path" entry in the "submodule.<name>" section of the
   * .gitmodules file. Return 0 only if a .gitmodules file was found, a section
@@@ -183,7 -176,7 +183,7 @@@ void set_diffopt_flags_from_submodule_c
  		if (ignore)
  			handle_ignore_submodules_arg(diffopt, ignore);
  		else if (is_gitmodules_unmerged(&the_index))
 -			DIFF_OPT_SET(diffopt, IGNORE_SUBMODULES);
 +			diffopt->flags.ignore_submodules = 1;
  	}
  }
  
@@@ -402,16 -395,16 +402,16 @@@ const char *submodule_strategy_to_strin
  void handle_ignore_submodules_arg(struct diff_options *diffopt,
  				  const char *arg)
  {
 -	DIFF_OPT_CLR(diffopt, IGNORE_SUBMODULES);
 -	DIFF_OPT_CLR(diffopt, IGNORE_UNTRACKED_IN_SUBMODULES);
 -	DIFF_OPT_CLR(diffopt, IGNORE_DIRTY_SUBMODULES);
 +	diffopt->flags.ignore_submodules = 0;
 +	diffopt->flags.ignore_untracked_in_submodules = 0;
 +	diffopt->flags.ignore_dirty_submodules = 0;
  
  	if (!strcmp(arg, "all"))
 -		DIFF_OPT_SET(diffopt, IGNORE_SUBMODULES);
 +		diffopt->flags.ignore_submodules = 1;
  	else if (!strcmp(arg, "untracked"))
 -		DIFF_OPT_SET(diffopt, IGNORE_UNTRACKED_IN_SUBMODULES);
 +		diffopt->flags.ignore_untracked_in_submodules = 1;
  	else if (!strcmp(arg, "dirty"))
 -		DIFF_OPT_SET(diffopt, IGNORE_DIRTY_SUBMODULES);
 +		diffopt->flags.ignore_dirty_submodules = 1;
  	else if (strcmp(arg, "none"))
  		die("bad --ignore-submodules argument: %s", arg);
  }
@@@ -503,7 -496,7 +503,7 @@@ static void show_submodule_header(struc
  
  	if (add_submodule_odb(path)) {
  		if (!message)
 -			message = "(not initialized)";
 +			message = "(commits not present)";
  		goto output_header;
  	}
  
@@@ -616,7 -609,7 +616,7 @@@ void show_submodule_inline_diff(struct 
  	argv_array_pushf(&cp.args, "--color=%s", want_color(o->use_color) ?
  			 "always" : "never");
  
 -	if (DIFF_OPT_TST(o, REVERSE_DIFF)) {
 +	if (o->flags.reverse_diff) {
  		argv_array_pushf(&cp.args, "--src-prefix=%s%s/",
  				 o->b_prefix, path);
  		argv_array_pushf(&cp.args, "--dst-prefix=%s%s/",
@@@ -674,11 -667,11 +674,11 @@@ const struct submodule *submodule_from_
  }
  
  static struct oid_array *submodule_commits(struct string_list *submodules,
 -					   const char *path)
 +					   const char *name)
  {
  	struct string_list_item *item;
  
 -	item = string_list_insert(submodules, path);
 +	item = string_list_insert(submodules, name);
  	if (item->util)
  		return (struct oid_array *) item->util;
  
@@@ -687,67 -680,39 +687,67 @@@
  	return (struct oid_array *) item->util;
  }
  
 +struct collect_changed_submodules_cb_data {
 +	struct string_list *changed;
 +	const struct object_id *commit_oid;
 +};
 +
 +/*
 + * this would normally be two functions: default_name_from_path() and
 + * path_from_default_name(). Since the default name is the same as
 + * the submodule path we can get away with just one function which only
 + * checks whether there is a submodule in the working directory at that
 + * location.
 + */
 +static const char *default_name_or_path(const char *path_or_name)
 +{
 +	int error_code;
 +
 +	if (!is_submodule_populated_gently(path_or_name, &error_code))
 +		return NULL;
 +
 +	return path_or_name;
 +}
 +
  static void collect_changed_submodules_cb(struct diff_queue_struct *q,
  					  struct diff_options *options,
  					  void *data)
  {
 +	struct collect_changed_submodules_cb_data *me = data;
 +	struct string_list *changed = me->changed;
 +	const struct object_id *commit_oid = me->commit_oid;
  	int i;
 -	struct string_list *changed = data;
  
  	for (i = 0; i < q->nr; i++) {
  		struct diff_filepair *p = q->queue[i];
  		struct oid_array *commits;
 +		const struct submodule *submodule;
 +		const char *name;
 +
  		if (!S_ISGITLINK(p->two->mode))
  			continue;
  
 -		if (S_ISGITLINK(p->one->mode)) {
 -			/*
 -			 * NEEDSWORK: We should honor the name configured in
 -			 * the .gitmodules file of the commit we are examining
 -			 * here to be able to correctly follow submodules
 -			 * being moved around.
 -			 */
 -			commits = submodule_commits(changed, p->two->path);
 -			oid_array_append(commits, &p->two->oid);
 -		} else {
 -			/* Submodule is new or was moved here */
 -			/*
 -			 * NEEDSWORK: When the .git directories of submodules
 -			 * live inside the superprojects .git directory some
 -			 * day we should fetch new submodules directly into
 -			 * that location too when config or options request
 -			 * that so they can be checked out from there.
 -			 */
 -			continue;
 +		submodule = submodule_from_path(commit_oid, p->two->path);
 +		if (submodule)
 +			name = submodule->name;
 +		else {
 +			name = default_name_or_path(p->two->path);
 +			/* make sure name does not collide with existing one */
 +			submodule = submodule_from_name(commit_oid, name);
 +			if (submodule) {
 +				warning("Submodule in commit %s at path: "
 +					"'%s' collides with a submodule named "
 +					"the same. Skipping it.",
 +					oid_to_hex(commit_oid), name);
 +				name = NULL;
 +			}
  		}
 +
 +		if (!name)
 +			continue;
 +
 +		commits = submodule_commits(changed, name);
 +		oid_array_append(commits, &p->two->oid);
  	}
  }
  
@@@ -770,14 -735,11 +770,14 @@@ static void collect_changed_submodules(
  
  	while ((commit = get_revision(&rev))) {
  		struct rev_info diff_rev;
 +		struct collect_changed_submodules_cb_data data;
 +		data.changed = changed;
 +		data.commit_oid = &commit->object.oid;
  
  		init_revisions(&diff_rev, NULL);
  		diff_rev.diffopt.output_format |= DIFF_FORMAT_CALLBACK;
  		diff_rev.diffopt.format_callback = collect_changed_submodules_cb;
 -		diff_rev.diffopt.format_callback_data = changed;
 +		diff_rev.diffopt.format_callback_data = &data;
  		diff_tree_combined_merge(commit, 1, &diff_rev);
  	}
  
@@@ -805,36 -767,19 +805,36 @@@ static int append_oid_to_argv(const str
  	return 0;
  }
  
 +struct has_commit_data {
 +	int result;
 +	const char *path;
 +};
 +
  static int check_has_commit(const struct object_id *oid, void *data)
  {
 -	int *has_commit = data;
 +	struct has_commit_data *cb = data;
  
 -	if (!lookup_commit_reference(oid))
 -		*has_commit = 0;
 +	enum object_type type = sha1_object_info(oid->hash, NULL);
  
 -	return 0;
 +	switch (type) {
 +	case OBJ_COMMIT:
 +		return 0;
 +	case OBJ_BAD:
 +		/*
 +		 * Object is missing or invalid. If invalid, an error message
 +		 * has already been printed.
 +		 */
 +		cb->result = 0;
 +		return 0;
 +	default:
 +		die(_("submodule entry '%s' (%s) is a %s, not a commit"),
 +		    cb->path, oid_to_hex(oid), typename(type));
 +	}
  }
  
  static int submodule_has_commits(const char *path, struct oid_array *commits)
  {
 -	int has_commit = 1;
 +	struct has_commit_data has_commit = { 1, path };
  
  	/*
  	 * Perform a cheap, but incorrect check for the existence of 'commits'.
@@@ -850,7 -795,7 +850,7 @@@
  
  	oid_array_for_each_unique(commits, check_has_commit, &has_commit);
  
 -	if (has_commit) {
 +	if (has_commit.result) {
  		/*
  		 * Even if the submodule is checked out and the commit is
  		 * present, make sure it exists in the submodule's object store
@@@ -869,12 -814,12 +869,12 @@@
  		cp.dir = path;
  
  		if (capture_command(&cp, &out, GIT_MAX_HEXSZ + 1) || out.len)
 -			has_commit = 0;
 +			has_commit.result = 0;
  
  		strbuf_release(&out);
  	}
  
 -	return has_commit;
 +	return has_commit.result;
  }
  
  static int submodule_needs_pushing(const char *path, struct oid_array *commits)
@@@ -925,7 -870,7 +925,7 @@@ int find_unpushed_submodules(struct oid
  		const char *remotes_name, struct string_list *needs_pushing)
  {
  	struct string_list submodules = STRING_LIST_INIT_DUP;
 -	struct string_list_item *submodule;
 +	struct string_list_item *name;
  	struct argv_array argv = ARGV_ARRAY_INIT;
  
  	/* argv.argv[0] will be ignored by setup_revisions */
@@@ -936,19 -881,9 +936,19 @@@
  
  	collect_changed_submodules(&submodules, &argv);
  
 -	for_each_string_list_item(submodule, &submodules) {
 -		struct oid_array *commits = submodule->util;
 -		const char *path = submodule->string;
 +	for_each_string_list_item(name, &submodules) {
 +		struct oid_array *commits = name->util;
 +		const struct submodule *submodule;
 +		const char *path = NULL;
 +
 +		submodule = submodule_from_name(&null_oid, name->string);
 +		if (submodule)
 +			path = submodule->path;
 +		else
 +			path = default_name_or_path(name->string);
 +
 +		if (!path)
 +			continue;
  
  		if (submodule_needs_pushing(path, commits))
  			string_list_insert(needs_pushing, path);
@@@ -1057,7 -992,7 +1057,7 @@@ int push_unpushed_submodules(struct oid
  		char *head;
  		struct object_id head_oid;
  
 -		head = resolve_refdup("HEAD", 0, head_oid.hash, NULL);
 +		head = resolve_refdup("HEAD", 0, &head_oid, NULL);
  		if (!head)
  			die(_("Failed to resolve HEAD as a valid ref."));
  
@@@ -1106,7 -1041,7 +1106,7 @@@ static void calculate_changed_submodule
  {
  	struct argv_array argv = ARGV_ARRAY_INIT;
  	struct string_list changed_submodules = STRING_LIST_INIT_DUP;
 -	const struct string_list_item *item;
 +	const struct string_list_item *name;
  
  	/* No need to check if there are no submodules configured */
  	if (!submodule_from_path(NULL, NULL))
@@@ -1121,26 -1056,16 +1121,26 @@@
  
  	/*
  	 * Collect all submodules (whether checked out or not) for which new
 -	 * commits have been recorded upstream in "changed_submodule_paths".
 +	 * commits have been recorded upstream in "changed_submodule_names".
  	 */
  	collect_changed_submodules(&changed_submodules, &argv);
  
 -	for_each_string_list_item(item, &changed_submodules) {
 -		struct oid_array *commits = item->util;
 -		const char *path = item->string;
 +	for_each_string_list_item(name, &changed_submodules) {
 +		struct oid_array *commits = name->util;
 +		const struct submodule *submodule;
 +		const char *path = NULL;
 +
 +		submodule = submodule_from_name(&null_oid, name->string);
 +		if (submodule)
 +			path = submodule->path;
 +		else
 +			path = default_name_or_path(name->string);
 +
 +		if (!path)
 +			continue;
  
  		if (!submodule_has_commits(path, commits))
 -			string_list_append(&changed_submodule_paths, path);
 +			string_list_append(&changed_submodule_names, name->string);
  	}
  
  	free_submodules_oids(&changed_submodules);
@@@ -1187,31 -1112,6 +1187,31 @@@ struct submodule_parallel_fetch 
  };
  #define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0, 0}
  
 +static int get_fetch_recurse_config(const struct submodule *submodule,
 +				    struct submodule_parallel_fetch *spf)
 +{
 +	if (spf->command_line_option != RECURSE_SUBMODULES_DEFAULT)
 +		return spf->command_line_option;
 +
 +	if (submodule) {
 +		char *key;
 +		const char *value;
 +
 +		int fetch_recurse = submodule->fetch_recurse;
 +		key = xstrfmt("submodule.%s.fetchRecurseSubmodules", submodule->name);
 +		if (!repo_config_get_string_const(the_repository, key, &value)) {
 +			fetch_recurse = parse_fetch_recurse_submodules_arg(key, value);
 +		}
 +		free(key);
 +
 +		if (fetch_recurse != RECURSE_SUBMODULES_NONE)
 +			/* local config overrules everything except commandline */
 +			return fetch_recurse;
 +	}
 +
 +	return spf->default_option;
 +}
 +
  static int get_next_submodule(struct child_process *cp,
  			      struct strbuf *err, void *data, void **task_cb)
  {
@@@ -1225,35 -1125,49 +1225,35 @@@
  		const struct cache_entry *ce = active_cache[spf->count];
  		const char *git_dir, *default_argv;
  		const struct submodule *submodule;
 +		struct submodule default_submodule = SUBMODULE_INIT;
  
  		if (!S_ISGITLINK(ce->ce_mode))
  			continue;
  
  		submodule = submodule_from_path(&null_oid, ce->name);
 -
 -		default_argv = "yes";
 -		if (spf->command_line_option == RECURSE_SUBMODULES_DEFAULT) {
 -			int fetch_recurse = RECURSE_SUBMODULES_NONE;
 -
 -			if (submodule) {
 -				char *key;
 -				const char *value;
 -
 -				fetch_recurse = submodule->fetch_recurse;
 -				key = xstrfmt("submodule.%s.fetchRecurseSubmodules", submodule->name);
 -				if (!repo_config_get_string_const(the_repository, key, &value)) {
 -					fetch_recurse = parse_fetch_recurse_submodules_arg(key, value);
 -				}
 -				free(key);
 +		if (!submodule) {
 +			const char *name = default_name_or_path(ce->name);
 +			if (name) {
 +				default_submodule.path = default_submodule.name = name;
 +				submodule = &default_submodule;
  			}
 +		}
  
 -			if (fetch_recurse != RECURSE_SUBMODULES_NONE) {
 -				if (fetch_recurse == RECURSE_SUBMODULES_OFF)
 -					continue;
 -				if (fetch_recurse == RECURSE_SUBMODULES_ON_DEMAND) {
 -					if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
 -						continue;
 -					default_argv = "on-demand";
 -				}
 -			} else {
 -				if (spf->default_option == RECURSE_SUBMODULES_OFF)
 -					continue;
 -				if (spf->default_option == RECURSE_SUBMODULES_ON_DEMAND) {
 -					if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
 -						continue;
 -					default_argv = "on-demand";
 -				}
 -			}
 -		} else if (spf->command_line_option == RECURSE_SUBMODULES_ON_DEMAND) {
 -			if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
 +		switch (get_fetch_recurse_config(submodule, spf))
 +		{
 +		default:
 +		case RECURSE_SUBMODULES_DEFAULT:
 +		case RECURSE_SUBMODULES_ON_DEMAND:
 +			if (!submodule || !unsorted_string_list_lookup(&changed_submodule_names,
 +							 submodule->name))
  				continue;
  			default_argv = "on-demand";
 +			break;
 +		case RECURSE_SUBMODULES_ON:
 +			default_argv = "yes";
 +			break;
 +		case RECURSE_SUBMODULES_OFF:
 +			continue;
  		}
  
  		strbuf_addf(&submodule_path, "%s/%s", spf->work_tree, ce->name);
@@@ -1344,7 -1258,7 +1344,7 @@@ int fetch_populated_submodules(const st
  
  	argv_array_clear(&spf.args);
  out:
 -	string_list_clear(&changed_submodule_paths, 1);
 +	string_list_clear(&changed_submodule_names, 1);
  	return spf.result;
  }
  
@@@ -1713,8 -1627,6 +1713,8 @@@ static int find_first_merges(struct obj
  			oid_to_hex(&a->object.oid));
  	init_revisions(&revs, NULL);
  	rev_opts.submodule = path;
 +	/* FIXME: can't handle linked worktrees in submodules yet */
 +	revs.single_worktree = path != NULL;
  	setup_revisions(ARRAY_SIZE(rev_args)-1, rev_args, &revs, &rev_opts);
  
  	/* save all revisions from the above list that contain b */
@@@ -1747,7 -1659,7 +1747,7 @@@
  			add_object_array(merges.objects[i].item, NULL, result);
  	}
  
 -	free(merges.objects);
 +	object_array_clear(&merges);
  	return result->nr;
  }
  
@@@ -1852,7 -1764,7 +1852,7 @@@ int merge_submodule(struct object_id *r
  			print_commit((struct commit *) merges.objects[i].item);
  	}
  
 -	free(merges.objects);
 +	object_array_clear(&merges);
  	return 0;
  }
  
@@@ -2059,10 -1971,6 +2059,10 @@@ const char *get_superproject_working_tr
  	return ret;
  }
  
 +/*
 + * Put the gitdir for a submodule (given relative to the main
 + * repository worktree) into `buf`, or return -1 on error.
 + */
  int submodule_to_gitdir(struct strbuf *buf, const char *submodule)
  {
  	const struct submodule *sub;
diff --combined t/helper/.gitignore
index 7c9d28a834,87a648a7cf..d02f9b39ac
--- a/t/helper/.gitignore
+++ b/t/helper/.gitignore
@@@ -3,7 -3,9 +3,9 @@@
  /test-config
  /test-date
  /test-delta
+ /test-drop-caches
  /test-dump-cache-tree
+ /test-dump-fsmonitor
  /test-dump-split-index
  /test-dump-untracked-cache
  /test-fake-ssh
@@@ -35,4 -37,3 +37,4 @@@
  /test-svn-fe
  /test-urlmatch-normalization
  /test-wildmatch
 +/test-write-cache
diff --combined unpack-trees.c
index 25740cb593,1f5d371636..bf8b602901
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@@ -14,6 -14,7 +14,7 @@@
  #include "dir.h"
  #include "submodule.h"
  #include "submodule-config.h"
+ #include "fsmonitor.h"
  
  /*
   * Error messages expected by scripts out of plumbing commands such as
@@@ -408,6 -409,7 +409,7 @@@ static int apply_sparse_checkout(struc
  		ce->ce_flags &= ~CE_SKIP_WORKTREE;
  	if (was_skip_worktree != ce_skip_worktree(ce)) {
  		ce->ce_flags |= CE_UPDATE_IN_BASE;
+ 		mark_fsmonitor_invalid(istate, ce);
  		istate->cache_changed |= CE_ENTRY_CHANGED;
  	}
  
@@@ -1541,15 -1543,15 +1543,15 @@@ static int verify_clean_subdirectory(co
  	int cnt = 0;
  
  	if (S_ISGITLINK(ce->ce_mode)) {
 -		unsigned char sha1[20];
 -		int sub_head = resolve_gitlink_ref(ce->name, "HEAD", sha1);
 +		struct object_id oid;
 +		int sub_head = resolve_gitlink_ref(ce->name, "HEAD", &oid);
  		/*
  		 * If we are not going to update the submodule, then
  		 * we don't care.
  		 */
 -		if (!sub_head && !hashcmp(sha1, ce->oid.hash))
 +		if (!sub_head && !oidcmp(&oid, &ce->oid))
  			return 0;
 -		return verify_clean_submodule(sub_head ? NULL : sha1_to_hex(sha1),
 +		return verify_clean_submodule(sub_head ? NULL : oid_to_hex(&oid),
  					      ce, error_type, o);
  	}