From: Junio C Hamano <gitster@pobox.com>
Date: Tue, 21 Nov 2017 05:07:51 +0000 (+0900)
Subject: Merge branch 'av/fsmonitor'
X-Git-Tag: v2.16.0-rc0~111
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/c9fdbca92c66b93f95a19bf1f38ab9088db78540?hp=-c

Merge branch 'av/fsmonitor'

Various fixes to bp/fsmonitor topic.

* av/fsmonitor:
fsmonitor: simplify determining the git worktree under Windows
fsmonitor: store fsmonitor bitmap before splitting index
fsmonitor: read from getcwd(), not the PWD environment variable
fsmonitor: delay updating state until after split index is merged
fsmonitor: document GIT_TRACE_FSMONITOR
fsmonitor: don't bother pretty-printing JSON from watchman
fsmonitor: set the PWD to the top of the working tree
---

c9fdbca92c66b93f95a19bf1f38ab9088db78540
diff --combined Documentation/git.txt
index 463b0eb0f5,ba31d95a82..483a1f3547
--- a/Documentation/git.txt
+++ b/Documentation/git.txt
@@@ -75,7 -75,7 +75,7 @@@ example the following invocations are e
  Note that omitting the `=` in `git -c foo.bar ...` is allowed and sets
  `foo.bar` to the boolean true value (just like `[foo]bar` would in a
  config file). Including the equals but with an empty value (like `git -c
 -foo.bar= ...`) sets `foo.bar` to the empty string which ` git config
 +foo.bar= ...`) sets `foo.bar` to the empty string which `git config
  --bool` will convert to `false`.
  
  --exec-path[=<path>]::
@@@ -159,10 -159,6 +159,10 @@@
  	Add "icase" magic to all pathspec. This is equivalent to setting
  	the `GIT_ICASE_PATHSPECS` environment variable to `1`.
  
 +--no-optional-locks::
 +	Do not perform optional operations that require locks. This is
 +	equivalent to setting the `GIT_OPTIONAL_LOCKS` to `0`.
 +
  GIT COMMANDS
  ------------
  
@@@ -595,6 -591,10 +595,10 @@@ into it
  Unsetting the variable, or setting it to empty, "0" or
  "false" (case insensitive) disables trace messages.
  
+ `GIT_TRACE_FSMONITOR`::
+ 	Enables trace messages for the filesystem monitor extension.
+ 	See `GIT_TRACE` for available trace output options.
+ 
  `GIT_TRACE_PACK_ACCESS`::
  	Enables trace messages for all accesses to any packs. For each
  	access, the pack file name and an offset in the pack is
@@@ -701,32 -701,6 +705,32 @@@ of clones and fetches
  	which feed potentially-untrusted URLS to git commands.  See
  	linkgit:git-config[1] for more details.
  
 +`GIT_OPTIONAL_LOCKS`::
 +	If set to `0`, Git will complete any requested operation without
 +	performing any optional sub-operations that require taking a lock.
 +	For example, this will prevent `git status` from refreshing the
 +	index as a side effect. This is useful for processes running in
 +	the background which do not want to cause lock contention with
 +	other operations on the repository.  Defaults to `1`.
 +
 +`GIT_REDIRECT_STDIN`::
 +`GIT_REDIRECT_STDOUT`::
 +`GIT_REDIRECT_STDERR`::
 +	Windows-only: allow redirecting the standard input/output/error
 +	handles to paths specified by the environment variables. This is
 +	particularly useful in multi-threaded applications where the
 +	canonical way to pass standard handles via `CreateProcess()` is
 +	not an option because it would require the handles to be marked
 +	inheritable (and consequently *every* spawned process would
 +	inherit them, possibly blocking regular Git operations). The
 +	primary intended use case is to use named pipes for communication
 +	(e.g. `\\.\pipe\my-git-stdin-123`).
 ++
 +Two special values are supported: `off` will simply close the
 +corresponding standard handle, and if `GIT_REDIRECT_STDERR` is
 +`2>&1`, standard error will be redirected to the same handle as
 +standard output.
 +
  Discussion[[Discussion]]
  ------------------------
  
diff --combined cache.h
index 7c71681236,a15edc7e1d..f684c7933b
--- a/cache.h
+++ b/cache.h
@@@ -4,7 -4,6 +4,7 @@@
  #include "git-compat-util.h"
  #include "strbuf.h"
  #include "hashmap.h"
 +#include "mru.h"
  #include "advice.h"
  #include "gettext.h"
  #include "convert.h"
@@@ -348,6 -347,7 +348,7 @@@ struct index_state 
  	unsigned char sha1[20];
  	struct untracked_cache *untracked;
  	uint64_t fsmonitor_last_update;
+ 	struct ewah_bitmap *fsmonitor_dirty;
  };
  
  extern struct index_state the_index;
@@@ -447,7 -447,6 +448,7 @@@ static inline enum object_type object_t
  #define GIT_NOGLOB_PATHSPECS_ENVIRONMENT "GIT_NOGLOB_PATHSPECS"
  #define GIT_ICASE_PATHSPECS_ENVIRONMENT "GIT_ICASE_PATHSPECS"
  #define GIT_QUARANTINE_ENVIRONMENT "GIT_QUARANTINE_PATH"
 +#define GIT_OPTIONAL_LOCKS_ENVIRONMENT "GIT_OPTIONAL_LOCKS"
  
  /*
   * This environment variable is expected to contain a boolean indicating
@@@ -605,28 -604,9 +606,28 @@@ extern int do_read_index(struct index_s
  extern int read_index_from(struct index_state *, const char *path);
  extern int is_index_unborn(struct index_state *);
  extern int read_index_unmerged(struct index_state *);
 +
 +/* For use with `write_locked_index()`. */
  #define COMMIT_LOCK		(1 << 0)
 -#define CLOSE_LOCK		(1 << 1)
 +
 +/*
 + * Write the index while holding an already-taken lock. Close the lock,
 + * and if `COMMIT_LOCK` is given, commit it.
 + *
 + * Unless a split index is in use, write the index into the lockfile.
 + *
 + * With a split index, write the shared index to a temporary file,
 + * adjust its permissions and rename it into place, then write the
 + * split index to the lockfile. If the temporary file for the shared
 + * index cannot be created, fall back to the behavior described in
 + * the previous paragraph.
 + *
 + * With `COMMIT_LOCK`, the lock is always committed or rolled back.
 + * Without it, the lock is closed, but neither committed nor rolled
 + * back.
 + */
  extern int write_locked_index(struct index_state *, struct lock_file *lock, unsigned flags);
 +
  extern int discard_index(struct index_state *);
  extern void move_index_extensions(struct index_state *dst, struct index_state *src);
  extern int unmerged_index(const struct index_state *);
@@@ -740,17 -720,12 +741,17 @@@ extern void fill_stat_cache_info(struc
  extern int refresh_index(struct index_state *, unsigned int flags, const struct pathspec *pathspec, char *seen, const char *header_msg);
  extern struct cache_entry *refresh_cache_entry(struct cache_entry *, unsigned int);
  
 +/*
 + * Opportunistically update the index but do not complain if we can't.
 + * The lockfile is always committed or rolled back.
 + */
  extern void update_index_if_able(struct index_state *, struct lock_file *);
  
  extern int hold_locked_index(struct lock_file *, int);
  extern void set_alternate_index_output(const char *);
  
  extern int verify_index_checksum;
 +extern int verify_ce_order;
  
  /* Environment bits from configuration mechanism */
  extern int trust_executable_bit;
@@@ -814,11 -789,6 +815,11 @@@ extern const char *core_fsmonitor
   */
  extern int ref_paranoia;
  
 +/*
 + * Returns the boolean value of $GIT_OPTIONAL_LOCKS (or the default value).
 + */
 +int use_optional_locks(void);
 +
  /*
   * The character that begins a commented line in user-editable file
   * that is subject to stripspace.
@@@ -1280,8 -1250,8 +1281,8 @@@ static inline unsigned int hexval(unsig
   */
  static inline int hex2chr(const char *s)
  {
 -	int val = hexval(s[0]);
 -	return (val < 0) ? val : (val << 4) | hexval(s[1]);
 +	unsigned int val = hexval(s[0]);
 +	return (val & ~0xf) ? val : (val << 4) | hexval(s[1]);
  }
  
  /* Convert to/from hex/sha1 representation */
@@@ -1347,13 -1317,6 +1348,13 @@@ extern int set_disambiguate_hint_config
  extern int get_sha1_hex(const char *hex, unsigned char *sha1);
  extern int get_oid_hex(const char *hex, struct object_id *sha1);
  
 +/*
 + * Read `len` pairs of hexadecimal digits from `hex` and write the
 + * values to `binary` as `len` bytes. Return 0 on success, or -1 if
 + * the input does not consist of hex digits).
 + */
 +extern int hex_to_bytes(unsigned char *binary, const char *hex, size_t len);
 +
  /*
   * Convert a binary sha1 to its hex equivalent. The `_r` variant is reentrant,
   * and writes the NUL-terminated output to the buffer `out`, which must be at
@@@ -1633,7 -1596,8 +1634,7 @@@ extern struct packed_git 
   * A most-recently-used ordered version of the packed_git list, which can
   * be iterated instead of packed_git (and marked via mru_mark).
   */
 -struct mru;
 -extern struct mru *packed_git_mru;
 +extern struct mru packed_git_mru;
  
  struct pack_entry {
  	off_t offset;
diff --combined read-cache.c
index 87e88b2642,0724ea8c9b..fbb4967c81
--- a/read-cache.c
+++ b/read-cache.c
@@@ -196,7 -196,7 +196,7 @@@ static int ce_compare_link(const struc
  
  static int ce_compare_gitlink(const struct cache_entry *ce)
  {
 -	unsigned char sha1[20];
 +	struct object_id oid;
  
  	/*
  	 * We don't actually require that the .git directory
@@@ -206,9 -206,9 +206,9 @@@
  	 *
  	 * If so, we consider it always to match.
  	 */
 -	if (resolve_gitlink_ref(ce->name, "HEAD", sha1) < 0)
 +	if (resolve_gitlink_ref(ce->name, "HEAD", &oid) < 0)
  		return 0;
 -	return hashcmp(sha1, ce->oid.hash);
 +	return oidcmp(&oid, &ce->oid);
  }
  
  static int ce_modified_check_fs(const struct cache_entry *ce, struct stat *st)
@@@ -225,7 -225,6 +225,7 @@@
  	case S_IFDIR:
  		if (S_ISGITLINK(ce->ce_mode))
  			return ce_compare_gitlink(ce) ? DATA_CHANGED : 0;
 +		/* else fallthrough */
  	default:
  		return TYPE_CHANGED;
  	}
@@@ -1532,9 -1531,6 +1532,9 @@@ struct ondisk_cache_entry_extended 
  /* Allow fsck to force verification of the index checksum. */
  int verify_index_checksum;
  
 +/* Allow fsck to force verification of the cache entry order. */
 +int verify_ce_order;
 +
  static int verify_hdr(struct cache_header *hdr, unsigned long size)
  {
  	git_SHA_CTX c;
@@@ -1695,9 -1691,6 +1695,9 @@@ static void check_ce_order(struct index
  {
  	unsigned int i;
  
 +	if (!verify_ce_order)
 +		return;
 +
  	for (i = 1; i < istate->cache_nr; i++) {
  		struct cache_entry *ce = istate->cache[i - 1];
  		struct cache_entry *next_ce = istate->cache[i];
@@@ -1954,7 -1947,7 +1954,7 @@@ static int ce_write_flush(git_SHA_CTX *
  	unsigned int buffered = write_buffer_len;
  	if (buffered) {
  		git_SHA1_Update(context, write_buffer, buffered);
 -		if (write_in_full(fd, write_buffer, buffered) != buffered)
 +		if (write_in_full(fd, write_buffer, buffered) < 0)
  			return -1;
  		write_buffer_len = 0;
  	}
@@@ -2003,7 -1996,7 +2003,7 @@@ static int ce_flush(git_SHA_CTX *contex
  
  	/* Flush first if not enough space for SHA1 signature */
  	if (left + 20 > WRITE_BUFFER_SIZE) {
 -		if (write_in_full(fd, write_buffer, left) != left)
 +		if (write_in_full(fd, write_buffer, left) < 0)
  			return -1;
  		left = 0;
  	}
@@@ -2012,7 -2005,7 +2012,7 @@@
  	git_SHA1_Final(write_buffer + left, context);
  	hashcpy(sha1, write_buffer + left);
  	left += 20;
 -	return (write_in_full(fd, write_buffer, left) != left) ? -1 : 0;
 +	return (write_in_full(fd, write_buffer, left) < 0) ? -1 : 0;
  }
  
  static void ce_smudge_racily_clean_entry(struct cache_entry *ce)
@@@ -2135,9 -2128,7 +2135,9 @@@ static int ce_write_entry(git_SHA_CTX *
  		if (!result)
  			result = ce_write(c, fd, to_remove_vi, prefix_size);
  		if (!result)
 -			result = ce_write(c, fd, ce->name + common, ce_namelen(ce) - common + 1);
 +			result = ce_write(c, fd, ce->name + common, ce_namelen(ce) - common);
 +		if (!result)
 +			result = ce_write(c, fd, padding, 1);
  
  		strbuf_splice(previous_name, common, to_remove,
  			      ce->name + common, ce_namelen(ce) - common);
@@@ -2207,22 -2198,17 +2207,22 @@@ static int has_racy_timestamp(struct in
  	return 0;
  }
  
 -/*
 - * Opportunistically update the index but do not complain if we can't
 - */
  void update_index_if_able(struct index_state *istate, struct lock_file *lockfile)
  {
  	if ((istate->cache_changed || has_racy_timestamp(istate)) &&
 -	    verify_index(istate) &&
 -	    write_locked_index(istate, lockfile, COMMIT_LOCK))
 +	    verify_index(istate))
 +		write_locked_index(istate, lockfile, COMMIT_LOCK);
 +	else
  		rollback_lock_file(lockfile);
  }
  
 +/*
 + * On success, `tempfile` is closed. If it is the temporary file
 + * of a `struct lock_file`, we will therefore effectively perform
 + * a 'close_lock_file_gently()`. Since that is an implementation
 + * detail of lockfiles, callers of `do_write_index()` should not
 + * rely on it.
 + */
  static int do_write_index(struct index_state *istate, struct tempfile *tempfile,
  			  int strip_extensions)
  {
@@@ -2358,10 -2344,8 +2358,10 @@@
  
  	if (ce_flush(&c, newfd, istate->sha1))
  		return -1;
 -	if (close_tempfile(tempfile))
 -		return error(_("could not close '%s'"), tempfile->filename.buf);
 +	if (close_tempfile_gently(tempfile)) {
 +		error(_("could not close '%s'"), tempfile->filename.buf);
 +		return -1;
 +	}
  	if (stat(tempfile->filename.buf, &st))
  		return -1;
  	istate->timestamp.sec = (unsigned int)st.st_mtime;
@@@ -2385,12 -2369,17 +2385,12 @@@ static int commit_locked_index(struct l
  static int do_write_locked_index(struct index_state *istate, struct lock_file *lock,
  				 unsigned flags)
  {
 -	int ret = do_write_index(istate, &lock->tempfile, 0);
 +	int ret = do_write_index(istate, lock->tempfile, 0);
  	if (ret)
  		return ret;
 -	assert((flags & (COMMIT_LOCK | CLOSE_LOCK)) !=
 -	       (COMMIT_LOCK | CLOSE_LOCK));
  	if (flags & COMMIT_LOCK)
  		return commit_locked_index(lock);
 -	else if (flags & CLOSE_LOCK)
 -		return close_lock_file(lock);
 -	else
 -		return ret;
 +	return close_lock_file_gently(lock);
  }
  
  static int write_split_index(struct index_state *istate,
@@@ -2463,33 -2452,34 +2463,33 @@@ static int clean_shared_index_files(con
  	return 0;
  }
  
 -static struct tempfile temporary_sharedindex;
 -
  static int write_shared_index(struct index_state *istate,
  			      struct lock_file *lock, unsigned flags)
  {
 +	struct tempfile *temp;
  	struct split_index *si = istate->split_index;
 -	int fd, ret;
 +	int ret;
  
 -	fd = mks_tempfile(&temporary_sharedindex, git_path("sharedindex_XXXXXX"));
 -	if (fd < 0) {
 +	temp = mks_tempfile(git_path("sharedindex_XXXXXX"));
 +	if (!temp) {
  		hashclr(si->base_sha1);
  		return do_write_locked_index(istate, lock, flags);
  	}
  	move_cache_to_base_index(istate);
 -	ret = do_write_index(si->base, &temporary_sharedindex, 1);
 +	ret = do_write_index(si->base, temp, 1);
  	if (ret) {
 -		delete_tempfile(&temporary_sharedindex);
 +		delete_tempfile(&temp);
  		return ret;
  	}
 -	ret = adjust_shared_perm(get_tempfile_path(&temporary_sharedindex));
 +	ret = adjust_shared_perm(get_tempfile_path(temp));
  	if (ret) {
  		int save_errno = errno;
 -		error("cannot fix permission bits on %s", get_tempfile_path(&temporary_sharedindex));
 -		delete_tempfile(&temporary_sharedindex);
 +		error("cannot fix permission bits on %s", get_tempfile_path(temp));
 +		delete_tempfile(&temp);
  		errno = save_errno;
  		return ret;
  	}
 -	ret = rename_tempfile(&temporary_sharedindex,
 +	ret = rename_tempfile(&temp,
  			      git_path("sharedindex.%s", sha1_to_hex(si->base->sha1)));
  	if (!ret) {
  		hashcpy(si->base_sha1, si->base->sha1);
@@@ -2535,12 -2525,14 +2535,15 @@@ int write_locked_index(struct index_sta
  	int new_shared_index, ret;
  	struct split_index *si = istate->split_index;
  
+ 	if (istate->fsmonitor_last_update)
+ 		fill_fsmonitor_bitmap(istate);
+ 
  	if (!si || alternate_index_output ||
  	    (istate->cache_changed & ~EXTMASK)) {
  		if (si)
  			hashclr(si->base_sha1);
 -		return do_write_locked_index(istate, lock, flags);
 +		ret = do_write_locked_index(istate, lock, flags);
 +		goto out;
  	}
  
  	if (getenv("GIT_TEST_SPLIT_INDEX")) {
@@@ -2556,7 -2548,7 +2559,7 @@@
  	if (new_shared_index) {
  		ret = write_shared_index(istate, lock, flags);
  		if (ret)
 -			return ret;
 +			goto out;
  	}
  
  	ret = write_split_index(istate, lock, flags);
@@@ -2565,9 -2557,6 +2568,9 @@@
  	if (!ret && !new_shared_index)
  		freshen_shared_index(sha1_to_hex(si->base_sha1), 1);
  
 +out:
 +	if (flags & COMMIT_LOCK)
 +		rollback_lock_file(lock);
  	return ret;
  }