From: Junio C Hamano <gitster@pobox.com>
Date: Mon, 25 Jan 2010 01:35:58 +0000 (-0800)
Subject: Merge branch 'jc/fix-tree-walk'
X-Git-Tag: v1.7.0-rc0~1
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/026680f881e751311674d97d0f6ed87f06a92bfb?hp=-c

Merge branch 'jc/fix-tree-walk'

* jc/fix-tree-walk:
read-tree --debug-unpack
unpack-trees.c: look ahead in the index
unpack-trees.c: prepare for looking ahead in the index
Aggressive three-way merge: fix D/F case
traverse_trees(): handle D/F conflict case sanely
more D/F conflict tests
tests: move convenience regexp to match object names to test-lib.sh

Conflicts:
builtin-read-tree.c
unpack-trees.c
unpack-trees.h
---

026680f881e751311674d97d0f6ed87f06a92bfb
diff --combined builtin-read-tree.c
index 5fda9905fc,0c1f40d10d..8bdcab1138
--- a/builtin-read-tree.c
+++ b/builtin-read-tree.c
@@@ -13,7 -13,6 +13,7 @@@
  #include "dir.h"
  #include "builtin.h"
  #include "parse-options.h"
 +#include "resolve-undo.h"
  
  static int nr_trees;
  static struct tree *trees[MAX_UNPACK_TREES];
@@@ -32,7 -31,7 +32,7 @@@ static int list_tree(unsigned char *sha
  }
  
  static const char * const read_tree_usage[] = {
 -	"git read-tree [[-m [--trivial] [--aggressive] | --reset | --prefix=<prefix>] [-u [--exclude-per-directory=<gitignore>] | -i]]  [--index-output=<file>] <tree-ish1> [<tree-ish2> [<tree-ish3>]]",
 +	"git read-tree [[-m [--trivial] [--aggressive] | --reset | --prefix=<prefix>] [-u [--exclude-per-directory=<gitignore>] | -i]] [--no-sparse-checkout] [--index-output=<file>] <tree-ish1> [<tree-ish2> [<tree-ish3>]]",
  	NULL
  };
  
@@@ -65,6 -64,34 +65,34 @@@ static int exclude_per_directory_cb(con
  	return 0;
  }
  
+ static void debug_stage(const char *label, struct cache_entry *ce,
+ 			struct unpack_trees_options *o)
+ {
+ 	printf("%s ", label);
+ 	if (!ce)
+ 		printf("(missing)\n");
+ 	else if (ce == o->df_conflict_entry)
+ 		printf("(conflict)\n");
+ 	else
+ 		printf("%06o #%d %s %.8s\n",
+ 		       ce->ce_mode, ce_stage(ce), ce->name,
+ 		       sha1_to_hex(ce->sha1));
+ }
+ 
+ static int debug_merge(struct cache_entry **stages, struct unpack_trees_options *o)
+ {
+ 	int i;
+ 
+ 	printf("* %d-way merge\n", o->merge_size);
+ 	debug_stage("index", stages[0], o);
+ 	for (i = 1; i <= o->merge_size; i++) {
+ 		char buf[24];
+ 		sprintf(buf, "ent#%d", i);
+ 		debug_stage(buf, stages[i], o);
+ 	}
+ 	return 0;
+ }
+ 
  static struct lock_file lock_file;
  
  int cmd_read_tree(int argc, const char **argv, const char *unused_prefix)
@@@ -99,8 -126,8 +127,10 @@@
  		  PARSE_OPT_NONEG, exclude_per_directory_cb },
  		OPT_SET_INT('i', NULL, &opts.index_only,
  			    "don't check the working tree after merging", 1),
 +		OPT_SET_INT(0, "no-sparse-checkout", &opts.skip_sparse_checkout,
 +			    "skip applying sparse checkout filter", 1),
+ 		OPT_SET_INT(0, "debug-unpack", &opts.debug_unpack,
+ 			    "debug unpack-trees", 1),
  		OPT_END()
  	};
  
@@@ -125,7 -152,6 +155,7 @@@
  			die("You need to resolve your current index first");
  		stage = opts.merge = 1;
  	}
 +	resolve_undo_clear();
  
  	for (i = 0; i < argc; i++) {
  		const char *arg = argv[i];
@@@ -169,6 -195,9 +199,9 @@@
  			opts.head_idx = 1;
  	}
  
+ 	if (opts.debug_unpack)
+ 		opts.fn = debug_merge;
+ 
  	cache_tree_free(&active_cache_tree);
  	for (i = 0; i < nr_trees; i++) {
  		struct tree *tree = trees[i];
@@@ -178,6 -207,9 +211,9 @@@
  	if (unpack_trees(nr_trees, t, &opts))
  		return 128;
  
+ 	if (opts.debug_unpack)
+ 		return 0; /* do not write the index out */
+ 
  	/*
  	 * When reading only one tree (either the most basic form,
  	 * "-m ent" or "--reset ent" form), we can obtain a fully
diff --combined cache.h
index 7d5c21e3f1,9a9596386d..b3370eb41e
--- a/cache.h
+++ b/cache.h
@@@ -177,20 -177,17 +177,22 @@@ struct cache_entry 
  
  #define CE_HASHED    (0x100000)
  #define CE_UNHASHED  (0x200000)
 +#define CE_CONFLICTED (0x800000)
 +
 +/* Only remove in work directory, not index */
 +#define CE_WT_REMOVE (0x400000)
  
+ #define CE_UNPACKED  (0x1000000)
+ 
  /*
   * Extended on-disk flags
   */
  #define CE_INTENT_TO_ADD 0x20000000
 +#define CE_SKIP_WORKTREE 0x40000000
  /* CE_EXTENDED2 is for future extension */
  #define CE_EXTENDED2 0x80000000
  
 -#define CE_EXTENDED_FLAGS (CE_INTENT_TO_ADD)
 +#define CE_EXTENDED_FLAGS (CE_INTENT_TO_ADD | CE_SKIP_WORKTREE)
  
  /*
   * Safeguard to avoid saving wrong flags:
@@@ -239,7 -236,6 +241,7 @@@ static inline size_t ce_namelen(const s
  			    ondisk_cache_entry_size(ce_namelen(ce)))
  #define ce_stage(ce) ((CE_STAGEMASK & (ce)->ce_flags) >> CE_STAGESHIFT)
  #define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE)
 +#define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE)
  #define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE)
  
  #define ce_permissions(mode) (((mode) & 0100) ? 0755 : 0644)
@@@ -288,7 -284,6 +290,7 @@@ static inline int ce_to_dtype(const str
  struct index_state {
  	struct cache_entry **cache;
  	unsigned int cache_nr, cache_alloc, cache_changed;
 +	struct string_list *resolve_undo;
  	struct cache_tree *cache_tree;
  	struct cache_time timestamp;
  	void *alloc;
@@@ -343,9 -338,6 +345,9 @@@ static inline void remove_name_hash(str
  #define ce_modified(ce, st, options) ie_modified(&the_index, (ce), (st), (options))
  #define cache_name_exists(name, namelen, igncase) index_name_exists(&the_index, (name), (namelen), (igncase))
  #define cache_name_is_other(name, namelen) index_name_is_other(&the_index, (name), (namelen))
 +#define resolve_undo_clear() resolve_undo_clear_index(&the_index)
 +#define unmerge_cache_entry_at(at) unmerge_index_entry_at(&the_index, at)
 +#define unmerge_cache(pathspec) unmerge_index(&the_index, pathspec)
  #endif
  
  enum object_type {
@@@ -455,6 -447,7 +457,6 @@@ extern int index_name_pos(const struct 
  #define ADD_CACHE_JUST_APPEND 8		/* Append only; tree.c::read_tree() */
  #define ADD_CACHE_NEW_ONLY 16		/* Do not replace existing ones */
  extern int add_index_entry(struct index_state *, struct cache_entry *ce, int option);
 -extern struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really);
  extern void rename_index_entry_at(struct index_state *, int pos, const char *new_name);
  extern int remove_index_entry_at(struct index_state *, int pos);
  extern void remove_marked_cache_entries(struct index_state *istate);
@@@ -473,9 -466,7 +475,9 @@@ extern int index_name_is_other(const st
  /* do stat comparison even if CE_VALID is true */
  #define CE_MATCH_IGNORE_VALID		01
  /* do not check the contents but report dirty on racily-clean entries */
 -#define CE_MATCH_RACY_IS_DIRTY	02
 +#define CE_MATCH_RACY_IS_DIRTY		02
 +/* do stat comparison even if CE_SKIP_WORKTREE is true */
 +#define CE_MATCH_IGNORE_SKIP_WORKTREE	04
  extern int ie_match_stat(const struct index_state *, struct cache_entry *, struct stat *, unsigned int);
  extern int ie_modified(const struct index_state *, struct cache_entry *, struct stat *, unsigned int);
  
@@@ -484,6 -475,9 +486,6 @@@ extern int index_fd(unsigned char *sha1
  extern int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object);
  extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st);
  
 -/* "careful lstat()" */
 -extern int check_path(const char *path, int len, struct stat *st, int skiplen);
 -
  #define REFRESH_REALLY		0x0001	/* ignore_valid */
  #define REFRESH_UNMERGED	0x0002	/* allow unmerged */
  #define REFRESH_QUIET		0x0004	/* be quiet about it */
@@@ -537,7 -531,6 +539,7 @@@ extern int auto_crlf
  extern int read_replace_refs;
  extern int fsync_object_files;
  extern int core_preload_index;
 +extern int core_apply_sparse_checkout;
  
  enum safe_crlf {
  	SAFE_CRLF_FALSE = 0,
@@@ -553,7 -546,6 +555,7 @@@ enum branch_track 
  	BRANCH_TRACK_REMOTE,
  	BRANCH_TRACK_ALWAYS,
  	BRANCH_TRACK_EXPLICIT,
 +	BRANCH_TRACK_OVERRIDE,
  };
  
  enum rebase_setup_type {
@@@ -628,6 -620,7 +630,6 @@@ static inline void hashclr(unsigned cha
  {
  	memset(hash, 0, 20);
  }
 -extern int is_empty_blob_sha1(const unsigned char *sha1);
  
  #define EMPTY_TREE_SHA1_HEX \
  	"4b825dc642cb6eb9a060e54bf8d69288fbee4904"
@@@ -697,6 -690,7 +699,6 @@@ extern int has_sha1_pack(const unsigne
  extern int has_sha1_file(const unsigned char *sha1);
  extern int has_loose_object_nonlocal(const unsigned char *sha1);
  
 -extern int has_pack_file(const unsigned char *sha1);
  extern int has_pack_index(const unsigned char *sha1);
  
  extern const signed char hexval_table[256];
@@@ -710,11 -704,7 +712,11 @@@ static inline unsigned int hexval(unsig
  #define DEFAULT_ABBREV 7
  
  extern int get_sha1(const char *str, unsigned char *sha1);
 -extern int get_sha1_with_mode(const char *str, unsigned char *sha1, unsigned *mode);
 +extern int get_sha1_with_mode_1(const char *str, unsigned char *sha1, unsigned *mode, int gently, const char *prefix);
 +static inline int get_sha1_with_mode(const char *str, unsigned char *sha1, unsigned *mode)
 +{
 +	return get_sha1_with_mode_1(str, sha1, mode, 1, NULL);
 +}
  extern int get_sha1_hex(const char *hex, unsigned char *sha1);
  extern char *sha1_to_hex(const unsigned char *sha1);	/* static buffer result! */
  extern int read_ref(const char *filename, unsigned char *sha1);
@@@ -722,7 -712,6 +724,7 @@@ extern const char *resolve_ref(const ch
  extern int dwim_ref(const char *str, int len, unsigned char *sha1, char **ref);
  extern int dwim_log(const char *str, int len, unsigned char *sha1, char **ref);
  extern int interpret_branch_name(const char *str, struct strbuf *);
 +extern int get_sha1_mb(const char *str, unsigned char *sha1);
  
  extern int refname_match(const char *abbrev_name, const char *full_name, const char **rules);
  extern const char *ref_rev_parse_rules[];
@@@ -797,6 -786,8 +799,6 @@@ extern int has_symlink_leading_path(con
  extern int threaded_has_symlink_leading_path(struct cache_def *, const char *, int);
  extern int has_symlink_or_noent_leading_path(const char *name, int len);
  extern int has_dirs_only_path(const char *name, int len, int prefix_len);
 -extern void invalidate_lstat_cache(const char *name, int len);
 -extern void clear_lstat_cache(void);
  extern void schedule_dir_for_removal(const char *name, int len);
  extern void remove_scheduled_dirs(void);
  
@@@ -936,11 -927,7 +938,11 @@@ extern const char *config_exclusive_fil
  #define MAX_GITNAME (1000)
  extern char git_default_email[MAX_GITNAME];
  extern char git_default_name[MAX_GITNAME];
 +#define IDENT_NAME_GIVEN 01
 +#define IDENT_MAIL_GIVEN 02
 +#define IDENT_ALL_GIVEN (IDENT_NAME_GIVEN|IDENT_MAIL_GIVEN)
  extern int user_ident_explicitly_given;
 +extern int user_ident_sufficiently_given(void);
  
  extern const char *git_commit_encoding;
  extern const char *git_log_output_encoding;
@@@ -1008,7 -995,6 +1010,7 @@@ extern int diff_auto_refresh_index
  
  /* match-trees.c */
  void shift_tree(const unsigned char *, const unsigned char *, unsigned char *, int);
 +void shift_tree_by(const unsigned char *, const unsigned char *, unsigned char *, const char *);
  
  /*
   * whitespace rules.
diff --combined diff-lib.c
index 23e180eed1,c9998f4c91..ec2e2ac005
--- a/diff-lib.c
+++ b/diff-lib.c
@@@ -10,7 -10,6 +10,7 @@@
  #include "cache-tree.h"
  #include "unpack-trees.h"
  #include "refs.h"
 +#include "submodule.h"
  
  /*
   * diff-files
@@@ -73,9 -72,8 +73,9 @@@ int run_diff_files(struct rev_info *rev
  		unsigned int oldmode, newmode;
  		struct cache_entry *ce = active_cache[i];
  		int changed;
 +		unsigned dirty_submodule = 0;
  
 -		if (DIFF_OPT_TST(&revs->diffopt, QUIET) &&
 +		if (DIFF_OPT_TST(&revs->diffopt, QUICK) &&
  			DIFF_OPT_TST(&revs->diffopt, HAS_CHANGES))
  			break;
  
@@@ -161,7 -159,7 +161,7 @@@
  				continue;
  		}
  
 -		if (ce_uptodate(ce))
 +		if ((ce_uptodate(ce) && !S_ISGITLINK(ce->ce_mode)) || ce_skip_worktree(ce))
  			continue;
  
  		/* If CE_VALID is set, don't look at workdir for file removal */
@@@ -174,16 -172,10 +174,16 @@@
  			if (silent_on_removed)
  				continue;
  			diff_addremove(&revs->diffopt, '-', ce->ce_mode,
 -				       ce->sha1, ce->name);
 +				       ce->sha1, ce->name, 0);
  			continue;
  		}
  		changed = ce_match_stat(ce, &st, ce_option);
 +		if (S_ISGITLINK(ce->ce_mode)
 +		    && (!changed || (revs->diffopt.output_format & DIFF_FORMAT_PATCH))
 +		    && is_submodule_modified(ce->name)) {
 +			changed = 1;
 +			dirty_submodule = 1;
 +		}
  		if (!changed) {
  			ce_mark_uptodate(ce);
  			if (!DIFF_OPT_TST(&revs->diffopt, FIND_COPIES_HARDER))
@@@ -193,7 -185,7 +193,7 @@@
  		newmode = ce_mode_from_stat(ce, st.st_mode);
  		diff_change(&revs->diffopt, oldmode, newmode,
  			    ce->sha1, (changed ? null_sha1 : ce->sha1),
 -			    ce->name);
 +			    ce->name, 0, dirty_submodule);
  
  	}
  	diffcore_std(&revs->diffopt);
@@@ -209,18 -201,16 +209,18 @@@
  static void diff_index_show_file(struct rev_info *revs,
  				 const char *prefix,
  				 struct cache_entry *ce,
 -				 const unsigned char *sha1, unsigned int mode)
 +				 const unsigned char *sha1, unsigned int mode,
 +				 unsigned dirty_submodule)
  {
  	diff_addremove(&revs->diffopt, prefix[0], mode,
 -		       sha1, ce->name);
 +		       sha1, ce->name, dirty_submodule);
  }
  
  static int get_stat_data(struct cache_entry *ce,
  			 const unsigned char **sha1p,
  			 unsigned int *modep,
 -			 int cached, int match_missing)
 +			 int cached, int match_missing,
 +			 unsigned *dirty_submodule, int output_format)
  {
  	const unsigned char *sha1 = ce->sha1;
  	unsigned int mode = ce->ce_mode;
@@@ -240,12 -230,6 +240,12 @@@
  			return -1;
  		}
  		changed = ce_match_stat(ce, &st, 0);
 +		if (S_ISGITLINK(ce->ce_mode)
 +		    && (!changed || (output_format & DIFF_FORMAT_PATCH))
 +		    && is_submodule_modified(ce->name)) {
 +			changed = 1;
 +			*dirty_submodule = 1;
 +		}
  		if (changed) {
  			mode = ce_mode_from_stat(ce, st.st_mode);
  			sha1 = null_sha1;
@@@ -263,17 -247,15 +263,17 @@@ static void show_new_file(struct rev_in
  {
  	const unsigned char *sha1;
  	unsigned int mode;
 +	unsigned dirty_submodule = 0;
  
  	/*
  	 * New file in the index: it might actually be different in
  	 * the working copy.
  	 */
 -	if (get_stat_data(new, &sha1, &mode, cached, match_missing) < 0)
 +	if (get_stat_data(new, &sha1, &mode, cached, match_missing,
 +	    &dirty_submodule, revs->diffopt.output_format) < 0)
  		return;
  
 -	diff_index_show_file(revs, "+", new, sha1, mode);
 +	diff_index_show_file(revs, "+", new, sha1, mode, dirty_submodule);
  }
  
  static int show_modified(struct rev_info *revs,
@@@ -284,13 -266,11 +284,13 @@@
  {
  	unsigned int mode, oldmode;
  	const unsigned char *sha1;
 +	unsigned dirty_submodule = 0;
  
 -	if (get_stat_data(new, &sha1, &mode, cached, match_missing) < 0) {
 +	if (get_stat_data(new, &sha1, &mode, cached, match_missing,
 +			  &dirty_submodule, revs->diffopt.output_format) < 0) {
  		if (report_missing)
  			diff_index_show_file(revs, "-", old,
 -					     old->sha1, old->ce_mode);
 +					     old->sha1, old->ce_mode, 0);
  		return -1;
  	}
  
@@@ -325,7 -305,7 +325,7 @@@
  		return 0;
  
  	diff_change(&revs->diffopt, oldmode, mode,
 -		    old->sha1, sha1, old->name);
 +		    old->sha1, sha1, old->name, 0, dirty_submodule);
  	return 0;
  }
  
@@@ -343,8 -323,7 +343,8 @@@ static void do_oneway_diff(struct unpac
  	int match_missing, cached;
  
  	/* if the entry is not checked out, don't examine work tree */
 -	cached = o->index_only || (idx && (idx->ce_flags & CE_VALID));
 +	cached = o->index_only ||
 +		(idx && ((idx->ce_flags & CE_VALID) || ce_skip_worktree(idx)));
  	/*
  	 * Backward compatibility wart - "diff-index -m" does
  	 * not mean "do not ignore merges", but "match_missing".
@@@ -372,7 -351,7 +372,7 @@@
  	 * Something removed from the tree?
  	 */
  	if (!idx) {
 -		diff_index_show_file(revs, "-", tree, tree->sha1, tree->ce_mode);
 +		diff_index_show_file(revs, "-", tree, tree->sha1, tree->ce_mode, 0);
  		return;
  	}
  
@@@ -380,21 -359,6 +380,6 @@@
  	show_modified(revs, tree, idx, 1, cached, match_missing);
  }
  
- static inline void skip_same_name(struct cache_entry *ce, struct unpack_trees_options *o)
- {
- 	int len = ce_namelen(ce);
- 	const struct index_state *index = o->src_index;
- 
- 	while (o->pos < index->cache_nr) {
- 		struct cache_entry *next = index->cache[o->pos];
- 		if (len != ce_namelen(next))
- 			break;
- 		if (memcmp(ce->name, next->name, len))
- 			break;
- 		o->pos++;
- 	}
- }
- 
  /*
   * The unpack_trees() interface is designed for merging, so
   * the different source entries are designed primarily for
@@@ -416,9 -380,6 +401,6 @@@ static int oneway_diff(struct cache_ent
  	struct cache_entry *tree = src[1];
  	struct rev_info *revs = o->unpack_data;
  
- 	if (idx && ce_stage(idx))
- 		skip_same_name(idx, o);
- 
  	/*
  	 * Unpack-trees generates a DF/conflict entry if
  	 * there was a directory in the index and a tree
@@@ -464,6 -425,7 +446,7 @@@ int run_diff_index(struct rev_info *rev
  		exit(128);
  
  	diff_set_mnemonic_prefix(&revs->diffopt, "c/", cached ? "i/" : "w/");
+ 	diffcore_fix_diff_index(&revs->diffopt);
  	diffcore_std(&revs->diffopt);
  	diff_flush(&revs->diffopt);
  	return 0;
@@@ -528,7 -490,7 +511,7 @@@ int index_differs_from(const char *def
  
  	init_revisions(&rev, NULL);
  	setup_revisions(0, NULL, &rev, def);
 -	DIFF_OPT_SET(&rev.diffopt, QUIET);
 +	DIFF_OPT_SET(&rev.diffopt, QUICK);
  	DIFF_OPT_SET(&rev.diffopt, EXIT_WITH_STATUS);
  	rev.diffopt.flags |= diff_flags;
  	run_diff_index(&rev, 1);
diff --combined diff.c
index 160dbfd718,3bfb4a19d2..f130a367dc
--- a/diff.c
+++ b/diff.c
@@@ -194,7 -194,6 +194,7 @@@ struct emit_callback 
  	struct diff_words_data *diff_words;
  	int *found_changesp;
  	FILE *file;
 +	struct strbuf *header;
  };
  
  static int count_lines(const char *data, int size)
@@@ -798,11 -797,6 +798,11 @@@ static void fn_out_consume(void *priv, 
  	const char *plain = diff_get_color(ecbdata->color_diff, DIFF_PLAIN);
  	const char *reset = diff_get_color(ecbdata->color_diff, DIFF_RESET);
  
 +	if (ecbdata->header) {
 +		fprintf(ecbdata->file, "%s", ecbdata->header->buf);
 +		strbuf_reset(ecbdata->header);
 +		ecbdata->header = NULL;
 +	}
  	*(ecbdata->found_changesp) = 1;
  
  	if (ecbdata->label_path[0]) {
@@@ -1607,7 -1601,6 +1607,7 @@@ static void builtin_diff(const char *na
  	const char *reset = diff_get_color_opt(o, DIFF_RESET);
  	const char *a_prefix, *b_prefix;
  	const char *textconv_one = NULL, *textconv_two = NULL;
 +	struct strbuf header = STRBUF_INIT;
  
  	if (DIFF_OPT_TST(o, SUBMODULE_LOG) &&
  			(!one->mode || S_ISGITLINK(one->mode)) &&
@@@ -1642,26 -1635,25 +1642,26 @@@
  	b_two = quote_two(b_prefix, name_b + (*name_b == '/'));
  	lbl[0] = DIFF_FILE_VALID(one) ? a_one : "/dev/null";
  	lbl[1] = DIFF_FILE_VALID(two) ? b_two : "/dev/null";
 -	fprintf(o->file, "%sdiff --git %s %s%s\n", set, a_one, b_two, reset);
 +	strbuf_addf(&header, "%sdiff --git %s %s%s\n", set, a_one, b_two, reset);
  	if (lbl[0][0] == '/') {
  		/* /dev/null */
 -		fprintf(o->file, "%snew file mode %06o%s\n", set, two->mode, reset);
 +		strbuf_addf(&header, "%snew file mode %06o%s\n", set, two->mode, reset);
  		if (xfrm_msg && xfrm_msg[0])
 -			fprintf(o->file, "%s%s%s\n", set, xfrm_msg, reset);
 +			strbuf_addf(&header, "%s%s%s\n", set, xfrm_msg, reset);
  	}
  	else if (lbl[1][0] == '/') {
 -		fprintf(o->file, "%sdeleted file mode %06o%s\n", set, one->mode, reset);
 +		strbuf_addf(&header, "%sdeleted file mode %06o%s\n", set, one->mode, reset);
  		if (xfrm_msg && xfrm_msg[0])
 -			fprintf(o->file, "%s%s%s\n", set, xfrm_msg, reset);
 +			strbuf_addf(&header, "%s%s%s\n", set, xfrm_msg, reset);
  	}
  	else {
  		if (one->mode != two->mode) {
 -			fprintf(o->file, "%sold mode %06o%s\n", set, one->mode, reset);
 -			fprintf(o->file, "%snew mode %06o%s\n", set, two->mode, reset);
 +			strbuf_addf(&header, "%sold mode %06o%s\n", set, one->mode, reset);
 +			strbuf_addf(&header, "%snew mode %06o%s\n", set, two->mode, reset);
  		}
  		if (xfrm_msg && xfrm_msg[0])
 -			fprintf(o->file, "%s%s%s\n", set, xfrm_msg, reset);
 +			strbuf_addf(&header, "%s%s%s\n", set, xfrm_msg, reset);
 +
  		/*
  		 * we do not run diff between different kind
  		 * of objects.
@@@ -1671,8 -1663,6 +1671,8 @@@
  		if (complete_rewrite &&
  		    (textconv_one || !diff_filespec_is_binary(one)) &&
  		    (textconv_two || !diff_filespec_is_binary(two))) {
 +			fprintf(o->file, "%s", header.buf);
 +			strbuf_reset(&header);
  			emit_rewrite_diff(name_a, name_b, one, two,
  						textconv_one, textconv_two, o);
  			o->found_changes = 1;
@@@ -1690,8 -1680,6 +1690,8 @@@
  		if (mf1.size == mf2.size &&
  		    !memcmp(mf1.ptr, mf2.ptr, mf1.size))
  			goto free_ab_and_return;
 +		fprintf(o->file, "%s", header.buf);
 +		strbuf_reset(&header);
  		if (DIFF_OPT_TST(o, BINARY))
  			emit_binary_diff(o->file, &mf1, &mf2);
  		else
@@@ -1708,11 -1696,6 +1708,11 @@@
  		struct emit_callback ecbdata;
  		const struct userdiff_funcname *pe;
  
 +		if (!DIFF_XDL_TST(o, WHITESPACE_FLAGS)) {
 +			fprintf(o->file, "%s", header.buf);
 +			strbuf_reset(&header);
 +		}
 +
  		if (textconv_one) {
  			size_t size;
  			mf1.ptr = run_textconv(textconv_one, one, &size);
@@@ -1742,7 -1725,6 +1742,7 @@@
  		if (ecbdata.ws_rule & WS_BLANK_AT_EOF)
  			check_blank_at_eof(&mf1, &mf2, &ecbdata);
  		ecbdata.file = o->file;
 +		ecbdata.header = header.len ? &header : NULL;
  		xpp.flags = XDF_NEED_MINIMAL | o->xdl_opts;
  		xecfg.ctxlen = o->context;
  		xecfg.interhunkctxlen = o->interhunkcontext;
@@@ -1787,7 -1769,6 +1787,7 @@@
  	}
  
   free_ab_and_return:
 +	strbuf_release(&header);
  	diff_free_filespec_data(one);
  	diff_free_filespec_data(two);
  	free(a_one);
@@@ -1997,7 -1978,7 +1997,7 @@@ static int reuse_worktree_file(const ch
  	 * If ce is marked as "assume unchanged", there is no
  	 * guarantee that work tree matches what we are looking for.
  	 */
 -	if (ce->ce_flags & CE_VALID)
 +	if ((ce->ce_flags & CE_VALID) || ce_skip_worktree(ce))
  		return 0;
  
  	/*
@@@ -2029,14 -2010,9 +2029,14 @@@ static int populate_from_stdin(struct d
  static int diff_populate_gitlink(struct diff_filespec *s, int size_only)
  {
  	int len;
 -	char *data = xmalloc(100);
 +	char *data = xmalloc(100), *dirty = "";
 +
 +	/* Are we looking at the work tree? */
 +	if (!s->sha1_valid && s->dirty_submodule)
 +		dirty = "-dirty";
 +
  	len = snprintf(data, 100,
 -		"Subproject commit %s\n", sha1_to_hex(s->sha1));
 +		       "Subproject commit %s%s\n", sha1_to_hex(s->sha1), dirty);
  	s->data = data;
  	s->size = len;
  	s->should_free = 1;
@@@ -2299,7 -2275,7 +2299,7 @@@ static void run_external_diff(const cha
  	}
  	*arg = NULL;
  	fflush(NULL);
 -	retval = run_command_v_opt(spawn_arg, 0);
 +	retval = run_command_v_opt(spawn_arg, RUN_USING_SHELL);
  	remove_tempfile();
  	if (retval) {
  		fprintf(stderr, "external diff died, stopping at %s.\n", name);
@@@ -2575,20 -2551,6 +2575,20 @@@ int diff_setup_done(struct diff_option
  	if (count > 1)
  		die("--name-only, --name-status, --check and -s are mutually exclusive");
  
 +	/*
 +	 * Most of the time we can say "there are changes"
 +	 * only by checking if there are changed paths, but
 +	 * --ignore-whitespace* options force us to look
 +	 * inside contents.
 +	 */
 +
 +	if (DIFF_XDL_TST(options, IGNORE_WHITESPACE) ||
 +	    DIFF_XDL_TST(options, IGNORE_WHITESPACE_CHANGE) ||
 +	    DIFF_XDL_TST(options, IGNORE_WHITESPACE_AT_EOL))
 +		DIFF_OPT_SET(options, DIFF_FROM_CONTENTS);
 +	else
 +		DIFF_OPT_CLR(options, DIFF_FROM_CONTENTS);
 +
  	if (DIFF_OPT_TST(options, FIND_COPIES_HARDER))
  		options->detect_rename = DIFF_DETECT_COPY;
  
@@@ -2649,7 -2611,7 +2649,7 @@@
  	 * to have found.  It does not make sense not to return with
  	 * exit code in such a case either.
  	 */
 -	if (DIFF_OPT_TST(options, QUIET)) {
 +	if (DIFF_OPT_TST(options, QUICK)) {
  		options->output_format = DIFF_FORMAT_NO_OUTPUT;
  		DIFF_OPT_SET(options, EXIT_WITH_STATUS);
  	}
@@@ -2840,7 -2802,7 +2840,7 @@@ int diff_opt_parse(struct diff_options 
  	else if (!strcmp(arg, "--exit-code"))
  		DIFF_OPT_SET(options, EXIT_WITH_STATUS);
  	else if (!strcmp(arg, "--quiet"))
 -		DIFF_OPT_SET(options, QUIET);
 +		DIFF_OPT_SET(options, QUICK);
  	else if (!strcmp(arg, "--ext-diff"))
  		DIFF_OPT_SET(options, ALLOW_EXTERNAL);
  	else if (!strcmp(arg, "--no-ext-diff"))
@@@ -3547,18 -3509,6 +3547,18 @@@ free_queue
  	q->nr = q->alloc = 0;
  	if (options->close_file)
  		fclose(options->file);
 +
 +	/*
 +	 * Report the content-level differences with HAS_CHANGES;
 +	 * diff_addremove/diff_change does not set the bit when
 +	 * DIFF_FROM_CONTENTS is in effect (e.g. with -w).
 +	 */
 +	if (DIFF_OPT_TST(options, DIFF_FROM_CONTENTS)) {
 +		if (options->found_changes)
 +			DIFF_OPT_SET(options, HAS_CHANGES);
 +		else
 +			DIFF_OPT_CLR(options, HAS_CHANGES);
 +	}
  }
  
  static void diffcore_apply_filter(const char *filter)
@@@ -3678,6 -3628,23 +3678,23 @@@ static void diffcore_skip_stat_unmatch(
  	*q = outq;
  }
  
+ static int diffnamecmp(const void *a_, const void *b_)
+ {
+ 	const struct diff_filepair *a = *((const struct diff_filepair **)a_);
+ 	const struct diff_filepair *b = *((const struct diff_filepair **)b_);
+ 	const char *name_a, *name_b;
+ 
+ 	name_a = a->one ? a->one->path : a->two->path;
+ 	name_b = b->one ? b->one->path : b->two->path;
+ 	return strcmp(name_a, name_b);
+ }
+ 
+ void diffcore_fix_diff_index(struct diff_options *options)
+ {
+ 	struct diff_queue_struct *q = &diff_queued_diff;
+ 	qsort(q->queue, q->nr, sizeof(q->queue[0]), diffnamecmp);
+ }
+ 
  void diffcore_std(struct diff_options *options)
  {
  	if (options->skip_stat_unmatch)
@@@ -3695,7 -3662,7 +3712,7 @@@
  	diff_resolve_rename_copy();
  	diffcore_apply_filter(options->filter);
  
 -	if (diff_queued_diff.nr)
 +	if (diff_queued_diff.nr && !DIFF_OPT_TST(options, DIFF_FROM_CONTENTS))
  		DIFF_OPT_SET(options, HAS_CHANGES);
  	else
  		DIFF_OPT_CLR(options, HAS_CHANGES);
@@@ -3719,7 -3686,7 +3736,7 @@@ int diff_result_code(struct diff_option
  void diff_addremove(struct diff_options *options,
  		    int addremove, unsigned mode,
  		    const unsigned char *sha1,
 -		    const char *concatpath)
 +		    const char *concatpath, unsigned dirty_submodule)
  {
  	struct diff_filespec *one, *two;
  
@@@ -3751,22 -3718,18 +3768,22 @@@
  
  	if (addremove != '+')
  		fill_filespec(one, sha1, mode);
 -	if (addremove != '-')
 +	if (addremove != '-') {
  		fill_filespec(two, sha1, mode);
 +		two->dirty_submodule = dirty_submodule;
 +	}
  
  	diff_queue(&diff_queued_diff, one, two);
 -	DIFF_OPT_SET(options, HAS_CHANGES);
 +	if (!DIFF_OPT_TST(options, DIFF_FROM_CONTENTS))
 +		DIFF_OPT_SET(options, HAS_CHANGES);
  }
  
  void diff_change(struct diff_options *options,
  		 unsigned old_mode, unsigned new_mode,
  		 const unsigned char *old_sha1,
  		 const unsigned char *new_sha1,
 -		 const char *concatpath)
 +		 const char *concatpath,
 +		 unsigned old_dirty_submodule, unsigned new_dirty_submodule)
  {
  	struct diff_filespec *one, *two;
  
@@@ -3779,8 -3742,6 +3796,8 @@@
  		const unsigned char *tmp_c;
  		tmp = old_mode; old_mode = new_mode; new_mode = tmp;
  		tmp_c = old_sha1; old_sha1 = new_sha1; new_sha1 = tmp_c;
 +		tmp = old_dirty_submodule; old_dirty_submodule = new_dirty_submodule;
 +			new_dirty_submodule = tmp;
  	}
  
  	if (options->prefix &&
@@@ -3791,12 -3752,9 +3808,12 @@@
  	two = alloc_filespec(concatpath);
  	fill_filespec(one, old_sha1, old_mode);
  	fill_filespec(two, new_sha1, new_mode);
 +	one->dirty_submodule = old_dirty_submodule;
 +	two->dirty_submodule = new_dirty_submodule;
  
  	diff_queue(&diff_queued_diff, one, two);
 -	DIFF_OPT_SET(options, HAS_CHANGES);
 +	if (!DIFF_OPT_TST(options, DIFF_FROM_CONTENTS))
 +		DIFF_OPT_SET(options, HAS_CHANGES);
  }
  
  void diff_unmerge(struct diff_options *options,
@@@ -3830,19 -3788,16 +3847,19 @@@ static char *run_textconv(const char *p
  	*arg = NULL;
  
  	memset(&child, 0, sizeof(child));
 +	child.use_shell = 1;
  	child.argv = argv;
  	child.out = -1;
  	if (start_command(&child) != 0 ||
  	    strbuf_read(&buf, child.out, 0) < 0 ||
  	    finish_command(&child) != 0) {
 +		close(child.out);
  		strbuf_release(&buf);
  		remove_tempfile();
  		error("error running textconv command '%s'", pgm);
  		return NULL;
  	}
 +	close(child.out);
  	remove_tempfile();
  
  	return strbuf_detach(&buf, outsize);
diff --combined diff.h
index 968a8dce95,471f606a92..2ef3341fb0
--- a/diff.h
+++ b/diff.h
@@@ -14,13 -14,12 +14,13 @@@ typedef void (*change_fn_t)(struct diff
  		 unsigned old_mode, unsigned new_mode,
  		 const unsigned char *old_sha1,
  		 const unsigned char *new_sha1,
 -		 const char *fullpath);
 +		 const char *fullpath,
 +		 unsigned old_dirty_submodule, unsigned new_dirty_submodule);
  
  typedef void (*add_remove_fn_t)(struct diff_options *options,
  		    int addremove, unsigned mode,
  		    const unsigned char *sha1,
 -		    const char *fullpath);
 +		    const char *fullpath, unsigned dirty_submodule);
  
  typedef void (*diff_format_fn_t)(struct diff_queue_struct *q,
  		struct diff_options *options, void *data);
@@@ -56,7 -55,7 +56,7 @@@
  #define DIFF_OPT_COLOR_DIFF          (1 <<  8)
  #define DIFF_OPT_COLOR_DIFF_WORDS    (1 <<  9)
  #define DIFF_OPT_HAS_CHANGES         (1 << 10)
 -#define DIFF_OPT_QUIET               (1 << 11)
 +#define DIFF_OPT_QUICK               (1 << 11)
  #define DIFF_OPT_NO_INDEX            (1 << 12)
  #define DIFF_OPT_ALLOW_EXTERNAL      (1 << 13)
  #define DIFF_OPT_EXIT_WITH_STATUS    (1 << 14)
@@@ -67,7 -66,7 +67,7 @@@
  #define DIFF_OPT_DIRSTAT_CUMULATIVE  (1 << 19)
  #define DIFF_OPT_DIRSTAT_BY_FILE     (1 << 20)
  #define DIFF_OPT_ALLOW_TEXTCONV      (1 << 21)
 -
 +#define DIFF_OPT_DIFF_FROM_CONTENTS  (1 << 22)
  #define DIFF_OPT_SUBMODULE_LOG       (1 << 23)
  
  #define DIFF_OPT_TST(opts, flag)    ((opts)->flags & DIFF_OPT_##flag)
@@@ -178,14 -177,13 +178,14 @@@ extern void diff_addremove(struct diff_
  			   int addremove,
  			   unsigned mode,
  			   const unsigned char *sha1,
 -			   const char *fullpath);
 +			   const char *fullpath, unsigned dirty_submodule);
  
  extern void diff_change(struct diff_options *,
  			unsigned mode1, unsigned mode2,
  			const unsigned char *sha1,
  			const unsigned char *sha2,
 -			const char *fullpath);
 +			const char *fullpath,
 +			unsigned dirty_submodule1, unsigned dirty_submodule2);
  
  extern void diff_unmerge(struct diff_options *,
  			 const char *path,
@@@ -210,6 -208,7 +210,7 @@@ extern int diff_setup_done(struct diff_
  #define DIFF_PICKAXE_REGEX	2
  
  extern void diffcore_std(struct diff_options *);
+ extern void diffcore_fix_diff_index(struct diff_options *);
  
  #define COMMON_DIFF_OPTIONS_HELP \
  "\ncommon diff options:\n" \
diff --combined t/test-lib.sh
index c1476f9a23,05efe1faa3..baa4093073
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@@ -74,6 -74,12 +74,12 @@@ case $(echo $GIT_TRACE |tr "[A-Z]" "[a-
  		;;
  esac
  
+ # Convenience
+ #
+ # A regexp to match 5 and 40 hexdigits
+ _x05='[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]'
+ _x40="$_x05$_x05$_x05$_x05$_x05$_x05$_x05$_x05"
+ 
  # Each test should start with something like this, after copyright notices:
  #
  # test_description='Description of this test...
@@@ -105,8 -111,6 +111,8 @@@ d
  		verbose=t; shift ;;
  	-q|--q|--qu|--qui|--quie|--quiet)
  		quiet=t; shift ;;
 +	--with-dashes)
 +		with_dashes=t; shift ;;
  	--no-color)
  		color=; shift ;;
  	--no-python)
@@@ -213,17 -217,6 +219,17 @@@ test_set_editor () 
  	export EDITOR
  }
  
 +test_decode_color () {
 +	sed	-e 's/.\[1m/<WHITE>/g' \
 +		-e 's/.\[31m/<RED>/g' \
 +		-e 's/.\[32m/<GREEN>/g' \
 +		-e 's/.\[33m/<YELLOW>/g' \
 +		-e 's/.\[34m/<BLUE>/g' \
 +		-e 's/.\[35m/<MAGENTA>/g' \
 +		-e 's/.\[36m/<CYAN>/g' \
 +		-e 's/.\[m/<RESET>/g'
 +}
 +
  test_tick () {
  	if test -z "${test_tick+set}"
  	then
@@@ -564,8 -557,19 +570,8 @@@ test_done () 
  # Test the binaries we have just built.  The tests are kept in
  # t/ subdirectory and are run in 'trash directory' subdirectory.
  TEST_DIRECTORY=$(pwd)
 -if test -z "$valgrind"
 +if test -n "$valgrind"
  then
 -	if test -z "$GIT_TEST_INSTALLED"
 -	then
 -		PATH=$TEST_DIRECTORY/..:$PATH
 -		GIT_EXEC_PATH=$TEST_DIRECTORY/..
 -	else
 -		GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path)  ||
 -		error "Cannot run git from $GIT_TEST_INSTALLED."
 -		PATH=$GIT_TEST_INSTALLED:$TEST_DIRECTORY/..:$PATH
 -		GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH}
 -	fi
 -else
  	make_symlink () {
  		test -h "$2" &&
  		test "$1" = "$(readlink "$2")" || {
@@@ -627,24 -631,6 +633,24 @@@
  	PATH=$GIT_VALGRIND/bin:$PATH
  	GIT_EXEC_PATH=$GIT_VALGRIND/bin
  	export GIT_VALGRIND
 +elif test -n "$GIT_TEST_INSTALLED" ; then
 +	GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path)  ||
 +	error "Cannot run git from $GIT_TEST_INSTALLED."
 +	PATH=$GIT_TEST_INSTALLED:$TEST_DIRECTORY/..:$PATH
 +	GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH}
 +else # normal case, use ../bin-wrappers only unless $with_dashes:
 +	git_bin_dir="$TEST_DIRECTORY/../bin-wrappers"
 +	if ! test -x "$git_bin_dir/git" ; then
 +		if test -z "$with_dashes" ; then
 +			say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH"
 +		fi
 +		with_dashes=t
 +	fi
 +	PATH="$git_bin_dir:$PATH"
 +	GIT_EXEC_PATH=$TEST_DIRECTORY/..
 +	if test -n "$with_dashes" ; then
 +		PATH="$TEST_DIRECTORY/..:$PATH"
 +	fi
  fi
  GIT_TEMPLATE_DIR=$(pwd)/../templates/blt
  unset GIT_CONFIG
@@@ -652,29 -638,20 +658,29 @@@ GIT_CONFIG_NOSYSTEM=
  GIT_CONFIG_NOGLOBAL=1
  export PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR GIT_CONFIG_NOSYSTEM GIT_CONFIG_NOGLOBAL
  
 +. ../GIT-BUILD-OPTIONS
 +
  GITPERLLIB=$(pwd)/../perl/blib/lib:$(pwd)/../perl/blib/arch/auto/Git
  export GITPERLLIB
  test -d ../templates/blt || {
  	error "You haven't built things yet, have you?"
  }
  
 +if test -z "$GIT_TEST_INSTALLED" && test -z "$NO_PYTHON"
 +then
 +	GITPYTHONLIB="$(pwd)/../git_remote_helpers/build/lib"
 +	export GITPYTHONLIB
 +	test -d ../git_remote_helpers/build || {
 +		error "You haven't built git_remote_helpers yet, have you?"
 +	}
 +fi
 +
  if ! test -x ../test-chmtime; then
  	echo >&2 'You need to build test-chmtime:'
  	echo >&2 'Run "make test-chmtime" in the source (toplevel) directory'
  	exit 1
  fi
  
 -. ../GIT-BUILD-OPTIONS
 -
  # Test repository
  test="trash directory.$(basename "$0" .sh)"
  test -n "$root" && test="$root/$test"
@@@ -758,7 -735,6 +764,7 @@@ case $(uname -s) i
  esac
  
  test -z "$NO_PERL" && test_set_prereq PERL
 +test -z "$NO_PYTHON" && test_set_prereq PYTHON
  
  # test whether the filesystem supports symbolic links
  ln -s x y 2>/dev/null && test -h y 2>/dev/null && test_set_prereq SYMLINKS
diff --combined unpack-trees.c
index 0ddbef3e63,10e8871a74..75f54cac97
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@@ -32,12 -32,6 +32,12 @@@ static struct unpack_trees_error_msgs u
  
  	/* bind_overlap */
  	"Entry '%s' overlaps with '%s'.  Cannot bind.",
 +
 +	/* sparse_not_uptodate_file */
 +	"Entry '%s' not uptodate. Cannot update sparse checkout.",
 +
 +	/* would_lose_orphaned */
 +	"Working tree file '%s' would be %s by sparse checkout update.",
  };
  
  #define ERRORMSG(o,fld) \
@@@ -67,16 -61,8 +67,16 @@@ static void unlink_entry(struct cache_e
  {
  	if (has_symlink_or_noent_leading_path(ce->name, ce_namelen(ce)))
  		return;
 -	if (unlink_or_warn(ce->name))
 -		return;
 +	if (S_ISGITLINK(ce->ce_mode)) {
 +		if (rmdir(ce->name)) {
 +			warning("unable to rmdir %s: %s",
 +				ce->name, strerror(errno));
 +			return;
 +		}
 +	}
 +	else
 +		if (unlink_or_warn(ce->name))
 +			return;
  	schedule_dir_for_removal(ce->name, ce_namelen(ce));
  }
  
@@@ -92,7 -78,7 +92,7 @@@ static int check_updates(struct unpack_
  	if (o->update && o->verbose_update) {
  		for (total = cnt = 0; cnt < index->cache_nr; cnt++) {
  			struct cache_entry *ce = index->cache[cnt];
 -			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE))
 +			if (ce->ce_flags & (CE_UPDATE | CE_REMOVE | CE_WT_REMOVE))
  				total++;
  		}
  
@@@ -106,13 -92,6 +106,13 @@@
  	for (i = 0; i < index->cache_nr; i++) {
  		struct cache_entry *ce = index->cache[i];
  
 +		if (ce->ce_flags & CE_WT_REMOVE) {
 +			display_progress(progress, ++cnt);
 +			if (o->update)
 +				unlink_entry(ce);
 +			continue;
 +		}
 +
  		if (ce->ce_flags & CE_REMOVE) {
  			display_progress(progress, ++cnt);
  			if (o->update)
@@@ -139,57 -118,6 +139,57 @@@
  	return errs != 0;
  }
  
 +static int verify_uptodate_sparse(struct cache_entry *ce, struct unpack_trees_options *o);
 +static int verify_absent_sparse(struct cache_entry *ce, const char *action, struct unpack_trees_options *o);
 +
 +static int will_have_skip_worktree(const struct cache_entry *ce, struct unpack_trees_options *o)
 +{
 +	const char *basename;
 +
 +	if (ce_stage(ce))
 +		return 0;
 +
 +	basename = strrchr(ce->name, '/');
 +	basename = basename ? basename+1 : ce->name;
 +	return excluded_from_list(ce->name, ce_namelen(ce), basename, NULL, o->el) <= 0;
 +}
 +
 +static int apply_sparse_checkout(struct cache_entry *ce, struct unpack_trees_options *o)
 +{
 +	int was_skip_worktree = ce_skip_worktree(ce);
 +
 +	if (will_have_skip_worktree(ce, o))
 +		ce->ce_flags |= CE_SKIP_WORKTREE;
 +	else
 +		ce->ce_flags &= ~CE_SKIP_WORKTREE;
 +
 +	/*
 +	 * We only care about files getting into the checkout area
 +	 * If merge strategies want to remove some, go ahead, this
 +	 * flag will be removed eventually in unpack_trees() if it's
 +	 * outside checkout area.
 +	 */
 +	if (ce->ce_flags & CE_REMOVE)
 +		return 0;
 +
 +	if (!was_skip_worktree && ce_skip_worktree(ce)) {
 +		/*
 +		 * If CE_UPDATE is set, verify_uptodate() must be called already
 +		 * also stat info may have lost after merged_entry() so calling
 +		 * verify_uptodate() again may fail
 +		 */
 +		if (!(ce->ce_flags & CE_UPDATE) && verify_uptodate_sparse(ce, o))
 +			return -1;
 +		ce->ce_flags |= CE_WT_REMOVE;
 +	}
 +	if (was_skip_worktree && !ce_skip_worktree(ce)) {
 +		if (verify_absent_sparse(ce, "overwritten", o))
 +			return -1;
 +		ce->ce_flags |= CE_UPDATE;
 +	}
 +	return 0;
 +}
 +
  static inline int call_unpack_fn(struct cache_entry **src, struct unpack_trees_options *o)
  {
  	int ret = o->fn(src, o);
@@@ -198,23 -126,142 +198,142 @@@
  	return ret;
  }
  
- static int unpack_index_entry(struct cache_entry *ce, struct unpack_trees_options *o)
+ static void mark_ce_used(struct cache_entry *ce, struct unpack_trees_options *o)
+ {
+ 	ce->ce_flags |= CE_UNPACKED;
+ 
+ 	if (o->cache_bottom < o->src_index->cache_nr &&
+ 	    o->src_index->cache[o->cache_bottom] == ce) {
+ 		int bottom = o->cache_bottom;
+ 		while (bottom < o->src_index->cache_nr &&
+ 		       o->src_index->cache[bottom]->ce_flags & CE_UNPACKED)
+ 			bottom++;
+ 		o->cache_bottom = bottom;
+ 	}
+ }
+ 
+ static void mark_all_ce_unused(struct index_state *index)
+ {
+ 	int i;
+ 	for (i = 0; i < index->cache_nr; i++)
+ 		index->cache[i]->ce_flags &= ~CE_UNPACKED;
+ }
+ 
+ static int locate_in_src_index(struct cache_entry *ce,
+ 			       struct unpack_trees_options *o)
+ {
+ 	struct index_state *index = o->src_index;
+ 	int len = ce_namelen(ce);
+ 	int pos = index_name_pos(index, ce->name, len);
+ 	if (pos < 0)
+ 		pos = -1 - pos;
+ 	return pos;
+ }
+ 
+ /*
+  * We call unpack_index_entry() with an unmerged cache entry
+  * only in diff-index, and it wants a single callback.  Skip
+  * the other unmerged entry with the same name.
+  */
+ static void mark_ce_used_same_name(struct cache_entry *ce,
+ 				   struct unpack_trees_options *o)
+ {
+ 	struct index_state *index = o->src_index;
+ 	int len = ce_namelen(ce);
+ 	int pos;
+ 
+ 	for (pos = locate_in_src_index(ce, o); pos < index->cache_nr; pos++) {
+ 		struct cache_entry *next = index->cache[pos];
+ 		if (len != ce_namelen(next) ||
+ 		    memcmp(ce->name, next->name, len))
+ 			break;
+ 		mark_ce_used(next, o);
+ 	}
+ }
+ 
+ static struct cache_entry *next_cache_entry(struct unpack_trees_options *o)
+ {
+ 	const struct index_state *index = o->src_index;
+ 	int pos = o->cache_bottom;
+ 
+ 	while (pos < index->cache_nr) {
+ 		struct cache_entry *ce = index->cache[pos];
+ 		if (!(ce->ce_flags & CE_UNPACKED))
+ 			return ce;
+ 		pos++;
+ 	}
+ 	return NULL;
+ }
+ 
+ static void add_same_unmerged(struct cache_entry *ce,
+ 			      struct unpack_trees_options *o)
+ {
+ 	struct index_state *index = o->src_index;
+ 	int len = ce_namelen(ce);
+ 	int pos = index_name_pos(index, ce->name, len);
+ 
+ 	if (0 <= pos)
+ 		die("programming error in a caller of mark_ce_used_same_name");
+ 	for (pos = -pos - 1; pos < index->cache_nr; pos++) {
+ 		struct cache_entry *next = index->cache[pos];
+ 		if (len != ce_namelen(next) ||
+ 		    memcmp(ce->name, next->name, len))
+ 			break;
+ 		add_entry(o, next, 0, 0);
+ 		mark_ce_used(next, o);
+ 	}
+ }
+ 
+ static int unpack_index_entry(struct cache_entry *ce,
+ 			      struct unpack_trees_options *o)
  {
  	struct cache_entry *src[5] = { ce, NULL, };
+ 	int ret;
  
- 	o->pos++;
+ 	mark_ce_used(ce, o);
  	if (ce_stage(ce)) {
  		if (o->skip_unmerged) {
  			add_entry(o, ce, 0, 0);
  			return 0;
  		}
  	}
- 	return call_unpack_fn(src, o);
+ 	ret = call_unpack_fn(src, o);
+ 	if (ce_stage(ce))
+ 		mark_ce_used_same_name(ce, o);
+ 	return ret;
+ }
+ 
+ static int find_cache_pos(struct traverse_info *, const struct name_entry *);
+ 
+ static void restore_cache_bottom(struct traverse_info *info, int bottom)
+ {
+ 	struct unpack_trees_options *o = info->data;
+ 
+ 	if (o->diff_index_cached)
+ 		return;
+ 	o->cache_bottom = bottom;
+ }
+ 
+ static int switch_cache_bottom(struct traverse_info *info)
+ {
+ 	struct unpack_trees_options *o = info->data;
+ 	int ret, pos;
+ 
+ 	if (o->diff_index_cached)
+ 		return 0;
+ 	ret = o->cache_bottom;
+ 	pos = find_cache_pos(info->prev, &info->name);
+ 
+ 	if (pos < -1)
+ 		o->cache_bottom = -2 - pos;
+ 	else if (pos < 0)
+ 		o->cache_bottom = o->src_index->cache_nr;
+ 	return ret;
  }
  
  static int traverse_trees_recursive(int n, unsigned long dirmask, unsigned long df_conflicts, struct name_entry *names, struct traverse_info *info)
  {
- 	int i;
+ 	int i, ret, bottom;
  	struct tree_desc t[MAX_UNPACK_TREES];
  	struct traverse_info newinfo;
  	struct name_entry *p;
@@@ -235,7 -282,11 +354,11 @@@
  			sha1 = names[i].sha1;
  		fill_tree_descriptor(t+i, sha1);
  	}
- 	return traverse_trees(n, t, &newinfo);
+ 
+ 	bottom = switch_cache_bottom(&newinfo);
+ 	ret = traverse_trees(n, t, &newinfo);
+ 	restore_cache_bottom(&newinfo, bottom);
+ 	return ret;
  }
  
  /*
@@@ -284,6 -335,20 +407,20 @@@ static int compare_entry(const struct c
  	return ce_namelen(ce) > traverse_path_len(info, n);
  }
  
+ static int ce_in_traverse_path(const struct cache_entry *ce,
+ 			       const struct traverse_info *info)
+ {
+ 	if (!info->prev)
+ 		return 1;
+ 	if (do_compare_entry(ce, info->prev, &info->name))
+ 		return 0;
+ 	/*
+ 	 * If ce (blob) is the same name as the path (which is a tree
+ 	 * we will be descending into), it won't be inside it.
+ 	 */
+ 	return (info->pathlen < ce_namelen(ce));
+ }
+ 
  static struct cache_entry *create_ce_entry(const struct traverse_info *info, const struct name_entry *n, int stage)
  {
  	int len = traverse_path_len(info, n);
@@@ -360,6 -425,114 +497,114 @@@ static int unpack_failed(struct unpack_
  	return -1;
  }
  
+ /* NEEDSWORK: give this a better name and share with tree-walk.c */
+ static int name_compare(const char *a, int a_len,
+ 			const char *b, int b_len)
+ {
+ 	int len = (a_len < b_len) ? a_len : b_len;
+ 	int cmp = memcmp(a, b, len);
+ 	if (cmp)
+ 		return cmp;
+ 	return (a_len - b_len);
+ }
+ 
+ /*
+  * The tree traversal is looking at name p.  If we have a matching entry,
+  * return it.  If name p is a directory in the index, do not return
+  * anything, as we will want to match it when the traversal descends into
+  * the directory.
+  */
+ static int find_cache_pos(struct traverse_info *info,
+ 			  const struct name_entry *p)
+ {
+ 	int pos;
+ 	struct unpack_trees_options *o = info->data;
+ 	struct index_state *index = o->src_index;
+ 	int pfxlen = info->pathlen;
+ 	int p_len = tree_entry_len(p->path, p->sha1);
+ 
+ 	for (pos = o->cache_bottom; pos < index->cache_nr; pos++) {
+ 		struct cache_entry *ce = index->cache[pos];
+ 		const char *ce_name, *ce_slash;
+ 		int cmp, ce_len;
+ 
+ 		if (!ce_in_traverse_path(ce, info))
+ 			continue;
+ 		if (ce->ce_flags & CE_UNPACKED)
+ 			continue;
+ 		ce_name = ce->name + pfxlen;
+ 		ce_slash = strchr(ce_name, '/');
+ 		if (ce_slash)
+ 			ce_len = ce_slash - ce_name;
+ 		else
+ 			ce_len = ce_namelen(ce) - pfxlen;
+ 		cmp = name_compare(p->path, p_len, ce_name, ce_len);
+ 		/*
+ 		 * Exact match; if we have a directory we need to
+ 		 * delay returning it.
+ 		 */
+ 		if (!cmp)
+ 			return ce_slash ? -2 - pos : pos;
+ 		if (0 < cmp)
+ 			continue; /* keep looking */
+ 		/*
+ 		 * ce_name sorts after p->path; could it be that we
+ 		 * have files under p->path directory in the index?
+ 		 * E.g.  ce_name == "t-i", and p->path == "t"; we may
+ 		 * have "t/a" in the index.
+ 		 */
+ 		if (p_len < ce_len && !memcmp(ce_name, p->path, p_len) &&
+ 		    ce_name[p_len] < '/')
+ 			continue; /* keep looking */
+ 		break;
+ 	}
+ 	return -1;
+ }
+ 
+ static struct cache_entry *find_cache_entry(struct traverse_info *info,
+ 					    const struct name_entry *p)
+ {
+ 	int pos = find_cache_pos(info, p);
+ 	struct unpack_trees_options *o = info->data;
+ 
+ 	if (0 <= pos)
+ 		return o->src_index->cache[pos];
+ 	else
+ 		return NULL;
+ }
+ 
+ static void debug_path(struct traverse_info *info)
+ {
+ 	if (info->prev) {
+ 		debug_path(info->prev);
+ 		if (*info->prev->name.path)
+ 			putchar('/');
+ 	}
+ 	printf("%s", info->name.path);
+ }
+ 
+ static void debug_name_entry(int i, struct name_entry *n)
+ {
+ 	printf("ent#%d %06o %s\n", i,
+ 	       n->path ? n->mode : 0,
+ 	       n->path ? n->path : "(missing)");
+ }
+ 
+ static void debug_unpack_callback(int n,
+ 				  unsigned long mask,
+ 				  unsigned long dirmask,
+ 				  struct name_entry *names,
+ 				  struct traverse_info *info)
+ {
+ 	int i;
+ 	printf("* unpack mask %lu, dirmask %lu, cnt %d ",
+ 	       mask, dirmask, n);
+ 	debug_path(info);
+ 	putchar('\n');
+ 	for (i = 0; i < n; i++)
+ 		debug_name_entry(i, names + i);
+ }
+ 
  static int unpack_callback(int n, unsigned long mask, unsigned long dirmask, struct name_entry *names, struct traverse_info *info)
  {
  	struct cache_entry *src[MAX_UNPACK_TREES + 1] = { NULL, };
@@@ -370,25 -543,38 +615,38 @@@
  	while (!p->mode)
  		p++;
  
+ 	if (o->debug_unpack)
+ 		debug_unpack_callback(n, mask, dirmask, names, info);
+ 
  	/* Are we supposed to look at the index too? */
  	if (o->merge) {
- 		while (o->pos < o->src_index->cache_nr) {
- 			struct cache_entry *ce = o->src_index->cache[o->pos];
- 			int cmp = compare_entry(ce, info, p);
+ 		while (1) {
+ 			int cmp;
+ 			struct cache_entry *ce;
+ 
+ 			if (o->diff_index_cached)
+ 				ce = next_cache_entry(o);
+ 			else
+ 				ce = find_cache_entry(info, p);
+ 
+ 			if (!ce)
+ 				break;
+ 			cmp = compare_entry(ce, info, p);
  			if (cmp < 0) {
  				if (unpack_index_entry(ce, o) < 0)
  					return unpack_failed(o, NULL);
  				continue;
  			}
  			if (!cmp) {
- 				o->pos++;
  				if (ce_stage(ce)) {
  					/*
- 					 * If we skip unmerged index entries, we'll skip this
- 					 * entry *and* the tree entries associated with it!
+ 					 * If we skip unmerged index
+ 					 * entries, we'll skip this
+ 					 * entry *and* the tree
+ 					 * entries associated with it!
  					 */
  					if (o->skip_unmerged) {
- 						add_entry(o, ce, 0, 0);
+ 						add_same_unmerged(ce, o);
  						return mask;
  					}
  				}
@@@ -401,6 -587,13 +659,13 @@@
  	if (unpack_nondirectories(n, mask, dirmask, src, names, info) < 0)
  		return -1;
  
+ 	if (src[0]) {
+ 		if (ce_stage(src[0]))
+ 			mark_ce_used_same_name(src[0], o);
+ 		else
+ 			mark_ce_used(src[0], o);
+ 	}
+ 
  	/* Now handle any directories.. */
  	if (dirmask) {
  		unsigned long conflicts = mask & ~dirmask;
@@@ -417,11 -610,13 +682,13 @@@
  			matches = cache_tree_matches_traversal(o->src_index->cache_tree,
  							       names, info);
  			/*
- 			 * Everything under the name matches.  Adjust o->pos to
- 			 * skip the entire hierarchy.
+ 			 * Everything under the name matches; skip the
+ 			 * entire hierarchy.  diff_index_cached codepath
+ 			 * special cases D/F conflicts in such a way that
+ 			 * it does not do any look-ahead, so this is safe.
  			 */
  			if (matches) {
- 				o->pos += matches;
+ 				o->cache_bottom += matches;
  				return mask;
  			}
  		}
@@@ -441,9 -636,8 +708,9 @@@
   */
  int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options *o)
  {
 -	int ret;
 +	int i, ret;
  	static struct cache_entry *dfc;
 +	struct exclude_list el;
  
  	if (len > MAX_UNPACK_TREES)
  		die("unpack_trees takes at most %d trees", MAX_UNPACK_TREES);
@@@ -453,23 -647,12 +720,22 @@@
  	state.quiet = 1;
  	state.refresh_cache = 1;
  
 +	memset(&el, 0, sizeof(el));
 +	if (!core_apply_sparse_checkout || !o->update)
 +		o->skip_sparse_checkout = 1;
 +	if (!o->skip_sparse_checkout) {
 +		if (add_excludes_from_file_to_list(git_path("info/sparse-checkout"), "", 0, NULL, &el, 0) < 0)
 +			o->skip_sparse_checkout = 1;
 +		else
 +			o->el = &el;
 +	}
 +
  	memset(&o->result, 0, sizeof(o->result));
  	o->result.initialized = 1;
- 	if (o->src_index) {
- 		o->result.timestamp.sec = o->src_index->timestamp.sec;
- 		o->result.timestamp.nsec = o->src_index->timestamp.nsec;
- 	}
+ 	o->result.timestamp.sec = o->src_index->timestamp.sec;
+ 	o->result.timestamp.nsec = o->src_index->timestamp.nsec;
  	o->merge_size = len;
+ 	mark_all_ce_unused(o->src_index);
  
  	if (!dfc)
  		dfc = xcalloc(1, cache_entry_size(0));
@@@ -483,66 -666,51 +749,87 @@@
  		info.fn = unpack_callback;
  		info.data = o;
  
- 		if (traverse_trees(len, t, &info) < 0) {
- 			ret = unpack_failed(o, NULL);
- 			goto done;
+ 		if (o->prefix) {
+ 			/*
+ 			 * Unpack existing index entries that sort before the
+ 			 * prefix the tree is spliced into.  Note that o->merge
+ 			 * is always true in this case.
+ 			 */
+ 			while (1) {
+ 				struct cache_entry *ce = next_cache_entry(o);
+ 				if (!ce)
+ 					break;
+ 				if (ce_in_traverse_path(ce, &info))
+ 					break;
+ 				if (unpack_index_entry(ce, o) < 0)
+ 					goto return_failed;
+ 			}
  		}
+ 
+ 		if (traverse_trees(len, t, &info) < 0)
+ 			goto return_failed;
  	}
  
  	/* Any left-over entries in the index? */
  	if (o->merge) {
- 		while (o->pos < o->src_index->cache_nr) {
- 			struct cache_entry *ce = o->src_index->cache[o->pos];
- 			if (unpack_index_entry(ce, o) < 0) {
- 				ret = unpack_failed(o, NULL);
- 				goto done;
- 			}
+ 		while (1) {
+ 			struct cache_entry *ce = next_cache_entry(o);
+ 			if (!ce)
+ 				break;
+ 			if (unpack_index_entry(ce, o) < 0)
+ 				goto return_failed;
  		}
  	}
+ 	mark_all_ce_unused(o->src_index);
  
 -	if (o->trivial_merges_only && o->nontrivial_merge)
 -		return unpack_failed(o, "Merge requires file-level merging");
 +	if (o->trivial_merges_only && o->nontrivial_merge) {
 +		ret = unpack_failed(o, "Merge requires file-level merging");
 +		goto done;
 +	}
 +
 +	if (!o->skip_sparse_checkout) {
 +		int empty_worktree = 1;
 +		for (i = 0;i < o->result.cache_nr;i++) {
 +			struct cache_entry *ce = o->result.cache[i];
 +
 +			if (apply_sparse_checkout(ce, o)) {
 +				ret = -1;
 +				goto done;
 +			}
 +			/*
 +			 * Merge strategies may set CE_UPDATE|CE_REMOVE outside checkout
 +			 * area as a result of ce_skip_worktree() shortcuts in
 +			 * verify_absent() and verify_uptodate(). Clear them.
 +			 */
 +			if (ce_skip_worktree(ce))
 +				ce->ce_flags &= ~(CE_UPDATE | CE_REMOVE);
 +			else
 +				empty_worktree = 0;
 +
 +		}
 +		if (o->result.cache_nr && empty_worktree) {
 +			ret = unpack_failed(o, "Sparse checkout leaves no entry on working directory");
 +			goto done;
 +		}
 +	}
  
  	o->src_index = NULL;
  	ret = check_updates(o) ? (-2) : 0;
  	if (o->dst_index)
  		*o->dst_index = o->result;
 +
 +done:
 +	for (i = 0;i < el.nr;i++)
 +		free(el.excludes[i]);
 +	if (el.excludes)
 +		free(el.excludes);
 +
  	return ret;
+ 
+ return_failed:
+ 	mark_all_ce_unused(o->src_index);
 -	return unpack_failed(o, NULL);
++	ret = unpack_failed(o, NULL);
++	goto done;
  }
  
  /* Here come the merge functions */
@@@ -558,8 -726,6 +845,8 @@@ static int same(struct cache_entry *a, 
  		return 0;
  	if (!a && !b)
  		return 1;
 +	if ((a->ce_flags | b->ce_flags) & CE_CONFLICTED)
 +		return 0;
  	return a->ce_mode == b->ce_mode &&
  	       !hashcmp(a->sha1, b->sha1);
  }
@@@ -569,17 -735,16 +856,17 @@@
   * When a CE gets turned into an unmerged entry, we
   * want it to be up-to-date
   */
 -static int verify_uptodate(struct cache_entry *ce,
 -		struct unpack_trees_options *o)
 +static int verify_uptodate_1(struct cache_entry *ce,
 +				   struct unpack_trees_options *o,
 +				   const char *error_msg)
  {
  	struct stat st;
  
 -	if (o->index_only || o->reset || ce_uptodate(ce))
 +	if (o->index_only || (!ce_skip_worktree(ce) && (o->reset || ce_uptodate(ce))))
  		return 0;
  
  	if (!lstat(ce->name, &st)) {
 -		unsigned changed = ie_match_stat(o->src_index, ce, &st, CE_MATCH_IGNORE_VALID);
 +		unsigned changed = ie_match_stat(o->src_index, ce, &st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE);
  		if (!changed)
  			return 0;
  		/*
@@@ -596,21 -761,7 +883,21 @@@
  	if (errno == ENOENT)
  		return 0;
  	return o->gently ? -1 :
 -		error(ERRORMSG(o, not_uptodate_file), ce->name);
 +		error(error_msg, ce->name);
 +}
 +
 +static int verify_uptodate(struct cache_entry *ce,
 +			   struct unpack_trees_options *o)
 +{
 +	if (!o->skip_sparse_checkout && will_have_skip_worktree(ce, o))
 +		return 0;
 +	return verify_uptodate_1(ce, o, ERRORMSG(o, not_uptodate_file));
 +}
 +
 +static int verify_uptodate_sparse(struct cache_entry *ce,
 +				  struct unpack_trees_options *o)
 +{
 +	return verify_uptodate_1(ce, o, ERRORMSG(o, sparse_not_uptodate_file));
  }
  
  static void invalidate_ce_path(struct cache_entry *ce, struct unpack_trees_options *o)
@@@ -661,7 -812,9 +948,9 @@@ static int verify_clean_subdirectory(st
  	 * in that directory.
  	 */
  	namelen = strlen(ce->name);
- 	for (i = o->pos; i < o->src_index->cache_nr; i++) {
+ 	for (i = locate_in_src_index(ce, o);
+ 	     i < o->src_index->cache_nr;
+ 	     i++) {
  		struct cache_entry *ce2 = o->src_index->cache[i];
  		int len = ce_namelen(ce2);
  		if (len < namelen ||
@@@ -669,12 -822,14 +958,14 @@@
  		    ce2->name[namelen] != '/')
  			break;
  		/*
- 		 * ce2->name is an entry in the subdirectory.
+ 		 * ce2->name is an entry in the subdirectory to be
+ 		 * removed.
  		 */
  		if (!ce_stage(ce2)) {
  			if (verify_uptodate(ce2, o))
  				return -1;
  			add_entry(o, ce2, CE_REMOVE, 0);
+ 			mark_ce_used(ce2, o);
  		}
  		cnt++;
  	}
@@@ -711,16 -866,15 +1002,16 @@@ static int icase_exists(struct unpack_t
  	struct cache_entry *src;
  
  	src = index_name_exists(o->src_index, dst->name, ce_namelen(dst), 1);
 -	return src && !ie_match_stat(o->src_index, src, st, CE_MATCH_IGNORE_VALID);
 +	return src && !ie_match_stat(o->src_index, src, st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE);
  }
  
  /*
   * We do not want to remove or overwrite a working tree file that
   * is not tracked, unless it is ignored.
   */
 -static int verify_absent(struct cache_entry *ce, const char *action,
 -			 struct unpack_trees_options *o)
 +static int verify_absent_1(struct cache_entry *ce, const char *action,
 +				 struct unpack_trees_options *o,
 +				 const char *error_msg)
  {
  	struct stat st;
  
@@@ -731,7 -885,6 +1022,6 @@@
  		return 0;
  
  	if (!lstat(ce->name, &st)) {
- 		int ret;
  		int dtype = ce_to_dtype(ce);
  		struct cache_entry *result;
  
@@@ -759,28 -912,8 +1049,8 @@@
  			 * files that are in "foo/" we would lose
  			 * them.
  			 */
- 			ret = verify_clean_subdirectory(ce, action, o);
- 			if (ret < 0)
- 				return ret;
- 
- 			/*
- 			 * If this removed entries from the index,
- 			 * what that means is:
- 			 *
- 			 * (1) the caller unpack_callback() saw path/foo
- 			 * in the index, and it has not removed it because
- 			 * it thinks it is handling 'path' as blob with
- 			 * D/F conflict;
- 			 * (2) we will return "ok, we placed a merged entry
- 			 * in the index" which would cause o->pos to be
- 			 * incremented by one;
- 			 * (3) however, original o->pos now has 'path/foo'
- 			 * marked with "to be removed".
- 			 *
- 			 * We need to increment it by the number of
- 			 * deleted entries here.
- 			 */
- 			o->pos += ret;
+ 			if (verify_clean_subdirectory(ce, action, o) < 0)
+ 				return -1;
  			return 0;
  		}
  
@@@ -800,30 -933,13 +1070,30 @@@
  	}
  	return 0;
  }
 +static int verify_absent(struct cache_entry *ce, const char *action,
 +			 struct unpack_trees_options *o)
 +{
 +	if (!o->skip_sparse_checkout && will_have_skip_worktree(ce, o))
 +		return 0;
 +	return verify_absent_1(ce, action, o, ERRORMSG(o, would_lose_untracked));
 +}
 +
 +static int verify_absent_sparse(struct cache_entry *ce, const char *action,
 +			 struct unpack_trees_options *o)
 +{
 +	return verify_absent_1(ce, action, o, ERRORMSG(o, would_lose_orphaned));
 +}
  
  static int merged_entry(struct cache_entry *merge, struct cache_entry *old,
  		struct unpack_trees_options *o)
  {
  	int update = CE_UPDATE;
  
 -	if (old) {
 +	if (!old) {
 +		if (verify_absent(merge, "overwritten", o))
 +			return -1;
 +		invalidate_ce_path(merge, o);
 +	} else if (!(old->ce_flags & CE_CONFLICTED)) {
  		/*
  		 * See if we can re-use the old CE directly?
  		 * That way we get the uptodate stat info.
@@@ -837,16 -953,13 +1107,16 @@@
  		} else {
  			if (verify_uptodate(old, o))
  				return -1;
 +			if (ce_skip_worktree(old))
 +				update |= CE_SKIP_WORKTREE;
  			invalidate_ce_path(old, o);
  		}
 -	}
 -	else {
 -		if (verify_absent(merge, "overwritten", o))
 -			return -1;
 -		invalidate_ce_path(merge, o);
 +	} else {
 +		/*
 +		 * Previously unmerged entry left as an existence
 +		 * marker by read_index_unmerged();
 +		 */
 +		invalidate_ce_path(old, o);
  	}
  
  	add_entry(o, merge, update, CE_STAGEMASK);
@@@ -862,7 -975,7 +1132,7 @@@ static int deleted_entry(struct cache_e
  			return -1;
  		return 0;
  	}
 -	if (verify_uptodate(old, o))
 +	if (!(old->ce_flags & CE_CONFLICTED) && verify_uptodate(old, o))
  		return -1;
  	add_entry(o, ce, CE_REMOVE, 0);
  	invalidate_ce_path(ce, o);
@@@ -927,7 -1040,8 +1197,8 @@@ int threeway_merge(struct cache_entry *
  		remote = NULL;
  	}
  
- 	/* First, if there's a #16 situation, note that to prevent #13
+ 	/*
+ 	 * First, if there's a #16 situation, note that to prevent #13
  	 * and #14.
  	 */
  	if (!same(remote, head)) {
@@@ -941,7 -1055,8 +1212,8 @@@
  		}
  	}
  
- 	/* We start with cases where the index is allowed to match
+ 	/*
+ 	 * We start with cases where the index is allowed to match
  	 * something other than the head: #14(ALT) and #2ALT, where it
  	 * is permitted to match the result instead.
  	 */
@@@ -971,12 -1086,13 +1243,13 @@@
  	if (!head && !remote && any_anc_missing)
  		return 0;
  
- 	/* Under the new "aggressive" rule, we resolve mostly trivial
+ 	/*
+ 	 * Under the "aggressive" rule, we resolve mostly trivial
  	 * cases that we historically had git-merge-one-file resolve.
  	 */
  	if (o->aggressive) {
- 		int head_deleted = !head && !df_conflict_head;
- 		int remote_deleted = !remote && !df_conflict_remote;
+ 		int head_deleted = !head;
+ 		int remote_deleted = !remote;
  		struct cache_entry *ce = NULL;
  
  		if (index)
@@@ -1164,10 -1280,10 +1437,10 @@@ int oneway_merge(struct cache_entry **s
  
  	if (old && same(old, a)) {
  		int update = 0;
 -		if (o->reset && !ce_uptodate(old)) {
 +		if (o->reset && !ce_uptodate(old) && !ce_skip_worktree(old)) {
  			struct stat st;
  			if (lstat(old->name, &st) ||
 -			    ie_match_stat(o->src_index, old, &st, CE_MATCH_IGNORE_VALID))
 +			    ie_match_stat(o->src_index, old, &st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE))
  				update |= CE_UPDATE;
  		}
  		add_entry(o, old, update, 0);
diff --combined unpack-trees.h
index 95ff36c824,701dca59a8..ef70eab390
--- a/unpack-trees.h
+++ b/unpack-trees.h
@@@ -4,7 -4,6 +4,7 @@@
  #define MAX_UNPACK_TREES 8
  
  struct unpack_trees_options;
 +struct exclude_list;
  
  typedef int (*merge_fn_t)(struct cache_entry **src,
  		struct unpack_trees_options *options);
@@@ -15,8 -14,6 +15,8 @@@ struct unpack_trees_error_msgs 
  	const char *not_uptodate_dir;
  	const char *would_lose_untracked;
  	const char *bind_overlap;
 +	const char *sparse_not_uptodate_file;
 +	const char *would_lose_orphaned;
  };
  
  struct unpack_trees_options {
@@@ -31,10 -28,10 +31,11 @@@
  		     skip_unmerged,
  		     initial_checkout,
  		     diff_index_cached,
+ 		     debug_unpack,
 +		     skip_sparse_checkout,
  		     gently;
  	const char *prefix;
- 	int pos;
+ 	int cache_bottom;
  	struct dir_struct *dir;
  	merge_fn_t fn;
  	struct unpack_trees_error_msgs msgs;
@@@ -48,8 -45,6 +49,8 @@@
  	struct index_state *dst_index;
  	struct index_state *src_index;
  	struct index_state result;
 +
 +	struct exclude_list *el; /* for internal use */
  };
  
  extern int unpack_trees(unsigned n, struct tree_desc *t,