From: Junio C Hamano <gitster@pobox.com>
Date: Mon, 25 Sep 2017 06:24:06 +0000 (+0900)
Subject: Merge branch 'jk/write-in-full-fix'
X-Git-Tag: v2.15.0-rc0~69
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/c50424a6f07f17ff9b06927606df650cd73a09a3?hp=-c

Merge branch 'jk/write-in-full-fix'

Many codepaths did not diagnose write failures correctly when disks
go full, due to their misuse of write_in_full() helper function,
which have been corrected.

* jk/write-in-full-fix:
read_pack_header: handle signed/unsigned comparison in read result
config: flip return value of store_write_*()
notes-merge: use ssize_t for write_in_full() return value
pkt-line: check write_in_full() errors against "< 0"
convert less-trivial versions of "write_in_full() != len"
avoid "write_in_full(fd, buf, len) != len" pattern
get-tar-commit-id: check write_in_full() return against 0
config: avoid "write_in_full(fd, buf, len) < len" pattern
---

c50424a6f07f17ff9b06927606df650cd73a09a3
diff --combined builtin/receive-pack.c
index 52c63ebfdc,01dea59f58..dd06b3fb4f
--- a/builtin/receive-pack.c
+++ b/builtin/receive-pack.c
@@@ -23,7 -23,6 +23,7 @@@
  #include "fsck.h"
  #include "tmp-objdir.h"
  #include "oidset.h"
 +#include "packfile.h"
  
  static const char * const receive_pack_usage[] = {
  	N_("git receive-pack <git-dir>"),
@@@ -743,7 -742,7 +743,7 @@@ static int run_and_feed_hook(const cha
  		size_t n;
  		if (feed(feed_state, &buf, &n))
  			break;
- 		if (write_in_full(proc.in, buf, n) != n)
+ 		if (write_in_full(proc.in, buf, n) < 0)
  			break;
  	}
  	close(proc.in);
@@@ -920,9 -919,9 +920,9 @@@ static int update_shallow_ref(struct co
   */
  static int head_has_history(void)
  {
 -	unsigned char sha1[20];
 +	struct object_id oid;
  
 -	return !get_sha1("HEAD", sha1);
 +	return !get_oid("HEAD", &oid);
  }
  
  static const char *push_to_deploy(unsigned char *sha1,
@@@ -1139,7 -1138,7 +1139,7 @@@ static const char *update(struct comman
  		}
  		if (ref_transaction_delete(transaction,
  					   namespaced_name,
 -					   old_oid->hash,
 +					   old_oid ? old_oid->hash : NULL,
  					   0, "push", &err)) {
  			rp_error("%s", err.buf);
  			strbuf_release(&err);
diff --combined builtin/unpack-file.c
index 281ca1db6c,672a54fcdf..32e0155577
--- a/builtin/unpack-file.c
+++ b/builtin/unpack-file.c
@@@ -1,7 -1,7 +1,7 @@@
  #include "builtin.h"
  #include "config.h"
  
 -static char *create_temp_file(unsigned char *sha1)
 +static char *create_temp_file(struct object_id *oid)
  {
  	static char path[50];
  	void *buf;
@@@ -9,13 -9,13 +9,13 @@@
  	unsigned long size;
  	int fd;
  
 -	buf = read_sha1_file(sha1, &type, &size);
 +	buf = read_sha1_file(oid->hash, &type, &size);
  	if (!buf || type != OBJ_BLOB)
 -		die("unable to read blob object %s", sha1_to_hex(sha1));
 +		die("unable to read blob object %s", oid_to_hex(oid));
  
  	xsnprintf(path, sizeof(path), ".merge_file_XXXXXX");
  	fd = xmkstemp(path);
- 	if (write_in_full(fd, buf, size) != size)
+ 	if (write_in_full(fd, buf, size) < 0)
  		die_errno("unable to write temp-file");
  	close(fd);
  	return path;
@@@ -23,15 -23,15 +23,15 @@@
  
  int cmd_unpack_file(int argc, const char **argv, const char *prefix)
  {
 -	unsigned char sha1[20];
 +	struct object_id oid;
  
  	if (argc != 2 || !strcmp(argv[1], "-h"))
  		usage("git unpack-file <sha1>");
 -	if (get_sha1(argv[1], sha1))
 +	if (get_oid(argv[1], &oid))
  		die("Not a valid object name %s", argv[1]);
  
  	git_config(git_default_config, NULL);
  
 -	puts(create_temp_file(sha1));
 +	puts(create_temp_file(&oid));
  	return 0;
  }
diff --combined config.c
index cd5a69e630,d70fe71273..a77cae56ed
--- a/config.c
+++ b/config.c
@@@ -929,7 -929,7 +929,7 @@@ ssize_t git_config_ssize_t(const char *
  	return ret;
  }
  
 -int git_parse_maybe_bool(const char *value)
 +static int git_parse_maybe_bool_text(const char *value)
  {
  	if (!value)
  		return 1;
@@@ -946,9 -946,9 +946,9 @@@
  	return -1;
  }
  
 -int git_config_maybe_bool(const char *name, const char *value)
 +int git_parse_maybe_bool(const char *value)
  {
 -	int v = git_parse_maybe_bool(value);
 +	int v = git_parse_maybe_bool_text(value);
  	if (0 <= v)
  		return v;
  	if (git_parse_int(value, &v))
@@@ -958,7 -958,7 +958,7 @@@
  
  int git_config_bool_or_int(const char *name, const char *value, int *is_bool)
  {
 -	int v = git_parse_maybe_bool(value);
 +	int v = git_parse_maybe_bool_text(value);
  	if (0 <= v) {
  		*is_bool = 1;
  		return v;
@@@ -1464,9 -1464,9 +1464,9 @@@ int git_config_from_mem(config_fn_t fn
  	return do_config_from(&top, fn, data);
  }
  
 -int git_config_from_blob_sha1(config_fn_t fn,
 +int git_config_from_blob_oid(config_fn_t fn,
  			      const char *name,
 -			      const unsigned char *sha1,
 +			      const struct object_id *oid,
  			      void *data)
  {
  	enum object_type type;
@@@ -1474,7 -1474,7 +1474,7 @@@
  	unsigned long size;
  	int ret;
  
 -	buf = read_sha1_file(sha1, &type, &size);
 +	buf = read_sha1_file(oid->hash, &type, &size);
  	if (!buf)
  		return error("unable to load config blob object '%s'", name);
  	if (type != OBJ_BLOB) {
@@@ -1492,11 -1492,11 +1492,11 @@@ static int git_config_from_blob_ref(con
  				    const char *name,
  				    void *data)
  {
 -	unsigned char sha1[20];
 +	struct object_id oid;
  
 -	if (get_sha1(name, sha1) < 0)
 +	if (get_oid(name, &oid) < 0)
  		return error("unable to resolve config blob '%s'", name);
 -	return git_config_from_blob_sha1(fn, name, sha1, data);
 +	return git_config_from_blob_oid(fn, name, &oid, data);
  }
  
  const char *git_etc_gitconfig(void)
@@@ -1719,19 -1719,17 +1719,19 @@@ static int configset_add_value(struct c
  }
  
  static int config_set_element_cmp(const void *unused_cmp_data,
 -				  const struct config_set_element *e1,
 -				  const struct config_set_element *e2,
 +				  const void *entry,
 +				  const void *entry_or_key,
  				  const void *unused_keydata)
  {
 +	const struct config_set_element *e1 = entry;
 +	const struct config_set_element *e2 = entry_or_key;
 +
  	return strcmp(e1->key, e2->key);
  }
  
  void git_configset_init(struct config_set *cs)
  {
 -	hashmap_init(&cs->config_hash, (hashmap_cmp_fn)config_set_element_cmp,
 -		     NULL, 0);
 +	hashmap_init(&cs->config_hash, config_set_element_cmp, NULL, 0);
  	cs->hash_initialized = 1;
  	cs->list.nr = 0;
  	cs->list.alloc = 0;
@@@ -1852,7 -1850,7 +1852,7 @@@ int git_configset_get_maybe_bool(struc
  {
  	const char *value;
  	if (!git_configset_get_value(cs, key, &value)) {
 -		*dest = git_config_maybe_bool(key, value);
 +		*dest = git_parse_maybe_bool(value);
  		if (*dest == -1)
  			return -1;
  		return 0;
@@@ -2059,23 -2057,6 +2059,23 @@@ int git_config_get_pathname(const char 
  	return repo_config_get_pathname(the_repository, key, dest);
  }
  
 +/*
 + * Note: This function exists solely to maintain backward compatibility with
 + * 'fetch' and 'update_clone' storing configuration in '.gitmodules' and should
 + * NOT be used anywhere else.
 + *
 + * Runs the provided config function on the '.gitmodules' file found in the
 + * working directory.
 + */
 +void config_from_gitmodules(config_fn_t fn, void *data)
 +{
 +	if (the_repository->worktree) {
 +		char *file = repo_worktree_path(the_repository, GITMODULES_FILE);
 +		git_config_from_file(fn, file, data);
 +		free(file);
 +	}
 +}
 +
  int git_config_get_expiry(const char *key, const char **output)
  {
  	int ret = git_config_get_string_const(key, output);
@@@ -2089,28 -2070,6 +2089,28 @@@
  	return ret;
  }
  
 +int git_config_get_expiry_in_days(const char *key, timestamp_t *expiry, timestamp_t now)
 +{
 +	char *expiry_string;
 +	intmax_t days;
 +	timestamp_t when;
 +
 +	if (git_config_get_string(key, &expiry_string))
 +		return 1; /* no such thing */
 +
 +	if (git_parse_signed(expiry_string, &days, maximum_signed_value_of_type(int))) {
 +		const int scale = 86400;
 +		*expiry = now - days * scale;
 +		return 0;
 +	}
 +
 +	if (!parse_expiry_date(expiry_string, &when)) {
 +		*expiry = when;
 +		return 0;
 +	}
 +	return -1; /* thing exists but cannot be parsed */
 +}
 +
  int git_config_get_untracked_cache(void)
  {
  	int val = -1;
@@@ -2292,10 -2251,11 +2292,11 @@@ static int write_error(const char *file
  	return 4;
  }
  
- static int store_write_section(int fd, const char *key)
+ static ssize_t write_section(int fd, const char *key)
  {
  	const char *dot;
- 	int i, success;
+ 	int i;
+ 	ssize_t ret;
  	struct strbuf sb = STRBUF_INIT;
  
  	dot = memchr(key, '.', store.baselen);
@@@ -2311,15 -2271,16 +2312,16 @@@
  		strbuf_addf(&sb, "[%.*s]\n", store.baselen, key);
  	}
  
- 	success = write_in_full(fd, sb.buf, sb.len) == sb.len;
+ 	ret = write_in_full(fd, sb.buf, sb.len);
  	strbuf_release(&sb);
  
- 	return success;
+ 	return ret;
  }
  
- static int store_write_pair(int fd, const char *key, const char *value)
+ static ssize_t write_pair(int fd, const char *key, const char *value)
  {
- 	int i, success;
+ 	int i;
+ 	ssize_t ret;
  	int length = strlen(key + store.baselen + 1);
  	const char *quote = "";
  	struct strbuf sb = STRBUF_INIT;
@@@ -2359,10 -2320,10 +2361,10 @@@
  		}
  	strbuf_addf(&sb, "%s\n", quote);
  
- 	success = write_in_full(fd, sb.buf, sb.len) == sb.len;
+ 	ret = write_in_full(fd, sb.buf, sb.len);
  	strbuf_release(&sb);
  
- 	return success;
+ 	return ret;
  }
  
  static ssize_t find_beginning_of_line(const char *contents, size_t size,
@@@ -2445,7 -2406,7 +2447,7 @@@ int git_config_set_multivar_in_file_gen
  {
  	int fd = -1, in_fd = -1;
  	int ret;
 -	struct lock_file *lock = NULL;
 +	struct lock_file lock = LOCK_INIT;
  	char *filename_buf = NULL;
  	char *contents = NULL;
  	size_t contents_sz;
@@@ -2464,7 -2425,8 +2466,7 @@@
  	 * The lock serves a purpose in addition to locking: the new
  	 * contents of .git/config will be written into it.
  	 */
 -	lock = xcalloc(1, sizeof(struct lock_file));
 -	fd = hold_lock_file_for_update(lock, config_filename, 0);
 +	fd = hold_lock_file_for_update(&lock, config_filename, 0);
  	if (fd < 0) {
  		error_errno("could not lock config file %s", config_filename);
  		free(store.key);
@@@ -2491,8 -2453,8 +2493,8 @@@
  		}
  
  		store.key = (char *)key;
- 		if (!store_write_section(fd, key) ||
- 		    !store_write_pair(fd, key, value))
+ 		if (write_section(fd, key) < 0 ||
+ 		    write_pair(fd, key, value) < 0)
  			goto write_err_out;
  	} else {
  		struct stat st;
@@@ -2577,8 -2539,8 +2579,8 @@@
  		close(in_fd);
  		in_fd = -1;
  
 -		if (chmod(get_lock_file_path(lock), st.st_mode & 07777) < 0) {
 -			error_errno("chmod on %s failed", get_lock_file_path(lock));
 +		if (chmod(get_lock_file_path(&lock), st.st_mode & 07777) < 0) {
 +			error_errno("chmod on %s failed", get_lock_file_path(&lock));
  			ret = CONFIG_NO_WRITE;
  			goto out_free;
  		}
@@@ -2602,11 -2564,10 +2604,10 @@@
  			/* write the first part of the config */
  			if (copy_end > copy_begin) {
  				if (write_in_full(fd, contents + copy_begin,
- 						  copy_end - copy_begin) <
- 				    copy_end - copy_begin)
+ 						  copy_end - copy_begin) < 0)
  					goto write_err_out;
  				if (new_line &&
- 				    write_str_in_full(fd, "\n") != 1)
+ 				    write_str_in_full(fd, "\n") < 0)
  					goto write_err_out;
  			}
  			copy_begin = store.offset[i];
@@@ -2615,37 -2576,45 +2616,36 @@@
  		/* write the pair (value == NULL means unset) */
  		if (value != NULL) {
  			if (store.state == START) {
- 				if (!store_write_section(fd, key))
+ 				if (write_section(fd, key) < 0)
  					goto write_err_out;
  			}
- 			if (!store_write_pair(fd, key, value))
+ 			if (write_pair(fd, key, value) < 0)
  				goto write_err_out;
  		}
  
  		/* write the rest of the config */
  		if (copy_begin < contents_sz)
  			if (write_in_full(fd, contents + copy_begin,
- 					  contents_sz - copy_begin) <
- 			    contents_sz - copy_begin)
+ 					  contents_sz - copy_begin) < 0)
  				goto write_err_out;
  
  		munmap(contents, contents_sz);
  		contents = NULL;
  	}
  
 -	if (commit_lock_file(lock) < 0) {
 +	if (commit_lock_file(&lock) < 0) {
  		error_errno("could not write config file %s", config_filename);
  		ret = CONFIG_NO_WRITE;
 -		lock = NULL;
  		goto out_free;
  	}
  
 -	/*
 -	 * lock is committed, so don't try to roll it back below.
 -	 * NOTE: Since lockfile.c keeps a linked list of all created
 -	 * lock_file structures, it isn't safe to free(lock).  It's
 -	 * better to just leave it hanging around.
 -	 */
 -	lock = NULL;
  	ret = 0;
  
  	/* Invalidate the config cache */
  	git_config_clear();
  
  out_free:
 -	if (lock)
 -		rollback_lock_file(lock);
 +	rollback_lock_file(&lock);
  	free(filename_buf);
  	if (contents)
  		munmap(contents, contents_sz);
@@@ -2654,7 -2623,7 +2654,7 @@@
  	return ret;
  
  write_err_out:
 -	ret = write_error(get_lock_file_path(lock));
 +	ret = write_error(get_lock_file_path(&lock));
  	goto out_free;
  
  }
@@@ -2803,7 -2772,7 +2803,7 @@@ int git_config_rename_section_in_file(c
  					continue;
  				}
  				store.baselen = strlen(new_name);
- 				if (!store_write_section(out_fd, new_name)) {
+ 				if (write_section(out_fd, new_name) < 0) {
  					ret = write_error(get_lock_file_path(lock));
  					goto out;
  				}
@@@ -2829,7 -2798,7 +2829,7 @@@
  		if (remove)
  			continue;
  		length = strlen(output);
- 		if (write_in_full(out_fd, output, length) != length) {
+ 		if (write_in_full(out_fd, output, length) < 0) {
  			ret = write_error(get_lock_file_path(lock));
  			goto out;
  		}
diff --combined diff.c
index ea7e5978bc,6f58bca2f9..3c6a3e0faa
--- a/diff.c
+++ b/diff.c
@@@ -16,12 -16,10 +16,12 @@@
  #include "userdiff.h"
  #include "submodule-config.h"
  #include "submodule.h"
 +#include "hashmap.h"
  #include "ll-merge.h"
  #include "string-list.h"
  #include "argv-array.h"
  #include "graph.h"
 +#include "packfile.h"
  
  #ifdef NO_FAST_WORKING_DIRECTORY
  #define FAST_WORKING_DIRECTORY 0
@@@ -34,7 -32,6 +34,7 @@@ static int diff_indent_heuristic = 1
  static int diff_rename_limit_default = 400;
  static int diff_suppress_blank_empty;
  static int diff_use_color_default = -1;
 +static int diff_color_moved_default;
  static int diff_context_default = 3;
  static int diff_interhunk_context_default;
  static const char *diff_word_regex_cfg;
@@@ -59,14 -56,6 +59,14 @@@ static char diff_colors[][COLOR_MAXLEN
  	GIT_COLOR_YELLOW,	/* COMMIT */
  	GIT_COLOR_BG_RED,	/* WHITESPACE */
  	GIT_COLOR_NORMAL,	/* FUNCINFO */
 +	GIT_COLOR_BOLD_MAGENTA,	/* OLD_MOVED */
 +	GIT_COLOR_BOLD_BLUE,	/* OLD_MOVED ALTERNATIVE */
 +	GIT_COLOR_FAINT,	/* OLD_MOVED_DIM */
 +	GIT_COLOR_FAINT_ITALIC,	/* OLD_MOVED_ALTERNATIVE_DIM */
 +	GIT_COLOR_BOLD_CYAN,	/* NEW_MOVED */
 +	GIT_COLOR_BOLD_YELLOW,	/* NEW_MOVED ALTERNATIVE */
 +	GIT_COLOR_FAINT,	/* NEW_MOVED_DIM */
 +	GIT_COLOR_FAINT_ITALIC,	/* NEW_MOVED_ALTERNATIVE_DIM */
  };
  
  static NORETURN void die_want_option(const char *option_name)
@@@ -92,22 -81,6 +92,22 @@@ static int parse_diff_color_slot(const 
  		return DIFF_WHITESPACE;
  	if (!strcasecmp(var, "func"))
  		return DIFF_FUNCINFO;
 +	if (!strcasecmp(var, "oldmoved"))
 +		return DIFF_FILE_OLD_MOVED;
 +	if (!strcasecmp(var, "oldmovedalternative"))
 +		return DIFF_FILE_OLD_MOVED_ALT;
 +	if (!strcasecmp(var, "oldmoveddimmed"))
 +		return DIFF_FILE_OLD_MOVED_DIM;
 +	if (!strcasecmp(var, "oldmovedalternativedimmed"))
 +		return DIFF_FILE_OLD_MOVED_ALT_DIM;
 +	if (!strcasecmp(var, "newmoved"))
 +		return DIFF_FILE_NEW_MOVED;
 +	if (!strcasecmp(var, "newmovedalternative"))
 +		return DIFF_FILE_NEW_MOVED_ALT;
 +	if (!strcasecmp(var, "newmoveddimmed"))
 +		return DIFF_FILE_NEW_MOVED_DIM;
 +	if (!strcasecmp(var, "newmovedalternativedimmed"))
 +		return DIFF_FILE_NEW_MOVED_ALT_DIM;
  	return -1;
  }
  
@@@ -256,44 -229,12 +256,44 @@@ int git_diff_heuristic_config(const cha
  	return 0;
  }
  
 +static int parse_color_moved(const char *arg)
 +{
 +	switch (git_parse_maybe_bool(arg)) {
 +	case 0:
 +		return COLOR_MOVED_NO;
 +	case 1:
 +		return COLOR_MOVED_DEFAULT;
 +	default:
 +		break;
 +	}
 +
 +	if (!strcmp(arg, "no"))
 +		return COLOR_MOVED_NO;
 +	else if (!strcmp(arg, "plain"))
 +		return COLOR_MOVED_PLAIN;
 +	else if (!strcmp(arg, "zebra"))
 +		return COLOR_MOVED_ZEBRA;
 +	else if (!strcmp(arg, "default"))
 +		return COLOR_MOVED_DEFAULT;
 +	else if (!strcmp(arg, "dimmed_zebra"))
 +		return COLOR_MOVED_ZEBRA_DIM;
 +	else
 +		return error(_("color moved setting must be one of 'no', 'default', 'zebra', 'dimmed_zebra', 'plain'"));
 +}
 +
  int git_diff_ui_config(const char *var, const char *value, void *cb)
  {
  	if (!strcmp(var, "diff.color") || !strcmp(var, "color.diff")) {
  		diff_use_color_default = git_config_colorbool(var, value);
  		return 0;
  	}
 +	if (!strcmp(var, "diff.colormoved")) {
 +		int cm = parse_color_moved(value);
 +		if (cm < 0)
 +			return -1;
 +		diff_color_moved_default = cm;
 +		return 0;
 +	}
  	if (!strcmp(var, "diff.context")) {
  		diff_context_default = git_config_int(var, value);
  		if (diff_context_default < 0)
@@@ -402,6 -343,9 +402,6 @@@ int git_diff_basic_config(const char *v
  		return 0;
  	}
  
 -	if (starts_with(var, "submodule."))
 -		return parse_submodule_config_option(var, value);
 -
  	if (git_diff_heuristic_config(var, value, cb) < 0)
  		return -1;
  
@@@ -459,9 -403,11 +459,9 @@@ static struct diff_tempfile 
  	 * If this diff_tempfile instance refers to a temporary file,
  	 * this tempfile object is used to manage its lifetime.
  	 */
 -	struct tempfile tempfile;
 +	struct tempfile *tempfile;
  } diff_temp[2];
  
 -typedef unsigned long (*sane_truncate_fn)(char *line, unsigned long len);
 -
  struct emit_callback {
  	int color_diff;
  	unsigned ws_rule;
@@@ -469,6 -415,7 +469,6 @@@
  	int blank_at_eof_in_postimage;
  	int lno_in_preimage;
  	int lno_in_postimage;
 -	sane_truncate_fn truncate;
  	const char **label_path;
  	struct diff_words_data *diff_words;
  	struct diff_options *opt;
@@@ -610,735 -557,68 +610,735 @@@ static void emit_line(struct diff_optio
  	emit_line_0(o, set, reset, line[0], line+1, len-1);
  }
  
 -static int new_blank_line_at_eof(struct emit_callback *ecbdata, const char *line, int len)
 +enum diff_symbol {
 +	DIFF_SYMBOL_BINARY_DIFF_HEADER,
 +	DIFF_SYMBOL_BINARY_DIFF_HEADER_DELTA,
 +	DIFF_SYMBOL_BINARY_DIFF_HEADER_LITERAL,
 +	DIFF_SYMBOL_BINARY_DIFF_BODY,
 +	DIFF_SYMBOL_BINARY_DIFF_FOOTER,
 +	DIFF_SYMBOL_STATS_SUMMARY_NO_FILES,
 +	DIFF_SYMBOL_STATS_SUMMARY_ABBREV,
 +	DIFF_SYMBOL_STATS_SUMMARY_INSERTS_DELETES,
 +	DIFF_SYMBOL_STATS_LINE,
 +	DIFF_SYMBOL_WORD_DIFF,
 +	DIFF_SYMBOL_STAT_SEP,
 +	DIFF_SYMBOL_SUMMARY,
 +	DIFF_SYMBOL_SUBMODULE_ADD,
 +	DIFF_SYMBOL_SUBMODULE_DEL,
 +	DIFF_SYMBOL_SUBMODULE_UNTRACKED,
 +	DIFF_SYMBOL_SUBMODULE_MODIFIED,
 +	DIFF_SYMBOL_SUBMODULE_HEADER,
 +	DIFF_SYMBOL_SUBMODULE_ERROR,
 +	DIFF_SYMBOL_SUBMODULE_PIPETHROUGH,
 +	DIFF_SYMBOL_REWRITE_DIFF,
 +	DIFF_SYMBOL_BINARY_FILES,
 +	DIFF_SYMBOL_HEADER,
 +	DIFF_SYMBOL_FILEPAIR_PLUS,
 +	DIFF_SYMBOL_FILEPAIR_MINUS,
 +	DIFF_SYMBOL_WORDS_PORCELAIN,
 +	DIFF_SYMBOL_WORDS,
 +	DIFF_SYMBOL_CONTEXT,
 +	DIFF_SYMBOL_CONTEXT_INCOMPLETE,
 +	DIFF_SYMBOL_PLUS,
 +	DIFF_SYMBOL_MINUS,
 +	DIFF_SYMBOL_NO_LF_EOF,
 +	DIFF_SYMBOL_CONTEXT_FRAGINFO,
 +	DIFF_SYMBOL_CONTEXT_MARKER,
 +	DIFF_SYMBOL_SEPARATOR
 +};
 +/*
 + * Flags for content lines:
 + * 0..12 are whitespace rules
 + * 13-15 are WSEH_NEW | WSEH_OLD | WSEH_CONTEXT
 + * 16 is marking if the line is blank at EOF
 + */
 +#define DIFF_SYMBOL_CONTENT_BLANK_LINE_EOF	(1<<16)
 +#define DIFF_SYMBOL_MOVED_LINE			(1<<17)
 +#define DIFF_SYMBOL_MOVED_LINE_ALT		(1<<18)
 +#define DIFF_SYMBOL_MOVED_LINE_UNINTERESTING	(1<<19)
 +#define DIFF_SYMBOL_CONTENT_WS_MASK (WSEH_NEW | WSEH_OLD | WSEH_CONTEXT | WS_RULE_MASK)
 +
 +/*
 + * This struct is used when we need to buffer the output of the diff output.
 + *
 + * NEEDSWORK: Instead of storing a copy of the line, add an offset pointer
 + * into the pre/post image file. This pointer could be a union with the
 + * line pointer. By storing an offset into the file instead of the literal line,
 + * we can decrease the memory footprint for the buffered output. At first we
 + * may want to only have indirection for the content lines, but we could also
 + * enhance the state for emitting prefabricated lines, e.g. the similarity
 + * score line or hunk/file headers would only need to store a number or path
 + * and then the output can be constructed later on depending on state.
 + */
 +struct emitted_diff_symbol {
 +	const char *line;
 +	int len;
 +	int flags;
 +	enum diff_symbol s;
 +};
 +#define EMITTED_DIFF_SYMBOL_INIT {NULL}
 +
 +struct emitted_diff_symbols {
 +	struct emitted_diff_symbol *buf;
 +	int nr, alloc;
 +};
 +#define EMITTED_DIFF_SYMBOLS_INIT {NULL, 0, 0}
 +
 +static void append_emitted_diff_symbol(struct diff_options *o,
 +				       struct emitted_diff_symbol *e)
  {
 -	if (!((ecbdata->ws_rule & WS_BLANK_AT_EOF) &&
 -	      ecbdata->blank_at_eof_in_preimage &&
 -	      ecbdata->blank_at_eof_in_postimage &&
 -	      ecbdata->blank_at_eof_in_preimage <= ecbdata->lno_in_preimage &&
 -	      ecbdata->blank_at_eof_in_postimage <= ecbdata->lno_in_postimage))
 -		return 0;
 -	return ws_blank_line(line, len, ecbdata->ws_rule);
 +	struct emitted_diff_symbol *f;
 +
 +	ALLOC_GROW(o->emitted_symbols->buf,
 +		   o->emitted_symbols->nr + 1,
 +		   o->emitted_symbols->alloc);
 +	f = &o->emitted_symbols->buf[o->emitted_symbols->nr++];
 +
 +	memcpy(f, e, sizeof(struct emitted_diff_symbol));
 +	f->line = e->line ? xmemdupz(e->line, e->len) : NULL;
  }
  
 -static void emit_line_checked(const char *reset,
 -			      struct emit_callback *ecbdata,
 -			      const char *line, int len,
 -			      enum color_diff color,
 -			      unsigned ws_error_highlight,
 -			      char sign)
 +struct moved_entry {
 +	struct hashmap_entry ent;
 +	const struct emitted_diff_symbol *es;
 +	struct moved_entry *next_line;
 +};
 +
 +static int next_byte(const char **cp, const char **endp,
 +		     const struct diff_options *diffopt)
 +{
 +	int retval;
 +
 +	if (*cp > *endp)
 +		return -1;
 +
 +	if (DIFF_XDL_TST(diffopt, IGNORE_WHITESPACE_CHANGE)) {
 +		while (*cp < *endp && isspace(**cp))
 +			(*cp)++;
 +		/*
 +		 * After skipping a couple of whitespaces, we still have to
 +		 * account for one space.
 +		 */
 +		return (int)' ';
 +	}
 +
 +	if (DIFF_XDL_TST(diffopt, IGNORE_WHITESPACE)) {
 +		while (*cp < *endp && isspace(**cp))
 +			(*cp)++;
 +		/* return the first non-ws character via the usual below */
 +	}
 +
 +	retval = (unsigned char)(**cp);
 +	(*cp)++;
 +	return retval;
 +}
 +
 +static int moved_entry_cmp(const struct diff_options *diffopt,
 +			   const struct moved_entry *a,
 +			   const struct moved_entry *b,
 +			   const void *keydata)
 +{
 +	const char *ap = a->es->line, *ae = a->es->line + a->es->len;
 +	const char *bp = b->es->line, *be = b->es->line + b->es->len;
 +
 +	if (!(diffopt->xdl_opts & XDF_WHITESPACE_FLAGS))
 +		return a->es->len != b->es->len  || memcmp(ap, bp, a->es->len);
 +
 +	if (DIFF_XDL_TST(diffopt, IGNORE_WHITESPACE_AT_EOL)) {
 +		while (ae > ap && isspace(*ae))
 +			ae--;
 +		while (be > bp && isspace(*be))
 +			be--;
 +	}
 +
 +	while (1) {
 +		int ca, cb;
 +		ca = next_byte(&ap, &ae, diffopt);
 +		cb = next_byte(&bp, &be, diffopt);
 +		if (ca != cb)
 +			return 1;
 +		if (ca < 0)
 +			return 0;
 +	}
 +}
 +
 +static unsigned get_string_hash(struct emitted_diff_symbol *es, struct diff_options *o)
 +{
 +	if (o->xdl_opts & XDF_WHITESPACE_FLAGS) {
 +		static struct strbuf sb = STRBUF_INIT;
 +		const char *ap = es->line, *ae = es->line + es->len;
 +		int c;
 +
 +		strbuf_reset(&sb);
 +		while (ae > ap && isspace(*ae))
 +			ae--;
 +		while ((c = next_byte(&ap, &ae, o)) > 0)
 +			strbuf_addch(&sb, c);
 +
 +		return memhash(sb.buf, sb.len);
 +	} else {
 +		return memhash(es->line, es->len);
 +	}
 +}
 +
 +static struct moved_entry *prepare_entry(struct diff_options *o,
 +					 int line_no)
 +{
 +	struct moved_entry *ret = xmalloc(sizeof(*ret));
 +	struct emitted_diff_symbol *l = &o->emitted_symbols->buf[line_no];
 +
 +	ret->ent.hash = get_string_hash(l, o);
 +	ret->es = l;
 +	ret->next_line = NULL;
 +
 +	return ret;
 +}
 +
 +static void add_lines_to_move_detection(struct diff_options *o,
 +					struct hashmap *add_lines,
 +					struct hashmap *del_lines)
 +{
 +	struct moved_entry *prev_line = NULL;
 +
 +	int n;
 +	for (n = 0; n < o->emitted_symbols->nr; n++) {
 +		struct hashmap *hm;
 +		struct moved_entry *key;
 +
 +		switch (o->emitted_symbols->buf[n].s) {
 +		case DIFF_SYMBOL_PLUS:
 +			hm = add_lines;
 +			break;
 +		case DIFF_SYMBOL_MINUS:
 +			hm = del_lines;
 +			break;
 +		default:
 +			prev_line = NULL;
 +			continue;
 +		}
 +
 +		key = prepare_entry(o, n);
 +		if (prev_line && prev_line->es->s == o->emitted_symbols->buf[n].s)
 +			prev_line->next_line = key;
 +
 +		hashmap_add(hm, key);
 +		prev_line = key;
 +	}
 +}
 +
 +static int shrink_potential_moved_blocks(struct moved_entry **pmb,
 +					 int pmb_nr)
 +{
 +	int lp, rp;
 +
 +	/* Shrink the set of potential block to the remaining running */
 +	for (lp = 0, rp = pmb_nr - 1; lp <= rp;) {
 +		while (lp < pmb_nr && pmb[lp])
 +			lp++;
 +		/* lp points at the first NULL now */
 +
 +		while (rp > -1 && !pmb[rp])
 +			rp--;
 +		/* rp points at the last non-NULL */
 +
 +		if (lp < pmb_nr && rp > -1 && lp < rp) {
 +			pmb[lp] = pmb[rp];
 +			pmb[rp] = NULL;
 +			rp--;
 +			lp++;
 +		}
 +	}
 +
 +	/* Remember the number of running sets */
 +	return rp + 1;
 +}
 +
 +/*
 + * If o->color_moved is COLOR_MOVED_PLAIN, this function does nothing.
 + *
 + * Otherwise, if the last block has fewer alphanumeric characters than
 + * COLOR_MOVED_MIN_ALNUM_COUNT, unset DIFF_SYMBOL_MOVED_LINE on all lines in
 + * that block.
 + *
 + * The last block consists of the (n - block_length)'th line up to but not
 + * including the nth line.
 + *
 + * NEEDSWORK: This uses the same heuristic as blame_entry_score() in blame.c.
 + * Think of a way to unify them.
 + */
 +static void adjust_last_block(struct diff_options *o, int n, int block_length)
 +{
 +	int i, alnum_count = 0;
 +	if (o->color_moved == COLOR_MOVED_PLAIN)
 +		return;
 +	for (i = 1; i < block_length + 1; i++) {
 +		const char *c = o->emitted_symbols->buf[n - i].line;
 +		for (; *c; c++) {
 +			if (!isalnum(*c))
 +				continue;
 +			alnum_count++;
 +			if (alnum_count >= COLOR_MOVED_MIN_ALNUM_COUNT)
 +				return;
 +		}
 +	}
 +	for (i = 1; i < block_length + 1; i++)
 +		o->emitted_symbols->buf[n - i].flags &= ~DIFF_SYMBOL_MOVED_LINE;
 +}
 +
 +/* Find blocks of moved code, delegate actual coloring decision to helper */
 +static void mark_color_as_moved(struct diff_options *o,
 +				struct hashmap *add_lines,
 +				struct hashmap *del_lines)
 +{
 +	struct moved_entry **pmb = NULL; /* potentially moved blocks */
 +	int pmb_nr = 0, pmb_alloc = 0;
 +	int n, flipped_block = 1, block_length = 0;
 +
 +
 +	for (n = 0; n < o->emitted_symbols->nr; n++) {
 +		struct hashmap *hm = NULL;
 +		struct moved_entry *key;
 +		struct moved_entry *match = NULL;
 +		struct emitted_diff_symbol *l = &o->emitted_symbols->buf[n];
 +		int i;
 +
 +		switch (l->s) {
 +		case DIFF_SYMBOL_PLUS:
 +			hm = del_lines;
 +			key = prepare_entry(o, n);
 +			match = hashmap_get(hm, key, o);
 +			free(key);
 +			break;
 +		case DIFF_SYMBOL_MINUS:
 +			hm = add_lines;
 +			key = prepare_entry(o, n);
 +			match = hashmap_get(hm, key, o);
 +			free(key);
 +			break;
 +		default:
 +			flipped_block = 1;
 +		}
 +
 +		if (!match) {
 +			adjust_last_block(o, n, block_length);
 +			pmb_nr = 0;
 +			block_length = 0;
 +			continue;
 +		}
 +
 +		l->flags |= DIFF_SYMBOL_MOVED_LINE;
 +
 +		if (o->color_moved == COLOR_MOVED_PLAIN)
 +			continue;
 +
 +		/* Check any potential block runs, advance each or nullify */
 +		for (i = 0; i < pmb_nr; i++) {
 +			struct moved_entry *p = pmb[i];
 +			struct moved_entry *pnext = (p && p->next_line) ?
 +					p->next_line : NULL;
 +			if (pnext && !hm->cmpfn(o, pnext, match, NULL)) {
 +				pmb[i] = p->next_line;
 +			} else {
 +				pmb[i] = NULL;
 +			}
 +		}
 +
 +		pmb_nr = shrink_potential_moved_blocks(pmb, pmb_nr);
 +
 +		if (pmb_nr == 0) {
 +			/*
 +			 * The current line is the start of a new block.
 +			 * Setup the set of potential blocks.
 +			 */
 +			for (; match; match = hashmap_get_next(hm, match)) {
 +				ALLOC_GROW(pmb, pmb_nr + 1, pmb_alloc);
 +				pmb[pmb_nr++] = match;
 +			}
 +
 +			flipped_block = (flipped_block + 1) % 2;
 +
 +			adjust_last_block(o, n, block_length);
 +			block_length = 0;
 +		}
 +
 +		block_length++;
 +
 +		if (flipped_block)
 +			l->flags |= DIFF_SYMBOL_MOVED_LINE_ALT;
 +	}
 +	adjust_last_block(o, n, block_length);
 +
 +	free(pmb);
 +}
 +
 +#define DIFF_SYMBOL_MOVED_LINE_ZEBRA_MASK \
 +  (DIFF_SYMBOL_MOVED_LINE | DIFF_SYMBOL_MOVED_LINE_ALT)
 +static void dim_moved_lines(struct diff_options *o)
 +{
 +	int n;
 +	for (n = 0; n < o->emitted_symbols->nr; n++) {
 +		struct emitted_diff_symbol *prev = (n != 0) ?
 +				&o->emitted_symbols->buf[n - 1] : NULL;
 +		struct emitted_diff_symbol *l = &o->emitted_symbols->buf[n];
 +		struct emitted_diff_symbol *next =
 +				(n < o->emitted_symbols->nr - 1) ?
 +				&o->emitted_symbols->buf[n + 1] : NULL;
 +
 +		/* Not a plus or minus line? */
 +		if (l->s != DIFF_SYMBOL_PLUS && l->s != DIFF_SYMBOL_MINUS)
 +			continue;
 +
 +		/* Not a moved line? */
 +		if (!(l->flags & DIFF_SYMBOL_MOVED_LINE))
 +			continue;
 +
 +		/*
 +		 * If prev or next are not a plus or minus line,
 +		 * pretend they don't exist
 +		 */
 +		if (prev && prev->s != DIFF_SYMBOL_PLUS &&
 +			    prev->s != DIFF_SYMBOL_MINUS)
 +			prev = NULL;
 +		if (next && next->s != DIFF_SYMBOL_PLUS &&
 +			    next->s != DIFF_SYMBOL_MINUS)
 +			next = NULL;
 +
 +		/* Inside a block? */
 +		if ((prev &&
 +		    (prev->flags & DIFF_SYMBOL_MOVED_LINE_ZEBRA_MASK) ==
 +		    (l->flags & DIFF_SYMBOL_MOVED_LINE_ZEBRA_MASK)) &&
 +		    (next &&
 +		    (next->flags & DIFF_SYMBOL_MOVED_LINE_ZEBRA_MASK) ==
 +		    (l->flags & DIFF_SYMBOL_MOVED_LINE_ZEBRA_MASK))) {
 +			l->flags |= DIFF_SYMBOL_MOVED_LINE_UNINTERESTING;
 +			continue;
 +		}
 +
 +		/* Check if we are at an interesting bound: */
 +		if (prev && (prev->flags & DIFF_SYMBOL_MOVED_LINE) &&
 +		    (prev->flags & DIFF_SYMBOL_MOVED_LINE_ALT) !=
 +		       (l->flags & DIFF_SYMBOL_MOVED_LINE_ALT))
 +			continue;
 +		if (next && (next->flags & DIFF_SYMBOL_MOVED_LINE) &&
 +		    (next->flags & DIFF_SYMBOL_MOVED_LINE_ALT) !=
 +		       (l->flags & DIFF_SYMBOL_MOVED_LINE_ALT))
 +			continue;
 +
 +		/*
 +		 * The boundary to prev and next are not interesting,
 +		 * so this line is not interesting as a whole
 +		 */
 +		l->flags |= DIFF_SYMBOL_MOVED_LINE_UNINTERESTING;
 +	}
 +}
 +
 +static void emit_line_ws_markup(struct diff_options *o,
 +				const char *set, const char *reset,
 +				const char *line, int len, char sign,
 +				unsigned ws_rule, int blank_at_eof)
  {
 -	const char *set = diff_get_color(ecbdata->color_diff, color);
  	const char *ws = NULL;
  
 -	if (ecbdata->opt->ws_error_highlight & ws_error_highlight) {
 -		ws = diff_get_color(ecbdata->color_diff, DIFF_WHITESPACE);
 +	if (o->ws_error_highlight & ws_rule) {
 +		ws = diff_get_color_opt(o, DIFF_WHITESPACE);
  		if (!*ws)
  			ws = NULL;
  	}
  
  	if (!ws)
 -		emit_line_0(ecbdata->opt, set, reset, sign, line, len);
 -	else if (sign == '+' && new_blank_line_at_eof(ecbdata, line, len))
 +		emit_line_0(o, set, reset, sign, line, len);
 +	else if (blank_at_eof)
  		/* Blank line at EOF - paint '+' as well */
 -		emit_line_0(ecbdata->opt, ws, reset, sign, line, len);
 +		emit_line_0(o, ws, reset, sign, line, len);
  	else {
  		/* Emit just the prefix, then the rest. */
 -		emit_line_0(ecbdata->opt, set, reset, sign, "", 0);
 -		ws_check_emit(line, len, ecbdata->ws_rule,
 -			      ecbdata->opt->file, set, reset, ws);
 +		emit_line_0(o, set, reset, sign, "", 0);
 +		ws_check_emit(line, len, ws_rule,
 +			      o->file, set, reset, ws);
  	}
  }
  
 +static void emit_diff_symbol_from_struct(struct diff_options *o,
 +					 struct emitted_diff_symbol *eds)
 +{
 +	static const char *nneof = " No newline at end of file\n";
 +	const char *context, *reset, *set, *meta, *fraginfo;
 +	struct strbuf sb = STRBUF_INIT;
 +
 +	enum diff_symbol s = eds->s;
 +	const char *line = eds->line;
 +	int len = eds->len;
 +	unsigned flags = eds->flags;
 +
 +	switch (s) {
 +	case DIFF_SYMBOL_NO_LF_EOF:
 +		context = diff_get_color_opt(o, DIFF_CONTEXT);
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		putc('\n', o->file);
 +		emit_line_0(o, context, reset, '\\',
 +			    nneof, strlen(nneof));
 +		break;
 +	case DIFF_SYMBOL_SUBMODULE_HEADER:
 +	case DIFF_SYMBOL_SUBMODULE_ERROR:
 +	case DIFF_SYMBOL_SUBMODULE_PIPETHROUGH:
 +	case DIFF_SYMBOL_STATS_SUMMARY_INSERTS_DELETES:
 +	case DIFF_SYMBOL_SUMMARY:
 +	case DIFF_SYMBOL_STATS_LINE:
 +	case DIFF_SYMBOL_BINARY_DIFF_BODY:
 +	case DIFF_SYMBOL_CONTEXT_FRAGINFO:
 +		emit_line(o, "", "", line, len);
 +		break;
 +	case DIFF_SYMBOL_CONTEXT_INCOMPLETE:
 +	case DIFF_SYMBOL_CONTEXT_MARKER:
 +		context = diff_get_color_opt(o, DIFF_CONTEXT);
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		emit_line(o, context, reset, line, len);
 +		break;
 +	case DIFF_SYMBOL_SEPARATOR:
 +		fprintf(o->file, "%s%c",
 +			diff_line_prefix(o),
 +			o->line_termination);
 +		break;
 +	case DIFF_SYMBOL_CONTEXT:
 +		set = diff_get_color_opt(o, DIFF_CONTEXT);
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		emit_line_ws_markup(o, set, reset, line, len, ' ',
 +				    flags & (DIFF_SYMBOL_CONTENT_WS_MASK), 0);
 +		break;
 +	case DIFF_SYMBOL_PLUS:
 +		switch (flags & (DIFF_SYMBOL_MOVED_LINE |
 +				 DIFF_SYMBOL_MOVED_LINE_ALT |
 +				 DIFF_SYMBOL_MOVED_LINE_UNINTERESTING)) {
 +		case DIFF_SYMBOL_MOVED_LINE |
 +		     DIFF_SYMBOL_MOVED_LINE_ALT |
 +		     DIFF_SYMBOL_MOVED_LINE_UNINTERESTING:
 +			set = diff_get_color_opt(o, DIFF_FILE_NEW_MOVED_ALT_DIM);
 +			break;
 +		case DIFF_SYMBOL_MOVED_LINE |
 +		     DIFF_SYMBOL_MOVED_LINE_ALT:
 +			set = diff_get_color_opt(o, DIFF_FILE_NEW_MOVED_ALT);
 +			break;
 +		case DIFF_SYMBOL_MOVED_LINE |
 +		     DIFF_SYMBOL_MOVED_LINE_UNINTERESTING:
 +			set = diff_get_color_opt(o, DIFF_FILE_NEW_MOVED_DIM);
 +			break;
 +		case DIFF_SYMBOL_MOVED_LINE:
 +			set = diff_get_color_opt(o, DIFF_FILE_NEW_MOVED);
 +			break;
 +		default:
 +			set = diff_get_color_opt(o, DIFF_FILE_NEW);
 +		}
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		emit_line_ws_markup(o, set, reset, line, len, '+',
 +				    flags & DIFF_SYMBOL_CONTENT_WS_MASK,
 +				    flags & DIFF_SYMBOL_CONTENT_BLANK_LINE_EOF);
 +		break;
 +	case DIFF_SYMBOL_MINUS:
 +		switch (flags & (DIFF_SYMBOL_MOVED_LINE |
 +				 DIFF_SYMBOL_MOVED_LINE_ALT |
 +				 DIFF_SYMBOL_MOVED_LINE_UNINTERESTING)) {
 +		case DIFF_SYMBOL_MOVED_LINE |
 +		     DIFF_SYMBOL_MOVED_LINE_ALT |
 +		     DIFF_SYMBOL_MOVED_LINE_UNINTERESTING:
 +			set = diff_get_color_opt(o, DIFF_FILE_OLD_MOVED_ALT_DIM);
 +			break;
 +		case DIFF_SYMBOL_MOVED_LINE |
 +		     DIFF_SYMBOL_MOVED_LINE_ALT:
 +			set = diff_get_color_opt(o, DIFF_FILE_OLD_MOVED_ALT);
 +			break;
 +		case DIFF_SYMBOL_MOVED_LINE |
 +		     DIFF_SYMBOL_MOVED_LINE_UNINTERESTING:
 +			set = diff_get_color_opt(o, DIFF_FILE_OLD_MOVED_DIM);
 +			break;
 +		case DIFF_SYMBOL_MOVED_LINE:
 +			set = diff_get_color_opt(o, DIFF_FILE_OLD_MOVED);
 +			break;
 +		default:
 +			set = diff_get_color_opt(o, DIFF_FILE_OLD);
 +		}
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		emit_line_ws_markup(o, set, reset, line, len, '-',
 +				    flags & DIFF_SYMBOL_CONTENT_WS_MASK, 0);
 +		break;
 +	case DIFF_SYMBOL_WORDS_PORCELAIN:
 +		context = diff_get_color_opt(o, DIFF_CONTEXT);
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		emit_line(o, context, reset, line, len);
 +		fputs("~\n", o->file);
 +		break;
 +	case DIFF_SYMBOL_WORDS:
 +		context = diff_get_color_opt(o, DIFF_CONTEXT);
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		/*
 +		 * Skip the prefix character, if any.  With
 +		 * diff_suppress_blank_empty, there may be
 +		 * none.
 +		 */
 +		if (line[0] != '\n') {
 +			line++;
 +			len--;
 +		}
 +		emit_line(o, context, reset, line, len);
 +		break;
 +	case DIFF_SYMBOL_FILEPAIR_PLUS:
 +		meta = diff_get_color_opt(o, DIFF_METAINFO);
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		fprintf(o->file, "%s%s+++ %s%s%s\n", diff_line_prefix(o), meta,
 +			line, reset,
 +			strchr(line, ' ') ? "\t" : "");
 +		break;
 +	case DIFF_SYMBOL_FILEPAIR_MINUS:
 +		meta = diff_get_color_opt(o, DIFF_METAINFO);
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		fprintf(o->file, "%s%s--- %s%s%s\n", diff_line_prefix(o), meta,
 +			line, reset,
 +			strchr(line, ' ') ? "\t" : "");
 +		break;
 +	case DIFF_SYMBOL_BINARY_FILES:
 +	case DIFF_SYMBOL_HEADER:
 +		fprintf(o->file, "%s", line);
 +		break;
 +	case DIFF_SYMBOL_BINARY_DIFF_HEADER:
 +		fprintf(o->file, "%sGIT binary patch\n", diff_line_prefix(o));
 +		break;
 +	case DIFF_SYMBOL_BINARY_DIFF_HEADER_DELTA:
 +		fprintf(o->file, "%sdelta %s\n", diff_line_prefix(o), line);
 +		break;
 +	case DIFF_SYMBOL_BINARY_DIFF_HEADER_LITERAL:
 +		fprintf(o->file, "%sliteral %s\n", diff_line_prefix(o), line);
 +		break;
 +	case DIFF_SYMBOL_BINARY_DIFF_FOOTER:
 +		fputs(diff_line_prefix(o), o->file);
 +		fputc('\n', o->file);
 +		break;
 +	case DIFF_SYMBOL_REWRITE_DIFF:
 +		fraginfo = diff_get_color(o->use_color, DIFF_FRAGINFO);
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		emit_line(o, fraginfo, reset, line, len);
 +		break;
 +	case DIFF_SYMBOL_SUBMODULE_ADD:
 +		set = diff_get_color_opt(o, DIFF_FILE_NEW);
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		emit_line(o, set, reset, line, len);
 +		break;
 +	case DIFF_SYMBOL_SUBMODULE_DEL:
 +		set = diff_get_color_opt(o, DIFF_FILE_OLD);
 +		reset = diff_get_color_opt(o, DIFF_RESET);
 +		emit_line(o, set, reset, line, len);
 +		break;
 +	case DIFF_SYMBOL_SUBMODULE_UNTRACKED:
 +		fprintf(o->file, "%sSubmodule %s contains untracked content\n",
 +			diff_line_prefix(o), line);
 +		break;
 +	case DIFF_SYMBOL_SUBMODULE_MODIFIED:
 +		fprintf(o->file, "%sSubmodule %s contains modified content\n",
 +			diff_line_prefix(o), line);
 +		break;
 +	case DIFF_SYMBOL_STATS_SUMMARY_NO_FILES:
 +		emit_line(o, "", "", " 0 files changed\n",
 +			  strlen(" 0 files changed\n"));
 +		break;
 +	case DIFF_SYMBOL_STATS_SUMMARY_ABBREV:
 +		emit_line(o, "", "", " ...\n", strlen(" ...\n"));
 +		break;
 +	case DIFF_SYMBOL_WORD_DIFF:
 +		fprintf(o->file, "%.*s", len, line);
 +		break;
 +	case DIFF_SYMBOL_STAT_SEP:
 +		fputs(o->stat_sep, o->file);
 +		break;
 +	default:
 +		die("BUG: unknown diff symbol");
 +	}
 +	strbuf_release(&sb);
 +}
 +
 +static void emit_diff_symbol(struct diff_options *o, enum diff_symbol s,
 +			     const char *line, int len, unsigned flags)
 +{
 +	struct emitted_diff_symbol e = {line, len, flags, s};
 +
 +	if (o->emitted_symbols)
 +		append_emitted_diff_symbol(o, &e);
 +	else
 +		emit_diff_symbol_from_struct(o, &e);
 +}
 +
 +void diff_emit_submodule_del(struct diff_options *o, const char *line)
 +{
 +	emit_diff_symbol(o, DIFF_SYMBOL_SUBMODULE_DEL, line, strlen(line), 0);
 +}
 +
 +void diff_emit_submodule_add(struct diff_options *o, const char *line)
 +{
 +	emit_diff_symbol(o, DIFF_SYMBOL_SUBMODULE_ADD, line, strlen(line), 0);
 +}
 +
 +void diff_emit_submodule_untracked(struct diff_options *o, const char *path)
 +{
 +	emit_diff_symbol(o, DIFF_SYMBOL_SUBMODULE_UNTRACKED,
 +			 path, strlen(path), 0);
 +}
 +
 +void diff_emit_submodule_modified(struct diff_options *o, const char *path)
 +{
 +	emit_diff_symbol(o, DIFF_SYMBOL_SUBMODULE_MODIFIED,
 +			 path, strlen(path), 0);
 +}
 +
 +void diff_emit_submodule_header(struct diff_options *o, const char *header)
 +{
 +	emit_diff_symbol(o, DIFF_SYMBOL_SUBMODULE_HEADER,
 +			 header, strlen(header), 0);
 +}
 +
 +void diff_emit_submodule_error(struct diff_options *o, const char *err)
 +{
 +	emit_diff_symbol(o, DIFF_SYMBOL_SUBMODULE_ERROR, err, strlen(err), 0);
 +}
 +
 +void diff_emit_submodule_pipethrough(struct diff_options *o,
 +				     const char *line, int len)
 +{
 +	emit_diff_symbol(o, DIFF_SYMBOL_SUBMODULE_PIPETHROUGH, line, len, 0);
 +}
 +
 +static int new_blank_line_at_eof(struct emit_callback *ecbdata, const char *line, int len)
 +{
 +	if (!((ecbdata->ws_rule & WS_BLANK_AT_EOF) &&
 +	      ecbdata->blank_at_eof_in_preimage &&
 +	      ecbdata->blank_at_eof_in_postimage &&
 +	      ecbdata->blank_at_eof_in_preimage <= ecbdata->lno_in_preimage &&
 +	      ecbdata->blank_at_eof_in_postimage <= ecbdata->lno_in_postimage))
 +		return 0;
 +	return ws_blank_line(line, len, ecbdata->ws_rule);
 +}
 +
  static void emit_add_line(const char *reset,
  			  struct emit_callback *ecbdata,
  			  const char *line, int len)
  {
 -	emit_line_checked(reset, ecbdata, line, len,
 -			  DIFF_FILE_NEW, WSEH_NEW, '+');
 +	unsigned flags = WSEH_NEW | ecbdata->ws_rule;
 +	if (new_blank_line_at_eof(ecbdata, line, len))
 +		flags |= DIFF_SYMBOL_CONTENT_BLANK_LINE_EOF;
 +
 +	emit_diff_symbol(ecbdata->opt, DIFF_SYMBOL_PLUS, line, len, flags);
  }
  
  static void emit_del_line(const char *reset,
  			  struct emit_callback *ecbdata,
  			  const char *line, int len)
  {
 -	emit_line_checked(reset, ecbdata, line, len,
 -			  DIFF_FILE_OLD, WSEH_OLD, '-');
 +	unsigned flags = WSEH_OLD | ecbdata->ws_rule;
 +	emit_diff_symbol(ecbdata->opt, DIFF_SYMBOL_MINUS, line, len, flags);
  }
  
  static void emit_context_line(const char *reset,
  			      struct emit_callback *ecbdata,
  			      const char *line, int len)
  {
 -	emit_line_checked(reset, ecbdata, line, len,
 -			  DIFF_CONTEXT, WSEH_CONTEXT, ' ');
 +	unsigned flags = WSEH_CONTEXT | ecbdata->ws_rule;
 +	emit_diff_symbol(ecbdata->opt, DIFF_SYMBOL_CONTEXT, line, len, flags);
  }
  
  static void emit_hunk_header(struct emit_callback *ecbdata,
@@@ -1361,8 -641,7 +1361,8 @@@
  	if (len < 10 ||
  	    memcmp(line, atat, 2) ||
  	    !(ep = memmem(line + 2, len - 2, atat, 2))) {
 -		emit_line(ecbdata->opt, context, reset, line, len);
 +		emit_diff_symbol(ecbdata->opt,
 +				 DIFF_SYMBOL_CONTEXT_MARKER, line, len, 0);
  		return;
  	}
  	ep += 2; /* skip over @@ */
@@@ -1396,9 -675,7 +1396,9 @@@
  	}
  
  	strbuf_add(&msgbuf, line + len, org_len - len);
 -	emit_line(ecbdata->opt, "", "", msgbuf.buf, msgbuf.len);
 +	strbuf_complete_line(&msgbuf);
 +	emit_diff_symbol(ecbdata->opt,
 +			 DIFF_SYMBOL_CONTEXT_FRAGINFO, msgbuf.buf, msgbuf.len, 0);
  	strbuf_release(&msgbuf);
  }
  
@@@ -1414,23 -691,23 +1414,23 @@@ static void remove_tempfile(void
  {
  	int i;
  	for (i = 0; i < ARRAY_SIZE(diff_temp); i++) {
 -		if (is_tempfile_active(&diff_temp[i].tempfile))
 +		if (is_tempfile_active(diff_temp[i].tempfile))
  			delete_tempfile(&diff_temp[i].tempfile);
  		diff_temp[i].name = NULL;
  	}
  }
  
 -static void print_line_count(FILE *file, int count)
 +static void add_line_count(struct strbuf *out, int count)
  {
  	switch (count) {
  	case 0:
 -		fprintf(file, "0,0");
 +		strbuf_addstr(out, "0,0");
  		break;
  	case 1:
 -		fprintf(file, "1");
 +		strbuf_addstr(out, "1");
  		break;
  	default:
 -		fprintf(file, "1,%d", count);
 +		strbuf_addf(out, "1,%d", count);
  		break;
  	}
  }
@@@ -1439,6 -716,7 +1439,6 @@@ static void emit_rewrite_lines(struct e
  			       int prefix, const char *data, int size)
  {
  	const char *endp = NULL;
 -	static const char *nneof = " No newline at end of file\n";
  	const char *reset = diff_get_color(ecb->color_diff, DIFF_RESET);
  
  	while (0 < size) {
@@@ -1456,8 -734,13 +1456,8 @@@
  		size -= len;
  		data += len;
  	}
 -	if (!endp) {
 -		const char *context = diff_get_color(ecb->color_diff,
 -						     DIFF_CONTEXT);
 -		putc('\n', ecb->opt->file);
 -		emit_line_0(ecb->opt, context, reset, '\\',
 -			    nneof, strlen(nneof));
 -	}
 +	if (!endp)
 +		emit_diff_symbol(ecb->opt, DIFF_SYMBOL_NO_LF_EOF, NULL, 0, 0);
  }
  
  static void emit_rewrite_diff(const char *name_a,
@@@ -1469,12 -752,16 +1469,12 @@@
  			      struct diff_options *o)
  {
  	int lc_a, lc_b;
 -	const char *name_a_tab, *name_b_tab;
 -	const char *metainfo = diff_get_color(o->use_color, DIFF_METAINFO);
 -	const char *fraginfo = diff_get_color(o->use_color, DIFF_FRAGINFO);
 -	const char *reset = diff_get_color(o->use_color, DIFF_RESET);
  	static struct strbuf a_name = STRBUF_INIT, b_name = STRBUF_INIT;
  	const char *a_prefix, *b_prefix;
  	char *data_one, *data_two;
  	size_t size_one, size_two;
  	struct emit_callback ecbdata;
 -	const char *line_prefix = diff_line_prefix(o);
 +	struct strbuf out = STRBUF_INIT;
  
  	if (diff_mnemonic_prefix && DIFF_OPT_TST(o, REVERSE_DIFF)) {
  		a_prefix = o->b_prefix;
@@@ -1486,6 -773,8 +1486,6 @@@
  
  	name_a += (*name_a == '/');
  	name_b += (*name_b == '/');
 -	name_a_tab = strchr(name_a, ' ') ? "\t" : "";
 -	name_b_tab = strchr(name_b, ' ') ? "\t" : "";
  
  	strbuf_reset(&a_name);
  	strbuf_reset(&b_name);
@@@ -1512,23 -801,18 +1512,23 @@@
  
  	lc_a = count_lines(data_one, size_one);
  	lc_b = count_lines(data_two, size_two);
 -	fprintf(o->file,
 -		"%s%s--- %s%s%s\n%s%s+++ %s%s%s\n%s%s@@ -",
 -		line_prefix, metainfo, a_name.buf, name_a_tab, reset,
 -		line_prefix, metainfo, b_name.buf, name_b_tab, reset,
 -		line_prefix, fraginfo);
 +
 +	emit_diff_symbol(o, DIFF_SYMBOL_FILEPAIR_MINUS,
 +			 a_name.buf, a_name.len, 0);
 +	emit_diff_symbol(o, DIFF_SYMBOL_FILEPAIR_PLUS,
 +			 b_name.buf, b_name.len, 0);
 +
 +	strbuf_addstr(&out, "@@ -");
  	if (!o->irreversible_delete)
 -		print_line_count(o->file, lc_a);
 +		add_line_count(&out, lc_a);
  	else
 -		fprintf(o->file, "?,?");
 -	fprintf(o->file, " +");
 -	print_line_count(o->file, lc_b);
 -	fprintf(o->file, " @@%s\n", reset);
 +		strbuf_addstr(&out, "?,?");
 +	strbuf_addstr(&out, " +");
 +	add_line_count(&out, lc_b);
 +	strbuf_addstr(&out, " @@\n");
 +	emit_diff_symbol(o, DIFF_SYMBOL_REWRITE_DIFF, out.buf, out.len, 0);
 +	strbuf_release(&out);
 +
  	if (lc_a && !o->irreversible_delete)
  		emit_rewrite_lines(&ecbdata, '-', data_one, size_one);
  	if (lc_b)
@@@ -1588,49 -872,37 +1588,49 @@@ struct diff_words_data 
  	struct diff_words_style *style;
  };
  
 -static int fn_out_diff_words_write_helper(FILE *fp,
 +static int fn_out_diff_words_write_helper(struct diff_options *o,
  					  struct diff_words_style_elem *st_el,
  					  const char *newline,
 -					  size_t count, const char *buf,
 -					  const char *line_prefix)
 +					  size_t count, const char *buf)
  {
  	int print = 0;
 +	struct strbuf sb = STRBUF_INIT;
  
  	while (count) {
  		char *p = memchr(buf, '\n', count);
  		if (print)
 -			fputs(line_prefix, fp);
 +			strbuf_addstr(&sb, diff_line_prefix(o));
 +
  		if (p != buf) {
 -			if (st_el->color && fputs(st_el->color, fp) < 0)
 -				return -1;
 -			if (fputs(st_el->prefix, fp) < 0 ||
 -			    fwrite(buf, p ? p - buf : count, 1, fp) != 1 ||
 -			    fputs(st_el->suffix, fp) < 0)
 -				return -1;
 -			if (st_el->color && *st_el->color
 -			    && fputs(GIT_COLOR_RESET, fp) < 0)
 -				return -1;
 +			const char *reset = st_el->color && *st_el->color ?
 +					    GIT_COLOR_RESET : NULL;
 +			if (st_el->color && *st_el->color)
 +				strbuf_addstr(&sb, st_el->color);
 +			strbuf_addstr(&sb, st_el->prefix);
 +			strbuf_add(&sb, buf, p ? p - buf : count);
 +			strbuf_addstr(&sb, st_el->suffix);
 +			if (reset)
 +				strbuf_addstr(&sb, reset);
  		}
  		if (!p)
 -			return 0;
 -		if (fputs(newline, fp) < 0)
 -			return -1;
 +			goto out;
 +
 +		strbuf_addstr(&sb, newline);
  		count -= p + 1 - buf;
  		buf = p + 1;
  		print = 1;
 +		if (count) {
 +			emit_diff_symbol(o, DIFF_SYMBOL_WORD_DIFF,
 +					 sb.buf, sb.len, 0);
 +			strbuf_reset(&sb);
 +		}
  	}
 +
 +out:
 +	if (sb.len)
 +		emit_diff_symbol(o, DIFF_SYMBOL_WORD_DIFF,
 +				 sb.buf, sb.len, 0);
 +	strbuf_release(&sb);
  	return 0;
  }
  
@@@ -1712,20 -984,24 +1712,20 @@@ static void fn_out_diff_words_aux(void 
  		fputs(line_prefix, diff_words->opt->file);
  	}
  	if (diff_words->current_plus != plus_begin) {
 -		fn_out_diff_words_write_helper(diff_words->opt->file,
 +		fn_out_diff_words_write_helper(diff_words->opt,
  				&style->ctx, style->newline,
  				plus_begin - diff_words->current_plus,
 -				diff_words->current_plus, line_prefix);
 -		if (*(plus_begin - 1) == '\n')
 -			fputs(line_prefix, diff_words->opt->file);
 +				diff_words->current_plus);
  	}
  	if (minus_begin != minus_end) {
 -		fn_out_diff_words_write_helper(diff_words->opt->file,
 +		fn_out_diff_words_write_helper(diff_words->opt,
  				&style->old, style->newline,
 -				minus_end - minus_begin, minus_begin,
 -				line_prefix);
 +				minus_end - minus_begin, minus_begin);
  	}
  	if (plus_begin != plus_end) {
 -		fn_out_diff_words_write_helper(diff_words->opt->file,
 +		fn_out_diff_words_write_helper(diff_words->opt,
  				&style->new, style->newline,
 -				plus_end - plus_begin, plus_begin,
 -				line_prefix);
 +				plus_end - plus_begin, plus_begin);
  	}
  
  	diff_words->current_plus = plus_end;
@@@ -1819,12 -1095,11 +1819,12 @@@ static void diff_words_show(struct diff
  
  	/* special case: only removal */
  	if (!diff_words->plus.text.size) {
 -		fputs(line_prefix, diff_words->opt->file);
 -		fn_out_diff_words_write_helper(diff_words->opt->file,
 +		emit_diff_symbol(diff_words->opt, DIFF_SYMBOL_WORD_DIFF,
 +				 line_prefix, strlen(line_prefix), 0);
 +		fn_out_diff_words_write_helper(diff_words->opt,
  			&style->old, style->newline,
  			diff_words->minus.text.size,
 -			diff_words->minus.text.ptr, line_prefix);
 +			diff_words->minus.text.ptr);
  		diff_words->minus.text.size = 0;
  		return;
  	}
@@@ -1847,12 -1122,12 +1847,12 @@@
  	if (diff_words->current_plus != diff_words->plus.text.ptr +
  			diff_words->plus.text.size) {
  		if (color_words_output_graph_prefix(diff_words))
 -			fputs(line_prefix, diff_words->opt->file);
 -		fn_out_diff_words_write_helper(diff_words->opt->file,
 +			emit_diff_symbol(diff_words->opt, DIFF_SYMBOL_WORD_DIFF,
 +					 line_prefix, strlen(line_prefix), 0);
 +		fn_out_diff_words_write_helper(diff_words->opt,
  			&style->ctx, style->newline,
  			diff_words->plus.text.ptr + diff_words->plus.text.size
 -			- diff_words->current_plus, diff_words->current_plus,
 -			line_prefix);
 +			- diff_words->current_plus, diff_words->current_plus);
  	}
  	diff_words->minus.text.size = diff_words->plus.text.size = 0;
  }
@@@ -1860,29 -1135,9 +1860,29 @@@
  /* In "color-words" mode, show word-diff of words accumulated in the buffer */
  static void diff_words_flush(struct emit_callback *ecbdata)
  {
 +	struct diff_options *wo = ecbdata->diff_words->opt;
 +
  	if (ecbdata->diff_words->minus.text.size ||
  	    ecbdata->diff_words->plus.text.size)
  		diff_words_show(ecbdata->diff_words);
 +
 +	if (wo->emitted_symbols) {
 +		struct diff_options *o = ecbdata->opt;
 +		struct emitted_diff_symbols *wol = wo->emitted_symbols;
 +		int i;
 +
 +		/*
 +		 * NEEDSWORK:
 +		 * Instead of appending each, concat all words to a line?
 +		 */
 +		for (i = 0; i < wol->nr; i++)
 +			append_emitted_diff_symbol(o, &wol->buf[i]);
 +
 +		for (i = 0; i < wol->nr; i++)
 +			free((void *)wol->buf[i].line);
 +
 +		wol->nr = 0;
 +	}
  }
  
  static void diff_filespec_load_driver(struct diff_filespec *one)
@@@ -1918,11 -1173,6 +1918,11 @@@ static void init_diff_words_data(struc
  		xcalloc(1, sizeof(struct diff_words_data));
  	ecbdata->diff_words->type = o->word_diff;
  	ecbdata->diff_words->opt = o;
 +
 +	if (orig_opts->emitted_symbols)
 +		o->emitted_symbols =
 +			xcalloc(1, sizeof(struct emitted_diff_symbols));
 +
  	if (!o->word_regex)
  		o->word_regex = userdiff_word_regex(one);
  	if (!o->word_regex)
@@@ -1957,7 -1207,6 +1957,7 @@@ static void free_diff_words_data(struc
  {
  	if (ecbdata->diff_words) {
  		diff_words_flush(ecbdata);
 +		free (ecbdata->diff_words->opt->emitted_symbols);
  		free (ecbdata->diff_words->opt);
  		free (ecbdata->diff_words->minus.text.ptr);
  		free (ecbdata->diff_words->minus.orig);
@@@ -1994,6 -1243,8 +1994,6 @@@ static unsigned long sane_truncate_line
  	unsigned long allot;
  	size_t l = len;
  
 -	if (ecb->truncate)
 -		return ecb->truncate(line, len);
  	cp = line;
  	allot = l;
  	while (0 < l) {
@@@ -2022,25 -1273,30 +2022,25 @@@ static void find_lno(const char *line, 
  static void fn_out_consume(void *priv, char *line, unsigned long len)
  {
  	struct emit_callback *ecbdata = priv;
 -	const char *meta = diff_get_color(ecbdata->color_diff, DIFF_METAINFO);
 -	const char *context = diff_get_color(ecbdata->color_diff, DIFF_CONTEXT);
  	const char *reset = diff_get_color(ecbdata->color_diff, DIFF_RESET);
  	struct diff_options *o = ecbdata->opt;
 -	const char *line_prefix = diff_line_prefix(o);
  
  	o->found_changes = 1;
  
  	if (ecbdata->header) {
 -		fprintf(o->file, "%s", ecbdata->header->buf);
 +		emit_diff_symbol(o, DIFF_SYMBOL_HEADER,
 +				 ecbdata->header->buf, ecbdata->header->len, 0);
  		strbuf_reset(ecbdata->header);
  		ecbdata->header = NULL;
  	}
  
  	if (ecbdata->label_path[0]) {
 -		const char *name_a_tab, *name_b_tab;
 -
 -		name_a_tab = strchr(ecbdata->label_path[0], ' ') ? "\t" : "";
 -		name_b_tab = strchr(ecbdata->label_path[1], ' ') ? "\t" : "";
 -
 -		fprintf(o->file, "%s%s--- %s%s%s\n",
 -			line_prefix, meta, ecbdata->label_path[0], reset, name_a_tab);
 -		fprintf(o->file, "%s%s+++ %s%s%s\n",
 -			line_prefix, meta, ecbdata->label_path[1], reset, name_b_tab);
 +		emit_diff_symbol(o, DIFF_SYMBOL_FILEPAIR_MINUS,
 +				 ecbdata->label_path[0],
 +				 strlen(ecbdata->label_path[0]), 0);
 +		emit_diff_symbol(o, DIFF_SYMBOL_FILEPAIR_PLUS,
 +				 ecbdata->label_path[1],
 +				 strlen(ecbdata->label_path[1]), 0);
  		ecbdata->label_path[0] = ecbdata->label_path[1] = NULL;
  	}
  
@@@ -2056,13 -1312,12 +2056,13 @@@
  		len = sane_truncate_line(ecbdata, line, len);
  		find_lno(line, ecbdata);
  		emit_hunk_header(ecbdata, line, len);
 -		if (line[len-1] != '\n')
 -			putc('\n', o->file);
  		return;
  	}
  
  	if (ecbdata->diff_words) {
 +		enum diff_symbol s =
 +			ecbdata->diff_words->type == DIFF_WORDS_PORCELAIN ?
 +			DIFF_SYMBOL_WORDS_PORCELAIN : DIFF_SYMBOL_WORDS;
  		if (line[0] == '-') {
  			diff_words_append(line, len,
  					  &ecbdata->diff_words->minus);
@@@ -2082,7 -1337,21 +2082,7 @@@
  			return;
  		}
  		diff_words_flush(ecbdata);
 -		if (ecbdata->diff_words->type == DIFF_WORDS_PORCELAIN) {
 -			emit_line(o, context, reset, line, len);
 -			fputs("~\n", o->file);
 -		} else {
 -			/*
 -			 * Skip the prefix character, if any.  With
 -			 * diff_suppress_blank_empty, there may be
 -			 * none.
 -			 */
 -			if (line[0] != '\n') {
 -			      line++;
 -			      len--;
 -			}
 -			emit_line(o, context, reset, line, len);
 -		}
 +		emit_diff_symbol(o, s, line, len, 0);
  		return;
  	}
  
@@@ -2103,8 -1372,8 +2103,8 @@@
  	default:
  		/* incomplete line at the end */
  		ecbdata->lno_in_preimage++;
 -		emit_line(o, diff_get_color(ecbdata->color_diff, DIFF_CONTEXT),
 -			  reset, line, len);
 +		emit_diff_symbol(o, DIFF_SYMBOL_CONTEXT_INCOMPLETE,
 +				 line, len, 0);
  		break;
  	}
  }
@@@ -2249,14 -1518,20 +2249,14 @@@ static int scale_linear(int it, int wid
  	return 1 + (it * (width - 1) / max_change);
  }
  
 -static void show_name(FILE *file,
 -		      const char *prefix, const char *name, int len)
 -{
 -	fprintf(file, " %s%-*s |", prefix, len, name);
 -}
 -
 -static void show_graph(FILE *file, char ch, int cnt, const char *set, const char *reset)
 +static void show_graph(struct strbuf *out, char ch, int cnt,
 +		       const char *set, const char *reset)
  {
  	if (cnt <= 0)
  		return;
 -	fprintf(file, "%s", set);
 -	while (cnt--)
 -		putc(ch, file);
 -	fprintf(file, "%s", reset);
 +	strbuf_addstr(out, set);
 +	strbuf_addchars(out, ch, cnt);
 +	strbuf_addstr(out, reset);
  }
  
  static void fill_print_name(struct diffstat_file *file)
@@@ -2280,16 -1555,14 +2280,16 @@@
  	file->print_name = pname;
  }
  
 -int print_stat_summary(FILE *fp, int files, int insertions, int deletions)
 +static void print_stat_summary_inserts_deletes(struct diff_options *options,
 +		int files, int insertions, int deletions)
  {
  	struct strbuf sb = STRBUF_INIT;
 -	int ret;
  
  	if (!files) {
  		assert(insertions == 0 && deletions == 0);
 -		return fprintf(fp, "%s\n", " 0 files changed");
 +		emit_diff_symbol(options, DIFF_SYMBOL_STATS_SUMMARY_NO_FILES,
 +				 NULL, 0, 0);
 +		return;
  	}
  
  	strbuf_addf(&sb,
@@@ -2316,19 -1589,9 +2316,19 @@@
  			    deletions);
  	}
  	strbuf_addch(&sb, '\n');
 -	ret = fputs(sb.buf, fp);
 +	emit_diff_symbol(options, DIFF_SYMBOL_STATS_SUMMARY_INSERTS_DELETES,
 +			 sb.buf, sb.len, 0);
  	strbuf_release(&sb);
 -	return ret;
 +}
 +
 +void print_stat_summary(FILE *fp, int files,
 +			int insertions, int deletions)
 +{
 +	struct diff_options o;
 +	memset(&o, 0, sizeof(o));
 +	o.file = fp;
 +
 +	print_stat_summary_inserts_deletes(&o, files, insertions, deletions);
  }
  
  static void show_stats(struct diffstat_t *data, struct diff_options *options)
@@@ -2338,13 -1601,13 +2338,13 @@@
  	int total_files = data->nr, count;
  	int width, name_width, graph_width, number_width = 0, bin_width = 0;
  	const char *reset, *add_c, *del_c;
 -	const char *line_prefix = "";
  	int extra_shown = 0;
 +	const char *line_prefix = diff_line_prefix(options);
 +	struct strbuf out = STRBUF_INIT;
  
  	if (data->nr == 0)
  		return;
  
 -	line_prefix = diff_line_prefix(options);
  	count = options->stat_count ? options->stat_count : data->nr;
  
  	reset = diff_get_color_opt(options, DIFF_RESET);
@@@ -2498,32 -1761,26 +2498,32 @@@
  		}
  
  		if (file->is_binary) {
 -			fprintf(options->file, "%s", line_prefix);
 -			show_name(options->file, prefix, name, len);
 -			fprintf(options->file, " %*s", number_width, "Bin");
 +			strbuf_addf(&out, " %s%-*s |", prefix, len, name);
 +			strbuf_addf(&out, " %*s", number_width, "Bin");
  			if (!added && !deleted) {
 -				putc('\n', options->file);
 +				strbuf_addch(&out, '\n');
 +				emit_diff_symbol(options, DIFF_SYMBOL_STATS_LINE,
 +						 out.buf, out.len, 0);
 +				strbuf_reset(&out);
  				continue;
  			}
 -			fprintf(options->file, " %s%"PRIuMAX"%s",
 +			strbuf_addf(&out, " %s%"PRIuMAX"%s",
  				del_c, deleted, reset);
 -			fprintf(options->file, " -> ");
 -			fprintf(options->file, "%s%"PRIuMAX"%s",
 +			strbuf_addstr(&out, " -> ");
 +			strbuf_addf(&out, "%s%"PRIuMAX"%s",
  				add_c, added, reset);
 -			fprintf(options->file, " bytes");
 -			fprintf(options->file, "\n");
 +			strbuf_addstr(&out, " bytes\n");
 +			emit_diff_symbol(options, DIFF_SYMBOL_STATS_LINE,
 +					 out.buf, out.len, 0);
 +			strbuf_reset(&out);
  			continue;
  		}
  		else if (file->is_unmerged) {
 -			fprintf(options->file, "%s", line_prefix);
 -			show_name(options->file, prefix, name, len);
 -			fprintf(options->file, " Unmerged\n");
 +			strbuf_addf(&out, " %s%-*s |", prefix, len, name);
 +			strbuf_addstr(&out, " Unmerged\n");
 +			emit_diff_symbol(options, DIFF_SYMBOL_STATS_LINE,
 +					 out.buf, out.len, 0);
 +			strbuf_reset(&out);
  			continue;
  		}
  
@@@ -2546,16 -1803,14 +2546,16 @@@
  				add = total - del;
  			}
  		}
 -		fprintf(options->file, "%s", line_prefix);
 -		show_name(options->file, prefix, name, len);
 -		fprintf(options->file, " %*"PRIuMAX"%s",
 +		strbuf_addf(&out, " %s%-*s |", prefix, len, name);
 +		strbuf_addf(&out, " %*"PRIuMAX"%s",
  			number_width, added + deleted,
  			added + deleted ? " " : "");
 -		show_graph(options->file, '+', add, add_c, reset);
 -		show_graph(options->file, '-', del, del_c, reset);
 -		fprintf(options->file, "\n");
 +		show_graph(&out, '+', add, add_c, reset);
 +		show_graph(&out, '-', del, del_c, reset);
 +		strbuf_addch(&out, '\n');
 +		emit_diff_symbol(options, DIFF_SYMBOL_STATS_LINE,
 +				 out.buf, out.len, 0);
 +		strbuf_reset(&out);
  	}
  
  	for (i = 0; i < data->nr; i++) {
@@@ -2576,14 -1831,11 +2576,14 @@@
  		if (i < count)
  			continue;
  		if (!extra_shown)
 -			fprintf(options->file, "%s ...\n", line_prefix);
 +			emit_diff_symbol(options,
 +					 DIFF_SYMBOL_STATS_SUMMARY_ABBREV,
 +					 NULL, 0, 0);
  		extra_shown = 1;
  	}
 -	fprintf(options->file, "%s", line_prefix);
 -	print_stat_summary(options->file, total_files, adds, dels);
 +
 +	print_stat_summary_inserts_deletes(options, total_files, adds, dels);
 +	strbuf_release(&out);
  }
  
  static void show_shortstats(struct diffstat_t *data, struct diff_options *options)
@@@ -2595,7 -1847,7 +2595,7 @@@
  
  	for (i = 0; i < data->nr; i++) {
  		int added = data->files[i]->added;
 -		int deleted= data->files[i]->deleted;
 +		int deleted = data->files[i]->deleted;
  
  		if (data->files[i]->is_unmerged ||
  		    (!data->files[i]->is_interesting && (added + deleted == 0))) {
@@@ -2605,7 -1857,8 +2605,7 @@@
  			dels += deleted;
  		}
  	}
 -	fprintf(options->file, "%s", diff_line_prefix(options));
 -	print_stat_summary(options->file, total_files, adds, dels);
 +	print_stat_summary_inserts_deletes(options, total_files, adds, dels);
  }
  
  static void show_numstat(struct diffstat_t *data, struct diff_options *options)
@@@ -2969,8 -2222,8 +2969,8 @@@ static unsigned char *deflate_it(char *
  	return deflated;
  }
  
 -static void emit_binary_diff_body(FILE *file, mmfile_t *one, mmfile_t *two,
 -				  const char *prefix)
 +static void emit_binary_diff_body(struct diff_options *o,
 +				  mmfile_t *one, mmfile_t *two)
  {
  	void *cp;
  	void *delta;
@@@ -2999,18 -2252,13 +2999,18 @@@
  	}
  
  	if (delta && delta_size < deflate_size) {
 -		fprintf(file, "%sdelta %lu\n", prefix, orig_size);
 +		char *s = xstrfmt("%lu", orig_size);
 +		emit_diff_symbol(o, DIFF_SYMBOL_BINARY_DIFF_HEADER_DELTA,
 +				 s, strlen(s), 0);
 +		free(s);
  		free(deflated);
  		data = delta;
  		data_size = delta_size;
 -	}
 -	else {
 -		fprintf(file, "%sliteral %lu\n", prefix, two->size);
 +	} else {
 +		char *s = xstrfmt("%lu", two->size);
 +		emit_diff_symbol(o, DIFF_SYMBOL_BINARY_DIFF_HEADER_LITERAL,
 +				 s, strlen(s), 0);
 +		free(s);
  		free(delta);
  		data = deflated;
  		data_size = deflate_size;
@@@ -3019,9 -2267,8 +3019,9 @@@
  	/* emit data encoded in base85 */
  	cp = data;
  	while (data_size) {
 +		int len;
  		int bytes = (52 < data_size) ? 52 : data_size;
 -		char line[70];
 +		char line[71];
  		data_size -= bytes;
  		if (bytes <= 26)
  			line[0] = bytes + 'A' - 1;
@@@ -3029,24 -2276,20 +3029,24 @@@
  			line[0] = bytes - 26 + 'a' - 1;
  		encode_85(line + 1, cp, bytes);
  		cp = (char *) cp + bytes;
 -		fprintf(file, "%s", prefix);
 -		fputs(line, file);
 -		fputc('\n', file);
 +
 +		len = strlen(line);
 +		line[len++] = '\n';
 +		line[len] = '\0';
 +
 +		emit_diff_symbol(o, DIFF_SYMBOL_BINARY_DIFF_BODY,
 +				 line, len, 0);
  	}
 -	fprintf(file, "%s\n", prefix);
 +	emit_diff_symbol(o, DIFF_SYMBOL_BINARY_DIFF_FOOTER, NULL, 0, 0);
  	free(data);
  }
  
 -static void emit_binary_diff(FILE *file, mmfile_t *one, mmfile_t *two,
 -			     const char *prefix)
 +static void emit_binary_diff(struct diff_options *o,
 +			     mmfile_t *one, mmfile_t *two)
  {
 -	fprintf(file, "%sGIT binary patch\n", prefix);
 -	emit_binary_diff_body(file, one, two, prefix);
 -	emit_binary_diff_body(file, two, one, prefix);
 +	emit_diff_symbol(o, DIFF_SYMBOL_BINARY_DIFF_HEADER, NULL, 0, 0);
 +	emit_binary_diff_body(o, one, two);
 +	emit_binary_diff_body(o, two, one);
  }
  
  int diff_filespec_is_binary(struct diff_filespec *one)
@@@ -3123,16 -2366,24 +3123,16 @@@ static void builtin_diff(const char *na
  	if (o->submodule_format == DIFF_SUBMODULE_LOG &&
  	    (!one->mode || S_ISGITLINK(one->mode)) &&
  	    (!two->mode || S_ISGITLINK(two->mode))) {
 -		const char *del = diff_get_color_opt(o, DIFF_FILE_OLD);
 -		const char *add = diff_get_color_opt(o, DIFF_FILE_NEW);
 -		show_submodule_summary(o->file, one->path ? one->path : two->path,
 -				line_prefix,
 +		show_submodule_summary(o, one->path ? one->path : two->path,
  				&one->oid, &two->oid,
 -				two->dirty_submodule,
 -				meta, del, add, reset);
 +				two->dirty_submodule);
  		return;
  	} else if (o->submodule_format == DIFF_SUBMODULE_INLINE_DIFF &&
  		   (!one->mode || S_ISGITLINK(one->mode)) &&
  		   (!two->mode || S_ISGITLINK(two->mode))) {
 -		const char *del = diff_get_color_opt(o, DIFF_FILE_OLD);
 -		const char *add = diff_get_color_opt(o, DIFF_FILE_NEW);
 -		show_submodule_inline_diff(o->file, one->path ? one->path : two->path,
 -				line_prefix,
 +		show_submodule_inline_diff(o, one->path ? one->path : two->path,
  				&one->oid, &two->oid,
 -				two->dirty_submodule,
 -				meta, del, add, reset, o);
 +				two->dirty_submodule);
  		return;
  	}
  
@@@ -3181,8 -2432,7 +3181,8 @@@
  		if (complete_rewrite &&
  		    (textconv_one || !diff_filespec_is_binary(one)) &&
  		    (textconv_two || !diff_filespec_is_binary(two))) {
 -			fprintf(o->file, "%s", header.buf);
 +			emit_diff_symbol(o, DIFF_SYMBOL_HEADER,
 +					 header.buf, header.len, 0);
  			strbuf_reset(&header);
  			emit_rewrite_diff(name_a, name_b, one, two,
  						textconv_one, textconv_two, o);
@@@ -3192,31 -2442,23 +3192,31 @@@
  	}
  
  	if (o->irreversible_delete && lbl[1][0] == '/') {
 -		fprintf(o->file, "%s", header.buf);
 +		emit_diff_symbol(o, DIFF_SYMBOL_HEADER, header.buf,
 +				 header.len, 0);
  		strbuf_reset(&header);
  		goto free_ab_and_return;
  	} else if (!DIFF_OPT_TST(o, TEXT) &&
  	    ( (!textconv_one && diff_filespec_is_binary(one)) ||
  	      (!textconv_two && diff_filespec_is_binary(two)) )) {
 +		struct strbuf sb = STRBUF_INIT;
  		if (!one->data && !two->data &&
  		    S_ISREG(one->mode) && S_ISREG(two->mode) &&
  		    !DIFF_OPT_TST(o, BINARY)) {
  			if (!oidcmp(&one->oid, &two->oid)) {
  				if (must_show_header)
 -					fprintf(o->file, "%s", header.buf);
 +					emit_diff_symbol(o, DIFF_SYMBOL_HEADER,
 +							 header.buf, header.len,
 +							 0);
  				goto free_ab_and_return;
  			}
 -			fprintf(o->file, "%s", header.buf);
 -			fprintf(o->file, "%sBinary files %s and %s differ\n",
 -				line_prefix, lbl[0], lbl[1]);
 +			emit_diff_symbol(o, DIFF_SYMBOL_HEADER,
 +					 header.buf, header.len, 0);
 +			strbuf_addf(&sb, "%sBinary files %s and %s differ\n",
 +				    diff_line_prefix(o), lbl[0], lbl[1]);
 +			emit_diff_symbol(o, DIFF_SYMBOL_BINARY_FILES,
 +					 sb.buf, sb.len, 0);
 +			strbuf_release(&sb);
  			goto free_ab_and_return;
  		}
  		if (fill_mmfile(&mf1, one) < 0 || fill_mmfile(&mf2, two) < 0)
@@@ -3225,21 -2467,16 +3225,21 @@@
  		if (mf1.size == mf2.size &&
  		    !memcmp(mf1.ptr, mf2.ptr, mf1.size)) {
  			if (must_show_header)
 -				fprintf(o->file, "%s", header.buf);
 +				emit_diff_symbol(o, DIFF_SYMBOL_HEADER,
 +						 header.buf, header.len, 0);
  			goto free_ab_and_return;
  		}
 -		fprintf(o->file, "%s", header.buf);
 +		emit_diff_symbol(o, DIFF_SYMBOL_HEADER, header.buf, header.len, 0);
  		strbuf_reset(&header);
  		if (DIFF_OPT_TST(o, BINARY))
 -			emit_binary_diff(o->file, &mf1, &mf2, line_prefix);
 -		else
 -			fprintf(o->file, "%sBinary files %s and %s differ\n",
 -				line_prefix, lbl[0], lbl[1]);
 +			emit_binary_diff(o, &mf1, &mf2);
 +		else {
 +			strbuf_addf(&sb, "%sBinary files %s and %s differ\n",
 +				    diff_line_prefix(o), lbl[0], lbl[1]);
 +			emit_diff_symbol(o, DIFF_SYMBOL_BINARY_FILES,
 +					 sb.buf, sb.len, 0);
 +			strbuf_release(&sb);
 +		}
  		o->found_changes = 1;
  	} else {
  		/* Crazy xdl interfaces.. */
@@@ -3251,8 -2488,7 +3251,8 @@@
  		const struct userdiff_funcname *pe;
  
  		if (must_show_header) {
 -			fprintf(o->file, "%s", header.buf);
 +			emit_diff_symbol(o, DIFF_SYMBOL_HEADER,
 +					 header.buf, header.len, 0);
  			strbuf_reset(&header);
  		}
  
@@@ -3721,6 -2957,7 +3721,6 @@@ static void prep_temp_blob(const char *
  			   const struct object_id *oid,
  			   int mode)
  {
 -	int fd;
  	struct strbuf buf = STRBUF_INIT;
  	struct strbuf template = STRBUF_INIT;
  	char *path_dup = xstrdup(path);
@@@ -3730,18 -2967,18 +3730,18 @@@
  	strbuf_addstr(&template, "XXXXXX_");
  	strbuf_addstr(&template, base);
  
 -	fd = mks_tempfile_ts(&temp->tempfile, template.buf, strlen(base) + 1);
 -	if (fd < 0)
 +	temp->tempfile = mks_tempfile_ts(template.buf, strlen(base) + 1);
 +	if (!temp->tempfile)
  		die_errno("unable to create temp-file");
  	if (convert_to_working_tree(path,
  			(const char *)blob, (size_t)size, &buf)) {
  		blob = buf.buf;
  		size = buf.len;
  	}
- 	if (write_in_full(temp->tempfile->fd, blob, size) != size ||
 -	if (write_in_full(fd, blob, size) < 0)
++	if (write_in_full(temp->tempfile->fd, blob, size) < 0 ||
 +	    close_tempfile_gently(temp->tempfile))
  		die_errno("unable to write temp-file");
 -	close_tempfile(&temp->tempfile);
 -	temp->name = get_tempfile_path(&temp->tempfile);
 +	temp->name = get_tempfile_path(temp->tempfile);
  	oid_to_hex_r(temp->hex, oid);
  	xsnprintf(temp->mode, sizeof(temp->mode), "%06o", mode);
  	strbuf_release(&buf);
@@@ -4009,7 -3246,7 +4009,7 @@@ static void diff_fill_oid_info(struct d
  			}
  			if (lstat(one->path, &st) < 0)
  				die_errno("stat '%s'", one->path);
 -			if (index_path(one->oid.hash, one->path, &st, 0))
 +			if (index_path(&one->oid, one->path, &st, 0))
  				die("cannot hash %s", one->path);
  		}
  	}
@@@ -4042,8 -3279,8 +4042,8 @@@ static void run_diff(struct diff_filepa
  	const char *other;
  	const char *attr_path;
  
 -	name  = p->one->path;
 -	other = (strcmp(name, p->two->path) ? p->two->path : NULL);
 +	name  = one->path;
 +	other = (strcmp(name, two->path) ? two->path : NULL);
  	attr_path = name;
  	if (o->prefix_length)
  		strip_prefix(o->prefix_length, &name, &other);
@@@ -4166,8 -3403,6 +4166,8 @@@ void diff_setup(struct diff_options *op
  		options->a_prefix = "a/";
  		options->b_prefix = "b/";
  	}
 +
 +	options->color_moved = diff_color_moved_default;
  }
  
  void diff_setup_done(struct diff_options *options)
@@@ -4277,9 -3512,6 +4277,9 @@@
  
  	if (DIFF_OPT_TST(options, FOLLOW_RENAMES) && options->pathspec.nr != 1)
  		die(_("--follow requires exactly one pathspec"));
 +
 +	if (!options->use_color || external_diff())
 +		options->color_moved = 0;
  }
  
  static int opt_arg(const char *arg, int arg_short, const char *arg_long, int *val)
@@@ -4704,19 -3936,7 +4704,19 @@@ int diff_opt_parse(struct diff_options 
  	}
  	else if (!strcmp(arg, "--no-color"))
  		options->use_color = 0;
 -	else if (!strcmp(arg, "--color-words")) {
 +	else if (!strcmp(arg, "--color-moved")) {
 +		if (diff_color_moved_default)
 +			options->color_moved = diff_color_moved_default;
 +		if (options->color_moved == COLOR_MOVED_NO)
 +			options->color_moved = COLOR_MOVED_DEFAULT;
 +	} else if (!strcmp(arg, "--no-color-moved"))
 +		options->color_moved = COLOR_MOVED_NO;
 +	else if (skip_prefix(arg, "--color-moved=", &arg)) {
 +		int cm = parse_color_moved(arg);
 +		if (cm < 0)
 +			die("bad --color-moved argument: %s", arg);
 +		options->color_moved = cm;
 +	} else if (!strcmp(arg, "--color-words")) {
  		options->use_color = 1;
  		options->word_diff = DIFF_WORDS_COLOR;
  	}
@@@ -5246,78 -4466,67 +5246,78 @@@ static void flush_one_pair(struct diff_
  	}
  }
  
 -static void show_file_mode_name(FILE *file, const char *newdelete, struct diff_filespec *fs)
 +static void show_file_mode_name(struct diff_options *opt, const char *newdelete, struct diff_filespec *fs)
  {
 +	struct strbuf sb = STRBUF_INIT;
  	if (fs->mode)
 -		fprintf(file, " %s mode %06o ", newdelete, fs->mode);
 +		strbuf_addf(&sb, " %s mode %06o ", newdelete, fs->mode);
  	else
 -		fprintf(file, " %s ", newdelete);
 -	write_name_quoted(fs->path, file, '\n');
 -}
 +		strbuf_addf(&sb, " %s ", newdelete);
  
 +	quote_c_style(fs->path, &sb, NULL, 0);
 +	strbuf_addch(&sb, '\n');
 +	emit_diff_symbol(opt, DIFF_SYMBOL_SUMMARY,
 +			 sb.buf, sb.len, 0);
 +	strbuf_release(&sb);
 +}
  
 -static void show_mode_change(FILE *file, struct diff_filepair *p, int show_name,
 -		const char *line_prefix)
 +static void show_mode_change(struct diff_options *opt, struct diff_filepair *p,
 +		int show_name)
  {
  	if (p->one->mode && p->two->mode && p->one->mode != p->two->mode) {
 -		fprintf(file, "%s mode change %06o => %06o%c", line_prefix, p->one->mode,
 -			p->two->mode, show_name ? ' ' : '\n');
 +		struct strbuf sb = STRBUF_INIT;
 +		strbuf_addf(&sb, " mode change %06o => %06o",
 +			    p->one->mode, p->two->mode);
  		if (show_name) {
 -			write_name_quoted(p->two->path, file, '\n');
 +			strbuf_addch(&sb, ' ');
 +			quote_c_style(p->two->path, &sb, NULL, 0);
  		}
 +		emit_diff_symbol(opt, DIFF_SYMBOL_SUMMARY,
 +				 sb.buf, sb.len, 0);
 +		strbuf_release(&sb);
  	}
  }
  
 -static void show_rename_copy(FILE *file, const char *renamecopy, struct diff_filepair *p,
 -			const char *line_prefix)
 +static void show_rename_copy(struct diff_options *opt, const char *renamecopy,
 +		struct diff_filepair *p)
  {
 +	struct strbuf sb = STRBUF_INIT;
  	char *names = pprint_rename(p->one->path, p->two->path);
 -
 -	fprintf(file, " %s %s (%d%%)\n", renamecopy, names, similarity_index(p));
 +	strbuf_addf(&sb, " %s %s (%d%%)\n",
 +			renamecopy, names, similarity_index(p));
  	free(names);
 -	show_mode_change(file, p, 0, line_prefix);
 +	emit_diff_symbol(opt, DIFF_SYMBOL_SUMMARY,
 +				 sb.buf, sb.len, 0);
 +	show_mode_change(opt, p, 0);
 +	strbuf_release(&sb);
  }
  
  static void diff_summary(struct diff_options *opt, struct diff_filepair *p)
  {
 -	FILE *file = opt->file;
 -	const char *line_prefix = diff_line_prefix(opt);
 -
  	switch(p->status) {
  	case DIFF_STATUS_DELETED:
 -		fputs(line_prefix, file);
 -		show_file_mode_name(file, "delete", p->one);
 +		show_file_mode_name(opt, "delete", p->one);
  		break;
  	case DIFF_STATUS_ADDED:
 -		fputs(line_prefix, file);
 -		show_file_mode_name(file, "create", p->two);
 +		show_file_mode_name(opt, "create", p->two);
  		break;
  	case DIFF_STATUS_COPIED:
 -		fputs(line_prefix, file);
 -		show_rename_copy(file, "copy", p, line_prefix);
 +		show_rename_copy(opt, "copy", p);
  		break;
  	case DIFF_STATUS_RENAMED:
 -		fputs(line_prefix, file);
 -		show_rename_copy(file, "rename", p, line_prefix);
 +		show_rename_copy(opt, "rename", p);
  		break;
  	default:
  		if (p->score) {
 -			fprintf(file, "%s rewrite ", line_prefix);
 -			write_name_quoted(p->two->path, file, ' ');
 -			fprintf(file, "(%d%%)\n", similarity_index(p));
 +			struct strbuf sb = STRBUF_INIT;
 +			strbuf_addstr(&sb, " rewrite ");
 +			quote_c_style(p->two->path, &sb, NULL, 0);
 +			strbuf_addf(&sb, " (%d%%)\n", similarity_index(p));
 +			emit_diff_symbol(opt, DIFF_SYMBOL_SUMMARY,
 +					 sb.buf, sb.len, 0);
 +			strbuf_release(&sb);
  		}
 -		show_mode_change(file, p, !p->score, line_prefix);
 +		show_mode_change(opt, p, !p->score);
  		break;
  	}
  }
@@@ -5522,51 -4731,6 +5522,51 @@@ void diff_warn_rename_limit(const char 
  		warning(_(rename_limit_advice), varname, needed);
  }
  
 +static void diff_flush_patch_all_file_pairs(struct diff_options *o)
 +{
 +	int i;
 +	static struct emitted_diff_symbols esm = EMITTED_DIFF_SYMBOLS_INIT;
 +	struct diff_queue_struct *q = &diff_queued_diff;
 +
 +	if (WSEH_NEW & WS_RULE_MASK)
 +		die("BUG: WS rules bit mask overlaps with diff symbol flags");
 +
 +	if (o->color_moved)
 +		o->emitted_symbols = &esm;
 +
 +	for (i = 0; i < q->nr; i++) {
 +		struct diff_filepair *p = q->queue[i];
 +		if (check_pair_status(p))
 +			diff_flush_patch(p, o);
 +	}
 +
 +	if (o->emitted_symbols) {
 +		if (o->color_moved) {
 +			struct hashmap add_lines, del_lines;
 +
 +			hashmap_init(&del_lines,
 +				     (hashmap_cmp_fn)moved_entry_cmp, o, 0);
 +			hashmap_init(&add_lines,
 +				     (hashmap_cmp_fn)moved_entry_cmp, o, 0);
 +
 +			add_lines_to_move_detection(o, &add_lines, &del_lines);
 +			mark_color_as_moved(o, &add_lines, &del_lines);
 +			if (o->color_moved == COLOR_MOVED_ZEBRA_DIM)
 +				dim_moved_lines(o);
 +
 +			hashmap_free(&add_lines, 0);
 +			hashmap_free(&del_lines, 0);
 +		}
 +
 +		for (i = 0; i < esm.nr; i++)
 +			emit_diff_symbol_from_struct(o, &esm.buf[i]);
 +
 +		for (i = 0; i < esm.nr; i++)
 +			free((void *)esm.buf[i].line);
 +	}
 +	esm.nr = 0;
 +}
 +
  void diff_flush(struct diff_options *options)
  {
  	struct diff_queue_struct *q = &diff_queued_diff;
@@@ -5639,7 -4803,6 +5639,7 @@@
  			fclose(options->file);
  		options->file = xfopen("/dev/null", "w");
  		options->close_file = 1;
 +		options->color_moved = 0;
  		for (i = 0; i < q->nr; i++) {
  			struct diff_filepair *p = q->queue[i];
  			if (check_pair_status(p))
@@@ -5651,14 -4814,20 +5651,14 @@@
  
  	if (output_format & DIFF_FORMAT_PATCH) {
  		if (separator) {
 -			fprintf(options->file, "%s%c",
 -				diff_line_prefix(options),
 -				options->line_termination);
 -			if (options->stat_sep) {
 +			emit_diff_symbol(options, DIFF_SYMBOL_SEPARATOR, NULL, 0, 0);
 +			if (options->stat_sep)
  				/* attach patch instead of inline */
 -				fputs(options->stat_sep, options->file);
 -			}
 +				emit_diff_symbol(options, DIFF_SYMBOL_STAT_SEP,
 +						 NULL, 0, 0);
  		}
  
 -		for (i = 0; i < q->nr; i++) {
 -			struct diff_filepair *p = q->queue[i];
 -			if (check_pair_status(p))
 -				diff_flush_patch(p, options);
 -		}
 +		diff_flush_patch_all_file_pairs(options);
  	}
  
  	if (output_format & DIFF_FORMAT_CALLBACK)
diff --combined entry.c
index cb291aa88b,3c1818f1d4..1c7e3c11d5
--- a/entry.c
+++ b/entry.c
@@@ -3,7 -3,6 +3,7 @@@
  #include "dir.h"
  #include "streaming.h"
  #include "submodule.h"
 +#include "progress.h"
  
  static void create_directories(const char *path, int path_len,
  			       const struct checkout *state)
@@@ -162,22 -161,16 +162,22 @@@ static int remove_available_paths(struc
  int finish_delayed_checkout(struct checkout *state)
  {
  	int errs = 0;
 +	unsigned delayed_object_count;
 +	off_t filtered_bytes = 0;
  	struct string_list_item *filter, *path;
 +	struct progress *progress;
  	struct delayed_checkout *dco = state->delayed_checkout;
  
  	if (!state->delayed_checkout)
  		return errs;
  
  	dco->state = CE_RETRY;
 +	delayed_object_count = dco->paths.nr;
 +	progress = start_delayed_progress(_("Filtering content"), delayed_object_count);
  	while (dco->filters.nr > 0) {
  		for_each_string_list_item(filter, &dco->filters) {
  			struct string_list available_paths = STRING_LIST_INIT_NODUP;
 +			display_progress(progress, delayed_object_count - dco->paths.nr);
  
  			if (!async_query_available_blobs(filter->string, &available_paths)) {
  				/* Filter reported an error */
@@@ -223,17 -216,11 +223,17 @@@
  				}
  				ce = index_file_exists(state->istate, path->string,
  						       strlen(path->string), 0);
 -				errs |= (ce ? checkout_entry(ce, state, NULL) : 1);
 +				if (ce) {
 +					errs |= checkout_entry(ce, state, NULL);
 +					filtered_bytes += ce->ce_stat_data.sd_size;
 +					display_throughput(progress, filtered_bytes);
 +				} else
 +					errs = 1;
  			}
  		}
  		string_list_remove_empty_items(&dco->filters, 0);
  	}
 +	stop_progress(&progress);
  	string_list_clear(&dco->filters, 0);
  
  	/* At this point we should not have any delayed paths anymore. */
@@@ -257,7 -244,8 +257,8 @@@ static int write_entry(struct cache_ent
  	char *new;
  	struct strbuf buf = STRBUF_INIT;
  	unsigned long size;
- 	size_t wrote, newsize = 0;
+ 	ssize_t wrote;
+ 	size_t newsize = 0;
  	struct stat st;
  	const struct submodule *sub;
  
@@@ -332,7 -320,7 +333,7 @@@
  			fstat_done = fstat_output(fd, state, &st);
  		close(fd);
  		free(new);
- 		if (wrote != size)
+ 		if (wrote < 0)
  			return error("unable to write file %s", path);
  		break;
  	case S_IFGITLINK:
diff --combined fast-import.c
index 49516d60e6,395524039b..35bf671f12
--- a/fast-import.c
+++ b/fast-import.c
@@@ -167,7 -167,6 +167,7 @@@ Format of STDIN stream
  #include "quote.h"
  #include "dir.h"
  #include "run-command.h"
 +#include "packfile.h"
  
  #define PACK_ID_BITS 16
  #define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)
@@@ -2952,7 -2951,7 +2952,7 @@@ static void parse_reset_branch(const ch
  
  static void cat_blob_write(const char *buf, unsigned long size)
  {
- 	if (write_in_full(cat_blob_fd, buf, size) != size)
+ 	if (write_in_full(cat_blob_fd, buf, size) < 0)
  		die_errno("Write to frontend failed");
  }
  
diff --combined http-backend.c
index 8076b1d5e5,5daff8ccab..e51c7805c8
--- a/http-backend.c
+++ b/http-backend.c
@@@ -9,7 -9,6 +9,7 @@@
  #include "string-list.h"
  #include "url.h"
  #include "argv-array.h"
 +#include "packfile.h"
  
  static const char content_type[] = "Content-Type";
  static const char content_length[] = "Content-Length";
@@@ -358,7 -357,7 +358,7 @@@ static void inflate_request(const char 
  				die("zlib error inflating request, result %d", ret);
  
  			n = stream.total_out - cnt;
- 			if (write_in_full(out, out_buf, n) != n)
+ 			if (write_in_full(out, out_buf, n) < 0)
  				die("%s aborted reading request", prog_name);
  			cnt += n;
  
@@@ -379,7 -378,7 +379,7 @@@ static void copy_request(const char *pr
  	ssize_t n = read_request(0, &buf);
  	if (n < 0)
  		die_errno("error reading request body");
- 	if (write_in_full(out, buf, n) != n)
+ 	if (write_in_full(out, buf, n) < 0)
  		die("%s aborted reading request", prog_name);
  	close(out);
  	free(buf);
diff --combined notes-merge.c
index b04d2f2131,01cecbdda3..597d43f65c
--- a/notes-merge.c
+++ b/notes-merge.c
@@@ -302,7 -302,7 +302,7 @@@ static void write_buf_to_worktree(cons
  	fd = xopen(path, O_WRONLY | O_EXCL | O_CREAT, 0666);
  
  	while (size > 0) {
- 		long ret = write_in_full(fd, buf, size);
+ 		ssize_t ret = write_in_full(fd, buf, size);
  		if (ret < 0) {
  			/* Ignore epipe */
  			if (errno == EPIPE)
@@@ -624,7 -624,7 +624,7 @@@ int notes_merge(struct notes_merge_opti
  	if (!oidcmp(&remote->object.oid, base_oid)) {
  		/* Already merged; result == local commit */
  		if (o->verbosity >= 2)
 -			printf("Already up-to-date!\n");
 +			printf("Already up to date!\n");
  		oidcpy(result_oid, &local->object.oid);
  		goto found_result;
  	}
@@@ -709,7 -709,7 +709,7 @@@ int notes_merge_commit(struct notes_mer
  		/* write file as blob, and add to partial_tree */
  		if (stat(path.buf, &st))
  			die_errno("Failed to stat '%s'", path.buf);
 -		if (index_path(blob_oid.hash, path.buf, &st, HASH_WRITE_OBJECT))
 +		if (index_path(&blob_oid, path.buf, &st, HASH_WRITE_OBJECT))
  			die("Failed to write blob object from '%s'", path.buf);
  		if (add_note(partial_tree, &obj_oid, &blob_oid, NULL))
  			die("Failed to add resolved note '%s' to notes tree",
diff --combined pkt-line.c
index f364944b93,4823d3bb9d..647bbd3bce
--- a/pkt-line.c
+++ b/pkt-line.c
@@@ -94,9 -94,9 +94,9 @@@ void packet_flush(int fd
  int packet_flush_gently(int fd)
  {
  	packet_trace("0000", 4, 1);
- 	if (write_in_full(fd, "0000", 4) == 4)
- 		return 0;
- 	return error("flush packet write failed");
+ 	if (write_in_full(fd, "0000", 4) < 0)
+ 		return error("flush packet write failed");
+ 	return 0;
  }
  
  void packet_buf_flush(struct strbuf *buf)
@@@ -136,20 -136,18 +136,19 @@@ static void format_packet(struct strbu
  static int packet_write_fmt_1(int fd, int gently,
  			      const char *fmt, va_list args)
  {
 -	struct strbuf buf = STRBUF_INIT;
 +	static struct strbuf buf = STRBUF_INIT;
- 	ssize_t count;
  
 +	strbuf_reset(&buf);
  	format_packet(&buf, fmt, args);
- 	count = write_in_full(fd, buf.buf, buf.len);
- 	if (count == buf.len)
- 		return 0;
- 
- 	if (!gently) {
- 		check_pipe(errno);
- 		die_errno("packet write with format failed");
+ 	if (write_in_full(fd, buf.buf, buf.len) < 0) {
+ 		if (!gently) {
+ 			check_pipe(errno);
+ 			die_errno("packet write with format failed");
+ 		}
+ 		return error("packet write with format failed");
  	}
- 	return error("packet write with format failed");
+ 
+ 	return 0;
  }
  
  void packet_write_fmt(int fd, const char *fmt, ...)
@@@ -184,9 -182,9 +183,9 @@@ static int packet_write_gently(const in
  	packet_size = size + 4;
  	set_packet_header(packet_write_buffer, packet_size);
  	memcpy(packet_write_buffer + 4, buf, size);
- 	if (write_in_full(fd_out, packet_write_buffer, packet_size) == packet_size)
- 		return 0;
- 	return error("packet write failed");
+ 	if (write_in_full(fd_out, packet_write_buffer, packet_size) < 0)
+ 		return error("packet write failed");
+ 	return 0;
  }
  
  void packet_buf_write(struct strbuf *buf, const char *fmt, ...)
diff --combined read-cache.c
index 7d00265c49,5e6d24d444..cdcd11c71e
--- a/read-cache.c
+++ b/read-cache.c
@@@ -160,9 -160,9 +160,9 @@@ static int ce_compare_data(const struc
  	int fd = git_open_cloexec(ce->name, O_RDONLY);
  
  	if (fd >= 0) {
 -		unsigned char sha1[20];
 -		if (!index_fd(sha1, fd, st, OBJ_BLOB, ce->name, 0))
 -			match = hashcmp(sha1, ce->oid.hash);
 +		struct object_id oid;
 +		if (!index_fd(&oid, fd, st, OBJ_BLOB, ce->name, 0))
 +			match = oidcmp(&oid, &ce->oid);
  		/* index_fd() closed the file descriptor already */
  	}
  	return match;
@@@ -689,7 -689,7 +689,7 @@@ int add_to_index(struct index_state *is
  		return 0;
  	}
  	if (!intent_only) {
 -		if (index_path(ce->oid.hash, path, st, HASH_WRITE_OBJECT)) {
 +		if (index_path(&ce->oid, path, st, HASH_WRITE_OBJECT)) {
  			free(ce);
  			return error("unable to index file %s", path);
  		}
@@@ -1499,7 -1499,6 +1499,7 @@@ struct ondisk_cache_entry_extended 
  };
  
  /* These are only used for v3 or lower */
 +#define align_padding_size(size, len) ((size + (len) + 8) & ~7) - (size + len)
  #define align_flex_name(STRUCT,len) ((offsetof(struct STRUCT,name) + (len) + 8) & ~7)
  #define ondisk_cache_entry_size(len) align_flex_name(ondisk_cache_entry,len)
  #define ondisk_cache_entry_extended_size(len) align_flex_name(ondisk_cache_entry_extended,len)
@@@ -1922,7 -1921,7 +1922,7 @@@ static int ce_write_flush(git_SHA_CTX *
  	unsigned int buffered = write_buffer_len;
  	if (buffered) {
  		git_SHA1_Update(context, write_buffer, buffered);
- 		if (write_in_full(fd, write_buffer, buffered) != buffered)
+ 		if (write_in_full(fd, write_buffer, buffered) < 0)
  			return -1;
  		write_buffer_len = 0;
  	}
@@@ -1971,7 -1970,7 +1971,7 @@@ static int ce_flush(git_SHA_CTX *contex
  
  	/* Flush first if not enough space for SHA1 signature */
  	if (left + 20 > WRITE_BUFFER_SIZE) {
- 		if (write_in_full(fd, write_buffer, left) != left)
+ 		if (write_in_full(fd, write_buffer, left) < 0)
  			return -1;
  		left = 0;
  	}
@@@ -1980,7 -1979,7 +1980,7 @@@
  	git_SHA1_Final(write_buffer + left, context);
  	hashcpy(sha1, write_buffer + left);
  	left += 20;
- 	return (write_in_full(fd, write_buffer, left) != left) ? -1 : 0;
+ 	return (write_in_full(fd, write_buffer, left) < 0) ? -1 : 0;
  }
  
  static void ce_smudge_racily_clean_entry(struct cache_entry *ce)
@@@ -2033,7 -2032,7 +2033,7 @@@
  }
  
  /* Copy miscellaneous fields but not the name */
 -static char *copy_cache_entry_to_ondisk(struct ondisk_cache_entry *ondisk,
 +static void copy_cache_entry_to_ondisk(struct ondisk_cache_entry *ondisk,
  				       struct cache_entry *ce)
  {
  	short flags;
@@@ -2057,35 -2056,32 +2057,35 @@@
  		struct ondisk_cache_entry_extended *ondisk2;
  		ondisk2 = (struct ondisk_cache_entry_extended *)ondisk;
  		ondisk2->flags2 = htons((ce->ce_flags & CE_EXTENDED_FLAGS) >> 16);
 -		return ondisk2->name;
 -	}
 -	else {
 -		return ondisk->name;
  	}
  }
  
  static int ce_write_entry(git_SHA_CTX *c, int fd, struct cache_entry *ce,
 -			  struct strbuf *previous_name)
 +			  struct strbuf *previous_name, struct ondisk_cache_entry *ondisk)
  {
  	int size;
 -	struct ondisk_cache_entry *ondisk;
  	int saved_namelen = saved_namelen; /* compiler workaround */
 -	char *name;
  	int result;
 +	static unsigned char padding[8] = { 0x00 };
  
  	if (ce->ce_flags & CE_STRIP_NAME) {
  		saved_namelen = ce_namelen(ce);
  		ce->ce_namelen = 0;
  	}
  
 +	if (ce->ce_flags & CE_EXTENDED)
 +		size = offsetof(struct ondisk_cache_entry_extended, name);
 +	else
 +		size = offsetof(struct ondisk_cache_entry, name);
 +
  	if (!previous_name) {
 -		size = ondisk_ce_size(ce);
 -		ondisk = xcalloc(1, size);
 -		name = copy_cache_entry_to_ondisk(ondisk, ce);
 -		memcpy(name, ce->name, ce_namelen(ce));
 +		int len = ce_namelen(ce);
 +		copy_cache_entry_to_ondisk(ondisk, ce);
 +		result = ce_write(c, fd, ondisk, size);
 +		if (!result)
 +			result = ce_write(c, fd, ce->name, len);
 +		if (!result)
 +			result = ce_write(c, fd, padding, align_padding_size(size, len));
  	} else {
  		int common, to_remove, prefix_size;
  		unsigned char to_remove_vi[16];
@@@ -2098,14 -2094,16 +2098,14 @@@
  		to_remove = previous_name->len - common;
  		prefix_size = encode_varint(to_remove, to_remove_vi);
  
 -		if (ce->ce_flags & CE_EXTENDED)
 -			size = offsetof(struct ondisk_cache_entry_extended, name);
 -		else
 -			size = offsetof(struct ondisk_cache_entry, name);
 -		size += prefix_size + (ce_namelen(ce) - common + 1);
 -
 -		ondisk = xcalloc(1, size);
 -		name = copy_cache_entry_to_ondisk(ondisk, ce);
 -		memcpy(name, to_remove_vi, prefix_size);
 -		memcpy(name + prefix_size, ce->name + common, ce_namelen(ce) - common);
 +		copy_cache_entry_to_ondisk(ondisk, ce);
 +		result = ce_write(c, fd, ondisk, size);
 +		if (!result)
 +			result = ce_write(c, fd, to_remove_vi, prefix_size);
 +		if (!result)
 +			result = ce_write(c, fd, ce->name + common, ce_namelen(ce) - common);
 +		if (!result)
 +			result = ce_write(c, fd, padding, 1);
  
  		strbuf_splice(previous_name, common, to_remove,
  			      ce->name + common, ce_namelen(ce) - common);
@@@ -2115,6 -2113,8 +2115,6 @@@
  		ce->ce_flags &= ~CE_STRIP_NAME;
  	}
  
 -	result = ce_write(c, fd, ondisk, size);
 -	free(ondisk);
  	return result;
  }
  
@@@ -2192,11 -2192,10 +2192,11 @@@ static int do_write_index(struct index_
  	int newfd = tempfile->fd;
  	git_SHA_CTX c;
  	struct cache_header hdr;
 -	int i, err, removed, extended, hdr_version;
 +	int i, err = 0, removed, extended, hdr_version;
  	struct cache_entry **cache = istate->cache;
  	int entries = istate->cache_nr;
  	struct stat st;
 +	struct ondisk_cache_entry_extended ondisk;
  	struct strbuf previous_name_buf = STRBUF_INIT, *previous_name;
  	int drop_cache_tree = 0;
  
@@@ -2233,7 -2232,6 +2233,7 @@@
  		return -1;
  
  	previous_name = (hdr_version == 4) ? &previous_name_buf : NULL;
 +
  	for (i = 0; i < entries; i++) {
  		struct cache_entry *ce = cache[i];
  		if (ce->ce_flags & CE_REMOVE)
@@@ -2249,21 -2247,15 +2249,21 @@@
  			if (allow)
  				warning(msg, ce->name);
  			else
 -				return error(msg, ce->name);
 +				err = error(msg, ce->name);
  
  			drop_cache_tree = 1;
  		}
 -		if (ce_write_entry(&c, newfd, ce, previous_name) < 0)
 -			return -1;
 +		if (ce_write_entry(&c, newfd, ce, previous_name, (struct ondisk_cache_entry *)&ondisk) < 0)
 +			err = -1;
 +
 +		if (err)
 +			break;
  	}
  	strbuf_release(&previous_name_buf);
  
 +	if (err)
 +		return err;
 +
  	/* Write extension data here */
  	if (!strip_extensions && istate->split_index) {
  		struct strbuf sb = STRBUF_INIT;
@@@ -2311,11 -2303,8 +2311,11 @@@
  
  	if (ce_flush(&c, newfd, istate->sha1))
  		return -1;
 -	if (close_tempfile(tempfile))
 -		return error(_("could not close '%s'"), tempfile->filename.buf);
 +	if (close_tempfile_gently(tempfile)) {
 +		error(_("could not close '%s'"), tempfile->filename.buf);
 +		delete_tempfile(&tempfile);
 +		return -1;
 +	}
  	if (stat(tempfile->filename.buf, &st))
  		return -1;
  	istate->timestamp.sec = (unsigned int)st.st_mtime;
@@@ -2339,7 -2328,7 +2339,7 @@@ static int commit_locked_index(struct l
  static int do_write_locked_index(struct index_state *istate, struct lock_file *lock,
  				 unsigned flags)
  {
 -	int ret = do_write_index(istate, &lock->tempfile, 0);
 +	int ret = do_write_index(istate, lock->tempfile, 0);
  	if (ret)
  		return ret;
  	assert((flags & (COMMIT_LOCK | CLOSE_LOCK)) !=
@@@ -2347,7 -2336,7 +2347,7 @@@
  	if (flags & COMMIT_LOCK)
  		return commit_locked_index(lock);
  	else if (flags & CLOSE_LOCK)
 -		return close_lock_file(lock);
 +		return close_lock_file_gently(lock);
  	else
  		return ret;
  }
@@@ -2422,33 -2411,34 +2422,33 @@@ static int clean_shared_index_files(con
  	return 0;
  }
  
 -static struct tempfile temporary_sharedindex;
 -
  static int write_shared_index(struct index_state *istate,
  			      struct lock_file *lock, unsigned flags)
  {
 +	struct tempfile *temp;
  	struct split_index *si = istate->split_index;
 -	int fd, ret;
 +	int ret;
  
 -	fd = mks_tempfile(&temporary_sharedindex, git_path("sharedindex_XXXXXX"));
 -	if (fd < 0) {
 +	temp = mks_tempfile(git_path("sharedindex_XXXXXX"));
 +	if (!temp) {
  		hashclr(si->base_sha1);
  		return do_write_locked_index(istate, lock, flags);
  	}
  	move_cache_to_base_index(istate);
 -	ret = do_write_index(si->base, &temporary_sharedindex, 1);
 +	ret = do_write_index(si->base, temp, 1);
  	if (ret) {
 -		delete_tempfile(&temporary_sharedindex);
 +		delete_tempfile(&temp);
  		return ret;
  	}
 -	ret = adjust_shared_perm(get_tempfile_path(&temporary_sharedindex));
 +	ret = adjust_shared_perm(get_tempfile_path(temp));
  	if (ret) {
  		int save_errno = errno;
 -		error("cannot fix permission bits on %s", get_tempfile_path(&temporary_sharedindex));
 -		delete_tempfile(&temporary_sharedindex);
 +		error("cannot fix permission bits on %s", get_tempfile_path(temp));
 +		delete_tempfile(&temp);
  		errno = save_errno;
  		return ret;
  	}
 -	ret = rename_tempfile(&temporary_sharedindex,
 +	ret = rename_tempfile(&temp,
  			      git_path("sharedindex.%s", sha1_to_hex(si->base->sha1)));
  	if (!ret) {
  		hashcpy(si->base_sha1, si->base->sha1);
diff --combined refs.c
index c30f4c36be,a5abca04b7..e8b67ddce8
--- a/refs.c
+++ b/refs.c
@@@ -174,24 -174,6 +174,24 @@@ int refname_is_safe(const char *refname
  	return 1;
  }
  
 +/*
 + * Return true if refname, which has the specified oid and flags, can
 + * be resolved to an object in the database. If the referred-to object
 + * does not exist, emit a warning and return false.
 + */
 +int ref_resolves_to_object(const char *refname,
 +			   const struct object_id *oid,
 +			   unsigned int flags)
 +{
 +	if (flags & REF_ISBROKEN)
 +		return 0;
 +	if (!has_sha1_file(oid->hash)) {
 +		error("%s does not point to a valid object!", refname);
 +		return 0;
 +	}
 +	return 1;
 +}
 +
  char *refs_resolve_refdup(struct ref_store *refs,
  			  const char *refname, int resolve_flags,
  			  unsigned char *sha1, int *flags)
@@@ -336,6 -318,12 +336,6 @@@ int for_each_tag_ref(each_ref_fn fn, vo
  	return refs_for_each_tag_ref(get_main_ref_store(), fn, cb_data);
  }
  
 -int for_each_tag_ref_submodule(const char *submodule, each_ref_fn fn, void *cb_data)
 -{
 -	return refs_for_each_tag_ref(get_submodule_ref_store(submodule),
 -				     fn, cb_data);
 -}
 -
  int refs_for_each_branch_ref(struct ref_store *refs, each_ref_fn fn, void *cb_data)
  {
  	return refs_for_each_ref_in(refs, "refs/heads/", fn, cb_data);
@@@ -346,6 -334,12 +346,6 @@@ int for_each_branch_ref(each_ref_fn fn
  	return refs_for_each_branch_ref(get_main_ref_store(), fn, cb_data);
  }
  
 -int for_each_branch_ref_submodule(const char *submodule, each_ref_fn fn, void *cb_data)
 -{
 -	return refs_for_each_branch_ref(get_submodule_ref_store(submodule),
 -					fn, cb_data);
 -}
 -
  int refs_for_each_remote_ref(struct ref_store *refs, each_ref_fn fn, void *cb_data)
  {
  	return refs_for_each_ref_in(refs, "refs/remotes/", fn, cb_data);
@@@ -356,6 -350,12 +356,6 @@@ int for_each_remote_ref(each_ref_fn fn
  	return refs_for_each_remote_ref(get_main_ref_store(), fn, cb_data);
  }
  
 -int for_each_remote_ref_submodule(const char *submodule, each_ref_fn fn, void *cb_data)
 -{
 -	return refs_for_each_remote_ref(get_submodule_ref_store(submodule),
 -					fn, cb_data);
 -}
 -
  int head_ref_namespaced(each_ref_fn fn, void *cb_data)
  {
  	struct strbuf buf = STRBUF_INIT;
@@@ -561,21 -561,6 +561,21 @@@ enum ref_type ref_type(const char *refn
         return REF_TYPE_NORMAL;
  }
  
 +long get_files_ref_lock_timeout_ms(void)
 +{
 +	static int configured = 0;
 +
 +	/* The default timeout is 100 ms: */
 +	static int timeout_ms = 100;
 +
 +	if (!configured) {
 +		git_config_get_int("core.filesreflocktimeout", &timeout_ms);
 +		configured = 1;
 +	}
 +
 +	return timeout_ms;
 +}
 +
  static int write_pseudoref(const char *pseudoref, const unsigned char *sha1,
  			   const unsigned char *old_sha1, struct strbuf *err)
  {
@@@ -588,13 -573,11 +588,13 @@@
  	strbuf_addf(&buf, "%s\n", sha1_to_hex(sha1));
  
  	filename = git_path("%s", pseudoref);
 -	fd = hold_lock_file_for_update(&lock, filename, LOCK_DIE_ON_ERROR);
 +	fd = hold_lock_file_for_update_timeout(&lock, filename,
 +					       LOCK_DIE_ON_ERROR,
 +					       get_files_ref_lock_timeout_ms());
  	if (fd < 0) {
  		strbuf_addf(err, "could not open '%s' for writing: %s",
  			    filename, strerror(errno));
 -		return -1;
 +		goto done;
  	}
  
  	if (old_sha1) {
@@@ -609,7 -592,7 +609,7 @@@
  		}
  	}
  
- 	if (write_in_full(fd, buf.buf, buf.len) != buf.len) {
+ 	if (write_in_full(fd, buf.buf, buf.len) < 0) {
  		strbuf_addf(err, "could not write to '%s'", filename);
  		rollback_lock_file(&lock);
  		goto done;
@@@ -633,9 -616,8 +633,9 @@@ static int delete_pseudoref(const char 
  		int fd;
  		unsigned char actual_old_sha1[20];
  
 -		fd = hold_lock_file_for_update(&lock, filename,
 -					       LOCK_DIE_ON_ERROR);
 +		fd = hold_lock_file_for_update_timeout(
 +				&lock, filename, LOCK_DIE_ON_ERROR,
 +				get_files_ref_lock_timeout_ms());
  		if (fd < 0)
  			die_errno(_("Could not open '%s' for writing"), filename);
  		if (read_ref(pseudoref, actual_old_sha1))
@@@ -836,7 -818,7 +836,7 @@@ int read_ref_at(const char *refname, un
  	for_each_reflog_ent_reverse(refname, read_ref_at_ent, &cb);
  
  	if (!cb.reccnt) {
 -		if (flags & GET_SHA1_QUIETLY)
 +		if (flags & GET_OID_QUIETLY)
  			exit(128);
  		else
  			die("Log for %s is empty.", refname);
@@@ -1248,13 -1230,19 +1248,13 @@@ int refs_rename_ref_available(struct re
  	return ok;
  }
  
 -int head_ref_submodule(const char *submodule, each_ref_fn fn, void *cb_data)
 +int refs_head_ref(struct ref_store *refs, each_ref_fn fn, void *cb_data)
  {
  	struct object_id oid;
  	int flag;
  
 -	if (submodule) {
 -		if (resolve_gitlink_ref(submodule, "HEAD", oid.hash) == 0)
 -			return fn("HEAD", &oid, 0, cb_data);
 -
 -		return 0;
 -	}
 -
 -	if (!read_ref_full("HEAD", RESOLVE_REF_READING, oid.hash, &flag))
 +	if (!refs_read_ref_full(refs, "HEAD", RESOLVE_REF_READING,
 +				oid.hash, &flag))
  		return fn("HEAD", &oid, flag, cb_data);
  
  	return 0;
@@@ -1262,7 -1250,7 +1262,7 @@@
  
  int head_ref(each_ref_fn fn, void *cb_data)
  {
 -	return head_ref_submodule(NULL, fn, cb_data);
 +	return refs_head_ref(get_main_ref_store(), fn, cb_data);
  }
  
  struct ref_iterator *refs_ref_iterator_begin(
@@@ -1320,6 -1308,11 +1320,6 @@@ int for_each_ref(each_ref_fn fn, void *
  	return refs_for_each_ref(get_main_ref_store(), fn, cb_data);
  }
  
 -int for_each_ref_submodule(const char *submodule, each_ref_fn fn, void *cb_data)
 -{
 -	return refs_for_each_ref(get_submodule_ref_store(submodule), fn, cb_data);
 -}
 -
  int refs_for_each_ref_in(struct ref_store *refs, const char *prefix,
  			 each_ref_fn fn, void *cb_data)
  {
@@@ -1341,15 -1334,23 +1341,15 @@@ int for_each_fullref_in(const char *pre
  			       prefix, fn, 0, flag, cb_data);
  }
  
 -int for_each_ref_in_submodule(const char *submodule, const char *prefix,
 -			      each_ref_fn fn, void *cb_data)
 -{
 -	return refs_for_each_ref_in(get_submodule_ref_store(submodule),
 -				    prefix, fn, cb_data);
 -}
 -
 -int for_each_fullref_in_submodule(const char *submodule, const char *prefix,
 -				  each_ref_fn fn, void *cb_data,
 -				  unsigned int broken)
 +int refs_for_each_fullref_in(struct ref_store *refs, const char *prefix,
 +			     each_ref_fn fn, void *cb_data,
 +			     unsigned int broken)
  {
  	unsigned int flag = 0;
  
  	if (broken)
  		flag = DO_FOR_EACH_INCLUDE_BROKEN;
 -	return do_for_each_ref(get_submodule_ref_store(submodule),
 -			       prefix, fn, 0, flag, cb_data);
 +	return do_for_each_ref(refs, prefix, fn, 0, flag, cb_data);
  }
  
  int for_each_replace_ref(each_ref_fn fn, void *cb_data)
@@@ -1484,10 -1485,25 +1484,10 @@@ const char *resolve_ref_unsafe(const ch
  int resolve_gitlink_ref(const char *submodule, const char *refname,
  			unsigned char *sha1)
  {
 -	size_t len = strlen(submodule);
  	struct ref_store *refs;
  	int flags;
  
 -	while (len && submodule[len - 1] == '/')
 -		len--;
 -
 -	if (!len)
 -		return -1;
 -
 -	if (submodule[len]) {
 -		/* We need to strip off one or more trailing slashes */
 -		char *stripped = xmemdupz(submodule, len);
 -
 -		refs = get_submodule_ref_store(stripped);
 -		free(stripped);
 -	} else {
 -		refs = get_submodule_ref_store(submodule);
 -	}
 +	refs = get_submodule_ref_store(submodule);
  
  	if (!refs)
  		return -1;
@@@ -1602,32 -1618,31 +1602,32 @@@ struct ref_store *get_submodule_ref_sto
  {
  	struct strbuf submodule_sb = STRBUF_INIT;
  	struct ref_store *refs;
 -	int ret;
 +	char *to_free = NULL;
 +	size_t len;
  
 -	if (!submodule || !*submodule) {
 -		/*
 -		 * FIXME: This case is ideally not allowed. But that
 -		 * can't happen until we clean up all the callers.
 -		 */
 -		return get_main_ref_store();
 -	}
 +	if (!submodule)
 +		return NULL;
 +
 +	len = strlen(submodule);
 +	while (len && is_dir_sep(submodule[len - 1]))
 +		len--;
 +	if (!len)
 +		return NULL;
 +
 +	if (submodule[len])
 +		/* We need to strip off one or more trailing slashes */
 +		submodule = to_free = xmemdupz(submodule, len);
  
  	refs = lookup_ref_store_map(&submodule_ref_stores, submodule);
  	if (refs)
 -		return refs;
 +		goto done;
  
  	strbuf_addstr(&submodule_sb, submodule);
 -	ret = is_nonbare_repository_dir(&submodule_sb);
 -	strbuf_release(&submodule_sb);
 -	if (!ret)
 -		return NULL;
 +	if (!is_nonbare_repository_dir(&submodule_sb))
 +		goto done;
  
 -	ret = submodule_to_gitdir(&submodule_sb, submodule);
 -	if (ret) {
 -		strbuf_release(&submodule_sb);
 -		return NULL;
 -	}
 +	if (submodule_to_gitdir(&submodule_sb, submodule))
 +		goto done;
  
  	/* assume that add_submodule_odb() has been called */
  	refs = ref_store_init(submodule_sb.buf,
@@@ -1635,10 -1650,7 +1635,10 @@@
  	register_ref_store_map(&submodule_ref_stores, "submodule",
  			       refs, submodule);
  
 +done:
  	strbuf_release(&submodule_sb);
 +	free(to_free);
 +
  	return refs;
  }
  
diff --combined refs/files-backend.c
index 32663a999e,f21a954ce7..dac33628b3
--- a/refs/files-backend.c
+++ b/refs/files-backend.c
@@@ -3,7 -3,6 +3,7 @@@
  #include "../refs.h"
  #include "refs-internal.h"
  #include "ref-cache.h"
 +#include "packed-backend.h"
  #include "../iterator.h"
  #include "../dir-iterator.h"
  #include "../lockfile.h"
@@@ -12,10 -11,43 +12,10 @@@
  
  struct ref_lock {
  	char *ref_name;
 -	struct lock_file *lk;
 +	struct lock_file lk;
  	struct object_id old_oid;
  };
  
 -/*
 - * Return true if refname, which has the specified oid and flags, can
 - * be resolved to an object in the database. If the referred-to object
 - * does not exist, emit a warning and return false.
 - */
 -static int ref_resolves_to_object(const char *refname,
 -				  const struct object_id *oid,
 -				  unsigned int flags)
 -{
 -	if (flags & REF_ISBROKEN)
 -		return 0;
 -	if (!has_sha1_file(oid->hash)) {
 -		error("%s does not point to a valid object!", refname);
 -		return 0;
 -	}
 -	return 1;
 -}
 -
 -struct packed_ref_cache {
 -	struct ref_cache *cache;
 -
 -	/*
 -	 * Count of references to the data structure in this instance,
 -	 * including the pointer from files_ref_store::packed if any.
 -	 * The data will not be freed as long as the reference count
 -	 * is nonzero.
 -	 */
 -	unsigned int referrers;
 -
 -	/* The metadata from when this packed-refs cache was read */
 -	struct stat_validity validity;
 -};
 -
  /*
   * Future: need to be in "struct repository"
   * when doing a full libification.
@@@ -26,12 -58,54 +26,12 @@@ struct files_ref_store 
  
  	char *gitdir;
  	char *gitcommondir;
 -	char *packed_refs_path;
  
  	struct ref_cache *loose;
 -	struct packed_ref_cache *packed;
  
 -	/*
 -	 * Lock used for the "packed-refs" file. Note that this (and
 -	 * thus the enclosing `files_ref_store`) must not be freed.
 -	 */
 -	struct lock_file packed_refs_lock;
 +	struct ref_store *packed_ref_store;
  };
  
 -/*
 - * Increment the reference count of *packed_refs.
 - */
 -static void acquire_packed_ref_cache(struct packed_ref_cache *packed_refs)
 -{
 -	packed_refs->referrers++;
 -}
 -
 -/*
 - * Decrease the reference count of *packed_refs.  If it goes to zero,
 - * free *packed_refs and return true; otherwise return false.
 - */
 -static int release_packed_ref_cache(struct packed_ref_cache *packed_refs)
 -{
 -	if (!--packed_refs->referrers) {
 -		free_ref_cache(packed_refs->cache);
 -		stat_validity_clear(&packed_refs->validity);
 -		free(packed_refs);
 -		return 1;
 -	} else {
 -		return 0;
 -	}
 -}
 -
 -static void clear_packed_ref_cache(struct files_ref_store *refs)
 -{
 -	if (refs->packed) {
 -		struct packed_ref_cache *packed_refs = refs->packed;
 -
 -		if (is_lock_file_locked(&refs->packed_refs_lock))
 -			die("BUG: packed-ref cache cleared while locked");
 -		refs->packed = NULL;
 -		release_packed_ref_cache(packed_refs);
 -	}
 -}
 -
  static void clear_loose_ref_cache(struct files_ref_store *refs)
  {
  	if (refs->loose) {
@@@ -58,8 -132,7 +58,8 @@@ static struct ref_store *files_ref_stor
  	get_common_dir_noenv(&sb, gitdir);
  	refs->gitcommondir = strbuf_detach(&sb, NULL);
  	strbuf_addf(&sb, "%s/packed-refs", refs->gitcommondir);
 -	refs->packed_refs_path = strbuf_detach(&sb, NULL);
 +	refs->packed_ref_store = packed_ref_store_create(sb.buf, flags);
 +	strbuf_release(&sb);
  
  	return ref_store;
  }
@@@ -102,10 -175,169 +102,10 @@@ static struct files_ref_store *files_do
  	return refs;
  }
  
 -/* The length of a peeled reference line in packed-refs, including EOL: */
 -#define PEELED_LINE_LENGTH 42
 -
 -/*
 - * The packed-refs header line that we write out.  Perhaps other
 - * traits will be added later.  The trailing space is required.
 - */
 -static const char PACKED_REFS_HEADER[] =
 -	"# pack-refs with: peeled fully-peeled \n";
 -
 -/*
 - * Parse one line from a packed-refs file.  Write the SHA1 to sha1.
 - * Return a pointer to the refname within the line (null-terminated),
 - * or NULL if there was a problem.
 - */
 -static const char *parse_ref_line(struct strbuf *line, struct object_id *oid)
 -{
 -	const char *ref;
 -
 -	if (parse_oid_hex(line->buf, oid, &ref) < 0)
 -		return NULL;
 -	if (!isspace(*ref++))
 -		return NULL;
 -
 -	if (isspace(*ref))
 -		return NULL;
 -
 -	if (line->buf[line->len - 1] != '\n')
 -		return NULL;
 -	line->buf[--line->len] = 0;
 -
 -	return ref;
 -}
 -
 -/*
 - * Read from `packed_refs_file` into a newly-allocated
 - * `packed_ref_cache` and return it. The return value will already
 - * have its reference count incremented.
 - *
 - * A comment line of the form "# pack-refs with: " may contain zero or
 - * more traits. We interpret the traits as follows:
 - *
 - *   No traits:
 - *
 - *      Probably no references are peeled. But if the file contains a
 - *      peeled value for a reference, we will use it.
 - *
 - *   peeled:
 - *
 - *      References under "refs/tags/", if they *can* be peeled, *are*
 - *      peeled in this file. References outside of "refs/tags/" are
 - *      probably not peeled even if they could have been, but if we find
 - *      a peeled value for such a reference we will use it.
 - *
 - *   fully-peeled:
 - *
 - *      All references in the file that can be peeled are peeled.
 - *      Inversely (and this is more important), any references in the
 - *      file for which no peeled value is recorded is not peelable. This
 - *      trait should typically be written alongside "peeled" for
 - *      compatibility with older clients, but we do not require it
 - *      (i.e., "peeled" is a no-op if "fully-peeled" is set).
 - */
 -static struct packed_ref_cache *read_packed_refs(const char *packed_refs_file)
 -{
 -	FILE *f;
 -	struct packed_ref_cache *packed_refs = xcalloc(1, sizeof(*packed_refs));
 -	struct ref_entry *last = NULL;
 -	struct strbuf line = STRBUF_INIT;
 -	enum { PEELED_NONE, PEELED_TAGS, PEELED_FULLY } peeled = PEELED_NONE;
 -	struct ref_dir *dir;
 -
 -	acquire_packed_ref_cache(packed_refs);
 -	packed_refs->cache = create_ref_cache(NULL, NULL);
 -	packed_refs->cache->root->flag &= ~REF_INCOMPLETE;
 -
 -	f = fopen(packed_refs_file, "r");
 -	if (!f) {
 -		if (errno == ENOENT) {
 -			/*
 -			 * This is OK; it just means that no
 -			 * "packed-refs" file has been written yet,
 -			 * which is equivalent to it being empty.
 -			 */
 -			return packed_refs;
 -		} else {
 -			die_errno("couldn't read %s", packed_refs_file);
 -		}
 -	}
 -
 -	stat_validity_update(&packed_refs->validity, fileno(f));
 -
 -	dir = get_ref_dir(packed_refs->cache->root);
 -	while (strbuf_getwholeline(&line, f, '\n') != EOF) {
 -		struct object_id oid;
 -		const char *refname;
 -		const char *traits;
 -
 -		if (skip_prefix(line.buf, "# pack-refs with:", &traits)) {
 -			if (strstr(traits, " fully-peeled "))
 -				peeled = PEELED_FULLY;
 -			else if (strstr(traits, " peeled "))
 -				peeled = PEELED_TAGS;
 -			/* perhaps other traits later as well */
 -			continue;
 -		}
 -
 -		refname = parse_ref_line(&line, &oid);
 -		if (refname) {
 -			int flag = REF_ISPACKED;
 -
 -			if (check_refname_format(refname, REFNAME_ALLOW_ONELEVEL)) {
 -				if (!refname_is_safe(refname))
 -					die("packed refname is dangerous: %s", refname);
 -				oidclr(&oid);
 -				flag |= REF_BAD_NAME | REF_ISBROKEN;
 -			}
 -			last = create_ref_entry(refname, &oid, flag);
 -			if (peeled == PEELED_FULLY ||
 -			    (peeled == PEELED_TAGS && starts_with(refname, "refs/tags/")))
 -				last->flag |= REF_KNOWS_PEELED;
 -			add_ref_entry(dir, last);
 -			continue;
 -		}
 -		if (last &&
 -		    line.buf[0] == '^' &&
 -		    line.len == PEELED_LINE_LENGTH &&
 -		    line.buf[PEELED_LINE_LENGTH - 1] == '\n' &&
 -		    !get_oid_hex(line.buf + 1, &oid)) {
 -			oidcpy(&last->u.value.peeled, &oid);
 -			/*
 -			 * Regardless of what the file header said,
 -			 * we definitely know the value of *this*
 -			 * reference:
 -			 */
 -			last->flag |= REF_KNOWS_PEELED;
 -		}
 -	}
 -
 -	fclose(f);
 -	strbuf_release(&line);
 -
 -	return packed_refs;
 -}
 -
 -static const char *files_packed_refs_path(struct files_ref_store *refs)
 -{
 -	return refs->packed_refs_path;
 -}
 -
  static void files_reflog_path(struct files_ref_store *refs,
  			      struct strbuf *sb,
  			      const char *refname)
  {
 -	if (!refname) {
 -		/*
 -		 * FIXME: of course this is wrong in multi worktree
 -		 * setting. To be fixed real soon.
 -		 */
 -		strbuf_addf(sb, "%s/logs", refs->gitcommondir);
 -		return;
 -	}
 -
  	switch (ref_type(refname)) {
  	case REF_TYPE_PER_WORKTREE:
  	case REF_TYPE_PSEUDOREF:
@@@ -138,6 -370,70 +138,6 @@@ static void files_ref_path(struct files
  	}
  }
  
 -/*
 - * Check that the packed refs cache (if any) still reflects the
 - * contents of the file. If not, clear the cache.
 - */
 -static void validate_packed_ref_cache(struct files_ref_store *refs)
 -{
 -	if (refs->packed &&
 -	    !stat_validity_check(&refs->packed->validity,
 -				 files_packed_refs_path(refs)))
 -		clear_packed_ref_cache(refs);
 -}
 -
 -/*
 - * Get the packed_ref_cache for the specified files_ref_store,
 - * creating and populating it if it hasn't been read before or if the
 - * file has been changed (according to its `validity` field) since it
 - * was last read. On the other hand, if we hold the lock, then assume
 - * that the file hasn't been changed out from under us, so skip the
 - * extra `stat()` call in `stat_validity_check()`.
 - */
 -static struct packed_ref_cache *get_packed_ref_cache(struct files_ref_store *refs)
 -{
 -	const char *packed_refs_file = files_packed_refs_path(refs);
 -
 -	if (!is_lock_file_locked(&refs->packed_refs_lock))
 -		validate_packed_ref_cache(refs);
 -
 -	if (!refs->packed)
 -		refs->packed = read_packed_refs(packed_refs_file);
 -
 -	return refs->packed;
 -}
 -
 -static struct ref_dir *get_packed_ref_dir(struct packed_ref_cache *packed_ref_cache)
 -{
 -	return get_ref_dir(packed_ref_cache->cache->root);
 -}
 -
 -static struct ref_dir *get_packed_refs(struct files_ref_store *refs)
 -{
 -	return get_packed_ref_dir(get_packed_ref_cache(refs));
 -}
 -
 -/*
 - * Add a reference to the in-memory packed reference cache.  This may
 - * only be called while the packed-refs file is locked (see
 - * lock_packed_refs()).  To actually write the packed-refs file, call
 - * commit_packed_refs().
 - */
 -static void add_packed_ref(struct files_ref_store *refs,
 -			   const char *refname, const struct object_id *oid)
 -{
 -	struct packed_ref_cache *packed_ref_cache = get_packed_ref_cache(refs);
 -
 -	if (!is_lock_file_locked(&refs->packed_refs_lock))
 -		die("BUG: packed refs not locked");
 -
 -	if (check_refname_format(refname, REFNAME_ALLOW_ONELEVEL))
 -		die("Reference has invalid format: '%s'", refname);
 -
 -	add_ref_entry(get_packed_ref_dir(packed_ref_cache),
 -		      create_ref_entry(refname, oid, REF_ISPACKED));
 -}
 -
  /*
   * Read the loose references from the namespace dirname into dir
   * (without recursing).  dirname must end with '/'.  dir must be the
@@@ -260,6 -556,39 +260,6 @@@ static struct ref_cache *get_loose_ref_
  	return refs->loose;
  }
  
 -/*
 - * Return the ref_entry for the given refname from the packed
 - * references.  If it does not exist, return NULL.
 - */
 -static struct ref_entry *get_packed_ref(struct files_ref_store *refs,
 -					const char *refname)
 -{
 -	return find_ref_entry(get_packed_refs(refs), refname);
 -}
 -
 -/*
 - * A loose ref file doesn't exist; check for a packed ref.
 - */
 -static int resolve_packed_ref(struct files_ref_store *refs,
 -			      const char *refname,
 -			      unsigned char *sha1, unsigned int *flags)
 -{
 -	struct ref_entry *entry;
 -
 -	/*
 -	 * The loose reference file does not exist; check for a packed
 -	 * reference.
 -	 */
 -	entry = get_packed_ref(refs, refname);
 -	if (entry) {
 -		hashcpy(sha1, entry->u.value.oid.hash);
 -		*flags |= REF_ISPACKED;
 -		return 0;
 -	}
 -	/* refname is not a packed reference. */
 -	return -1;
 -}
 -
  static int files_read_raw_ref(struct ref_store *ref_store,
  			      const char *refname, unsigned char *sha1,
  			      struct strbuf *referent, unsigned int *type)
@@@ -303,8 -632,7 +303,8 @@@ stat_ref
  	if (lstat(path, &st) < 0) {
  		if (errno != ENOENT)
  			goto out;
 -		if (resolve_packed_ref(refs, refname, sha1, type)) {
 +		if (refs_read_raw_ref(refs->packed_ref_store, refname,
 +				      sha1, referent, type)) {
  			errno = ENOENT;
  			goto out;
  		}
@@@ -343,8 -671,7 +343,8 @@@
  		 * ref is supposed to be, there could still be a
  		 * packed ref:
  		 */
 -		if (resolve_packed_ref(refs, refname, sha1, type)) {
 +		if (refs_read_raw_ref(refs->packed_ref_store, refname,
 +				      sha1, referent, type)) {
  			errno = EISDIR;
  			goto out;
  		}
@@@ -409,7 -736,9 +409,7 @@@ out
  
  static void unlock_ref(struct ref_lock *lock)
  {
 -	/* Do not free lock->lk -- atexit() still looks at them */
 -	if (lock->lk)
 -		rollback_lock_file(lock->lk);
 +	rollback_lock_file(&lock->lk);
  	free(lock->ref_name);
  	free(lock);
  }
@@@ -523,9 -852,10 +523,9 @@@ retry
  		goto error_return;
  	}
  
 -	if (!lock->lk)
 -		lock->lk = xcalloc(1, sizeof(struct lock_file));
 -
 -	if (hold_lock_file_for_update(lock->lk, ref_file.buf, LOCK_NO_DEREF) < 0) {
 +	if (hold_lock_file_for_update_timeout(
 +			    &lock->lk, ref_file.buf, LOCK_NO_DEREF,
 +			    get_files_ref_lock_timeout_ms()) < 0) {
  		if (errno == ENOENT && --attempts_remaining > 0) {
  			/*
  			 * Maybe somebody just deleted one of the
@@@ -620,11 -950,11 +620,11 @@@
  
  		/*
  		 * If the ref did not exist and we are creating it,
 -		 * make sure there is no existing ref that conflicts
 -		 * with refname:
 +		 * make sure there is no existing packed ref that
 +		 * conflicts with refname:
  		 */
  		if (refs_verify_refname_available(
 -				    &refs->base, refname,
 +				    refs->packed_ref_store, refname,
  				    extras, skip, err))
  			goto error_return;
  	}
@@@ -671,9 -1001,15 +671,9 @@@ static int files_peel_ref(struct ref_st
  	 * be expensive and (b) loose references anyway usually do not
  	 * have REF_KNOWS_PEELED.
  	 */
 -	if (flag & REF_ISPACKED) {
 -		struct ref_entry *r = get_packed_ref(refs, refname);
 -		if (r) {
 -			if (peel_entry(r, 0))
 -				return -1;
 -			hashcpy(sha1, r->u.value.peeled.hash);
 -			return 0;
 -		}
 -	}
 +	if (flag & REF_ISPACKED &&
 +	    !refs_peel_ref(refs->packed_ref_store, refname, sha1))
 +		return 0;
  
  	return peel_object(base, sha1);
  }
@@@ -681,6 -1017,7 +681,6 @@@
  struct files_ref_iterator {
  	struct ref_iterator base;
  
 -	struct packed_ref_cache *packed_ref_cache;
  	struct ref_iterator *iter0;
  	unsigned int flags;
  };
@@@ -733,6 -1070,7 +733,6 @@@ static int files_ref_iterator_abort(str
  	if (iter->iter0)
  		ok = ref_iterator_abort(iter->iter0);
  
 -	release_packed_ref_cache(iter->packed_ref_cache);
  	base_ref_iterator_free(ref_iterator);
  	return ok;
  }
@@@ -774,28 -1112,18 +774,28 @@@ static struct ref_iterator *files_ref_i
  	 * (If they've already been read, that's OK; we only need to
  	 * guarantee that they're read before the packed refs, not
  	 * *how much* before.) After that, we call
 -	 * get_packed_ref_cache(), which internally checks whether the
 -	 * packed-ref cache is up to date with what is on disk, and
 -	 * re-reads it if not.
 +	 * packed_ref_iterator_begin(), which internally checks
 +	 * whether the packed-ref cache is up to date with what is on
 +	 * disk, and re-reads it if not.
  	 */
  
  	loose_iter = cache_ref_iterator_begin(get_loose_ref_cache(refs),
  					      prefix, 1);
  
 -	iter->packed_ref_cache = get_packed_ref_cache(refs);
 -	acquire_packed_ref_cache(iter->packed_ref_cache);
 -	packed_iter = cache_ref_iterator_begin(iter->packed_ref_cache->cache,
 -					       prefix, 0);
 +	/*
 +	 * The packed-refs file might contain broken references, for
 +	 * example an old version of a reference that points at an
 +	 * object that has since been garbage-collected. This is OK as
 +	 * long as there is a corresponding loose reference that
 +	 * overrides it, and we don't want to emit an error message in
 +	 * this case. So ask the packed_ref_store for all of its
 +	 * references, and (if needed) do our own check for broken
 +	 * ones in files_ref_iterator_advance(), after we have merged
 +	 * the packed and loose references.
 +	 */
 +	packed_iter = refs_ref_iterator_begin(
 +			refs->packed_ref_store, prefix, 0,
 +			DO_FOR_EACH_INCLUDE_BROKEN);
  
  	iter->iter0 = overlay_ref_iterator_begin(loose_iter, packed_iter);
  	iter->flags = flags;
@@@ -853,9 -1181,7 +853,9 @@@ static int create_reflock(const char *p
  {
  	struct lock_file *lk = cb;
  
 -	return hold_lock_file_for_update(lk, path, LOCK_NO_DEREF) < 0 ? -1 : 0;
 +	return hold_lock_file_for_update_timeout(
 +			lk, path, LOCK_NO_DEREF,
 +			get_files_ref_lock_timeout_ms()) < 0 ? -1 : 0;
  }
  
  /*
@@@ -929,15 -1255,17 +929,15 @@@ static struct ref_lock *lock_ref_sha1_b
  	 * our refname.
  	 */
  	if (is_null_oid(&lock->old_oid) &&
 -	    refs_verify_refname_available(&refs->base, refname,
 +	    refs_verify_refname_available(refs->packed_ref_store, refname,
  					  extras, skip, err)) {
  		last_errno = ENOTDIR;
  		goto error_return;
  	}
  
 -	lock->lk = xcalloc(1, sizeof(struct lock_file));
 -
  	lock->ref_name = xstrdup(refname);
  
 -	if (raceproof_create_file(ref_file.buf, create_reflock, lock->lk)) {
 +	if (raceproof_create_file(ref_file.buf, create_reflock, &lock->lk)) {
  		last_errno = errno;
  		unable_to_lock_message(ref_file.buf, errno, err);
  		goto error_return;
@@@ -959,6 -1287,124 +959,6 @@@
  	return lock;
  }
  
 -/*
 - * Write an entry to the packed-refs file for the specified refname.
 - * If peeled is non-NULL, write it as the entry's peeled value.
 - */
 -static void write_packed_entry(FILE *fh, const char *refname,
 -			       const unsigned char *sha1,
 -			       const unsigned char *peeled)
 -{
 -	fprintf_or_die(fh, "%s %s\n", sha1_to_hex(sha1), refname);
 -	if (peeled)
 -		fprintf_or_die(fh, "^%s\n", sha1_to_hex(peeled));
 -}
 -
 -/*
 - * Lock the packed-refs file for writing. Flags is passed to
 - * hold_lock_file_for_update(). Return 0 on success. On errors, set
 - * errno appropriately and return a nonzero value.
 - */
 -static int lock_packed_refs(struct files_ref_store *refs, int flags)
 -{
 -	static int timeout_configured = 0;
 -	static int timeout_value = 1000;
 -	struct packed_ref_cache *packed_ref_cache;
 -
 -	files_assert_main_repository(refs, "lock_packed_refs");
 -
 -	if (!timeout_configured) {
 -		git_config_get_int("core.packedrefstimeout", &timeout_value);
 -		timeout_configured = 1;
 -	}
 -
 -	if (hold_lock_file_for_update_timeout(
 -			    &refs->packed_refs_lock, files_packed_refs_path(refs),
 -			    flags, timeout_value) < 0)
 -		return -1;
 -
 -	/*
 -	 * Now that we hold the `packed-refs` lock, make sure that our
 -	 * cache matches the current version of the file. Normally
 -	 * `get_packed_ref_cache()` does that for us, but that
 -	 * function assumes that when the file is locked, any existing
 -	 * cache is still valid. We've just locked the file, but it
 -	 * might have changed the moment *before* we locked it.
 -	 */
 -	validate_packed_ref_cache(refs);
 -
 -	packed_ref_cache = get_packed_ref_cache(refs);
 -	/* Increment the reference count to prevent it from being freed: */
 -	acquire_packed_ref_cache(packed_ref_cache);
 -	return 0;
 -}
 -
 -/*
 - * Write the current version of the packed refs cache from memory to
 - * disk. The packed-refs file must already be locked for writing (see
 - * lock_packed_refs()). Return zero on success. On errors, set errno
 - * and return a nonzero value
 - */
 -static int commit_packed_refs(struct files_ref_store *refs)
 -{
 -	struct packed_ref_cache *packed_ref_cache =
 -		get_packed_ref_cache(refs);
 -	int ok, error = 0;
 -	int save_errno = 0;
 -	FILE *out;
 -	struct ref_iterator *iter;
 -
 -	files_assert_main_repository(refs, "commit_packed_refs");
 -
 -	if (!is_lock_file_locked(&refs->packed_refs_lock))
 -		die("BUG: packed-refs not locked");
 -
 -	out = fdopen_lock_file(&refs->packed_refs_lock, "w");
 -	if (!out)
 -		die_errno("unable to fdopen packed-refs descriptor");
 -
 -	fprintf_or_die(out, "%s", PACKED_REFS_HEADER);
 -
 -	iter = cache_ref_iterator_begin(packed_ref_cache->cache, NULL, 0);
 -	while ((ok = ref_iterator_advance(iter)) == ITER_OK) {
 -		struct object_id peeled;
 -		int peel_error = ref_iterator_peel(iter, &peeled);
 -
 -		write_packed_entry(out, iter->refname, iter->oid->hash,
 -				   peel_error ? NULL : peeled.hash);
 -	}
 -
 -	if (ok != ITER_DONE)
 -		die("error while iterating over references");
 -
 -	if (commit_lock_file(&refs->packed_refs_lock)) {
 -		save_errno = errno;
 -		error = -1;
 -	}
 -	release_packed_ref_cache(packed_ref_cache);
 -	errno = save_errno;
 -	return error;
 -}
 -
 -/*
 - * Rollback the lockfile for the packed-refs file, and discard the
 - * in-memory packed reference cache.  (The packed-refs file will be
 - * read anew if it is needed again after this function is called.)
 - */
 -static void rollback_packed_refs(struct files_ref_store *refs)
 -{
 -	struct packed_ref_cache *packed_ref_cache =
 -		get_packed_ref_cache(refs);
 -
 -	files_assert_main_repository(refs, "rollback_packed_refs");
 -
 -	if (!is_lock_file_locked(&refs->packed_refs_lock))
 -		die("BUG: packed-refs not locked");
 -	rollback_lock_file(&refs->packed_refs_lock);
 -	release_packed_ref_cache(packed_ref_cache);
 -	clear_packed_ref_cache(refs);
 -}
 -
  struct ref_to_prune {
  	struct ref_to_prune *next;
  	unsigned char sha1[20];
@@@ -1041,17 -1487,11 +1041,17 @@@ static void prune_ref(struct files_ref_
  	strbuf_release(&err);
  }
  
 -static void prune_refs(struct files_ref_store *refs, struct ref_to_prune *r)
 +/*
 + * Prune the loose versions of the references in the linked list
 + * `*refs_to_prune`, freeing the entries in the list as we go.
 + */
 +static void prune_refs(struct files_ref_store *refs, struct ref_to_prune **refs_to_prune)
  {
 -	while (r) {
 +	while (*refs_to_prune) {
 +		struct ref_to_prune *r = *refs_to_prune;
 +		*refs_to_prune = r->next;
  		prune_ref(refs, r);
 -		r = r->next;
 +		free(r);
  	}
  }
  
@@@ -1087,16 -1527,12 +1087,16 @@@ static int files_pack_refs(struct ref_s
  		files_downcast(ref_store, REF_STORE_WRITE | REF_STORE_ODB,
  			       "pack_refs");
  	struct ref_iterator *iter;
 -	struct ref_dir *packed_refs;
  	int ok;
  	struct ref_to_prune *refs_to_prune = NULL;
 +	struct strbuf err = STRBUF_INIT;
 +	struct ref_transaction *transaction;
 +
 +	transaction = ref_store_transaction_begin(refs->packed_ref_store, &err);
 +	if (!transaction)
 +		return -1;
  
 -	lock_packed_refs(refs, LOCK_DIE_ON_ERROR);
 -	packed_refs = get_packed_refs(refs);
 +	packed_refs_lock(refs->packed_ref_store, LOCK_DIE_ON_ERROR, &err);
  
  	iter = cache_ref_iterator_begin(get_loose_ref_cache(refs), NULL, 0);
  	while ((ok = ref_iterator_advance(iter)) == ITER_OK) {
@@@ -1105,19 -1541,29 +1105,19 @@@
  		 * in the packed ref cache. If the reference should be
  		 * pruned, also add it to refs_to_prune.
  		 */
 -		struct ref_entry *packed_entry;
 -
  		if (!should_pack_ref(iter->refname, iter->oid, iter->flags,
  				     flags))
  			continue;
  
  		/*
 -		 * Create an entry in the packed-refs cache equivalent
 -		 * to the one from the loose ref cache, except that
 -		 * we don't copy the peeled status, because we want it
 -		 * to be re-peeled.
 +		 * Add a reference creation for this reference to the
 +		 * packed-refs transaction:
  		 */
 -		packed_entry = find_ref_entry(packed_refs, iter->refname);
 -		if (packed_entry) {
 -			/* Overwrite existing packed entry with info from loose entry */
 -			packed_entry->flag = REF_ISPACKED;
 -			oidcpy(&packed_entry->u.value.oid, iter->oid);
 -		} else {
 -			packed_entry = create_ref_entry(iter->refname, iter->oid,
 -							REF_ISPACKED);
 -			add_ref_entry(packed_refs, packed_entry);
 -		}
 -		oidclr(&packed_entry->u.value.peeled);
 +		if (ref_transaction_update(transaction, iter->refname,
 +					   iter->oid->hash, NULL,
 +					   REF_NODEREF, NULL, &err))
 +			die("failure preparing to create packed reference %s: %s",
 +			    iter->refname, err.buf);
  
  		/* Schedule the loose reference for pruning if requested. */
  		if ((flags & PACK_REFS_PRUNE)) {
@@@ -1131,16 -1577,67 +1131,16 @@@
  	if (ok != ITER_DONE)
  		die("error while iterating over references");
  
 -	if (commit_packed_refs(refs))
 -		die_errno("unable to overwrite old ref-pack file");
 -
 -	prune_refs(refs, refs_to_prune);
 -	return 0;
 -}
 -
 -/*
 - * Rewrite the packed-refs file, omitting any refs listed in
 - * 'refnames'. On error, leave packed-refs unchanged, write an error
 - * message to 'err', and return a nonzero value.
 - *
 - * The refs in 'refnames' needn't be sorted. `err` must not be NULL.
 - */
 -static int repack_without_refs(struct files_ref_store *refs,
 -			       struct string_list *refnames, struct strbuf *err)
 -{
 -	struct ref_dir *packed;
 -	struct string_list_item *refname;
 -	int ret, needs_repacking = 0, removed = 0;
 -
 -	files_assert_main_repository(refs, "repack_without_refs");
 -	assert(err);
 -
 -	/* Look for a packed ref */
 -	for_each_string_list_item(refname, refnames) {
 -		if (get_packed_ref(refs, refname->string)) {
 -			needs_repacking = 1;
 -			break;
 -		}
 -	}
 +	if (ref_transaction_commit(transaction, &err))
 +		die("unable to write new packed-refs: %s", err.buf);
  
 -	/* Avoid locking if we have nothing to do */
 -	if (!needs_repacking)
 -		return 0; /* no refname exists in packed refs */
 +	ref_transaction_free(transaction);
  
 -	if (lock_packed_refs(refs, 0)) {
 -		unable_to_lock_message(files_packed_refs_path(refs), errno, err);
 -		return -1;
 -	}
 -	packed = get_packed_refs(refs);
 +	packed_refs_unlock(refs->packed_ref_store);
  
 -	/* Remove refnames from the cache */
 -	for_each_string_list_item(refname, refnames)
 -		if (remove_entry_from_dir(packed, refname->string) != -1)
 -			removed = 1;
 -	if (!removed) {
 -		/*
 -		 * All packed entries disappeared while we were
 -		 * acquiring the lock.
 -		 */
 -		rollback_packed_refs(refs);
 -		return 0;
 -	}
 -
 -	/* Write what remains */
 -	ret = commit_packed_refs(refs);
 -	if (ret)
 -		strbuf_addf(err, "unable to overwrite old ref-pack file: %s",
 -			    strerror(errno));
 -	return ret;
 +	prune_refs(refs, &refs_to_prune);
 +	strbuf_release(&err);
 +	return 0;
  }
  
  static int files_delete_refs(struct ref_store *ref_store, const char *msg,
@@@ -1154,16 -1651,24 +1154,16 @@@
  	if (!refnames->nr)
  		return 0;
  
 -	result = repack_without_refs(refs, refnames, &err);
 -	if (result) {
 -		/*
 -		 * If we failed to rewrite the packed-refs file, then
 -		 * it is unsafe to try to remove loose refs, because
 -		 * doing so might expose an obsolete packed value for
 -		 * a reference that might even point at an object that
 -		 * has been garbage collected.
 -		 */
 -		if (refnames->nr == 1)
 -			error(_("could not delete reference %s: %s"),
 -			      refnames->items[0].string, err.buf);
 -		else
 -			error(_("could not delete references: %s"), err.buf);
 +	if (packed_refs_lock(refs->packed_ref_store, 0, &err))
 +		goto error;
  
 -		goto out;
 +	if (refs_delete_refs(refs->packed_ref_store, msg, refnames, flags)) {
 +		packed_refs_unlock(refs->packed_ref_store);
 +		goto error;
  	}
  
 +	packed_refs_unlock(refs->packed_ref_store);
 +
  	for (i = 0; i < refnames->nr; i++) {
  		const char *refname = refnames->items[i].string;
  
@@@ -1171,24 -1676,9 +1171,24 @@@
  			result |= error(_("could not remove reference %s"), refname);
  	}
  
 -out:
  	strbuf_release(&err);
  	return result;
 +
 +error:
 +	/*
 +	 * If we failed to rewrite the packed-refs file, then it is
 +	 * unsafe to try to remove loose refs, because doing so might
 +	 * expose an obsolete packed value for a reference that might
 +	 * even point at an object that has been garbage collected.
 +	 */
 +	if (refnames->nr == 1)
 +		error(_("could not delete reference %s: %s"),
 +		      refnames->items[0].string, err.buf);
 +	else
 +		error(_("could not delete references: %s"), err.buf);
 +
 +	strbuf_release(&err);
 +	return -1;
  }
  
  /*
@@@ -1402,16 -1892,16 +1402,16 @@@ static int files_rename_ref(struct ref_
  	return ret;
  }
  
 -static int close_ref(struct ref_lock *lock)
 +static int close_ref_gently(struct ref_lock *lock)
  {
 -	if (close_lock_file(lock->lk))
 +	if (close_lock_file_gently(&lock->lk))
  		return -1;
  	return 0;
  }
  
  static int commit_ref(struct ref_lock *lock)
  {
 -	char *path = get_locked_file_path(lock->lk);
 +	char *path = get_locked_file_path(&lock->lk);
  	struct stat st;
  
  	if (!lstat(path, &st) && S_ISDIR(st.st_mode)) {
@@@ -1435,7 -1925,7 +1435,7 @@@
  		free(path);
  	}
  
 -	if (commit_lock_file(lock->lk))
 +	if (commit_lock_file(&lock->lk))
  		return -1;
  	return 0;
  }
@@@ -1549,7 -2039,7 +1549,7 @@@ static int log_ref_write_fd(int fd, con
  
  	written = len <= maxlen ? write_in_full(fd, logrec, len) : -1;
  	free(logrec);
- 	if (written != len)
+ 	if (written < 0)
  		return -1;
  
  	return 0;
@@@ -1627,12 -2117,12 +1627,12 @@@ static int write_ref_to_lockfile(struc
  		unlock_ref(lock);
  		return -1;
  	}
 -	fd = get_lock_file_fd(lock->lk);
 +	fd = get_lock_file_fd(&lock->lk);
- 	if (write_in_full(fd, oid_to_hex(oid), GIT_SHA1_HEXSZ) != GIT_SHA1_HEXSZ ||
- 	    write_in_full(fd, &term, 1) != 1 ||
+ 	if (write_in_full(fd, oid_to_hex(oid), GIT_SHA1_HEXSZ) < 0 ||
+ 	    write_in_full(fd, &term, 1) < 0 ||
 -	    close_ref(lock) < 0) {
 +	    close_ref_gently(lock) < 0) {
  		strbuf_addf(err,
 -			    "couldn't write '%s'", get_lock_file_path(lock->lk));
 +			    "couldn't write '%s'", get_lock_file_path(&lock->lk));
  		unlock_ref(lock);
  		return -1;
  	}
@@@ -1709,7 -2199,7 +1709,7 @@@ static int create_ref_symlink(struct re
  {
  	int ret = -1;
  #ifndef NO_SYMLINK_HEAD
 -	char *ref_path = get_locked_file_path(lock->lk);
 +	char *ref_path = get_locked_file_path(&lock->lk);
  	unlink(ref_path);
  	ret = symlink(target, ref_path);
  	free(ref_path);
@@@ -1745,14 -2235,14 +1745,14 @@@ static int create_symref_locked(struct 
  		return 0;
  	}
  
 -	if (!fdopen_lock_file(lock->lk, "w"))
 +	if (!fdopen_lock_file(&lock->lk, "w"))
  		return error("unable to fdopen %s: %s",
 -			     lock->lk->tempfile.filename.buf, strerror(errno));
 +			     lock->lk.tempfile->filename.buf, strerror(errno));
  
  	update_symref_reflog(refs, lock, refname, target, logmsg);
  
  	/* no error check; commit_ref will check ferror */
 -	fprintf(lock->lk->tempfile.fp, "ref: %s\n", target);
 +	fprintf(lock->lk.tempfile->fp, "ref: %s\n", target);
  	if (commit_ref(lock) < 0)
  		return error("unable to write symref for %s: %s", refname,
  			     strerror(errno));
@@@ -2059,63 -2549,23 +2059,63 @@@ static struct ref_iterator_vtable files
  	files_reflog_iterator_abort
  };
  
 -static struct ref_iterator *files_reflog_iterator_begin(struct ref_store *ref_store)
 +static struct ref_iterator *reflog_iterator_begin(struct ref_store *ref_store,
 +						  const char *gitdir)
  {
 -	struct files_ref_store *refs =
 -		files_downcast(ref_store, REF_STORE_READ,
 -			       "reflog_iterator_begin");
  	struct files_reflog_iterator *iter = xcalloc(1, sizeof(*iter));
  	struct ref_iterator *ref_iterator = &iter->base;
  	struct strbuf sb = STRBUF_INIT;
  
  	base_ref_iterator_init(ref_iterator, &files_reflog_iterator_vtable);
 -	files_reflog_path(refs, &sb, NULL);
 +	strbuf_addf(&sb, "%s/logs", gitdir);
  	iter->dir_iterator = dir_iterator_begin(sb.buf);
  	iter->ref_store = ref_store;
  	strbuf_release(&sb);
 +
  	return ref_iterator;
  }
  
 +static enum iterator_selection reflog_iterator_select(
 +	struct ref_iterator *iter_worktree,
 +	struct ref_iterator *iter_common,
 +	void *cb_data)
 +{
 +	if (iter_worktree) {
 +		/*
 +		 * We're a bit loose here. We probably should ignore
 +		 * common refs if they are accidentally added as
 +		 * per-worktree refs.
 +		 */
 +		return ITER_SELECT_0;
 +	} else if (iter_common) {
 +		if (ref_type(iter_common->refname) == REF_TYPE_NORMAL)
 +			return ITER_SELECT_1;
 +
 +		/*
 +		 * The main ref store may contain main worktree's
 +		 * per-worktree refs, which should be ignored
 +		 */
 +		return ITER_SKIP_1;
 +	} else
 +		return ITER_DONE;
 +}
 +
 +static struct ref_iterator *files_reflog_iterator_begin(struct ref_store *ref_store)
 +{
 +	struct files_ref_store *refs =
 +		files_downcast(ref_store, REF_STORE_READ,
 +			       "reflog_iterator_begin");
 +
 +	if (!strcmp(refs->gitdir, refs->gitcommondir)) {
 +		return reflog_iterator_begin(ref_store, refs->gitcommondir);
 +	} else {
 +		return merge_ref_iterator_begin(
 +			reflog_iterator_begin(ref_store, refs->gitdir),
 +			reflog_iterator_begin(ref_store, refs->gitcommondir),
 +			reflog_iterator_select, refs);
 +	}
 +}
 +
  /*
   * If update is a direct update of head_ref (the reference pointed to
   * by HEAD), then add an extra REF_LOG_ONLY update for HEAD.
@@@ -2139,10 -2589,11 +2139,10 @@@ static int split_head_update(struct ref
  
  	/*
  	 * First make sure that HEAD is not already in the
 -	 * transaction. This insertion is O(N) in the transaction
 +	 * transaction. This check is O(lg N) in the transaction
  	 * size, but it happens at most once per transaction.
  	 */
 -	item = string_list_insert(affected_refnames, "HEAD");
 -	if (item->util) {
 +	if (string_list_has_string(affected_refnames, "HEAD")) {
  		/* An entry already existed */
  		strbuf_addf(err,
  			    "multiple updates for 'HEAD' (including one "
@@@ -2157,14 -2608,6 +2157,14 @@@
  			update->new_oid.hash, update->old_oid.hash,
  			update->msg);
  
 +	/*
 +	 * Add "HEAD". This insertion is O(N) in the transaction
 +	 * size, but it happens at most once per transaction.
 +	 * Add new_update->refname instead of a literal "HEAD".
 +	 */
 +	if (strcmp(new_update->refname, "HEAD"))
 +		BUG("%s unexpectedly not 'HEAD'", new_update->refname);
 +	item = string_list_insert(affected_refnames, new_update->refname);
  	item->util = new_update;
  
  	return 0;
@@@ -2191,12 -2634,13 +2191,12 @@@ static int split_symref_update(struct f
  
  	/*
  	 * First make sure that referent is not already in the
 -	 * transaction. This insertion is O(N) in the transaction
 +	 * transaction. This check is O(lg N) in the transaction
  	 * size, but it happens at most once per symref in a
  	 * transaction.
  	 */
 -	item = string_list_insert(affected_refnames, referent);
 -	if (item->util) {
 -		/* An entry already existed */
 +	if (string_list_has_string(affected_refnames, referent)) {
 +		/* An entry already exists */
  		strbuf_addf(err,
  			    "multiple updates for '%s' (including one "
  			    "via symref '%s') are not allowed",
@@@ -2231,17 -2675,6 +2231,17 @@@
  	update->flags |= REF_LOG_ONLY | REF_NODEREF;
  	update->flags &= ~REF_HAVE_OLD;
  
 +	/*
 +	 * Add the referent. This insertion is O(N) in the transaction
 +	 * size, but it happens at most once per symref in a
 +	 * transaction. Make sure to add new_update->refname, which will
 +	 * be valid as long as affected_refnames is in use, and NOT
 +	 * referent, which might soon be freed by our caller.
 +	 */
 +	item = string_list_insert(affected_refnames, new_update->refname);
 +	if (item->util)
 +		BUG("%s unexpectedly found in affected_refnames",
 +		    new_update->refname);
  	item->util = new_update;
  
  	return 0;
@@@ -2313,7 -2746,7 +2313,7 @@@ static int lock_ref_for_update(struct f
  	struct strbuf referent = STRBUF_INIT;
  	int mustexist = (update->flags & REF_HAVE_OLD) &&
  		!is_null_oid(&update->old_oid);
 -	int ret;
 +	int ret = 0;
  	struct ref_lock *lock;
  
  	files_assert_main_repository(refs, "lock_ref_for_update");
@@@ -2325,7 -2758,7 +2325,7 @@@
  		ret = split_head_update(update, transaction, head_ref,
  					affected_refnames, err);
  		if (ret)
 -			return ret;
 +			goto out;
  	}
  
  	ret = lock_raw_ref(refs, update->refname, mustexist,
@@@ -2339,7 -2772,7 +2339,7 @@@
  		strbuf_addf(err, "cannot lock ref '%s': %s",
  			    original_update_refname(update), reason);
  		free(reason);
 -		return ret;
 +		goto out;
  	}
  
  	update->backend_data = lock;
@@@ -2358,12 -2791,10 +2358,12 @@@
  					strbuf_addf(err, "cannot lock ref '%s': "
  						    "error reading reference",
  						    original_update_refname(update));
 -					return -1;
 +					ret = TRANSACTION_GENERIC_ERROR;
 +					goto out;
  				}
  			} else if (check_old_oid(update, &lock->old_oid, err)) {
 -				return TRANSACTION_GENERIC_ERROR;
 +				ret = TRANSACTION_GENERIC_ERROR;
 +				goto out;
  			}
  		} else {
  			/*
@@@ -2377,15 -2808,13 +2377,15 @@@
  						  referent.buf, transaction,
  						  affected_refnames, err);
  			if (ret)
 -				return ret;
 +				goto out;
  		}
  	} else {
  		struct ref_update *parent_update;
  
 -		if (check_old_oid(update, &lock->old_oid, err))
 -			return TRANSACTION_GENERIC_ERROR;
 +		if (check_old_oid(update, &lock->old_oid, err)) {
 +			ret = TRANSACTION_GENERIC_ERROR;
 +			goto out;
 +		}
  
  		/*
  		 * If this update is happening indirectly because of a
@@@ -2422,8 -2851,7 +2422,8 @@@
  				    "cannot update ref '%s': %s",
  				    update->refname, write_err);
  			free(write_err);
 -			return TRANSACTION_GENERIC_ERROR;
 +			ret = TRANSACTION_GENERIC_ERROR;
 +			goto out;
  		} else {
  			update->flags |= REF_NEEDS_COMMIT;
  		}
@@@ -2434,35 -2862,22 +2434,35 @@@
  		 * the lockfile is still open. Close it to
  		 * free up the file descriptor:
  		 */
 -		if (close_ref(lock)) {
 +		if (close_ref_gently(lock)) {
  			strbuf_addf(err, "couldn't close '%s.lock'",
  				    update->refname);
 -			return TRANSACTION_GENERIC_ERROR;
 +			ret = TRANSACTION_GENERIC_ERROR;
 +			goto out;
  		}
  	}
 -	return 0;
 +
 +out:
 +	strbuf_release(&referent);
 +	return ret;
  }
  
 +struct files_transaction_backend_data {
 +	struct ref_transaction *packed_transaction;
 +	int packed_refs_locked;
 +};
 +
  /*
   * Unlock any references in `transaction` that are still locked, and
   * mark the transaction closed.
   */
 -static void files_transaction_cleanup(struct ref_transaction *transaction)
 +static void files_transaction_cleanup(struct files_ref_store *refs,
 +				      struct ref_transaction *transaction)
  {
  	size_t i;
 +	struct files_transaction_backend_data *backend_data =
 +		transaction->backend_data;
 +	struct strbuf err = STRBUF_INIT;
  
  	for (i = 0; i < transaction->nr; i++) {
  		struct ref_update *update = transaction->updates[i];
@@@ -2474,17 -2889,6 +2474,17 @@@
  		}
  	}
  
 +	if (backend_data->packed_transaction &&
 +	    ref_transaction_abort(backend_data->packed_transaction, &err)) {
 +		error("error aborting transaction: %s", err.buf);
 +		strbuf_release(&err);
 +	}
 +
 +	if (backend_data->packed_refs_locked)
 +		packed_refs_unlock(refs->packed_ref_store);
 +
 +	free(backend_data);
 +
  	transaction->state = REF_TRANSACTION_CLOSED;
  }
  
@@@ -2501,17 -2905,12 +2501,17 @@@ static int files_transaction_prepare(st
  	char *head_ref = NULL;
  	int head_type;
  	struct object_id head_oid;
 +	struct files_transaction_backend_data *backend_data;
 +	struct ref_transaction *packed_transaction = NULL;
  
  	assert(err);
  
  	if (!transaction->nr)
  		goto cleanup;
  
 +	backend_data = xcalloc(1, sizeof(*backend_data));
 +	transaction->backend_data = backend_data;
 +
  	/*
  	 * Fail if a refname appears more than once in the
  	 * transaction. (If we end up splitting up any updates using
@@@ -2578,41 -2977,6 +2578,41 @@@
  					  head_ref, &affected_refnames, err);
  		if (ret)
  			break;
 +
 +		if (update->flags & REF_DELETING &&
 +		    !(update->flags & REF_LOG_ONLY) &&
 +		    !(update->flags & REF_ISPRUNING)) {
 +			/*
 +			 * This reference has to be deleted from
 +			 * packed-refs if it exists there.
 +			 */
 +			if (!packed_transaction) {
 +				packed_transaction = ref_store_transaction_begin(
 +						refs->packed_ref_store, err);
 +				if (!packed_transaction) {
 +					ret = TRANSACTION_GENERIC_ERROR;
 +					goto cleanup;
 +				}
 +
 +				backend_data->packed_transaction =
 +					packed_transaction;
 +			}
 +
 +			ref_transaction_add_update(
 +					packed_transaction, update->refname,
 +					update->flags & ~REF_HAVE_OLD,
 +					update->new_oid.hash, update->old_oid.hash,
 +					NULL);
 +		}
 +	}
 +
 +	if (packed_transaction) {
 +		if (packed_refs_lock(refs->packed_ref_store, 0, err)) {
 +			ret = TRANSACTION_GENERIC_ERROR;
 +			goto cleanup;
 +		}
 +		backend_data->packed_refs_locked = 1;
 +		ret = ref_transaction_prepare(packed_transaction, err);
  	}
  
  cleanup:
@@@ -2620,7 -2984,7 +2620,7 @@@
  	string_list_clear(&affected_refnames, 0);
  
  	if (ret)
 -		files_transaction_cleanup(transaction);
 +		files_transaction_cleanup(refs, transaction);
  	else
  		transaction->state = REF_TRANSACTION_PREPARED;
  
@@@ -2635,10 -2999,9 +2635,10 @@@ static int files_transaction_finish(str
  		files_downcast(ref_store, 0, "ref_transaction_finish");
  	size_t i;
  	int ret = 0;
 -	struct string_list refs_to_delete = STRING_LIST_INIT_NODUP;
 -	struct string_list_item *ref_to_delete;
  	struct strbuf sb = STRBUF_INIT;
 +	struct files_transaction_backend_data *backend_data;
 +	struct ref_transaction *packed_transaction;
 +
  
  	assert(err);
  
@@@ -2647,9 -3010,6 +2647,9 @@@
  		return 0;
  	}
  
 +	backend_data = transaction->backend_data;
 +	packed_transaction = backend_data->packed_transaction;
 +
  	/* Perform updates first so live commits remain referenced */
  	for (i = 0; i < transaction->nr; i++) {
  		struct ref_update *update = transaction->updates[i];
@@@ -2685,44 -3045,7 +2685,44 @@@
  			}
  		}
  	}
 -	/* Perform deletes now that updates are safely completed */
 +
 +	/*
 +	 * Now that updates are safely completed, we can perform
 +	 * deletes. First delete the reflogs of any references that
 +	 * will be deleted, since (in the unexpected event of an
 +	 * error) leaving a reference without a reflog is less bad
 +	 * than leaving a reflog without a reference (the latter is a
 +	 * mildly invalid repository state):
 +	 */
 +	for (i = 0; i < transaction->nr; i++) {
 +		struct ref_update *update = transaction->updates[i];
 +		if (update->flags & REF_DELETING &&
 +		    !(update->flags & REF_LOG_ONLY) &&
 +		    !(update->flags & REF_ISPRUNING)) {
 +			strbuf_reset(&sb);
 +			files_reflog_path(refs, &sb, update->refname);
 +			if (!unlink_or_warn(sb.buf))
 +				try_remove_empty_parents(refs, update->refname,
 +							 REMOVE_EMPTY_PARENTS_REFLOG);
 +		}
 +	}
 +
 +	/*
 +	 * Perform deletes now that updates are safely completed.
 +	 *
 +	 * First delete any packed versions of the references, while
 +	 * retaining the packed-refs lock:
 +	 */
 +	if (packed_transaction) {
 +		ret = ref_transaction_commit(packed_transaction, err);
 +		ref_transaction_free(packed_transaction);
 +		packed_transaction = NULL;
 +		backend_data->packed_transaction = NULL;
 +		if (ret)
 +			goto cleanup;
 +	}
 +
 +	/* Now delete the loose versions of the references: */
  	for (i = 0; i < transaction->nr; i++) {
  		struct ref_update *update = transaction->updates[i];
  		struct ref_lock *lock = update->backend_data;
@@@ -2740,13 -3063,31 +2740,13 @@@
  				}
  				update->flags |= REF_DELETED_LOOSE;
  			}
 -
 -			if (!(update->flags & REF_ISPRUNING))
 -				string_list_append(&refs_to_delete,
 -						   lock->ref_name);
  		}
  	}
  
 -	if (repack_without_refs(refs, &refs_to_delete, err)) {
 -		ret = TRANSACTION_GENERIC_ERROR;
 -		goto cleanup;
 -	}
 -
 -	/* Delete the reflogs of any references that were deleted: */
 -	for_each_string_list_item(ref_to_delete, &refs_to_delete) {
 -		strbuf_reset(&sb);
 -		files_reflog_path(refs, &sb, ref_to_delete->string);
 -		if (!unlink_or_warn(sb.buf))
 -			try_remove_empty_parents(refs, ref_to_delete->string,
 -						 REMOVE_EMPTY_PARENTS_REFLOG);
 -	}
 -
  	clear_loose_ref_cache(refs);
  
  cleanup:
 -	files_transaction_cleanup(transaction);
 +	files_transaction_cleanup(refs, transaction);
  
  	for (i = 0; i < transaction->nr; i++) {
  		struct ref_update *update = transaction->updates[i];
@@@ -2764,6 -3105,7 +2764,6 @@@
  	}
  
  	strbuf_release(&sb);
 -	string_list_clear(&refs_to_delete, 0);
  	return ret;
  }
  
@@@ -2771,10 -3113,7 +2771,10 @@@ static int files_transaction_abort(stru
  				   struct ref_transaction *transaction,
  				   struct strbuf *err)
  {
 -	files_transaction_cleanup(transaction);
 +	struct files_ref_store *refs =
 +		files_downcast(ref_store, 0, "ref_transaction_abort");
 +
 +	files_transaction_cleanup(refs, transaction);
  	return 0;
  }
  
@@@ -2796,7 -3135,6 +2796,7 @@@ static int files_initial_transaction_co
  	size_t i;
  	int ret = 0;
  	struct string_list affected_refnames = STRING_LIST_INIT_NODUP;
 +	struct ref_transaction *packed_transaction = NULL;
  
  	assert(err);
  
@@@ -2829,12 -3167,6 +2829,12 @@@
  				 &affected_refnames))
  		die("BUG: initial ref transaction called with existing refs");
  
 +	packed_transaction = ref_store_transaction_begin(refs->packed_ref_store, err);
 +	if (!packed_transaction) {
 +		ret = TRANSACTION_GENERIC_ERROR;
 +		goto cleanup;
 +	}
 +
  	for (i = 0; i < transaction->nr; i++) {
  		struct ref_update *update = transaction->updates[i];
  
@@@ -2847,31 -3179,32 +2847,31 @@@
  			ret = TRANSACTION_NAME_CONFLICT;
  			goto cleanup;
  		}
 +
 +		/*
 +		 * Add a reference creation for this reference to the
 +		 * packed-refs transaction:
 +		 */
 +		ref_transaction_add_update(packed_transaction, update->refname,
 +					   update->flags & ~REF_HAVE_OLD,
 +					   update->new_oid.hash, update->old_oid.hash,
 +					   NULL);
  	}
  
 -	if (lock_packed_refs(refs, 0)) {
 -		strbuf_addf(err, "unable to lock packed-refs file: %s",
 -			    strerror(errno));
 +	if (packed_refs_lock(refs->packed_ref_store, 0, err)) {
  		ret = TRANSACTION_GENERIC_ERROR;
  		goto cleanup;
  	}
  
 -	for (i = 0; i < transaction->nr; i++) {
 -		struct ref_update *update = transaction->updates[i];
 -
 -		if ((update->flags & REF_HAVE_NEW) &&
 -		    !is_null_oid(&update->new_oid))
 -			add_packed_ref(refs, update->refname,
 -				       &update->new_oid);
 -	}
 -
 -	if (commit_packed_refs(refs)) {
 -		strbuf_addf(err, "unable to commit packed-refs file: %s",
 -			    strerror(errno));
 +	if (initial_ref_transaction_commit(packed_transaction, err)) {
  		ret = TRANSACTION_GENERIC_ERROR;
  		goto cleanup;
  	}
  
  cleanup:
 +	if (packed_transaction)
 +		ref_transaction_free(packed_transaction);
 +	packed_refs_unlock(refs->packed_ref_store);
  	transaction->state = REF_TRANSACTION_CLOSED;
  	string_list_clear(&affected_refnames, 0);
  	return ret;
@@@ -3000,17 -3333,16 +3000,17 @@@ static int files_reflog_expire(struct r
  			!(type & REF_ISSYMREF) &&
  			!is_null_oid(&cb.last_kept_oid);
  
 -		if (close_lock_file(&reflog_lock)) {
 +		if (close_lock_file_gently(&reflog_lock)) {
  			status |= error("couldn't write %s: %s", log_file,
  					strerror(errno));
 +			rollback_lock_file(&reflog_lock);
  		} else if (update &&
 -			   (write_in_full(get_lock_file_fd(lock->lk),
 +			   (write_in_full(get_lock_file_fd(&lock->lk),
- 				oid_to_hex(&cb.last_kept_oid), GIT_SHA1_HEXSZ) != GIT_SHA1_HEXSZ ||
- 			    write_str_in_full(get_lock_file_fd(&lock->lk), "\n") != 1 ||
+ 				oid_to_hex(&cb.last_kept_oid), GIT_SHA1_HEXSZ) < 0 ||
 -			    write_str_in_full(get_lock_file_fd(lock->lk), "\n") < 0 ||
 -			    close_ref(lock) < 0)) {
++			    write_str_in_full(get_lock_file_fd(&lock->lk), "\n") < 1 ||
 +			    close_ref_gently(lock) < 0)) {
  			status |= error("couldn't write %s",
 -					get_lock_file_path(lock->lk));
 +					get_lock_file_path(&lock->lk));
  			rollback_lock_file(&reflog_lock);
  		} else if (commit_lock_file(&reflog_lock)) {
  			status |= error("unable to write reflog '%s' (%s)",
diff --combined rerere.c
index d77235645e,51376cf6da..1ce440f4bb
--- a/rerere.c
+++ b/rerere.c
@@@ -258,7 -258,7 +258,7 @@@ static int write_rr(struct string_list 
  				    rerere_id_hex(id),
  				    rr->items[i].string, 0);
  
- 		if (write_in_full(out_fd, buf.buf, buf.len) != buf.len)
+ 		if (write_in_full(out_fd, buf.buf, buf.len) < 0)
  			die("unable to write rerere record");
  
  		strbuf_release(&buf);
@@@ -1133,14 -1133,14 +1133,14 @@@ int rerere_forget(struct pathspec *path
   * Garbage collection support
   */
  
 -static time_t rerere_created_at(struct rerere_id *id)
 +static timestamp_t rerere_created_at(struct rerere_id *id)
  {
  	struct stat st;
  
  	return stat(rerere_path(id, "preimage"), &st) ? (time_t) 0 : st.st_mtime;
  }
  
 -static time_t rerere_last_used_at(struct rerere_id *id)
 +static timestamp_t rerere_last_used_at(struct rerere_id *id)
  {
  	struct stat st;
  
@@@ -1157,11 -1157,11 +1157,11 @@@ static void unlink_rr_item(struct rerer
  	id->collection->status[id->variant] = 0;
  }
  
 -static void prune_one(struct rerere_id *id, time_t now,
 -		      int cutoff_resolve, int cutoff_noresolve)
 +static void prune_one(struct rerere_id *id,
 +		      timestamp_t cutoff_resolve, timestamp_t cutoff_noresolve)
  {
 -	time_t then;
 -	int cutoff;
 +	timestamp_t then;
 +	timestamp_t cutoff;
  
  	then = rerere_last_used_at(id);
  	if (then)
@@@ -1172,7 -1172,7 +1172,7 @@@
  			return;
  		cutoff = cutoff_noresolve;
  	}
 -	if (then < now - cutoff * 86400)
 +	if (then < cutoff)
  		unlink_rr_item(id);
  }
  
@@@ -1182,15 -1182,15 +1182,15 @@@ void rerere_gc(struct string_list *rr
  	DIR *dir;
  	struct dirent *e;
  	int i;
 -	time_t now = time(NULL);
 -	int cutoff_noresolve = 15;
 -	int cutoff_resolve = 60;
 +	timestamp_t now = time(NULL);
 +	timestamp_t cutoff_noresolve = now - 15 * 86400;
 +	timestamp_t cutoff_resolve = now - 60 * 86400;
  
  	if (setup_rerere(rr, 0) < 0)
  		return;
  
 -	git_config_get_int("gc.rerereresolved", &cutoff_resolve);
 -	git_config_get_int("gc.rerereunresolved", &cutoff_noresolve);
 +	git_config_get_expiry_in_days("gc.rerereresolved", &cutoff_resolve, now);
 +	git_config_get_expiry_in_days("gc.rerereunresolved", &cutoff_noresolve, now);
  	git_config(git_default_config, NULL);
  	dir = opendir(git_path("rr-cache"));
  	if (!dir)
@@@ -1211,7 -1211,7 +1211,7 @@@
  		for (id.variant = 0, id.collection = rr_dir;
  		     id.variant < id.collection->status_nr;
  		     id.variant++) {
 -			prune_one(&id, now, cutoff_resolve, cutoff_noresolve);
 +			prune_one(&id, cutoff_resolve, cutoff_noresolve);
  			if (id.collection->status[id.variant])
  				now_empty = 0;
  		}
diff --combined sha1_file.c
index b4a67bb838,20a9d39c00..dd7cbe52ef
--- a/sha1_file.c
+++ b/sha1_file.c
@@@ -28,9 -28,11 +28,9 @@@
  #include "list.h"
  #include "mergesort.h"
  #include "quote.h"
 +#include "packfile.h"
  
 -#define SZ_FMT PRIuMAX
 -static inline uintmax_t sz_fmt(size_t s) { return s; }
 -
 -const unsigned char null_sha1[20];
 +const unsigned char null_sha1[GIT_MAX_RAWSZ];
  const struct object_id null_oid;
  const struct object_id empty_tree_oid = {
  	EMPTY_TREE_SHA1_BIN_LITERAL
@@@ -276,6 -278,28 +276,6 @@@ static const char *alt_sha1_path(struc
  	return buf->buf;
  }
  
 - char *odb_pack_name(struct strbuf *buf,
 -		     const unsigned char *sha1,
 -		     const char *ext)
 -{
 -	strbuf_reset(buf);
 -	strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(),
 -		    sha1_to_hex(sha1), ext);
 -	return buf->buf;
 -}
 -
 -char *sha1_pack_name(const unsigned char *sha1)
 -{
 -	static struct strbuf buf = STRBUF_INIT;
 -	return odb_pack_name(&buf, sha1, "pack");
 -}
 -
 -char *sha1_pack_index_name(const unsigned char *sha1)
 -{
 -	static struct strbuf buf = STRBUF_INIT;
 -	return odb_pack_name(&buf, sha1, "idx");
 -}
 -
  struct alternate_object_database *alt_odb_list;
  static struct alternate_object_database **alt_odb_tail;
  
@@@ -681,6 -705,213 +681,6 @@@ static int has_loose_object(const unsig
  	return check_and_freshen(sha1, 0);
  }
  
 -static unsigned int pack_used_ctr;
 -static unsigned int pack_mmap_calls;
 -static unsigned int peak_pack_open_windows;
 -static unsigned int pack_open_windows;
 -static unsigned int pack_open_fds;
 -static unsigned int pack_max_fds;
 -static size_t peak_pack_mapped;
 -static size_t pack_mapped;
 -struct packed_git *packed_git;
 -
 -static struct mru packed_git_mru_storage;
 -struct mru *packed_git_mru = &packed_git_mru_storage;
 -
 -void pack_report(void)
 -{
 -	fprintf(stderr,
 -		"pack_report: getpagesize()            = %10" SZ_FMT "\n"
 -		"pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
 -		"pack_report: core.packedGitLimit      = %10" SZ_FMT "\n",
 -		sz_fmt(getpagesize()),
 -		sz_fmt(packed_git_window_size),
 -		sz_fmt(packed_git_limit));
 -	fprintf(stderr,
 -		"pack_report: pack_used_ctr            = %10u\n"
 -		"pack_report: pack_mmap_calls          = %10u\n"
 -		"pack_report: pack_open_windows        = %10u / %10u\n"
 -		"pack_report: pack_mapped              = "
 -			"%10" SZ_FMT " / %10" SZ_FMT "\n",
 -		pack_used_ctr,
 -		pack_mmap_calls,
 -		pack_open_windows, peak_pack_open_windows,
 -		sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
 -}
 -
 -/*
 - * Open and mmap the index file at path, perform a couple of
 - * consistency checks, then record its information to p.  Return 0 on
 - * success.
 - */
 -static int check_packed_git_idx(const char *path, struct packed_git *p)
 -{
 -	void *idx_map;
 -	struct pack_idx_header *hdr;
 -	size_t idx_size;
 -	uint32_t version, nr, i, *index;
 -	int fd = git_open(path);
 -	struct stat st;
 -
 -	if (fd < 0)
 -		return -1;
 -	if (fstat(fd, &st)) {
 -		close(fd);
 -		return -1;
 -	}
 -	idx_size = xsize_t(st.st_size);
 -	if (idx_size < 4 * 256 + 20 + 20) {
 -		close(fd);
 -		return error("index file %s is too small", path);
 -	}
 -	idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
 -	close(fd);
 -
 -	hdr = idx_map;
 -	if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
 -		version = ntohl(hdr->idx_version);
 -		if (version < 2 || version > 2) {
 -			munmap(idx_map, idx_size);
 -			return error("index file %s is version %"PRIu32
 -				     " and is not supported by this binary"
 -				     " (try upgrading GIT to a newer version)",
 -				     path, version);
 -		}
 -	} else
 -		version = 1;
 -
 -	nr = 0;
 -	index = idx_map;
 -	if (version > 1)
 -		index += 2;  /* skip index header */
 -	for (i = 0; i < 256; i++) {
 -		uint32_t n = ntohl(index[i]);
 -		if (n < nr) {
 -			munmap(idx_map, idx_size);
 -			return error("non-monotonic index %s", path);
 -		}
 -		nr = n;
 -	}
 -
 -	if (version == 1) {
 -		/*
 -		 * Total size:
 -		 *  - 256 index entries 4 bytes each
 -		 *  - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
 -		 *  - 20-byte SHA1 of the packfile
 -		 *  - 20-byte SHA1 file checksum
 -		 */
 -		if (idx_size != 4*256 + nr * 24 + 20 + 20) {
 -			munmap(idx_map, idx_size);
 -			return error("wrong index v1 file size in %s", path);
 -		}
 -	} else if (version == 2) {
 -		/*
 -		 * Minimum size:
 -		 *  - 8 bytes of header
 -		 *  - 256 index entries 4 bytes each
 -		 *  - 20-byte sha1 entry * nr
 -		 *  - 4-byte crc entry * nr
 -		 *  - 4-byte offset entry * nr
 -		 *  - 20-byte SHA1 of the packfile
 -		 *  - 20-byte SHA1 file checksum
 -		 * And after the 4-byte offset table might be a
 -		 * variable sized table containing 8-byte entries
 -		 * for offsets larger than 2^31.
 -		 */
 -		unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
 -		unsigned long max_size = min_size;
 -		if (nr)
 -			max_size += (nr - 1)*8;
 -		if (idx_size < min_size || idx_size > max_size) {
 -			munmap(idx_map, idx_size);
 -			return error("wrong index v2 file size in %s", path);
 -		}
 -		if (idx_size != min_size &&
 -		    /*
 -		     * make sure we can deal with large pack offsets.
 -		     * 31-bit signed offset won't be enough, neither
 -		     * 32-bit unsigned one will be.
 -		     */
 -		    (sizeof(off_t) <= 4)) {
 -			munmap(idx_map, idx_size);
 -			return error("pack too large for current definition of off_t in %s", path);
 -		}
 -	}
 -
 -	p->index_version = version;
 -	p->index_data = idx_map;
 -	p->index_size = idx_size;
 -	p->num_objects = nr;
 -	return 0;
 -}
 -
 -int open_pack_index(struct packed_git *p)
 -{
 -	char *idx_name;
 -	size_t len;
 -	int ret;
 -
 -	if (p->index_data)
 -		return 0;
 -
 -	if (!strip_suffix(p->pack_name, ".pack", &len))
 -		die("BUG: pack_name does not end in .pack");
 -	idx_name = xstrfmt("%.*s.idx", (int)len, p->pack_name);
 -	ret = check_packed_git_idx(idx_name, p);
 -	free(idx_name);
 -	return ret;
 -}
 -
 -static void scan_windows(struct packed_git *p,
 -	struct packed_git **lru_p,
 -	struct pack_window **lru_w,
 -	struct pack_window **lru_l)
 -{
 -	struct pack_window *w, *w_l;
 -
 -	for (w_l = NULL, w = p->windows; w; w = w->next) {
 -		if (!w->inuse_cnt) {
 -			if (!*lru_w || w->last_used < (*lru_w)->last_used) {
 -				*lru_p = p;
 -				*lru_w = w;
 -				*lru_l = w_l;
 -			}
 -		}
 -		w_l = w;
 -	}
 -}
 -
 -static int unuse_one_window(struct packed_git *current)
 -{
 -	struct packed_git *p, *lru_p = NULL;
 -	struct pack_window *lru_w = NULL, *lru_l = NULL;
 -
 -	if (current)
 -		scan_windows(current, &lru_p, &lru_w, &lru_l);
 -	for (p = packed_git; p; p = p->next)
 -		scan_windows(p, &lru_p, &lru_w, &lru_l);
 -	if (lru_p) {
 -		munmap(lru_w->base, lru_w->len);
 -		pack_mapped -= lru_w->len;
 -		if (lru_l)
 -			lru_l->next = lru_w->next;
 -		else
 -			lru_p->windows = lru_w->next;
 -		free(lru_w);
 -		pack_open_windows--;
 -		return 1;
 -	}
 -	return 0;
 -}
 -
 -void release_pack_memory(size_t need)
 -{
 -	size_t cur = pack_mapped;
 -	while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
 -		; /* nothing */
 -}
 -
  static void mmap_limit_check(size_t length)
  {
  	static size_t limit = 0;
@@@ -719,360 -950,1959 +719,360 @@@ void *xmmap(void *start, size_t length
  	return ret;
  }
  
 -void close_pack_windows(struct packed_git *p)
 -{
 -	while (p->windows) {
 -		struct pack_window *w = p->windows;
 -
 -		if (w->inuse_cnt)
 -			die("pack '%s' still has open windows to it",
 -			    p->pack_name);
 -		munmap(w->base, w->len);
 -		pack_mapped -= w->len;
 -		pack_open_windows--;
 -		p->windows = w->next;
 -		free(w);
 -	}
 -}
 -
 -static int close_pack_fd(struct packed_git *p)
 -{
 -	if (p->pack_fd < 0)
 -		return 0;
 -
 -	close(p->pack_fd);
 -	pack_open_fds--;
 -	p->pack_fd = -1;
 -
 -	return 1;
 -}
 -
 -static void close_pack(struct packed_git *p)
 -{
 -	close_pack_windows(p);
 -	close_pack_fd(p);
 -	close_pack_index(p);
 -}
 -
 -void close_all_packs(void)
 -{
 -	struct packed_git *p;
 -
 -	for (p = packed_git; p; p = p->next)
 -		if (p->do_not_close)
 -			die("BUG: want to close pack marked 'do-not-close'");
 -		else
 -			close_pack(p);
 -}
 -
 -
  /*
 - * The LRU pack is the one with the oldest MRU window, preferring packs
 - * with no used windows, or the oldest mtime if it has no windows allocated.
 + * With an in-core object data in "map", rehash it to make sure the
 + * object name actually matches "sha1" to detect object corruption.
 + * With "map" == NULL, try reading the object named with "sha1" using
 + * the streaming interface and rehash it to do the same.
   */
 -static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struct pack_window **mru_w, int *accept_windows_inuse)
 -{
 -	struct pack_window *w, *this_mru_w;
 -	int has_windows_inuse = 0;
 -
 -	/*
 -	 * Reject this pack if it has windows and the previously selected
 -	 * one does not.  If this pack does not have windows, reject
 -	 * it if the pack file is newer than the previously selected one.
 -	 */
 -	if (*lru_p && !*mru_w && (p->windows || p->mtime > (*lru_p)->mtime))
 -		return;
 -
 -	for (w = this_mru_w = p->windows; w; w = w->next) {
 -		/*
 -		 * Reject this pack if any of its windows are in use,
 -		 * but the previously selected pack did not have any
 -		 * inuse windows.  Otherwise, record that this pack
 -		 * has windows in use.
 -		 */
 -		if (w->inuse_cnt) {
 -			if (*accept_windows_inuse)
 -				has_windows_inuse = 1;
 -			else
 -				return;
 -		}
 -
 -		if (w->last_used > this_mru_w->last_used)
 -			this_mru_w = w;
 -
 -		/*
 -		 * Reject this pack if it has windows that have been
 -		 * used more recently than the previously selected pack.
 -		 * If the previously selected pack had windows inuse and
 -		 * we have not encountered a window in this pack that is
 -		 * inuse, skip this check since we prefer a pack with no
 -		 * inuse windows to one that has inuse windows.
 -		 */
 -		if (*mru_w && *accept_windows_inuse == has_windows_inuse &&
 -		    this_mru_w->last_used > (*mru_w)->last_used)
 -			return;
 -	}
 -
 -	/*
 -	 * Select this pack.
 -	 */
 -	*mru_w = this_mru_w;
 -	*lru_p = p;
 -	*accept_windows_inuse = has_windows_inuse;
 -}
 -
 -static int close_one_pack(void)
 +int check_sha1_signature(const unsigned char *sha1, void *map,
 +			 unsigned long size, const char *type)
  {
 -	struct packed_git *p, *lru_p = NULL;
 -	struct pack_window *mru_w = NULL;
 -	int accept_windows_inuse = 1;
 +	unsigned char real_sha1[20];
 +	enum object_type obj_type;
 +	struct git_istream *st;
 +	git_SHA_CTX c;
 +	char hdr[32];
 +	int hdrlen;
  
 -	for (p = packed_git; p; p = p->next) {
 -		if (p->pack_fd == -1)
 -			continue;
 -		find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
 +	if (map) {
 +		hash_sha1_file(map, size, type, real_sha1);
 +		return hashcmp(sha1, real_sha1) ? -1 : 0;
  	}
  
 -	if (lru_p)
 -		return close_pack_fd(lru_p);
 +	st = open_istream(sha1, &obj_type, &size, NULL);
 +	if (!st)
 +		return -1;
  
 -	return 0;
 -}
 +	/* Generate the header */
 +	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1;
  
 -void unuse_pack(struct pack_window **w_cursor)
 -{
 -	struct pack_window *w = *w_cursor;
 -	if (w) {
 -		w->inuse_cnt--;
 -		*w_cursor = NULL;
 -	}
 -}
 +	/* Sha1.. */
 +	git_SHA1_Init(&c);
 +	git_SHA1_Update(&c, hdr, hdrlen);
 +	for (;;) {
 +		char buf[1024 * 16];
 +		ssize_t readlen = read_istream(st, buf, sizeof(buf));
  
 -void close_pack_index(struct packed_git *p)
 -{
 -	if (p->index_data) {
 -		munmap((void *)p->index_data, p->index_size);
 -		p->index_data = NULL;
 +		if (readlen < 0) {
 +			close_istream(st);
 +			return -1;
 +		}
 +		if (!readlen)
 +			break;
 +		git_SHA1_Update(&c, buf, readlen);
  	}
 +	git_SHA1_Final(real_sha1, &c);
 +	close_istream(st);
 +	return hashcmp(sha1, real_sha1) ? -1 : 0;
  }
  
 -static unsigned int get_max_fd_limit(void)
 +int git_open_cloexec(const char *name, int flags)
  {
 -#ifdef RLIMIT_NOFILE
 -	{
 -		struct rlimit lim;
 +	int fd;
 +	static int o_cloexec = O_CLOEXEC;
  
 -		if (!getrlimit(RLIMIT_NOFILE, &lim))
 -			return lim.rlim_cur;
 +	fd = open(name, flags | o_cloexec);
 +	if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) {
 +		/* Try again w/o O_CLOEXEC: the kernel might not support it */
 +		o_cloexec &= ~O_CLOEXEC;
 +		fd = open(name, flags | o_cloexec);
  	}
 -#endif
  
 -#ifdef _SC_OPEN_MAX
 +#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC)
  	{
 -		long open_max = sysconf(_SC_OPEN_MAX);
 -		if (0 < open_max)
 -			return open_max;
 -		/*
 -		 * Otherwise, we got -1 for one of the two
 -		 * reasons:
 -		 *
 -		 * (1) sysconf() did not understand _SC_OPEN_MAX
 -		 *     and signaled an error with -1; or
 -		 * (2) sysconf() said there is no limit.
 -		 *
 -		 * We _could_ clear errno before calling sysconf() to
 -		 * tell these two cases apart and return a huge number
 -		 * in the latter case to let the caller cap it to a
 -		 * value that is not so selfish, but letting the
 -		 * fallback OPEN_MAX codepath take care of these cases
 -		 * is a lot simpler.
 -		 */
 -	}
 -#endif
 +		static int fd_cloexec = FD_CLOEXEC;
  
 -#ifdef OPEN_MAX
 -	return OPEN_MAX;
 -#else
 -	return 1; /* see the caller ;-) */
 +		if (!o_cloexec && 0 <= fd && fd_cloexec) {
 +			/* Opened w/o O_CLOEXEC?  try with fcntl(2) to add it */
 +			int flags = fcntl(fd, F_GETFD);
 +			if (fcntl(fd, F_SETFD, flags | fd_cloexec))
 +				fd_cloexec = 0;
 +		}
 +	}
  #endif
 +	return fd;
  }
  
  /*
 - * Do not call this directly as this leaks p->pack_fd on error return;
 - * call open_packed_git() instead.
 + * Find "sha1" as a loose object in the local repository or in an alternate.
 + * Returns 0 on success, negative on failure.
 + *
 + * The "path" out-parameter will give the path of the object we found (if any).
 + * Note that it may point to static storage and is only valid until another
 + * call to sha1_file_name(), etc.
   */
 -static int open_packed_git_1(struct packed_git *p)
 +static int stat_sha1_file(const unsigned char *sha1, struct stat *st,
 +			  const char **path)
  {
 -	struct stat st;
 -	struct pack_header hdr;
 -	unsigned char sha1[20];
 -	unsigned char *idx_sha1;
 -	long fd_flag;
 -
 -	if (!p->index_data && open_pack_index(p))
 -		return error("packfile %s index unavailable", p->pack_name);
 +	struct alternate_object_database *alt;
  
 -	if (!pack_max_fds) {
 -		unsigned int max_fds = get_max_fd_limit();
 +	*path = sha1_file_name(sha1);
 +	if (!lstat(*path, st))
 +		return 0;
  
 -		/* Save 3 for stdin/stdout/stderr, 22 for work */
 -		if (25 < max_fds)
 -			pack_max_fds = max_fds - 25;
 -		else
 -			pack_max_fds = 1;
 +	prepare_alt_odb();
 +	errno = ENOENT;
 +	for (alt = alt_odb_list; alt; alt = alt->next) {
 +		*path = alt_sha1_path(alt, sha1);
 +		if (!lstat(*path, st))
 +			return 0;
  	}
  
 -	while (pack_max_fds <= pack_open_fds && close_one_pack())
 -		; /* nothing */
 -
 -	p->pack_fd = git_open(p->pack_name);
 -	if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
 -		return -1;
 -	pack_open_fds++;
 -
 -	/* If we created the struct before we had the pack we lack size. */
 -	if (!p->pack_size) {
 -		if (!S_ISREG(st.st_mode))
 -			return error("packfile %s not a regular file", p->pack_name);
 -		p->pack_size = st.st_size;
 -	} else if (p->pack_size != st.st_size)
 -		return error("packfile %s size changed", p->pack_name);
 -
 -	/* We leave these file descriptors open with sliding mmap;
 -	 * there is no point keeping them open across exec(), though.
 -	 */
 -	fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
 -	if (fd_flag < 0)
 -		return error("cannot determine file descriptor flags");
 -	fd_flag |= FD_CLOEXEC;
 -	if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
 -		return error("cannot set FD_CLOEXEC");
 -
 -	/* Verify we recognize this pack file format. */
 -	if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
 -		return error("file %s is far too short to be a packfile", p->pack_name);
 -	if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
 -		return error("file %s is not a GIT packfile", p->pack_name);
 -	if (!pack_version_ok(hdr.hdr_version))
 -		return error("packfile %s is version %"PRIu32" and not"
 -			" supported (try upgrading GIT to a newer version)",
 -			p->pack_name, ntohl(hdr.hdr_version));
 -
 -	/* Verify the pack matches its index. */
 -	if (p->num_objects != ntohl(hdr.hdr_entries))
 -		return error("packfile %s claims to have %"PRIu32" objects"
 -			     " while index indicates %"PRIu32" objects",
 -			     p->pack_name, ntohl(hdr.hdr_entries),
 -			     p->num_objects);
 -	if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
 -		return error("end of packfile %s is unavailable", p->pack_name);
 -	if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
 -		return error("packfile %s signature is unavailable", p->pack_name);
 -	idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
 -	if (hashcmp(sha1, idx_sha1))
 -		return error("packfile %s does not match index", p->pack_name);
 -	return 0;
 -}
 -
 -static int open_packed_git(struct packed_git *p)
 -{
 -	if (!open_packed_git_1(p))
 -		return 0;
 -	close_pack_fd(p);
  	return -1;
  }
  
 -static int in_window(struct pack_window *win, off_t offset)
 +/*
 + * Like stat_sha1_file(), but actually open the object and return the
 + * descriptor. See the caveats on the "path" parameter above.
 + */
 +static int open_sha1_file(const unsigned char *sha1, const char **path)
  {
 -	/* We must promise at least 20 bytes (one hash) after the
 -	 * offset is available from this window, otherwise the offset
 -	 * is not actually in this window and a different window (which
 -	 * has that one hash excess) must be used.  This is to support
 -	 * the object header and delta base parsing routines below.
 -	 */
 -	off_t win_off = win->offset;
 -	return win_off <= offset
 -		&& (offset + 20) <= (win_off + win->len);
 -}
 +	int fd;
 +	struct alternate_object_database *alt;
 +	int most_interesting_errno;
  
 -unsigned char *use_pack(struct packed_git *p,
 -		struct pack_window **w_cursor,
 -		off_t offset,
 -		unsigned long *left)
 -{
 -	struct pack_window *win = *w_cursor;
 +	*path = sha1_file_name(sha1);
 +	fd = git_open(*path);
 +	if (fd >= 0)
 +		return fd;
 +	most_interesting_errno = errno;
  
 -	/* Since packfiles end in a hash of their content and it's
 -	 * pointless to ask for an offset into the middle of that
 -	 * hash, and the in_window function above wouldn't match
 -	 * don't allow an offset too close to the end of the file.
 -	 */
 -	if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
 -		die("packfile %s cannot be accessed", p->pack_name);
 -	if (offset > (p->pack_size - 20))
 -		die("offset beyond end of packfile (truncated pack?)");
 -	if (offset < 0)
 -		die(_("offset before end of packfile (broken .idx?)"));
 -
 -	if (!win || !in_window(win, offset)) {
 -		if (win)
 -			win->inuse_cnt--;
 -		for (win = p->windows; win; win = win->next) {
 -			if (in_window(win, offset))
 -				break;
 -		}
 -		if (!win) {
 -			size_t window_align = packed_git_window_size / 2;
 -			off_t len;
 -
 -			if (p->pack_fd == -1 && open_packed_git(p))
 -				die("packfile %s cannot be accessed", p->pack_name);
 -
 -			win = xcalloc(1, sizeof(*win));
 -			win->offset = (offset / window_align) * window_align;
 -			len = p->pack_size - win->offset;
 -			if (len > packed_git_window_size)
 -				len = packed_git_window_size;
 -			win->len = (size_t)len;
 -			pack_mapped += win->len;
 -			while (packed_git_limit < pack_mapped
 -				&& unuse_one_window(p))
 -				; /* nothing */
 -			win->base = xmmap(NULL, win->len,
 -				PROT_READ, MAP_PRIVATE,
 -				p->pack_fd, win->offset);
 -			if (win->base == MAP_FAILED)
 -				die_errno("packfile %s cannot be mapped",
 -					  p->pack_name);
 -			if (!win->offset && win->len == p->pack_size
 -				&& !p->do_not_close)
 -				close_pack_fd(p);
 -			pack_mmap_calls++;
 -			pack_open_windows++;
 -			if (pack_mapped > peak_pack_mapped)
 -				peak_pack_mapped = pack_mapped;
 -			if (pack_open_windows > peak_pack_open_windows)
 -				peak_pack_open_windows = pack_open_windows;
 -			win->next = p->windows;
 -			p->windows = win;
 -		}
 -	}
 -	if (win != *w_cursor) {
 -		win->last_used = pack_used_ctr++;
 -		win->inuse_cnt++;
 -		*w_cursor = win;
 +	prepare_alt_odb();
 +	for (alt = alt_odb_list; alt; alt = alt->next) {
 +		*path = alt_sha1_path(alt, sha1);
 +		fd = git_open(*path);
 +		if (fd >= 0)
 +			return fd;
 +		if (most_interesting_errno == ENOENT)
 +			most_interesting_errno = errno;
  	}
 -	offset -= win->offset;
 -	if (left)
 -		*left = win->len - xsize_t(offset);
 -	return win->base + offset;
 +	errno = most_interesting_errno;
 +	return -1;
  }
  
 -static struct packed_git *alloc_packed_git(int extra)
 +/*
 + * Map the loose object at "path" if it is not NULL, or the path found by
 + * searching for a loose object named "sha1".
 + */
 +static void *map_sha1_file_1(const char *path,
 +			     const unsigned char *sha1,
 +			     unsigned long *size)
  {
 -	struct packed_git *p = xmalloc(st_add(sizeof(*p), extra));
 -	memset(p, 0, sizeof(*p));
 -	p->pack_fd = -1;
 -	return p;
 -}
 -
 -static void try_to_free_pack_memory(size_t size)
 -{
 -	release_pack_memory(size);
 -}
 -
 -struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
 -{
 -	static int have_set_try_to_free_routine;
 -	struct stat st;
 -	size_t alloc;
 -	struct packed_git *p;
 -
 -	if (!have_set_try_to_free_routine) {
 -		have_set_try_to_free_routine = 1;
 -		set_try_to_free_routine(try_to_free_pack_memory);
 -	}
 -
 -	/*
 -	 * Make sure a corresponding .pack file exists and that
 -	 * the index looks sane.
 -	 */
 -	if (!strip_suffix_mem(path, &path_len, ".idx"))
 -		return NULL;
 -
 -	/*
 -	 * ".pack" is long enough to hold any suffix we're adding (and
 -	 * the use xsnprintf double-checks that)
 -	 */
 -	alloc = st_add3(path_len, strlen(".pack"), 1);
 -	p = alloc_packed_git(alloc);
 -	memcpy(p->pack_name, path, path_len);
 -
 -	xsnprintf(p->pack_name + path_len, alloc - path_len, ".keep");
 -	if (!access(p->pack_name, F_OK))
 -		p->pack_keep = 1;
 -
 -	xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
 -	if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
 -		free(p);
 -		return NULL;
 -	}
 -
 -	/* ok, it looks sane as far as we can check without
 -	 * actually mapping the pack file.
 -	 */
 -	p->pack_size = st.st_size;
 -	p->pack_local = local;
 -	p->mtime = st.st_mtime;
 -	if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
 -		hashclr(p->sha1);
 -	return p;
 -}
 -
 -struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
 -{
 -	const char *path = sha1_pack_name(sha1);
 -	size_t alloc = st_add(strlen(path), 1);
 -	struct packed_git *p = alloc_packed_git(alloc);
 -
 -	memcpy(p->pack_name, path, alloc); /* includes NUL */
 -	hashcpy(p->sha1, sha1);
 -	if (check_packed_git_idx(idx_path, p)) {
 -		free(p);
 -		return NULL;
 -	}
 -
 -	return p;
 -}
 -
 -void install_packed_git(struct packed_git *pack)
 -{
 -	if (pack->pack_fd != -1)
 -		pack_open_fds++;
 -
 -	pack->next = packed_git;
 -	packed_git = pack;
 -}
 -
 -void (*report_garbage)(unsigned seen_bits, const char *path);
 -
 -static void report_helper(const struct string_list *list,
 -			  int seen_bits, int first, int last)
 -{
 -	if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX))
 -		return;
 -
 -	for (; first < last; first++)
 -		report_garbage(seen_bits, list->items[first].string);
 -}
 -
 -static void report_pack_garbage(struct string_list *list)
 -{
 -	int i, baselen = -1, first = 0, seen_bits = 0;
 -
 -	if (!report_garbage)
 -		return;
 -
 -	string_list_sort(list);
 -
 -	for (i = 0; i < list->nr; i++) {
 -		const char *path = list->items[i].string;
 -		if (baselen != -1 &&
 -		    strncmp(path, list->items[first].string, baselen)) {
 -			report_helper(list, seen_bits, first, i);
 -			baselen = -1;
 -			seen_bits = 0;
 -		}
 -		if (baselen == -1) {
 -			const char *dot = strrchr(path, '.');
 -			if (!dot) {
 -				report_garbage(PACKDIR_FILE_GARBAGE, path);
 -				continue;
 -			}
 -			baselen = dot - path + 1;
 -			first = i;
 -		}
 -		if (!strcmp(path + baselen, "pack"))
 -			seen_bits |= 1;
 -		else if (!strcmp(path + baselen, "idx"))
 -			seen_bits |= 2;
 -	}
 -	report_helper(list, seen_bits, first, list->nr);
 -}
 -
 -static void prepare_packed_git_one(char *objdir, int local)
 -{
 -	struct strbuf path = STRBUF_INIT;
 -	size_t dirnamelen;
 -	DIR *dir;
 -	struct dirent *de;
 -	struct string_list garbage = STRING_LIST_INIT_DUP;
 -
 -	strbuf_addstr(&path, objdir);
 -	strbuf_addstr(&path, "/pack");
 -	dir = opendir(path.buf);
 -	if (!dir) {
 -		if (errno != ENOENT)
 -			error_errno("unable to open object pack directory: %s",
 -				    path.buf);
 -		strbuf_release(&path);
 -		return;
 -	}
 -	strbuf_addch(&path, '/');
 -	dirnamelen = path.len;
 -	while ((de = readdir(dir)) != NULL) {
 -		struct packed_git *p;
 -		size_t base_len;
 -
 -		if (is_dot_or_dotdot(de->d_name))
 -			continue;
 -
 -		strbuf_setlen(&path, dirnamelen);
 -		strbuf_addstr(&path, de->d_name);
 -
 -		base_len = path.len;
 -		if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
 -			/* Don't reopen a pack we already have. */
 -			for (p = packed_git; p; p = p->next) {
 -				size_t len;
 -				if (strip_suffix(p->pack_name, ".pack", &len) &&
 -				    len == base_len &&
 -				    !memcmp(p->pack_name, path.buf, len))
 -					break;
 -			}
 -			if (p == NULL &&
 -			    /*
 -			     * See if it really is a valid .idx file with
 -			     * corresponding .pack file that we can map.
 -			     */
 -			    (p = add_packed_git(path.buf, path.len, local)) != NULL)
 -				install_packed_git(p);
 -		}
 -
 -		if (!report_garbage)
 -			continue;
 -
 -		if (ends_with(de->d_name, ".idx") ||
 -		    ends_with(de->d_name, ".pack") ||
 -		    ends_with(de->d_name, ".bitmap") ||
 -		    ends_with(de->d_name, ".keep"))
 -			string_list_append(&garbage, path.buf);
 -		else
 -			report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
 -	}
 -	closedir(dir);
 -	report_pack_garbage(&garbage);
 -	string_list_clear(&garbage, 0);
 -	strbuf_release(&path);
 -}
 -
 -static int approximate_object_count_valid;
 -
 -/*
 - * Give a fast, rough count of the number of objects in the repository. This
 - * ignores loose objects completely. If you have a lot of them, then either
 - * you should repack because your performance will be awful, or they are
 - * all unreachable objects about to be pruned, in which case they're not really
 - * interesting as a measure of repo size in the first place.
 - */
 -unsigned long approximate_object_count(void)
 -{
 -	static unsigned long count;
 -	if (!approximate_object_count_valid) {
 -		struct packed_git *p;
 -
 -		prepare_packed_git();
 -		count = 0;
 -		for (p = packed_git; p; p = p->next) {
 -			if (open_pack_index(p))
 -				continue;
 -			count += p->num_objects;
 -		}
 -	}
 -	return count;
 -}
 -
 -static void *get_next_packed_git(const void *p)
 -{
 -	return ((const struct packed_git *)p)->next;
 -}
 -
 -static void set_next_packed_git(void *p, void *next)
 -{
 -	((struct packed_git *)p)->next = next;
 -}
 -
 -static int sort_pack(const void *a_, const void *b_)
 -{
 -	const struct packed_git *a = a_;
 -	const struct packed_git *b = b_;
 -	int st;
 -
 -	/*
 -	 * Local packs tend to contain objects specific to our
 -	 * variant of the project than remote ones.  In addition,
 -	 * remote ones could be on a network mounted filesystem.
 -	 * Favor local ones for these reasons.
 -	 */
 -	st = a->pack_local - b->pack_local;
 -	if (st)
 -		return -st;
 -
 -	/*
 -	 * Younger packs tend to contain more recent objects,
 -	 * and more recent objects tend to get accessed more
 -	 * often.
 -	 */
 -	if (a->mtime < b->mtime)
 -		return 1;
 -	else if (a->mtime == b->mtime)
 -		return 0;
 -	return -1;
 -}
 -
 -static void rearrange_packed_git(void)
 -{
 -	packed_git = llist_mergesort(packed_git, get_next_packed_git,
 -				     set_next_packed_git, sort_pack);
 -}
 -
 -static void prepare_packed_git_mru(void)
 -{
 -	struct packed_git *p;
 -
 -	mru_clear(packed_git_mru);
 -	for (p = packed_git; p; p = p->next)
 -		mru_append(packed_git_mru, p);
 -}
 -
 -static int prepare_packed_git_run_once = 0;
 -void prepare_packed_git(void)
 -{
 -	struct alternate_object_database *alt;
 -
 -	if (prepare_packed_git_run_once)
 -		return;
 -	prepare_packed_git_one(get_object_directory(), 1);
 -	prepare_alt_odb();
 -	for (alt = alt_odb_list; alt; alt = alt->next)
 -		prepare_packed_git_one(alt->path, 0);
 -	rearrange_packed_git();
 -	prepare_packed_git_mru();
 -	prepare_packed_git_run_once = 1;
 -}
 -
 -void reprepare_packed_git(void)
 -{
 -	approximate_object_count_valid = 0;
 -	prepare_packed_git_run_once = 0;
 -	prepare_packed_git();
 -}
 -
 -static void mark_bad_packed_object(struct packed_git *p,
 -				   const unsigned char *sha1)
 -{
 -	unsigned i;
 -	for (i = 0; i < p->num_bad_objects; i++)
 -		if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
 -			return;
 -	p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
 -				      st_mult(GIT_MAX_RAWSZ,
 -					      st_add(p->num_bad_objects, 1)));
 -	hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
 -	p->num_bad_objects++;
 -}
 -
 -static const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
 -{
 -	struct packed_git *p;
 -	unsigned i;
 -
 -	for (p = packed_git; p; p = p->next)
 -		for (i = 0; i < p->num_bad_objects; i++)
 -			if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
 -				return p;
 -	return NULL;
 -}
 -
 -/*
 - * With an in-core object data in "map", rehash it to make sure the
 - * object name actually matches "sha1" to detect object corruption.
 - * With "map" == NULL, try reading the object named with "sha1" using
 - * the streaming interface and rehash it to do the same.
 - */
 -int check_sha1_signature(const unsigned char *sha1, void *map,
 -			 unsigned long size, const char *type)
 -{
 -	unsigned char real_sha1[20];
 -	enum object_type obj_type;
 -	struct git_istream *st;
 -	git_SHA_CTX c;
 -	char hdr[32];
 -	int hdrlen;
 -
 -	if (map) {
 -		hash_sha1_file(map, size, type, real_sha1);
 -		return hashcmp(sha1, real_sha1) ? -1 : 0;
 -	}
 -
 -	st = open_istream(sha1, &obj_type, &size, NULL);
 -	if (!st)
 -		return -1;
 -
 -	/* Generate the header */
 -	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1;
 -
 -	/* Sha1.. */
 -	git_SHA1_Init(&c);
 -	git_SHA1_Update(&c, hdr, hdrlen);
 -	for (;;) {
 -		char buf[1024 * 16];
 -		ssize_t readlen = read_istream(st, buf, sizeof(buf));
 -
 -		if (readlen < 0) {
 -			close_istream(st);
 -			return -1;
 -		}
 -		if (!readlen)
 -			break;
 -		git_SHA1_Update(&c, buf, readlen);
 -	}
 -	git_SHA1_Final(real_sha1, &c);
 -	close_istream(st);
 -	return hashcmp(sha1, real_sha1) ? -1 : 0;
 -}
 -
 -int git_open_cloexec(const char *name, int flags)
 -{
 -	int fd;
 -	static int o_cloexec = O_CLOEXEC;
 -
 -	fd = open(name, flags | o_cloexec);
 -	if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) {
 -		/* Try again w/o O_CLOEXEC: the kernel might not support it */
 -		o_cloexec &= ~O_CLOEXEC;
 -		fd = open(name, flags | o_cloexec);
 -	}
 -
 -#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC)
 -	{
 -		static int fd_cloexec = FD_CLOEXEC;
 -
 -		if (!o_cloexec && 0 <= fd && fd_cloexec) {
 -			/* Opened w/o O_CLOEXEC?  try with fcntl(2) to add it */
 -			int flags = fcntl(fd, F_GETFD);
 -			if (fcntl(fd, F_SETFD, flags | fd_cloexec))
 -				fd_cloexec = 0;
 -		}
 -	}
 -#endif
 -	return fd;
 -}
 -
 -/*
 - * Find "sha1" as a loose object in the local repository or in an alternate.
 - * Returns 0 on success, negative on failure.
 - *
 - * The "path" out-parameter will give the path of the object we found (if any).
 - * Note that it may point to static storage and is only valid until another
 - * call to sha1_file_name(), etc.
 - */
 -static int stat_sha1_file(const unsigned char *sha1, struct stat *st,
 -			  const char **path)
 -{
 -	struct alternate_object_database *alt;
 -
 -	*path = sha1_file_name(sha1);
 -	if (!lstat(*path, st))
 -		return 0;
 -
 -	prepare_alt_odb();
 -	errno = ENOENT;
 -	for (alt = alt_odb_list; alt; alt = alt->next) {
 -		*path = alt_sha1_path(alt, sha1);
 -		if (!lstat(*path, st))
 -			return 0;
 -	}
 -
 -	return -1;
 -}
 -
 -/*
 - * Like stat_sha1_file(), but actually open the object and return the
 - * descriptor. See the caveats on the "path" parameter above.
 - */
 -static int open_sha1_file(const unsigned char *sha1, const char **path)
 -{
 -	int fd;
 -	struct alternate_object_database *alt;
 -	int most_interesting_errno;
 -
 -	*path = sha1_file_name(sha1);
 -	fd = git_open(*path);
 -	if (fd >= 0)
 -		return fd;
 -	most_interesting_errno = errno;
 -
 -	prepare_alt_odb();
 -	for (alt = alt_odb_list; alt; alt = alt->next) {
 -		*path = alt_sha1_path(alt, sha1);
 -		fd = git_open(*path);
 -		if (fd >= 0)
 -			return fd;
 -		if (most_interesting_errno == ENOENT)
 -			most_interesting_errno = errno;
 -	}
 -	errno = most_interesting_errno;
 -	return -1;
 -}
 -
 -/*
 - * Map the loose object at "path" if it is not NULL, or the path found by
 - * searching for a loose object named "sha1".
 - */
 -static void *map_sha1_file_1(const char *path,
 -			     const unsigned char *sha1,
 -			     unsigned long *size)
 -{
 -	void *map;
 -	int fd;
 -
 -	if (path)
 -		fd = git_open(path);
 -	else
 -		fd = open_sha1_file(sha1, &path);
 -	map = NULL;
 -	if (fd >= 0) {
 -		struct stat st;
 -
 -		if (!fstat(fd, &st)) {
 -			*size = xsize_t(st.st_size);
 -			if (!*size) {
 -				/* mmap() is forbidden on empty files */
 -				error("object file %s is empty", path);
 -				return NULL;
 -			}
 -			map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
 -		}
 -		close(fd);
 -	}
 -	return map;
 -}
 -
 -void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
 -{
 -	return map_sha1_file_1(NULL, sha1, size);
 -}
 -
 -unsigned long unpack_object_header_buffer(const unsigned char *buf,
 -		unsigned long len, enum object_type *type, unsigned long *sizep)
 -{
 -	unsigned shift;
 -	unsigned long size, c;
 -	unsigned long used = 0;
 -
 -	c = buf[used++];
 -	*type = (c >> 4) & 7;
 -	size = c & 15;
 -	shift = 4;
 -	while (c & 0x80) {
 -		if (len <= used || bitsizeof(long) <= shift) {
 -			error("bad object header");
 -			size = used = 0;
 -			break;
 -		}
 -		c = buf[used++];
 -		size += (c & 0x7f) << shift;
 -		shift += 7;
 -	}
 -	*sizep = size;
 -	return used;
 -}
 -
 -static int unpack_sha1_short_header(git_zstream *stream,
 -				    unsigned char *map, unsigned long mapsize,
 -				    void *buffer, unsigned long bufsiz)
 -{
 -	/* Get the data stream */
 -	memset(stream, 0, sizeof(*stream));
 -	stream->next_in = map;
 -	stream->avail_in = mapsize;
 -	stream->next_out = buffer;
 -	stream->avail_out = bufsiz;
 -
 -	git_inflate_init(stream);
 -	return git_inflate(stream, 0);
 -}
 -
 -int unpack_sha1_header(git_zstream *stream,
 -		       unsigned char *map, unsigned long mapsize,
 -		       void *buffer, unsigned long bufsiz)
 -{
 -	int status = unpack_sha1_short_header(stream, map, mapsize,
 -					      buffer, bufsiz);
 -
 -	if (status < Z_OK)
 -		return status;
 -
 -	/* Make sure we have the terminating NUL */
 -	if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
 -		return -1;
 -	return 0;
 -}
 -
 -static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map,
 -					unsigned long mapsize, void *buffer,
 -					unsigned long bufsiz, struct strbuf *header)
 -{
 -	int status;
 -
 -	status = unpack_sha1_short_header(stream, map, mapsize, buffer, bufsiz);
 -	if (status < Z_OK)
 -		return -1;
 -
 -	/*
 -	 * Check if entire header is unpacked in the first iteration.
 -	 */
 -	if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
 -		return 0;
 -
 -	/*
 -	 * buffer[0..bufsiz] was not large enough.  Copy the partial
 -	 * result out to header, and then append the result of further
 -	 * reading the stream.
 -	 */
 -	strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
 -	stream->next_out = buffer;
 -	stream->avail_out = bufsiz;
 -
 -	do {
 -		status = git_inflate(stream, 0);
 -		strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
 -		if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
 -			return 0;
 -		stream->next_out = buffer;
 -		stream->avail_out = bufsiz;
 -	} while (status != Z_STREAM_END);
 -	return -1;
 -}
 -
 -static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
 -{
 -	int bytes = strlen(buffer) + 1;
 -	unsigned char *buf = xmallocz(size);
 -	unsigned long n;
 -	int status = Z_OK;
 -
 -	n = stream->total_out - bytes;
 -	if (n > size)
 -		n = size;
 -	memcpy(buf, (char *) buffer + bytes, n);
 -	bytes = n;
 -	if (bytes <= size) {
 -		/*
 -		 * The above condition must be (bytes <= size), not
 -		 * (bytes < size).  In other words, even though we
 -		 * expect no more output and set avail_out to zero,
 -		 * the input zlib stream may have bytes that express
 -		 * "this concludes the stream", and we *do* want to
 -		 * eat that input.
 -		 *
 -		 * Otherwise we would not be able to test that we
 -		 * consumed all the input to reach the expected size;
 -		 * we also want to check that zlib tells us that all
 -		 * went well with status == Z_STREAM_END at the end.
 -		 */
 -		stream->next_out = buf + bytes;
 -		stream->avail_out = size - bytes;
 -		while (status == Z_OK)
 -			status = git_inflate(stream, Z_FINISH);
 -	}
 -	if (status == Z_STREAM_END && !stream->avail_in) {
 -		git_inflate_end(stream);
 -		return buf;
 -	}
 -
 -	if (status < 0)
 -		error("corrupt loose object '%s'", sha1_to_hex(sha1));
 -	else if (stream->avail_in)
 -		error("garbage at end of loose object '%s'",
 -		      sha1_to_hex(sha1));
 -	free(buf);
 -	return NULL;
 -}
 -
 -/*
 - * We used to just use "sscanf()", but that's actually way
 - * too permissive for what we want to check. So do an anal
 - * object header parse by hand.
 - */
 -static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
 -			       unsigned int flags)
 -{
 -	const char *type_buf = hdr;
 -	unsigned long size;
 -	int type, type_len = 0;
 -
 -	/*
 -	 * The type can be of any size but is followed by
 -	 * a space.
 -	 */
 -	for (;;) {
 -		char c = *hdr++;
 -		if (!c)
 -			return -1;
 -		if (c == ' ')
 -			break;
 -		type_len++;
 -	}
 -
 -	type = type_from_string_gently(type_buf, type_len, 1);
 -	if (oi->typename)
 -		strbuf_add(oi->typename, type_buf, type_len);
 -	/*
 -	 * Set type to 0 if its an unknown object and
 -	 * we're obtaining the type using '--allow-unknown-type'
 -	 * option.
 -	 */
 -	if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE) && (type < 0))
 -		type = 0;
 -	else if (type < 0)
 -		die("invalid object type");
 -	if (oi->typep)
 -		*oi->typep = type;
 -
 -	/*
 -	 * The length must follow immediately, and be in canonical
 -	 * decimal format (ie "010" is not valid).
 -	 */
 -	size = *hdr++ - '0';
 -	if (size > 9)
 -		return -1;
 -	if (size) {
 -		for (;;) {
 -			unsigned long c = *hdr - '0';
 -			if (c > 9)
 -				break;
 -			hdr++;
 -			size = size * 10 + c;
 -		}
 -	}
 -
 -	if (oi->sizep)
 -		*oi->sizep = size;
 -
 -	/*
 -	 * The length must be followed by a zero byte
 -	 */
 -	return *hdr ? -1 : type;
 -}
 -
 -int parse_sha1_header(const char *hdr, unsigned long *sizep)
 -{
 -	struct object_info oi = OBJECT_INFO_INIT;
 -
 -	oi.sizep = sizep;
 -	return parse_sha1_header_extended(hdr, &oi, 0);
 -}
 -
 -unsigned long get_size_from_delta(struct packed_git *p,
 -				  struct pack_window **w_curs,
 -			          off_t curpos)
 -{
 -	const unsigned char *data;
 -	unsigned char delta_head[20], *in;
 -	git_zstream stream;
 -	int st;
 -
 -	memset(&stream, 0, sizeof(stream));
 -	stream.next_out = delta_head;
 -	stream.avail_out = sizeof(delta_head);
 -
 -	git_inflate_init(&stream);
 -	do {
 -		in = use_pack(p, w_curs, curpos, &stream.avail_in);
 -		stream.next_in = in;
 -		st = git_inflate(&stream, Z_FINISH);
 -		curpos += stream.next_in - in;
 -	} while ((st == Z_OK || st == Z_BUF_ERROR) &&
 -		 stream.total_out < sizeof(delta_head));
 -	git_inflate_end(&stream);
 -	if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) {
 -		error("delta data unpack-initial failed");
 -		return 0;
 -	}
 -
 -	/* Examine the initial part of the delta to figure out
 -	 * the result size.
 -	 */
 -	data = delta_head;
 -
 -	/* ignore base size */
 -	get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
 -
 -	/* Read the result size */
 -	return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
 -}
 -
 -static off_t get_delta_base(struct packed_git *p,
 -				    struct pack_window **w_curs,
 -				    off_t *curpos,
 -				    enum object_type type,
 -				    off_t delta_obj_offset)
 -{
 -	unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
 -	off_t base_offset;
 -
 -	/* use_pack() assured us we have [base_info, base_info + 20)
 -	 * as a range that we can look at without walking off the
 -	 * end of the mapped window.  Its actually the hash size
 -	 * that is assured.  An OFS_DELTA longer than the hash size
 -	 * is stupid, as then a REF_DELTA would be smaller to store.
 -	 */
 -	if (type == OBJ_OFS_DELTA) {
 -		unsigned used = 0;
 -		unsigned char c = base_info[used++];
 -		base_offset = c & 127;
 -		while (c & 128) {
 -			base_offset += 1;
 -			if (!base_offset || MSB(base_offset, 7))
 -				return 0;  /* overflow */
 -			c = base_info[used++];
 -			base_offset = (base_offset << 7) + (c & 127);
 -		}
 -		base_offset = delta_obj_offset - base_offset;
 -		if (base_offset <= 0 || base_offset >= delta_obj_offset)
 -			return 0;  /* out of bound */
 -		*curpos += used;
 -	} else if (type == OBJ_REF_DELTA) {
 -		/* The base entry _must_ be in the same pack */
 -		base_offset = find_pack_entry_one(base_info, p);
 -		*curpos += 20;
 -	} else
 -		die("I am totally screwed");
 -	return base_offset;
 -}
 -
 -/*
 - * Like get_delta_base above, but we return the sha1 instead of the pack
 - * offset. This means it is cheaper for REF deltas (we do not have to do
 - * the final object lookup), but more expensive for OFS deltas (we
 - * have to load the revidx to convert the offset back into a sha1).
 - */
 -static const unsigned char *get_delta_base_sha1(struct packed_git *p,
 -						struct pack_window **w_curs,
 -						off_t curpos,
 -						enum object_type type,
 -						off_t delta_obj_offset)
 -{
 -	if (type == OBJ_REF_DELTA) {
 -		unsigned char *base = use_pack(p, w_curs, curpos, NULL);
 -		return base;
 -	} else if (type == OBJ_OFS_DELTA) {
 -		struct revindex_entry *revidx;
 -		off_t base_offset = get_delta_base(p, w_curs, &curpos,
 -						   type, delta_obj_offset);
 -
 -		if (!base_offset)
 -			return NULL;
 -
 -		revidx = find_pack_revindex(p, base_offset);
 -		if (!revidx)
 -			return NULL;
 -
 -		return nth_packed_object_sha1(p, revidx->nr);
 -	} else
 -		return NULL;
 -}
 -
 -int unpack_object_header(struct packed_git *p,
 -			 struct pack_window **w_curs,
 -			 off_t *curpos,
 -			 unsigned long *sizep)
 -{
 -	unsigned char *base;
 -	unsigned long left;
 -	unsigned long used;
 -	enum object_type type;
 -
 -	/* use_pack() assures us we have [base, base + 20) available
 -	 * as a range that we can look at.  (Its actually the hash
 -	 * size that is assured.)  With our object header encoding
 -	 * the maximum deflated object size is 2^137, which is just
 -	 * insane, so we know won't exceed what we have been given.
 -	 */
 -	base = use_pack(p, w_curs, *curpos, &left);
 -	used = unpack_object_header_buffer(base, left, &type, sizep);
 -	if (!used) {
 -		type = OBJ_BAD;
 -	} else
 -		*curpos += used;
 -
 -	return type;
 -}
 -
 -static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
 -{
 -	int type;
 -	struct revindex_entry *revidx;
 -	const unsigned char *sha1;
 -	revidx = find_pack_revindex(p, obj_offset);
 -	if (!revidx)
 -		return OBJ_BAD;
 -	sha1 = nth_packed_object_sha1(p, revidx->nr);
 -	mark_bad_packed_object(p, sha1);
 -	type = sha1_object_info(sha1, NULL);
 -	if (type <= OBJ_NONE)
 -		return OBJ_BAD;
 -	return type;
 -}
 -
 -#define POI_STACK_PREALLOC 64
 -
 -static enum object_type packed_to_object_type(struct packed_git *p,
 -					      off_t obj_offset,
 -					      enum object_type type,
 -					      struct pack_window **w_curs,
 -					      off_t curpos)
 -{
 -	off_t small_poi_stack[POI_STACK_PREALLOC];
 -	off_t *poi_stack = small_poi_stack;
 -	int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC;
 -
 -	while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
 -		off_t base_offset;
 -		unsigned long size;
 -		/* Push the object we're going to leave behind */
 -		if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
 -			poi_stack_alloc = alloc_nr(poi_stack_nr);
 -			ALLOC_ARRAY(poi_stack, poi_stack_alloc);
 -			memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
 -		} else {
 -			ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
 -		}
 -		poi_stack[poi_stack_nr++] = obj_offset;
 -		/* If parsing the base offset fails, just unwind */
 -		base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
 -		if (!base_offset)
 -			goto unwind;
 -		curpos = obj_offset = base_offset;
 -		type = unpack_object_header(p, w_curs, &curpos, &size);
 -		if (type <= OBJ_NONE) {
 -			/* If getting the base itself fails, we first
 -			 * retry the base, otherwise unwind */
 -			type = retry_bad_packed_offset(p, base_offset);
 -			if (type > OBJ_NONE)
 -				goto out;
 -			goto unwind;
 -		}
 -	}
 -
 -	switch (type) {
 -	case OBJ_BAD:
 -	case OBJ_COMMIT:
 -	case OBJ_TREE:
 -	case OBJ_BLOB:
 -	case OBJ_TAG:
 -		break;
 -	default:
 -		error("unknown object type %i at offset %"PRIuMAX" in %s",
 -		      type, (uintmax_t)obj_offset, p->pack_name);
 -		type = OBJ_BAD;
 -	}
 -
 -out:
 -	if (poi_stack != small_poi_stack)
 -		free(poi_stack);
 -	return type;
 -
 -unwind:
 -	while (poi_stack_nr) {
 -		obj_offset = poi_stack[--poi_stack_nr];
 -		type = retry_bad_packed_offset(p, obj_offset);
 -		if (type > OBJ_NONE)
 -			goto out;
 -	}
 -	type = OBJ_BAD;
 -	goto out;
 -}
 -
 -static struct hashmap delta_base_cache;
 -static size_t delta_base_cached;
 -
 -static LIST_HEAD(delta_base_cache_lru);
 -
 -struct delta_base_cache_key {
 -	struct packed_git *p;
 -	off_t base_offset;
 -};
 -
 -struct delta_base_cache_entry {
 -	struct hashmap hash;
 -	struct delta_base_cache_key key;
 -	struct list_head lru;
 -	void *data;
 -	unsigned long size;
 -	enum object_type type;
 -};
 -
 -static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset)
 -{
 -	unsigned int hash;
 -
 -	hash = (unsigned int)(intptr_t)p + (unsigned int)base_offset;
 -	hash += (hash >> 8) + (hash >> 16);
 -	return hash;
 -}
 -
 -static struct delta_base_cache_entry *
 -get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
 -{
 -	struct hashmap_entry entry;
 -	struct delta_base_cache_key key;
 -
 -	if (!delta_base_cache.cmpfn)
 -		return NULL;
 -
 -	hashmap_entry_init(&entry, pack_entry_hash(p, base_offset));
 -	key.p = p;
 -	key.base_offset = base_offset;
 -	return hashmap_get(&delta_base_cache, &entry, &key);
 -}
 -
 -static int delta_base_cache_key_eq(const struct delta_base_cache_key *a,
 -				   const struct delta_base_cache_key *b)
 -{
 -	return a->p == b->p && a->base_offset == b->base_offset;
 -}
 +	void *map;
 +	int fd;
  
 -static int delta_base_cache_hash_cmp(const void *unused_cmp_data,
 -				     const void *va, const void *vb,
 -				     const void *vkey)
 -{
 -	const struct delta_base_cache_entry *a = va, *b = vb;
 -	const struct delta_base_cache_key *key = vkey;
 -	if (key)
 -		return !delta_base_cache_key_eq(&a->key, key);
 +	if (path)
 +		fd = git_open(path);
  	else
 -		return !delta_base_cache_key_eq(&a->key, &b->key);
 -}
 -
 -static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
 -{
 -	return !!get_delta_base_cache_entry(p, base_offset);
 -}
 -
 -/*
 - * Remove the entry from the cache, but do _not_ free the associated
 - * entry data. The caller takes ownership of the "data" buffer, and
 - * should copy out any fields it wants before detaching.
 - */
 -static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent)
 -{
 -	hashmap_remove(&delta_base_cache, ent, &ent->key);
 -	list_del(&ent->lru);
 -	delta_base_cached -= ent->size;
 -	free(ent);
 -}
 -
 -static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
 -	unsigned long *base_size, enum object_type *type)
 -{
 -	struct delta_base_cache_entry *ent;
 -
 -	ent = get_delta_base_cache_entry(p, base_offset);
 -	if (!ent)
 -		return unpack_entry(p, base_offset, type, base_size);
 -
 -	if (type)
 -		*type = ent->type;
 -	if (base_size)
 -		*base_size = ent->size;
 -	return xmemdupz(ent->data, ent->size);
 -}
 -
 -static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
 -{
 -	free(ent->data);
 -	detach_delta_base_cache_entry(ent);
 -}
 -
 -void clear_delta_base_cache(void)
 -{
 -	struct list_head *lru, *tmp;
 -	list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
 -		struct delta_base_cache_entry *entry =
 -			list_entry(lru, struct delta_base_cache_entry, lru);
 -		release_delta_base_cache(entry);
 -	}
 -}
 -
 -static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
 -	void *base, unsigned long base_size, enum object_type type)
 -{
 -	struct delta_base_cache_entry *ent = xmalloc(sizeof(*ent));
 -	struct list_head *lru, *tmp;
 -
 -	delta_base_cached += base_size;
 -
 -	list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
 -		struct delta_base_cache_entry *f =
 -			list_entry(lru, struct delta_base_cache_entry, lru);
 -		if (delta_base_cached <= delta_base_cache_limit)
 -			break;
 -		release_delta_base_cache(f);
 -	}
 -
 -	ent->key.p = p;
 -	ent->key.base_offset = base_offset;
 -	ent->type = type;
 -	ent->data = base;
 -	ent->size = base_size;
 -	list_add_tail(&ent->lru, &delta_base_cache_lru);
 -
 -	if (!delta_base_cache.cmpfn)
 -		hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, NULL, 0);
 -	hashmap_entry_init(ent, pack_entry_hash(p, base_offset));
 -	hashmap_add(&delta_base_cache, ent);
 -}
 -
 -int packed_object_info(struct packed_git *p, off_t obj_offset,
 -		       struct object_info *oi)
 -{
 -	struct pack_window *w_curs = NULL;
 -	unsigned long size;
 -	off_t curpos = obj_offset;
 -	enum object_type type;
 -
 -	/*
 -	 * We always get the representation type, but only convert it to
 -	 * a "real" type later if the caller is interested.
 -	 */
 -	if (oi->contentp) {
 -		*oi->contentp = cache_or_unpack_entry(p, obj_offset, oi->sizep,
 -						      &type);
 -		if (!*oi->contentp)
 -			type = OBJ_BAD;
 -	} else {
 -		type = unpack_object_header(p, &w_curs, &curpos, &size);
 -	}
 -
 -	if (!oi->contentp && oi->sizep) {
 -		if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
 -			off_t tmp_pos = curpos;
 -			off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
 -							   type, obj_offset);
 -			if (!base_offset) {
 -				type = OBJ_BAD;
 -				goto out;
 -			}
 -			*oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
 -			if (*oi->sizep == 0) {
 -				type = OBJ_BAD;
 -				goto out;
 -			}
 -		} else {
 -			*oi->sizep = size;
 -		}
 -	}
 -
 -	if (oi->disk_sizep) {
 -		struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
 -		*oi->disk_sizep = revidx[1].offset - obj_offset;
 -	}
 -
 -	if (oi->typep || oi->typename) {
 -		enum object_type ptot;
 -		ptot = packed_to_object_type(p, obj_offset, type, &w_curs,
 -					     curpos);
 -		if (oi->typep)
 -			*oi->typep = ptot;
 -		if (oi->typename) {
 -			const char *tn = typename(ptot);
 -			if (tn)
 -				strbuf_addstr(oi->typename, tn);
 -		}
 -		if (ptot < 0) {
 -			type = OBJ_BAD;
 -			goto out;
 -		}
 -	}
 -
 -	if (oi->delta_base_sha1) {
 -		if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
 -			const unsigned char *base;
 -
 -			base = get_delta_base_sha1(p, &w_curs, curpos,
 -						   type, obj_offset);
 -			if (!base) {
 -				type = OBJ_BAD;
 -				goto out;
 -			}
 -
 -			hashcpy(oi->delta_base_sha1, base);
 -		} else
 -			hashclr(oi->delta_base_sha1);
 -	}
 -
 -out:
 -	unuse_pack(&w_curs);
 -	return type;
 -}
 -
 -static void *unpack_compressed_entry(struct packed_git *p,
 -				    struct pack_window **w_curs,
 -				    off_t curpos,
 -				    unsigned long size)
 -{
 -	int st;
 -	git_zstream stream;
 -	unsigned char *buffer, *in;
 -
 -	buffer = xmallocz_gently(size);
 -	if (!buffer)
 -		return NULL;
 -	memset(&stream, 0, sizeof(stream));
 -	stream.next_out = buffer;
 -	stream.avail_out = size + 1;
 -
 -	git_inflate_init(&stream);
 -	do {
 -		in = use_pack(p, w_curs, curpos, &stream.avail_in);
 -		stream.next_in = in;
 -		st = git_inflate(&stream, Z_FINISH);
 -		if (!stream.avail_out)
 -			break; /* the payload is larger than it should be */
 -		curpos += stream.next_in - in;
 -	} while (st == Z_OK || st == Z_BUF_ERROR);
 -	git_inflate_end(&stream);
 -	if ((st != Z_STREAM_END) || stream.total_out != size) {
 -		free(buffer);
 -		return NULL;
 -	}
 -
 -	return buffer;
 -}
 -
 -static void *read_object(const unsigned char *sha1, enum object_type *type,
 -			 unsigned long *size);
 -
 -static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
 -{
 -	static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
 -	trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
 -			 p->pack_name, (uintmax_t)obj_offset);
 -}
 -
 -int do_check_packed_object_crc;
 -
 -#define UNPACK_ENTRY_STACK_PREALLOC 64
 -struct unpack_entry_stack_ent {
 -	off_t obj_offset;
 -	off_t curpos;
 -	unsigned long size;
 -};
 -
 -void *unpack_entry(struct packed_git *p, off_t obj_offset,
 -		   enum object_type *final_type, unsigned long *final_size)
 -{
 -	struct pack_window *w_curs = NULL;
 -	off_t curpos = obj_offset;
 -	void *data = NULL;
 -	unsigned long size;
 -	enum object_type type;
 -	struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
 -	struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
 -	int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
 -	int base_from_cache = 0;
 -
 -	write_pack_access_log(p, obj_offset);
 -
 -	/* PHASE 1: drill down to the innermost base object */
 -	for (;;) {
 -		off_t base_offset;
 -		int i;
 -		struct delta_base_cache_entry *ent;
 -
 -		ent = get_delta_base_cache_entry(p, curpos);
 -		if (ent) {
 -			type = ent->type;
 -			data = ent->data;
 -			size = ent->size;
 -			detach_delta_base_cache_entry(ent);
 -			base_from_cache = 1;
 -			break;
 -		}
 -
 -		if (do_check_packed_object_crc && p->index_version > 1) {
 -			struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
 -			off_t len = revidx[1].offset - obj_offset;
 -			if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
 -				const unsigned char *sha1 =
 -					nth_packed_object_sha1(p, revidx->nr);
 -				error("bad packed object CRC for %s",
 -				      sha1_to_hex(sha1));
 -				mark_bad_packed_object(p, sha1);
 -				data = NULL;
 -				goto out;
 -			}
 -		}
 -
 -		type = unpack_object_header(p, &w_curs, &curpos, &size);
 -		if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
 -			break;
 -
 -		base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
 -		if (!base_offset) {
 -			error("failed to validate delta base reference "
 -			      "at offset %"PRIuMAX" from %s",
 -			      (uintmax_t)curpos, p->pack_name);
 -			/* bail to phase 2, in hopes of recovery */
 -			data = NULL;
 -			break;
 -		}
 -
 -		/* push object, proceed to base */
 -		if (delta_stack_nr >= delta_stack_alloc
 -		    && delta_stack == small_delta_stack) {
 -			delta_stack_alloc = alloc_nr(delta_stack_nr);
 -			ALLOC_ARRAY(delta_stack, delta_stack_alloc);
 -			memcpy(delta_stack, small_delta_stack,
 -			       sizeof(*delta_stack)*delta_stack_nr);
 -		} else {
 -			ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
 -		}
 -		i = delta_stack_nr++;
 -		delta_stack[i].obj_offset = obj_offset;
 -		delta_stack[i].curpos = curpos;
 -		delta_stack[i].size = size;
 -
 -		curpos = obj_offset = base_offset;
 -	}
 -
 -	/* PHASE 2: handle the base */
 -	switch (type) {
 -	case OBJ_OFS_DELTA:
 -	case OBJ_REF_DELTA:
 -		if (data)
 -			die("BUG: unpack_entry: left loop at a valid delta");
 -		break;
 -	case OBJ_COMMIT:
 -	case OBJ_TREE:
 -	case OBJ_BLOB:
 -	case OBJ_TAG:
 -		if (!base_from_cache)
 -			data = unpack_compressed_entry(p, &w_curs, curpos, size);
 -		break;
 -	default:
 -		data = NULL;
 -		error("unknown object type %i at offset %"PRIuMAX" in %s",
 -		      type, (uintmax_t)obj_offset, p->pack_name);
 -	}
 -
 -	/* PHASE 3: apply deltas in order */
 -
 -	/* invariants:
 -	 *   'data' holds the base data, or NULL if there was corruption
 -	 */
 -	while (delta_stack_nr) {
 -		void *delta_data;
 -		void *base = data;
 -		void *external_base = NULL;
 -		unsigned long delta_size, base_size = size;
 -		int i;
 -
 -		data = NULL;
 -
 -		if (base)
 -			add_delta_base_cache(p, obj_offset, base, base_size, type);
 -
 -		if (!base) {
 -			/*
 -			 * We're probably in deep shit, but let's try to fetch
 -			 * the required base anyway from another pack or loose.
 -			 * This is costly but should happen only in the presence
 -			 * of a corrupted pack, and is better than failing outright.
 -			 */
 -			struct revindex_entry *revidx;
 -			const unsigned char *base_sha1;
 -			revidx = find_pack_revindex(p, obj_offset);
 -			if (revidx) {
 -				base_sha1 = nth_packed_object_sha1(p, revidx->nr);
 -				error("failed to read delta base object %s"
 -				      " at offset %"PRIuMAX" from %s",
 -				      sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
 -				      p->pack_name);
 -				mark_bad_packed_object(p, base_sha1);
 -				base = read_object(base_sha1, &type, &base_size);
 -				external_base = base;
 -			}
 -		}
 -
 -		i = --delta_stack_nr;
 -		obj_offset = delta_stack[i].obj_offset;
 -		curpos = delta_stack[i].curpos;
 -		delta_size = delta_stack[i].size;
 -
 -		if (!base)
 -			continue;
 -
 -		delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
 -
 -		if (!delta_data) {
 -			error("failed to unpack compressed delta "
 -			      "at offset %"PRIuMAX" from %s",
 -			      (uintmax_t)curpos, p->pack_name);
 -			data = NULL;
 -			free(external_base);
 -			continue;
 -		}
 -
 -		data = patch_delta(base, base_size,
 -				   delta_data, delta_size,
 -				   &size);
 -
 -		/*
 -		 * We could not apply the delta; warn the user, but keep going.
 -		 * Our failure will be noticed either in the next iteration of
 -		 * the loop, or if this is the final delta, in the caller when
 -		 * we return NULL. Those code paths will take care of making
 -		 * a more explicit warning and retrying with another copy of
 -		 * the object.
 -		 */
 -		if (!data)
 -			error("failed to apply delta");
 -
 -		free(delta_data);
 -		free(external_base);
 -	}
 -
 -	if (final_type)
 -		*final_type = type;
 -	if (final_size)
 -		*final_size = size;
 -
 -out:
 -	unuse_pack(&w_curs);
 -
 -	if (delta_stack != small_delta_stack)
 -		free(delta_stack);
 -
 -	return data;
 -}
 +		fd = open_sha1_file(sha1, &path);
 +	map = NULL;
 +	if (fd >= 0) {
 +		struct stat st;
  
 -const unsigned char *nth_packed_object_sha1(struct packed_git *p,
 -					    uint32_t n)
 -{
 -	const unsigned char *index = p->index_data;
 -	if (!index) {
 -		if (open_pack_index(p))
 -			return NULL;
 -		index = p->index_data;
 -	}
 -	if (n >= p->num_objects)
 -		return NULL;
 -	index += 4 * 256;
 -	if (p->index_version == 1) {
 -		return index + 24 * n + 4;
 -	} else {
 -		index += 8;
 -		return index + 20 * n;
 +		if (!fstat(fd, &st)) {
 +			*size = xsize_t(st.st_size);
 +			if (!*size) {
 +				/* mmap() is forbidden on empty files */
 +				error("object file %s is empty", path);
 +				return NULL;
 +			}
 +			map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
 +		}
 +		close(fd);
  	}
 +	return map;
  }
  
 -const struct object_id *nth_packed_object_oid(struct object_id *oid,
 -					      struct packed_git *p,
 -					      uint32_t n)
 +void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
  {
 -	const unsigned char *hash = nth_packed_object_sha1(p, n);
 -	if (!hash)
 -		return NULL;
 -	hashcpy(oid->hash, hash);
 -	return oid;
 +	return map_sha1_file_1(NULL, sha1, size);
  }
  
 -void check_pack_index_ptr(const struct packed_git *p, const void *vptr)
 +static int unpack_sha1_short_header(git_zstream *stream,
 +				    unsigned char *map, unsigned long mapsize,
 +				    void *buffer, unsigned long bufsiz)
  {
 -	const unsigned char *ptr = vptr;
 -	const unsigned char *start = p->index_data;
 -	const unsigned char *end = start + p->index_size;
 -	if (ptr < start)
 -		die(_("offset before start of pack index for %s (corrupt index?)"),
 -		    p->pack_name);
 -	/* No need to check for underflow; .idx files must be at least 8 bytes */
 -	if (ptr >= end - 8)
 -		die(_("offset beyond end of pack index for %s (truncated index?)"),
 -		    p->pack_name);
 -}
 +	/* Get the data stream */
 +	memset(stream, 0, sizeof(*stream));
 +	stream->next_in = map;
 +	stream->avail_in = mapsize;
 +	stream->next_out = buffer;
 +	stream->avail_out = bufsiz;
  
 -off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
 -{
 -	const unsigned char *index = p->index_data;
 -	index += 4 * 256;
 -	if (p->index_version == 1) {
 -		return ntohl(*((uint32_t *)(index + 24 * n)));
 -	} else {
 -		uint32_t off;
 -		index += 8 + p->num_objects * (20 + 4);
 -		off = ntohl(*((uint32_t *)(index + 4 * n)));
 -		if (!(off & 0x80000000))
 -			return off;
 -		index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
 -		check_pack_index_ptr(p, index);
 -		return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
 -				   ntohl(*((uint32_t *)(index + 4)));
 -	}
 +	git_inflate_init(stream);
 +	return git_inflate(stream, 0);
  }
  
 -off_t find_pack_entry_one(const unsigned char *sha1,
 -				  struct packed_git *p)
 +int unpack_sha1_header(git_zstream *stream,
 +		       unsigned char *map, unsigned long mapsize,
 +		       void *buffer, unsigned long bufsiz)
  {
 -	const uint32_t *level1_ofs = p->index_data;
 -	const unsigned char *index = p->index_data;
 -	unsigned hi, lo, stride;
 -	static int use_lookup = -1;
 -	static int debug_lookup = -1;
 -
 -	if (debug_lookup < 0)
 -		debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
 -
 -	if (!index) {
 -		if (open_pack_index(p))
 -			return 0;
 -		level1_ofs = p->index_data;
 -		index = p->index_data;
 -	}
 -	if (p->index_version > 1) {
 -		level1_ofs += 2;
 -		index += 8;
 -	}
 -	index += 4 * 256;
 -	hi = ntohl(level1_ofs[*sha1]);
 -	lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
 -	if (p->index_version > 1) {
 -		stride = 20;
 -	} else {
 -		stride = 24;
 -		index += 4;
 -	}
 -
 -	if (debug_lookup)
 -		printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
 -		       sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
 +	int status = unpack_sha1_short_header(stream, map, mapsize,
 +					      buffer, bufsiz);
  
 -	if (use_lookup < 0)
 -		use_lookup = !!getenv("GIT_USE_LOOKUP");
 -	if (use_lookup) {
 -		int pos = sha1_entry_pos(index, stride, 0,
 -					 lo, hi, p->num_objects, sha1);
 -		if (pos < 0)
 -			return 0;
 -		return nth_packed_object_offset(p, pos);
 -	}
 +	if (status < Z_OK)
 +		return status;
  
 -	while (lo < hi) {
 -		unsigned mi = (lo + hi) / 2;
 -		int cmp = hashcmp(index + mi * stride, sha1);
 -
 -		if (debug_lookup)
 -			printf("lo %u hi %u rg %u mi %u\n",
 -			       lo, hi, hi - lo, mi);
 -		if (!cmp)
 -			return nth_packed_object_offset(p, mi);
 -		if (cmp > 0)
 -			hi = mi;
 -		else
 -			lo = mi+1;
 -	}
 +	/* Make sure we have the terminating NUL */
 +	if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
 +		return -1;
  	return 0;
  }
  
 -int is_pack_valid(struct packed_git *p)
 +static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map,
 +					unsigned long mapsize, void *buffer,
 +					unsigned long bufsiz, struct strbuf *header)
  {
 -	/* An already open pack is known to be valid. */
 -	if (p->pack_fd != -1)
 -		return 1;
 +	int status;
 +
 +	status = unpack_sha1_short_header(stream, map, mapsize, buffer, bufsiz);
 +	if (status < Z_OK)
 +		return -1;
  
 -	/* If the pack has one window completely covering the
 -	 * file size, the pack is known to be valid even if
 -	 * the descriptor is not currently open.
 +	/*
 +	 * Check if entire header is unpacked in the first iteration.
  	 */
 -	if (p->windows) {
 -		struct pack_window *w = p->windows;
 +	if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
 +		return 0;
  
 -		if (!w->offset && w->len == p->pack_size)
 -			return 1;
 -	}
 +	/*
 +	 * buffer[0..bufsiz] was not large enough.  Copy the partial
 +	 * result out to header, and then append the result of further
 +	 * reading the stream.
 +	 */
 +	strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
 +	stream->next_out = buffer;
 +	stream->avail_out = bufsiz;
  
 -	/* Force the pack to open to prove its valid. */
 -	return !open_packed_git(p);
 +	do {
 +		status = git_inflate(stream, 0);
 +		strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
 +		if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
 +			return 0;
 +		stream->next_out = buffer;
 +		stream->avail_out = bufsiz;
 +	} while (status != Z_STREAM_END);
 +	return -1;
  }
  
 -static int fill_pack_entry(const unsigned char *sha1,
 -			   struct pack_entry *e,
 -			   struct packed_git *p)
 +static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
  {
 -	off_t offset;
 +	int bytes = strlen(buffer) + 1;
 +	unsigned char *buf = xmallocz(size);
 +	unsigned long n;
 +	int status = Z_OK;
  
 -	if (p->num_bad_objects) {
 -		unsigned i;
 -		for (i = 0; i < p->num_bad_objects; i++)
 -			if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
 -				return 0;
 +	n = stream->total_out - bytes;
 +	if (n > size)
 +		n = size;
 +	memcpy(buf, (char *) buffer + bytes, n);
 +	bytes = n;
 +	if (bytes <= size) {
 +		/*
 +		 * The above condition must be (bytes <= size), not
 +		 * (bytes < size).  In other words, even though we
 +		 * expect no more output and set avail_out to zero,
 +		 * the input zlib stream may have bytes that express
 +		 * "this concludes the stream", and we *do* want to
 +		 * eat that input.
 +		 *
 +		 * Otherwise we would not be able to test that we
 +		 * consumed all the input to reach the expected size;
 +		 * we also want to check that zlib tells us that all
 +		 * went well with status == Z_STREAM_END at the end.
 +		 */
 +		stream->next_out = buf + bytes;
 +		stream->avail_out = size - bytes;
 +		while (status == Z_OK)
 +			status = git_inflate(stream, Z_FINISH);
 +	}
 +	if (status == Z_STREAM_END && !stream->avail_in) {
 +		git_inflate_end(stream);
 +		return buf;
  	}
  
 -	offset = find_pack_entry_one(sha1, p);
 -	if (!offset)
 -		return 0;
 -
 -	/*
 -	 * We are about to tell the caller where they can locate the
 -	 * requested object.  We better make sure the packfile is
 -	 * still here and can be accessed before supplying that
 -	 * answer, as it may have been deleted since the index was
 -	 * loaded!
 -	 */
 -	if (!is_pack_valid(p))
 -		return 0;
 -	e->offset = offset;
 -	e->p = p;
 -	hashcpy(e->sha1, sha1);
 -	return 1;
 +	if (status < 0)
 +		error("corrupt loose object '%s'", sha1_to_hex(sha1));
 +	else if (stream->avail_in)
 +		error("garbage at end of loose object '%s'",
 +		      sha1_to_hex(sha1));
 +	free(buf);
 +	return NULL;
  }
  
  /*
 - * Iff a pack file contains the object named by sha1, return true and
 - * store its location to e.
 + * We used to just use "sscanf()", but that's actually way
 + * too permissive for what we want to check. So do an anal
 + * object header parse by hand.
   */
 -static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
 +static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
 +			       unsigned int flags)
  {
 -	struct mru_entry *p;
 +	const char *type_buf = hdr;
 +	unsigned long size;
 +	int type, type_len = 0;
  
 -	prepare_packed_git();
 -	if (!packed_git)
 -		return 0;
 +	/*
 +	 * The type can be of any size but is followed by
 +	 * a space.
 +	 */
 +	for (;;) {
 +		char c = *hdr++;
 +		if (!c)
 +			return -1;
 +		if (c == ' ')
 +			break;
 +		type_len++;
 +	}
  
 -	for (p = packed_git_mru->head; p; p = p->next) {
 -		if (fill_pack_entry(sha1, e, p->item)) {
 -			mru_mark(packed_git_mru, p);
 -			return 1;
 +	type = type_from_string_gently(type_buf, type_len, 1);
 +	if (oi->typename)
 +		strbuf_add(oi->typename, type_buf, type_len);
 +	/*
 +	 * Set type to 0 if its an unknown object and
 +	 * we're obtaining the type using '--allow-unknown-type'
 +	 * option.
 +	 */
 +	if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE) && (type < 0))
 +		type = 0;
 +	else if (type < 0)
 +		die("invalid object type");
 +	if (oi->typep)
 +		*oi->typep = type;
 +
 +	/*
 +	 * The length must follow immediately, and be in canonical
 +	 * decimal format (ie "010" is not valid).
 +	 */
 +	size = *hdr++ - '0';
 +	if (size > 9)
 +		return -1;
 +	if (size) {
 +		for (;;) {
 +			unsigned long c = *hdr - '0';
 +			if (c > 9)
 +				break;
 +			hdr++;
 +			size = size * 10 + c;
  		}
  	}
 -	return 0;
 +
 +	if (oi->sizep)
 +		*oi->sizep = size;
 +
 +	/*
 +	 * The length must be followed by a zero byte
 +	 */
 +	return *hdr ? -1 : type;
  }
  
 -struct packed_git *find_sha1_pack(const unsigned char *sha1,
 -				  struct packed_git *packs)
 +int parse_sha1_header(const char *hdr, unsigned long *sizep)
  {
 -	struct packed_git *p;
 -
 -	for (p = packs; p; p = p->next) {
 -		if (find_pack_entry_one(sha1, p))
 -			return p;
 -	}
 -	return NULL;
 +	struct object_info oi = OBJECT_INFO_INIT;
  
 +	oi.sizep = sizep;
 +	return parse_sha1_header_extended(hdr, &oi, 0);
  }
  
  static int sha1_loose_object_info(const unsigned char *sha1,
@@@ -1145,7 -2975,6 +1145,7 @@@
  	if (oi->sizep == &size_scratch)
  		oi->sizep = NULL;
  	strbuf_release(&hdrbuf);
 +	oi->whence = OI_LOOSE;
  	return (status < 0) ? status : 0;
  }
  
@@@ -1183,8 -3012,10 +1183,8 @@@ int sha1_object_info_extended(const uns
  
  	if (!find_pack_entry(real, &e)) {
  		/* Most likely it's a loose object. */
 -		if (!sha1_loose_object_info(real, oi, flags)) {
 -			oi->whence = OI_LOOSE;
 +		if (!sha1_loose_object_info(real, oi, flags))
  			return 0;
 -		}
  
  		/* Not a loose object; someone else may have just packed it. */
  		if (flags & OBJECT_INFO_QUICK) {
@@@ -1207,7 -3038,10 +1207,7 @@@
  	if (rtype < 0) {
  		mark_bad_packed_object(e.p, real);
  		return sha1_object_info_extended(real, oi, 0);
 -	} else if (in_delta_base_cache(e.p, e.offset)) {
 -		oi->whence = OI_DBCACHED;
 -	} else {
 -		oi->whence = OI_PACKED;
 +	} else if (oi->whence == OI_PACKED) {
  		oi->u.packed.offset = e.offset;
  		oi->u.packed.pack = e.p;
  		oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
@@@ -1231,18 -3065,28 +1231,18 @@@ int sha1_object_info(const unsigned cha
  	return type;
  }
  
 -static void *read_packed_sha1(const unsigned char *sha1,
 -			      enum object_type *type, unsigned long *size)
 +static void *read_object(const unsigned char *sha1, enum object_type *type,
 +			 unsigned long *size)
  {
 -	struct pack_entry e;
 -	void *data;
 +	struct object_info oi = OBJECT_INFO_INIT;
 +	void *content;
 +	oi.typep = type;
 +	oi.sizep = size;
 +	oi.contentp = &content;
  
 -	if (!find_pack_entry(sha1, &e))
 +	if (sha1_object_info_extended(sha1, &oi, 0) < 0)
  		return NULL;
 -	data = cache_or_unpack_entry(e.p, e.offset, size, type);
 -	if (!data) {
 -		/*
 -		 * We're probably in deep shit, but let's try to fetch
 -		 * the required object anyway from another pack or loose.
 -		 * This should happen only in the presence of a corrupted
 -		 * pack, and is better than failing outright.
 -		 */
 -		error("failed to read object %s at offset %"PRIuMAX" from %s",
 -		      sha1_to_hex(sha1), (uintmax_t)e.offset, e.p->pack_name);
 -		mark_bad_packed_object(e.p, sha1);
 -		data = read_object(sha1, type, size);
 -	}
 -	return data;
 +	return content;
  }
  
  int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
@@@ -1263,6 -3107,20 +1263,6 @@@
  	return 0;
  }
  
 -static void *read_object(const unsigned char *sha1, enum object_type *type,
 -			 unsigned long *size)
 -{
 -	struct object_info oi = OBJECT_INFO_INIT;
 -	void *content;
 -	oi.typep = type;
 -	oi.sizep = size;
 -	oi.contentp = &content;
 -
 -	if (sha1_object_info_extended(sha1, &oi, 0) < 0)
 -		return NULL;
 -	return content;
 -}
 -
  /*
   * This function dies on corrupt objects; the callers who want to
   * deal with them should arrange to call read_object() and give error
@@@ -1581,7 -3439,7 +1581,7 @@@ int write_sha1_file(const void *buf, un
  }
  
  int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
 -			     unsigned char *sha1, unsigned flags)
 +			     struct object_id *oid, unsigned flags)
  {
  	char *header;
  	int hdrlen, status = 0;
@@@ -1589,13 -3447,13 +1589,13 @@@
  	/* type string, SP, %lu of the length plus NUL must fit this */
  	hdrlen = strlen(type) + 32;
  	header = xmalloc(hdrlen);
 -	write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen);
 +	write_sha1_file_prepare(buf, len, type, oid->hash, header, &hdrlen);
  
  	if (!(flags & HASH_WRITE_OBJECT))
  		goto cleanup;
 -	if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
 +	if (freshen_packed_object(oid->hash) || freshen_loose_object(oid->hash))
  		goto cleanup;
 -	status = write_loose_object(sha1, header, hdrlen, buf, len, 0);
 +	status = write_loose_object(oid->hash, header, hdrlen, buf, len, 0);
  
  cleanup:
  	free(header);
@@@ -1613,7 -3471,7 +1613,7 @@@ int force_object_loose(const unsigned c
  
  	if (has_loose_object(sha1))
  		return 0;
 -	buf = read_packed_sha1(sha1, &type, &len);
 +	buf = read_object(sha1, &type, &len);
  	if (!buf)
  		return error("cannot read sha1_file for %s", sha1_to_hex(sha1));
  	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), len) + 1;
@@@ -1623,6 -3481,20 +1623,6 @@@
  	return ret;
  }
  
 -int has_pack_index(const unsigned char *sha1)
 -{
 -	struct stat st;
 -	if (stat(sha1_pack_index_name(sha1), &st))
 -		return 0;
 -	return 1;
 -}
 -
 -int has_sha1_pack(const unsigned char *sha1)
 -{
 -	struct pack_entry e;
 -	return find_pack_entry(sha1, &e);
 -}
 -
  int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
  {
  	if (!startup_info->have_repository)
@@@ -1785,14 -3657,14 +1785,14 @@@ static int index_core(unsigned char *sh
   * binary blobs, they generally do not want to get any conversion, and
   * callers should avoid this code path when filters are requested.
   */
 -static int index_stream(unsigned char *sha1, int fd, size_t size,
 +static int index_stream(struct object_id *oid, int fd, size_t size,
  			enum object_type type, const char *path,
  			unsigned flags)
  {
 -	return index_bulk_checkin(sha1, fd, size, type, path, flags);
 +	return index_bulk_checkin(oid->hash, fd, size, type, path, flags);
  }
  
 -int index_fd(unsigned char *sha1, int fd, struct stat *st,
 +int index_fd(struct object_id *oid, int fd, struct stat *st,
  	     enum object_type type, const char *path, unsigned flags)
  {
  	int ret;
@@@ -1802,32 -3674,31 +1802,32 @@@
  	 * die() for large files.
  	 */
  	if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path))
 -		ret = index_stream_convert_blob(sha1, fd, path, flags);
 +		ret = index_stream_convert_blob(oid->hash, fd, path, flags);
  	else if (!S_ISREG(st->st_mode))
 -		ret = index_pipe(sha1, fd, type, path, flags);
 +		ret = index_pipe(oid->hash, fd, type, path, flags);
  	else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
  		 (path && would_convert_to_git(&the_index, path)))
 -		ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
 +		ret = index_core(oid->hash, fd, xsize_t(st->st_size), type, path,
  				 flags);
  	else
 -		ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path,
 +		ret = index_stream(oid, fd, xsize_t(st->st_size), type, path,
  				   flags);
  	close(fd);
  	return ret;
  }
  
 -int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags)
 +int index_path(struct object_id *oid, const char *path, struct stat *st, unsigned flags)
  {
  	int fd;
  	struct strbuf sb = STRBUF_INIT;
 +	int rc = 0;
  
  	switch (st->st_mode & S_IFMT) {
  	case S_IFREG:
  		fd = open(path, O_RDONLY);
  		if (fd < 0)
  			return error_errno("open(\"%s\")", path);
 -		if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
 +		if (index_fd(oid, fd, st, OBJ_BLOB, path, flags) < 0)
  			return error("%s: failed to insert into database",
  				     path);
  		break;
@@@ -1835,22 -3706,23 +1835,22 @@@
  		if (strbuf_readlink(&sb, path, st->st_size))
  			return error_errno("readlink(\"%s\")", path);
  		if (!(flags & HASH_WRITE_OBJECT))
 -			hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
 -		else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
 -			return error("%s: failed to insert into database",
 -				     path);
 +			hash_sha1_file(sb.buf, sb.len, blob_type, oid->hash);
 +		else if (write_sha1_file(sb.buf, sb.len, blob_type, oid->hash))
 +			rc = error("%s: failed to insert into database", path);
  		strbuf_release(&sb);
  		break;
  	case S_IFDIR:
 -		return resolve_gitlink_ref(path, "HEAD", sha1);
 +		return resolve_gitlink_ref(path, "HEAD", oid->hash);
  	default:
  		return error("%s: unsupported file type", path);
  	}
 -	return 0;
 +	return rc;
  }
  
  int read_pack_header(int fd, struct pack_header *header)
  {
- 	if (read_in_full(fd, header, sizeof(*header)) < sizeof(*header))
+ 	if (read_in_full(fd, header, sizeof(*header)) != sizeof(*header))
  		/* "eof before pack header was fully read" */
  		return PH_ERROR_EOF;
  
@@@ -2015,6 -3887,46 +2015,6 @@@ int for_each_loose_object(each_loose_ob
  	return foreach_alt_odb(loose_from_alt_odb, &alt);
  }
  
 -static int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn cb, void *data)
 -{
 -	uint32_t i;
 -	int r = 0;
 -
 -	for (i = 0; i < p->num_objects; i++) {
 -		struct object_id oid;
 -
 -		if (!nth_packed_object_oid(&oid, p, i))
 -			return error("unable to get sha1 of object %u in %s",
 -				     i, p->pack_name);
 -
 -		r = cb(&oid, p, i, data);
 -		if (r)
 -			break;
 -	}
 -	return r;
 -}
 -
 -int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
 -{
 -	struct packed_git *p;
 -	int r = 0;
 -	int pack_errors = 0;
 -
 -	prepare_packed_git();
 -	for (p = packed_git; p; p = p->next) {
 -		if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
 -			continue;
 -		if (open_pack_index(p)) {
 -			pack_errors = 1;
 -			continue;
 -		}
 -		r = for_each_object_in_pack(p, cb, data);
 -		if (r)
 -			break;
 -	}
 -	return r ? r : pack_errors;
 -}
 -
  static int check_stream_sha1(git_zstream *stream,
  			     const char *hdr,
  			     unsigned long size,
diff --combined shallow.c
index 1cc1c76415,e8429a9a84..eabb65d3a7
--- a/shallow.c
+++ b/shallow.c
@@@ -107,7 -107,7 +107,7 @@@ struct commit_list *get_shallow_commits
  		cur_depth++;
  		if ((depth != INFINITE_DEPTH && cur_depth >= depth) ||
  		    (is_repository_shallow() && !commit->parents &&
 -		     (graft = lookup_commit_graft(commit->object.oid.hash)) != NULL &&
 +		     (graft = lookup_commit_graft(&commit->object.oid)) != NULL &&
  		     graft->nr_parent < 0)) {
  			commit_list_insert(commit, &result);
  			commit->object.flags |= shallow_flag;
@@@ -286,26 -286,28 +286,26 @@@ int write_shallow_commits(struct strbu
  	return write_shallow_commits_1(out, use_pack_protocol, extra, 0);
  }
  
 -static struct tempfile temporary_shallow;
 -
  const char *setup_temporary_shallow(const struct oid_array *extra)
  {
 +	struct tempfile *temp;
  	struct strbuf sb = STRBUF_INIT;
 -	int fd;
  
  	if (write_shallow_commits(&sb, 0, extra)) {
 -		fd = xmks_tempfile(&temporary_shallow, git_path("shallow_XXXXXX"));
 +		temp = xmks_tempfile(git_path("shallow_XXXXXX"));
  
- 		if (write_in_full(temp->fd, sb.buf, sb.len) != sb.len ||
 -		if (write_in_full(fd, sb.buf, sb.len) < 0)
++		if (write_in_full(temp->fd, sb.buf, sb.len) < 0 ||
 +		    close_tempfile_gently(temp) < 0)
  			die_errno("failed to write to %s",
 -				  get_tempfile_path(&temporary_shallow));
 -		close_tempfile(&temporary_shallow);
 +				  get_tempfile_path(temp));
  		strbuf_release(&sb);
 -		return get_tempfile_path(&temporary_shallow);
 +		return get_tempfile_path(temp);
  	}
  	/*
  	 * is_repository_shallow() sees empty string as "no shallow
  	 * file".
  	 */
 -	return get_tempfile_path(&temporary_shallow);
 +	return "";
  }
  
  void setup_alternate_shallow(struct lock_file *shallow_lock,
@@@ -319,7 -321,7 +319,7 @@@
  				       LOCK_DIE_ON_ERROR);
  	check_shallow_file_for_update();
  	if (write_shallow_commits(&sb, 0, extra)) {
- 		if (write_in_full(fd, sb.buf, sb.len) != sb.len)
+ 		if (write_in_full(fd, sb.buf, sb.len) < 0)
  			die_errno("failed to write to %s",
  				  get_lock_file_path(shallow_lock));
  		*alternate_shallow_file = get_lock_file_path(shallow_lock);
@@@ -366,7 -368,7 +366,7 @@@ void prune_shallow(int show_only
  				       LOCK_DIE_ON_ERROR);
  	check_shallow_file_for_update();
  	if (write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY)) {
- 		if (write_in_full(fd, sb.buf, sb.len) != sb.len)
+ 		if (write_in_full(fd, sb.buf, sb.len) < 0)
  			die_errno("failed to write to %s",
  				  get_lock_file_path(&shallow_lock));
  		commit_lock_file(&shallow_lock);
@@@ -396,7 -398,7 +396,7 @@@ void prepare_shallow_info(struct shallo
  	for (i = 0; i < sa->nr; i++) {
  		if (has_object_file(sa->oid + i)) {
  			struct commit_graft *graft;
 -			graft = lookup_commit_graft(sa->oid[i].hash);
 +			graft = lookup_commit_graft(&sa->oid[i]);
  			if (graft && graft->nr_parent < 0)
  				continue;
  			info->ours[info->nr_ours++] = i;
diff --combined streaming.c
index 6f1c60f12b,c8b85e4498..5892b50bd8
--- a/streaming.c
+++ b/streaming.c
@@@ -3,7 -3,6 +3,7 @@@
   */
  #include "cache.h"
  #include "streaming.h"
 +#include "packfile.h"
  
  enum input_source {
  	stream_error = -1,
@@@ -540,7 -539,7 +540,7 @@@ int stream_blob_to_fd(int fd, const str
  			kept = 0;
  		wrote = write_in_full(fd, buf, readlen);
  
- 		if (wrote != readlen)
+ 		if (wrote < 0)
  			goto close_and_exit;
  	}
  	if (kept && (lseek(fd, kept - 1, SEEK_CUR) == (off_t) -1 ||
diff --combined transport-helper.c
index 42b960ff86,2128a32f1c..c948d5215c
--- a/transport-helper.c
+++ b/transport-helper.c
@@@ -44,8 -44,7 +44,7 @@@ static void sendline(struct helper_dat
  {
  	if (debug)
  		fprintf(stderr, "Debug: Remote helper: -> %s", buffer->buf);
- 	if (write_in_full(helper->helper->in, buffer->buf, buffer->len)
- 		!= buffer->len)
+ 	if (write_in_full(helper->helper->in, buffer->buf, buffer->len) < 0)
  		die_errno("Full write to remote helper failed");
  }
  
@@@ -74,7 -73,7 +73,7 @@@ static void write_constant(int fd, cons
  {
  	if (debug)
  		fprintf(stderr, "Debug: Remote helper: -> %s", str);
- 	if (write_in_full(fd, str, strlen(str)) != strlen(str))
+ 	if (write_in_full(fd, str, strlen(str)) < 0)
  		die_errno("Full write to remote helper failed");
  }
  
@@@ -604,7 -603,6 +603,7 @@@ static int process_connect_service(stru
  			cmdbuf.buf);
  
  exit:
 +	strbuf_release(&cmdbuf);
  	fclose(input);
  	return ret;
  }
@@@ -928,7 -926,7 +927,7 @@@ static int push_refs_with_export(struc
  		struct object_id oid;
  
  		private = apply_refspecs(data->refspecs, data->refspec_nr, ref->name);
 -		if (private && !get_sha1(private, oid.hash)) {
 +		if (private && !get_oid(private, &oid)) {
  			strbuf_addf(&buf, "^%s", private);
  			string_list_append(&revlist_args, strbuf_detach(&buf, NULL));
  			oidcpy(&ref->old_oid, &oid);
@@@ -1118,13 -1116,6 +1117,13 @@@ int transport_helper_init(struct transp
  __attribute__((format (printf, 1, 2)))
  static void transfer_debug(const char *fmt, ...)
  {
 +	/*
 +	 * NEEDSWORK: This function is sometimes used from multiple threads, and
 +	 * we end up using debug_enabled racily. That "should not matter" since
 +	 * we always write the same value, but it's still wrong. This function
 +	 * is listed in .tsan-suppressions for the time being.
 +	 */
 +
  	va_list args;
  	char msgbuf[PBUFFERSIZE];
  	static int debug_enabled = -1;