From: Junio C Hamano <gitster@pobox.com>
Date: Wed, 19 Aug 2015 21:48:55 +0000 (-0700)
Subject: Merge branch 'jc/finalize-temp-file'
X-Git-Tag: v2.6.0-rc0~58
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/51a22ce1477b991793c20d3143d72f0ae6e38c76?ds=inline;hp=-c

Merge branch 'jc/finalize-temp-file'

Long overdue micro clean-up.

* jc/finalize-temp-file:
sha1_file.c: rename move_temp_to_file() to finalize_object_file()
---

51a22ce1477b991793c20d3143d72f0ae6e38c76
diff --combined builtin/index-pack.c
index 3f10840441,4acce92227..3431de2362
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@@ -18,14 -18,16 +18,14 @@@ static const char index_pack_usage[] 
  struct object_entry {
  	struct pack_idx_entry idx;
  	unsigned long size;
 -	unsigned int hdr_size;
 -	enum object_type type;
 -	enum object_type real_type;
 -	unsigned delta_depth;
 -	int base_object_no;
 +	unsigned char hdr_size;
 +	signed char type;
 +	signed char real_type;
  };
  
 -union delta_base {
 -	unsigned char sha1[20];
 -	off_t offset;
 +struct object_stat {
 +	unsigned delta_depth;
 +	int base_object_no;
  };
  
  struct base_data {
@@@ -47,35 -49,31 +47,35 @@@ struct thread_local 
  	int pack_fd;
  };
  
 -/*
 - * Even if sizeof(union delta_base) == 24 on 64-bit archs, we really want
 - * to memcmp() only the first 20 bytes.
 - */
 -#define UNION_BASE_SZ	20
 -
  #define FLAG_LINK (1u<<20)
  #define FLAG_CHECKED (1u<<21)
  
 -struct delta_entry {
 -	union delta_base base;
 +struct ofs_delta_entry {
 +	off_t offset;
 +	int obj_no;
 +};
 +
 +struct ref_delta_entry {
 +	unsigned char sha1[20];
  	int obj_no;
  };
  
  static struct object_entry *objects;
 -static struct delta_entry *deltas;
 +static struct object_stat *obj_stat;
 +static struct ofs_delta_entry *ofs_deltas;
 +static struct ref_delta_entry *ref_deltas;
  static struct thread_local nothread_data;
  static int nr_objects;
 -static int nr_deltas;
 +static int nr_ofs_deltas;
 +static int nr_ref_deltas;
 +static int ref_deltas_alloc;
  static int nr_resolved_deltas;
  static int nr_threads;
  
  static int from_stdin;
  static int strict;
  static int do_fsck_object;
 +static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
  static int verbose;
  static int show_stat;
  static int check_self_contained_and_connected;
@@@ -114,10 -112,6 +114,10 @@@ static pthread_mutex_t deepest_delta_mu
  #define deepest_delta_lock()	lock_mutex(&deepest_delta_mutex)
  #define deepest_delta_unlock()	unlock_mutex(&deepest_delta_mutex)
  
 +static pthread_mutex_t type_cas_mutex;
 +#define type_cas_lock()		lock_mutex(&type_cas_mutex)
 +#define type_cas_unlock()	unlock_mutex(&type_cas_mutex)
 +
  static pthread_key_t key;
  
  static inline void lock_mutex(pthread_mutex_t *mutex)
@@@ -141,7 -135,6 +141,7 @@@ static void init_thread(void
  	init_recursive_mutex(&read_mutex);
  	pthread_mutex_init(&counter_mutex, NULL);
  	pthread_mutex_init(&work_mutex, NULL);
 +	pthread_mutex_init(&type_cas_mutex, NULL);
  	if (show_stat)
  		pthread_mutex_init(&deepest_delta_mutex, NULL);
  	pthread_key_create(&key, NULL);
@@@ -164,7 -157,6 +164,7 @@@ static void cleanup_thread(void
  	pthread_mutex_destroy(&read_mutex);
  	pthread_mutex_destroy(&counter_mutex);
  	pthread_mutex_destroy(&work_mutex);
 +	pthread_mutex_destroy(&type_cas_mutex);
  	if (show_stat)
  		pthread_mutex_destroy(&deepest_delta_mutex);
  	for (i = 0; i < nr_threads; i++)
@@@ -187,13 -179,10 +187,13 @@@
  #define deepest_delta_lock()
  #define deepest_delta_unlock()
  
 +#define type_cas_lock()
 +#define type_cas_unlock()
 +
  #endif
  
  
 -static int mark_link(struct object *obj, int type, void *data)
 +static int mark_link(struct object *obj, int type, void *data, struct fsck_options *options)
  {
  	if (!obj)
  		return -1;
@@@ -449,7 -438,7 +449,7 @@@ static void *unpack_entry_data(unsigne
  	if (type == OBJ_BLOB && size > big_file_threshold)
  		buf = fixed_buf;
  	else
 -		buf = xmalloc(size);
 +		buf = xmallocz(size);
  
  	memset(&stream, 0, sizeof(stream));
  	git_inflate_init(&stream);
@@@ -478,8 -467,7 +478,8 @@@
  }
  
  static void *unpack_raw_entry(struct object_entry *obj,
 -			      union delta_base *delta_base,
 +			      off_t *ofs_offset,
 +			      unsigned char *ref_sha1,
  			      unsigned char *sha1)
  {
  	unsigned char *p;
@@@ -508,10 -496,11 +508,10 @@@
  
  	switch (obj->type) {
  	case OBJ_REF_DELTA:
 -		hashcpy(delta_base->sha1, fill(20));
 +		hashcpy(ref_sha1, fill(20));
  		use(20);
  		break;
  	case OBJ_OFS_DELTA:
 -		memset(delta_base, 0, sizeof(*delta_base));
  		p = fill(1);
  		c = *p;
  		use(1);
@@@ -525,8 -514,8 +525,8 @@@
  			use(1);
  			base_offset = (base_offset << 7) + (c & 127);
  		}
 -		delta_base->offset = obj->idx.offset - base_offset;
 -		if (delta_base->offset <= 0 || delta_base->offset >= obj->idx.offset)
 +		*ofs_offset = obj->idx.offset - base_offset;
 +		if (*ofs_offset <= 0 || *ofs_offset >= obj->idx.offset)
  			bad_object(obj->idx.offset, _("delta base offset is out of bound"));
  		break;
  	case OBJ_COMMIT:
@@@ -554,7 -543,7 +554,7 @@@ static void *unpack_data(struct object_
  	git_zstream stream;
  	int status;
  
 -	data = xmalloc(consume ? 64*1024 : obj->size);
 +	data = xmallocz(consume ? 64*1024 : obj->size);
  	inbuf = xmalloc((len < 64*1024) ? len : 64*1024);
  
  	memset(&stream, 0, sizeof(stream));
@@@ -610,110 -599,55 +610,110 @@@ static void *get_data_from_pack(struct 
  	return unpack_data(obj, NULL, NULL);
  }
  
 -static int compare_delta_bases(const union delta_base *base1,
 -			       const union delta_base *base2,
 -			       enum object_type type1,
 -			       enum object_type type2)
 +static int compare_ofs_delta_bases(off_t offset1, off_t offset2,
 +				   enum object_type type1,
 +				   enum object_type type2)
  {
  	int cmp = type1 - type2;
  	if (cmp)
  		return cmp;
 -	return memcmp(base1, base2, UNION_BASE_SZ);
 +	return offset1 < offset2 ? -1 :
 +	       offset1 > offset2 ?  1 :
 +	       0;
  }
  
 -static int find_delta(const union delta_base *base, enum object_type type)
 +static int find_ofs_delta(const off_t offset, enum object_type type)
  {
 -	int first = 0, last = nr_deltas;
 -
 -        while (first < last) {
 -                int next = (first + last) / 2;
 -                struct delta_entry *delta = &deltas[next];
 -                int cmp;
 -
 -		cmp = compare_delta_bases(base, &delta->base,
 -					  type, objects[delta->obj_no].type);
 -                if (!cmp)
 -                        return next;
 -                if (cmp < 0) {
 -                        last = next;
 -                        continue;
 -                }
 -                first = next+1;
 -        }
 -        return -first-1;
 +	int first = 0, last = nr_ofs_deltas;
 +
 +	while (first < last) {
 +		int next = (first + last) / 2;
 +		struct ofs_delta_entry *delta = &ofs_deltas[next];
 +		int cmp;
 +
 +		cmp = compare_ofs_delta_bases(offset, delta->offset,
 +					      type, objects[delta->obj_no].type);
 +		if (!cmp)
 +			return next;
 +		if (cmp < 0) {
 +			last = next;
 +			continue;
 +		}
 +		first = next+1;
 +	}
 +	return -first-1;
  }
  
 -static void find_delta_children(const union delta_base *base,
 -				int *first_index, int *last_index,
 -				enum object_type type)
 +static void find_ofs_delta_children(off_t offset,
 +				    int *first_index, int *last_index,
 +				    enum object_type type)
  {
 -	int first = find_delta(base, type);
 +	int first = find_ofs_delta(offset, type);
  	int last = first;
 -	int end = nr_deltas - 1;
 +	int end = nr_ofs_deltas - 1;
  
  	if (first < 0) {
  		*first_index = 0;
  		*last_index = -1;
  		return;
  	}
 -	while (first > 0 && !memcmp(&deltas[first - 1].base, base, UNION_BASE_SZ))
 +	while (first > 0 && ofs_deltas[first - 1].offset == offset)
  		--first;
 -	while (last < end && !memcmp(&deltas[last + 1].base, base, UNION_BASE_SZ))
 +	while (last < end && ofs_deltas[last + 1].offset == offset)
 +		++last;
 +	*first_index = first;
 +	*last_index = last;
 +}
 +
 +static int compare_ref_delta_bases(const unsigned char *sha1,
 +				   const unsigned char *sha2,
 +				   enum object_type type1,
 +				   enum object_type type2)
 +{
 +	int cmp = type1 - type2;
 +	if (cmp)
 +		return cmp;
 +	return hashcmp(sha1, sha2);
 +}
 +
 +static int find_ref_delta(const unsigned char *sha1, enum object_type type)
 +{
 +	int first = 0, last = nr_ref_deltas;
 +
 +	while (first < last) {
 +		int next = (first + last) / 2;
 +		struct ref_delta_entry *delta = &ref_deltas[next];
 +		int cmp;
 +
 +		cmp = compare_ref_delta_bases(sha1, delta->sha1,
 +					      type, objects[delta->obj_no].type);
 +		if (!cmp)
 +			return next;
 +		if (cmp < 0) {
 +			last = next;
 +			continue;
 +		}
 +		first = next+1;
 +	}
 +	return -first-1;
 +}
 +
 +static void find_ref_delta_children(const unsigned char *sha1,
 +				    int *first_index, int *last_index,
 +				    enum object_type type)
 +{
 +	int first = find_ref_delta(sha1, type);
 +	int last = first;
 +	int end = nr_ref_deltas - 1;
 +
 +	if (first < 0) {
 +		*first_index = 0;
 +		*last_index = -1;
 +		return;
 +	}
 +	while (first > 0 && !hashcmp(ref_deltas[first - 1].sha1, sha1))
 +		--first;
 +	while (last < end && !hashcmp(ref_deltas[last + 1].sha1, sha1))
  		++last;
  	*first_index = first;
  	*last_index = last;
@@@ -787,7 -721,7 +787,7 @@@ static void sha1_object(const void *dat
  	assert(data || obj_entry);
  
  	read_lock();
 -	collision_test_needed = has_sha1_file(sha1);
 +	collision_test_needed = has_sha1_file_with_flags(sha1, HAS_SHA1_QUICK);
  	read_unlock();
  
  	if (collision_test_needed && !data) {
@@@ -839,9 -773,9 +839,9 @@@
  			if (!obj)
  				die(_("invalid %s"), typename(type));
  			if (do_fsck_object &&
 -			    fsck_object(obj, 1, fsck_error_function))
 +			    fsck_object(obj, buf, size, &fsck_options))
  				die(_("Error in object"));
 -			if (fsck_walk(obj, mark_link, NULL))
 +			if (fsck_walk(obj, NULL, &fsck_options))
  				die(_("Not all child objects of %s are reachable"), sha1_to_hex(obj->sha1));
  
  			if (obj->type == OBJ_TREE) {
@@@ -928,16 -862,15 +928,16 @@@ static void resolve_delta(struct object
  {
  	void *base_data, *delta_data;
  
 -	delta_obj->real_type = base->obj->real_type;
  	if (show_stat) {
 -		delta_obj->delta_depth = base->obj->delta_depth + 1;
 +		int i = delta_obj - objects;
 +		int j = base->obj - objects;
 +		obj_stat[i].delta_depth = obj_stat[j].delta_depth + 1;
  		deepest_delta_lock();
 -		if (deepest_delta < delta_obj->delta_depth)
 -			deepest_delta = delta_obj->delta_depth;
 +		if (deepest_delta < obj_stat[i].delta_depth)
 +			deepest_delta = obj_stat[i].delta_depth;
  		deepest_delta_unlock();
 +		obj_stat[i].base_object_no = j;
  	}
 -	delta_obj->base_object_no = base->obj - objects;
  	delta_data = get_data_from_pack(delta_obj);
  	base_data = get_base_data(base);
  	result->obj = delta_obj;
@@@ -955,37 -888,20 +955,37 @@@
  	counter_unlock();
  }
  
 +/*
 + * Standard boolean compare-and-swap: atomically check whether "*type" is
 + * "want"; if so, swap in "set" and return true. Otherwise, leave it untouched
 + * and return false.
 + */
 +static int compare_and_swap_type(signed char *type,
 +				 enum object_type want,
 +				 enum object_type set)
 +{
 +	enum object_type old;
 +
 +	type_cas_lock();
 +	old = *type;
 +	if (old == want)
 +		*type = set;
 +	type_cas_unlock();
 +
 +	return old == want;
 +}
 +
  static struct base_data *find_unresolved_deltas_1(struct base_data *base,
  						  struct base_data *prev_base)
  {
  	if (base->ref_last == -1 && base->ofs_last == -1) {
 -		union delta_base base_spec;
 +		find_ref_delta_children(base->obj->idx.sha1,
 +					&base->ref_first, &base->ref_last,
 +					OBJ_REF_DELTA);
  
 -		hashcpy(base_spec.sha1, base->obj->idx.sha1);
 -		find_delta_children(&base_spec,
 -				    &base->ref_first, &base->ref_last, OBJ_REF_DELTA);
 -
 -		memset(&base_spec, 0, sizeof(base_spec));
 -		base_spec.offset = base->obj->idx.offset;
 -		find_delta_children(&base_spec,
 -				    &base->ofs_first, &base->ofs_last, OBJ_OFS_DELTA);
 +		find_ofs_delta_children(base->obj->idx.offset,
 +					&base->ofs_first, &base->ofs_last,
 +					OBJ_OFS_DELTA);
  
  		if (base->ref_last == -1 && base->ofs_last == -1) {
  			free(base->data);
@@@ -996,13 -912,10 +996,13 @@@
  	}
  
  	if (base->ref_first <= base->ref_last) {
 -		struct object_entry *child = objects + deltas[base->ref_first].obj_no;
 +		struct object_entry *child = objects + ref_deltas[base->ref_first].obj_no;
  		struct base_data *result = alloc_base_data();
  
 -		assert(child->real_type == OBJ_REF_DELTA);
 +		if (!compare_and_swap_type(&child->real_type, OBJ_REF_DELTA,
 +					   base->obj->real_type))
 +			die("BUG: child->real_type != OBJ_REF_DELTA");
 +
  		resolve_delta(child, base, result);
  		if (base->ref_first == base->ref_last && base->ofs_last == -1)
  			free_base_data(base);
@@@ -1012,11 -925,10 +1012,11 @@@
  	}
  
  	if (base->ofs_first <= base->ofs_last) {
 -		struct object_entry *child = objects + deltas[base->ofs_first].obj_no;
 +		struct object_entry *child = objects + ofs_deltas[base->ofs_first].obj_no;
  		struct base_data *result = alloc_base_data();
  
  		assert(child->real_type == OBJ_OFS_DELTA);
 +		child->real_type = base->obj->real_type;
  		resolve_delta(child, base, result);
  		if (base->ofs_first == base->ofs_last)
  			free_base_data(base);
@@@ -1048,22 -960,15 +1048,22 @@@ static void find_unresolved_deltas(stru
  	}
  }
  
 -static int compare_delta_entry(const void *a, const void *b)
 +static int compare_ofs_delta_entry(const void *a, const void *b)
 +{
 +	const struct ofs_delta_entry *delta_a = a;
 +	const struct ofs_delta_entry *delta_b = b;
 +
 +	return delta_a->offset < delta_b->offset ? -1 :
 +	       delta_a->offset > delta_b->offset ?  1 :
 +	       0;
 +}
 +
 +static int compare_ref_delta_entry(const void *a, const void *b)
  {
 -	const struct delta_entry *delta_a = a;
 -	const struct delta_entry *delta_b = b;
 +	const struct ref_delta_entry *delta_a = a;
 +	const struct ref_delta_entry *delta_b = b;
  
 -	/* group by type (ref vs ofs) and then by value (sha-1 or offset) */
 -	return compare_delta_bases(&delta_a->base, &delta_b->base,
 -				   objects[delta_a->obj_no].type,
 -				   objects[delta_b->obj_no].type);
 +	return hashcmp(delta_a->sha1, delta_b->sha1);
  }
  
  static void resolve_base(struct object_entry *obj)
@@@ -1109,8 -1014,7 +1109,8 @@@ static void *threaded_second_pass(void 
  static void parse_pack_objects(unsigned char *sha1)
  {
  	int i, nr_delays = 0;
 -	struct delta_entry *delta = deltas;
 +	struct ofs_delta_entry *ofs_delta = ofs_deltas;
 +	unsigned char ref_delta_sha1[20];
  	struct stat st;
  
  	if (verbose)
@@@ -1119,18 -1023,12 +1119,18 @@@
  				nr_objects);
  	for (i = 0; i < nr_objects; i++) {
  		struct object_entry *obj = &objects[i];
 -		void *data = unpack_raw_entry(obj, &delta->base, obj->idx.sha1);
 +		void *data = unpack_raw_entry(obj, &ofs_delta->offset,
 +					      ref_delta_sha1, obj->idx.sha1);
  		obj->real_type = obj->type;
 -		if (is_delta_type(obj->type)) {
 -			nr_deltas++;
 -			delta->obj_no = i;
 -			delta++;
 +		if (obj->type == OBJ_OFS_DELTA) {
 +			nr_ofs_deltas++;
 +			ofs_delta->obj_no = i;
 +			ofs_delta++;
 +		} else if (obj->type == OBJ_REF_DELTA) {
 +			ALLOC_GROW(ref_deltas, nr_ref_deltas + 1, ref_deltas_alloc);
 +			hashcpy(ref_deltas[nr_ref_deltas].sha1, ref_delta_sha1);
 +			ref_deltas[nr_ref_deltas].obj_no = i;
 +			nr_ref_deltas++;
  		} else if (!data) {
  			/* large blobs, check later */
  			obj->real_type = OBJ_BAD;
@@@ -1181,18 -1079,15 +1181,18 @@@ static void resolve_deltas(void
  {
  	int i;
  
 -	if (!nr_deltas)
 +	if (!nr_ofs_deltas && !nr_ref_deltas)
  		return;
  
  	/* Sort deltas by base SHA1/offset for fast searching */
 -	qsort(deltas, nr_deltas, sizeof(struct delta_entry),
 -	      compare_delta_entry);
 +	qsort(ofs_deltas, nr_ofs_deltas, sizeof(struct ofs_delta_entry),
 +	      compare_ofs_delta_entry);
 +	qsort(ref_deltas, nr_ref_deltas, sizeof(struct ref_delta_entry),
 +	      compare_ref_delta_entry);
  
  	if (verbose)
 -		progress = start_progress(_("Resolving deltas"), nr_deltas);
 +		progress = start_progress(_("Resolving deltas"),
 +					  nr_ref_deltas + nr_ofs_deltas);
  
  #ifndef NO_PTHREADS
  	nr_dispatched = 0;
@@@ -1227,10 -1122,10 +1227,10 @@@
   * - append objects to convert thin pack to full pack if required
   * - write the final 20-byte SHA-1
   */
 -static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved);
 +static void fix_unresolved_deltas(struct sha1file *f);
  static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned char *pack_sha1)
  {
 -	if (nr_deltas == nr_resolved_deltas) {
 +	if (nr_ref_deltas + nr_ofs_deltas == nr_resolved_deltas) {
  		stop_progress(&progress);
  		/* Flush remaining pack final 20-byte SHA1. */
  		flush();
@@@ -1241,15 -1136,17 +1241,15 @@@
  		struct sha1file *f;
  		unsigned char read_sha1[20], tail_sha1[20];
  		struct strbuf msg = STRBUF_INIT;
 -		int nr_unresolved = nr_deltas - nr_resolved_deltas;
 +		int nr_unresolved = nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas;
  		int nr_objects_initial = nr_objects;
  		if (nr_unresolved <= 0)
  			die(_("confusion beyond insanity"));
 -		objects = xrealloc(objects,
 -				   (nr_objects + nr_unresolved + 1)
 -				   * sizeof(*objects));
 +		REALLOC_ARRAY(objects, nr_objects + nr_unresolved + 1);
  		memset(objects + nr_objects + 1, 0,
  		       nr_unresolved * sizeof(*objects));
  		f = sha1fd(output_fd, curr_pack);
 -		fix_unresolved_deltas(f, nr_unresolved);
 +		fix_unresolved_deltas(f);
  		strbuf_addf(&msg, _("completed with %d local objects"),
  			    nr_objects - nr_objects_initial);
  		stop_progress_msg(&progress, msg.buf);
@@@ -1263,11 -1160,11 +1263,11 @@@
  			die(_("Unexpected tail checksum for %s "
  			      "(disk corruption?)"), curr_pack);
  	}
 -	if (nr_deltas != nr_resolved_deltas)
 +	if (nr_ofs_deltas + nr_ref_deltas != nr_resolved_deltas)
  		die(Q_("pack has %d unresolved delta",
  		       "pack has %d unresolved deltas",
 -		       nr_deltas - nr_resolved_deltas),
 -		    nr_deltas - nr_resolved_deltas);
 +		       nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas),
 +		    nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas);
  }
  
  static int write_compressed(struct sha1file *f, void *in, unsigned int size)
@@@ -1276,6 -1173,7 +1276,6 @@@
  	int status;
  	unsigned char outbuf[4096];
  
 -	memset(&stream, 0, sizeof(stream));
  	git_deflate_init(&stream, zlib_compression_level);
  	stream.next_in = in;
  	stream.avail_in = size;
@@@ -1326,15 -1224,15 +1326,15 @@@ static struct object_entry *append_obj_
  
  static int delta_pos_compare(const void *_a, const void *_b)
  {
 -	struct delta_entry *a = *(struct delta_entry **)_a;
 -	struct delta_entry *b = *(struct delta_entry **)_b;
 +	struct ref_delta_entry *a = *(struct ref_delta_entry **)_a;
 +	struct ref_delta_entry *b = *(struct ref_delta_entry **)_b;
  	return a->obj_no - b->obj_no;
  }
  
 -static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved)
 +static void fix_unresolved_deltas(struct sha1file *f)
  {
 -	struct delta_entry **sorted_by_pos;
 -	int i, n = 0;
 +	struct ref_delta_entry **sorted_by_pos;
 +	int i;
  
  	/*
  	 * Since many unresolved deltas may well be themselves base objects
@@@ -1346,26 -1244,29 +1346,26 @@@
  	 * before deltas depending on them, a good heuristic is to start
  	 * resolving deltas in the same order as their position in the pack.
  	 */
 -	sorted_by_pos = xmalloc(nr_unresolved * sizeof(*sorted_by_pos));
 -	for (i = 0; i < nr_deltas; i++) {
 -		if (objects[deltas[i].obj_no].real_type != OBJ_REF_DELTA)
 -			continue;
 -		sorted_by_pos[n++] = &deltas[i];
 -	}
 -	qsort(sorted_by_pos, n, sizeof(*sorted_by_pos), delta_pos_compare);
 +	sorted_by_pos = xmalloc(nr_ref_deltas * sizeof(*sorted_by_pos));
 +	for (i = 0; i < nr_ref_deltas; i++)
 +		sorted_by_pos[i] = &ref_deltas[i];
 +	qsort(sorted_by_pos, nr_ref_deltas, sizeof(*sorted_by_pos), delta_pos_compare);
  
 -	for (i = 0; i < n; i++) {
 -		struct delta_entry *d = sorted_by_pos[i];
 +	for (i = 0; i < nr_ref_deltas; i++) {
 +		struct ref_delta_entry *d = sorted_by_pos[i];
  		enum object_type type;
  		struct base_data *base_obj = alloc_base_data();
  
  		if (objects[d->obj_no].real_type != OBJ_REF_DELTA)
  			continue;
 -		base_obj->data = read_sha1_file(d->base.sha1, &type, &base_obj->size);
 +		base_obj->data = read_sha1_file(d->sha1, &type, &base_obj->size);
  		if (!base_obj->data)
  			continue;
  
 -		if (check_sha1_signature(d->base.sha1, base_obj->data,
 +		if (check_sha1_signature(d->sha1, base_obj->data,
  				base_obj->size, typename(type)))
 -			die(_("local object %s is corrupt"), sha1_to_hex(d->base.sha1));
 -		base_obj->obj = append_obj_to_pack(f, d->base.sha1,
 +			die(_("local object %s is corrupt"), sha1_to_hex(d->sha1));
 +		base_obj->obj = append_obj_to_pack(f, d->sha1,
  					base_obj->data, base_obj->size, type);
  		find_unresolved_deltas(base_obj);
  		display_progress(progress, nr_resolved_deltas);
@@@ -1421,7 -1322,7 +1421,7 @@@ static void final(const char *final_pac
  				 get_object_directory(), sha1_to_hex(sha1));
  			final_pack_name = name;
  		}
- 		if (move_temp_to_file(curr_pack_name, final_pack_name))
+ 		if (finalize_object_file(curr_pack_name, final_pack_name))
  			die(_("cannot store pack file"));
  	} else if (from_stdin)
  		chmod(final_pack_name, 0444);
@@@ -1432,7 -1333,7 +1432,7 @@@
  				 get_object_directory(), sha1_to_hex(sha1));
  			final_index_name = name;
  		}
- 		if (move_temp_to_file(curr_index_name, final_index_name))
+ 		if (finalize_object_file(curr_index_name, final_index_name))
  			die(_("cannot store index file"));
  	} else
  		chmod(final_index_name, 0444);
@@@ -1557,7 -1458,7 +1557,7 @@@ static void read_idx_option(struct pack
  
  static void show_pack_info(int stat_only)
  {
 -	int i, baseobjects = nr_objects - nr_deltas;
 +	int i, baseobjects = nr_objects - nr_ref_deltas - nr_ofs_deltas;
  	unsigned long *chain_histogram = NULL;
  
  	if (deepest_delta)
@@@ -1567,7 -1468,7 +1567,7 @@@
  		struct object_entry *obj = &objects[i];
  
  		if (is_delta_type(obj->type))
 -			chain_histogram[obj->delta_depth - 1]++;
 +			chain_histogram[obj_stat[i].delta_depth - 1]++;
  		if (stat_only)
  			continue;
  		printf("%s %-6s %lu %lu %"PRIuMAX,
@@@ -1576,8 -1477,8 +1576,8 @@@
  		       (unsigned long)(obj[1].idx.offset - obj->idx.offset),
  		       (uintmax_t)obj->idx.offset);
  		if (is_delta_type(obj->type)) {
 -			struct object_entry *bobj = &objects[obj->base_object_no];
 -			printf(" %u %s", obj->delta_depth, sha1_to_hex(bobj->idx.sha1));
 +			struct object_entry *bobj = &objects[obj_stat[i].base_object_no];
 +			printf(" %u %s", obj_stat[i].delta_depth, sha1_to_hex(bobj->idx.sha1));
  		}
  		putchar('\n');
  	}
@@@ -1604,8 -1505,7 +1604,8 @@@ int cmd_index_pack(int argc, const cha
  	const char *curr_index;
  	const char *index_name = NULL, *pack_name = NULL;
  	const char *keep_name = NULL, *keep_msg = NULL;
 -	char *index_name_buf = NULL, *keep_name_buf = NULL;
 +	struct strbuf index_name_buf = STRBUF_INIT,
 +		      keep_name_buf = STRBUF_INIT;
  	struct pack_idx_entry **idx_objects;
  	struct pack_idx_option opts;
  	unsigned char pack_sha1[20];
@@@ -1615,7 -1515,6 +1615,7 @@@
  		usage(index_pack_usage);
  
  	check_replace_refs = 0;
 +	fsck_options.walk = mark_link;
  
  	reset_pack_idx_option(&opts);
  	git_config(git_index_pack_config, &opts);
@@@ -1633,10 -1532,6 +1633,10 @@@
  			} else if (!strcmp(arg, "--strict")) {
  				strict = 1;
  				do_fsck_object = 1;
 +			} else if (skip_prefix(arg, "--strict=", &arg)) {
 +				strict = 1;
 +				do_fsck_object = 1;
 +				fsck_set_msg_types(&fsck_options, arg);
  			} else if (!strcmp(arg, "--check-self-contained-and-connected")) {
  				strict = 1;
  				check_self_contained_and_connected = 1;
@@@ -1707,22 -1602,24 +1707,22 @@@
  	if (fix_thin_pack && !from_stdin)
  		die(_("--fix-thin cannot be used without --stdin"));
  	if (!index_name && pack_name) {
 -		int len = strlen(pack_name);
 -		if (!has_extension(pack_name, ".pack"))
 +		size_t len;
 +		if (!strip_suffix(pack_name, ".pack", &len))
  			die(_("packfile name '%s' does not end with '.pack'"),
  			    pack_name);
 -		index_name_buf = xmalloc(len);
 -		memcpy(index_name_buf, pack_name, len - 5);
 -		strcpy(index_name_buf + len - 5, ".idx");
 -		index_name = index_name_buf;
 +		strbuf_add(&index_name_buf, pack_name, len);
 +		strbuf_addstr(&index_name_buf, ".idx");
 +		index_name = index_name_buf.buf;
  	}
  	if (keep_msg && !keep_name && pack_name) {
 -		int len = strlen(pack_name);
 -		if (!has_extension(pack_name, ".pack"))
 +		size_t len;
 +		if (!strip_suffix(pack_name, ".pack", &len))
  			die(_("packfile name '%s' does not end with '.pack'"),
  			    pack_name);
 -		keep_name_buf = xmalloc(len);
 -		memcpy(keep_name_buf, pack_name, len - 5);
 -		strcpy(keep_name_buf + len - 5, ".keep");
 -		keep_name = keep_name_buf;
 +		strbuf_add(&keep_name_buf, pack_name, len);
 +		strbuf_addstr(&keep_name_buf, ".idx");
 +		keep_name = keep_name_buf.buf;
  	}
  	if (verify) {
  		if (!index_name)
@@@ -1745,14 -1642,11 +1745,14 @@@
  	curr_pack = open_pack_file(pack_name);
  	parse_pack_header();
  	objects = xcalloc(nr_objects + 1, sizeof(struct object_entry));
 -	deltas = xcalloc(nr_objects, sizeof(struct delta_entry));
 +	if (show_stat)
 +		obj_stat = xcalloc(nr_objects + 1, sizeof(struct object_stat));
 +	ofs_deltas = xcalloc(nr_objects, sizeof(struct ofs_delta_entry));
  	parse_pack_objects(pack_sha1);
  	resolve_deltas();
  	conclude_pack(fix_thin_pack, curr_pack, pack_sha1);
 -	free(deltas);
 +	free(ofs_deltas);
 +	free(ref_deltas);
  	if (strict)
  		foreign_nr = check_objects();
  
@@@ -1773,8 -1667,8 +1773,8 @@@
  	else
  		close(input_fd);
  	free(objects);
 -	free(index_name_buf);
 -	free(keep_name_buf);
 +	strbuf_release(&index_name_buf);
 +	strbuf_release(&keep_name_buf);
  	if (pack_name == NULL)
  		free((void *) curr_pack);
  	if (index_name == NULL)
diff --combined cache.h
index 6bb7119032,428a2e0135..14f070ef34
--- a/cache.h
+++ b/cache.h
@@@ -7,8 -7,6 +7,8 @@@
  #include "advice.h"
  #include "gettext.h"
  #include "convert.h"
 +#include "trace.h"
 +#include "string-list.h"
  
  #include SHA1_HEADER
  #ifndef git_SHA_CTX
@@@ -43,14 -41,6 +43,14 @@@ int git_deflate_end_gently(git_zstream 
  int git_deflate(git_zstream *, int flush);
  unsigned long git_deflate_bound(git_zstream *, unsigned long);
  
 +/* The length in bytes and in hex digits of an object name (SHA-1 value). */
 +#define GIT_SHA1_RAWSZ 20
 +#define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ)
 +
 +struct object_id {
 +	unsigned char hash[GIT_SHA1_RAWSZ];
 +};
 +
  #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT)
  #define DTYPE(de)	((de)->d_type)
  #else
@@@ -73,25 -63,17 +73,25 @@@
   *
   * The value 0160000 is not normally a valid mode, and
   * also just happens to be S_IFDIR + S_IFLNK
 - *
 - * NOTE! We *really* shouldn't depend on the S_IFxxx macros
 - * always having the same values everywhere. We should use
 - * our internal git values for these things, and then we can
 - * translate that to the OS-specific value. It just so
 - * happens that everybody shares the same bit representation
 - * in the UNIX world (and apparently wider too..)
   */
  #define S_IFGITLINK	0160000
  #define S_ISGITLINK(m)	(((m) & S_IFMT) == S_IFGITLINK)
  
 +/*
 + * Some mode bits are also used internally for computations.
 + *
 + * They *must* not overlap with any valid modes, and they *must* not be emitted
 + * to outside world - i.e. appear on disk or network. In other words, it's just
 + * temporary fields, which we internally use, but they have to stay in-house.
 + *
 + * ( such approach is valid, as standard S_IF* fits into 16 bits, and in Git
 + *   codebase mode is `unsigned int` which is assumed to be at least 32 bits )
 + */
 +
 +/* used internally in tree-diff */
 +#define S_DIFFTREE_IFXMIN_NEQ	0x80000000
 +
 +
  /*
   * Intensive research over the course of many years has shown that
   * port 9418 is totally unused by anything else. Or
@@@ -153,7 -135,6 +153,7 @@@ struct cache_entry 
  	unsigned int ce_mode;
  	unsigned int ce_flags;
  	unsigned int ce_namelen;
 +	unsigned int index;	/* for link extension */
  	unsigned char sha1[20];
  	char name[FLEX_ARRAY]; /* more */
  };
@@@ -164,7 -145,7 +164,7 @@@
  #define CE_STAGESHIFT 12
  
  /*
 - * Range 0xFFFF0000 in ce_flags is divided into
 + * Range 0xFFFF0FFF in ce_flags is divided into
   * two parts: in-memory flags and on-disk ones.
   * Flags in CE_EXTENDED_FLAGS will get saved on-disk
   * if you want to save a new flag, add it in
@@@ -187,9 -168,6 +187,9 @@@
  /* used to temporarily mark paths matched by pathspecs */
  #define CE_MATCHED           (1 << 26)
  
 +#define CE_UPDATE_IN_BASE    (1 << 27)
 +#define CE_STRIP_NAME        (1 << 28)
 +
  /*
   * Extended on-disk flags
   */
@@@ -290,32 -268,18 +290,32 @@@ static inline unsigned int canon_mode(u
  
  #define cache_entry_size(len) (offsetof(struct cache_entry,name) + (len) + 1)
  
 +#define SOMETHING_CHANGED	(1 << 0) /* unclassified changes go here */
 +#define CE_ENTRY_CHANGED	(1 << 1)
 +#define CE_ENTRY_REMOVED	(1 << 2)
 +#define CE_ENTRY_ADDED		(1 << 3)
 +#define RESOLVE_UNDO_CHANGED	(1 << 4)
 +#define CACHE_TREE_CHANGED	(1 << 5)
 +#define SPLIT_INDEX_ORDERED	(1 << 6)
 +#define UNTRACKED_CHANGED	(1 << 7)
 +
 +struct split_index;
 +struct untracked_cache;
 +
  struct index_state {
  	struct cache_entry **cache;
  	unsigned int version;
  	unsigned int cache_nr, cache_alloc, cache_changed;
  	struct string_list *resolve_undo;
  	struct cache_tree *cache_tree;
 +	struct split_index *split_index;
  	struct cache_time timestamp;
  	unsigned name_hash_initialized : 1,
  		 initialized : 1;
  	struct hashmap name_hash;
  	struct hashmap dir_hash;
  	unsigned char sha1[20];
 +	struct untracked_cache *untracked;
  };
  
  extern struct index_state the_index;
@@@ -338,6 -302,7 +338,6 @@@ extern void free_name_hash(struct index
  #define read_cache_preload(pathspec) read_index_preload(&the_index, (pathspec))
  #define is_cache_unborn() is_index_unborn(&the_index)
  #define read_cache_unmerged() read_index_unmerged(&the_index)
 -#define write_cache(newfd, cache, entries) write_index(&the_index, (newfd))
  #define discard_cache() discard_index(&the_index)
  #define unmerged_cache() unmerged_index(&the_index)
  #define cache_name_pos(name, namelen) index_name_pos(&the_index,(name),(namelen))
@@@ -382,7 -347,6 +382,7 @@@ static inline enum object_type object_t
  
  /* Double-check local_repo_env below if you add to this list. */
  #define GIT_DIR_ENVIRONMENT "GIT_DIR"
 +#define GIT_COMMON_DIR_ENVIRONMENT "GIT_COMMON_DIR"
  #define GIT_NAMESPACE_ENVIRONMENT "GIT_NAMESPACE"
  #define GIT_WORK_TREE_ENVIRONMENT "GIT_WORK_TREE"
  #define GIT_PREFIX_ENVIRONMENT "GIT_PREFIX"
@@@ -397,7 -361,6 +397,7 @@@
  #define EXEC_PATH_ENVIRONMENT "GIT_EXEC_PATH"
  #define CEILING_DIRECTORIES_ENVIRONMENT "GIT_CEILING_DIRECTORIES"
  #define NO_REPLACE_OBJECTS_ENVIRONMENT "GIT_NO_REPLACE_OBJECTS"
 +#define GIT_REPLACE_REF_BASE_ENVIRONMENT "GIT_REPLACE_REF_BASE"
  #define GITATTRIBUTES_FILE ".gitattributes"
  #define INFOATTRIBUTES_FILE "info/attributes"
  #define ATTRIBUTE_MACRO_PREFIX "[attr]"
@@@ -437,27 -400,15 +437,27 @@@ extern int is_inside_git_dir(void)
  extern char *git_work_tree_cfg;
  extern int is_inside_work_tree(void);
  extern const char *get_git_dir(void);
 +extern const char *get_git_common_dir(void);
  extern int is_git_directory(const char *path);
  extern char *get_object_directory(void);
  extern char *get_index_file(void);
  extern char *get_graft_file(void);
  extern int set_git_dir(const char *path);
 +extern int get_common_dir(struct strbuf *sb, const char *gitdir);
  extern const char *get_git_namespace(void);
  extern const char *strip_namespace(const char *namespaced_ref);
  extern const char *get_git_work_tree(void);
 -extern const char *read_gitfile(const char *path);
 +
 +#define READ_GITFILE_ERR_STAT_FAILED 1
 +#define READ_GITFILE_ERR_NOT_A_FILE 2
 +#define READ_GITFILE_ERR_OPEN_FAILED 3
 +#define READ_GITFILE_ERR_READ_FAILED 4
 +#define READ_GITFILE_ERR_INVALID_FORMAT 5
 +#define READ_GITFILE_ERR_NO_PATH 6
 +#define READ_GITFILE_ERR_NOT_A_REPO 7
 +#define READ_GITFILE_ERR_TOO_LARGE 8
 +extern const char *read_gitfile_gently(const char *path, int *return_error_code);
 +#define read_gitfile(path) read_gitfile_gently((path), NULL)
  extern const char *resolve_gitdir(const char *suspect);
  extern void set_git_work_tree(const char *tree);
  
@@@ -501,22 -452,17 +501,22 @@@ extern int daemonize(void)
  				alloc = (nr); \
  			else \
  				alloc = alloc_nr(alloc); \
 -			x = xrealloc((x), alloc * sizeof(*(x))); \
 +			REALLOC_ARRAY(x, alloc); \
  		} \
  	} while (0)
  
  /* Initialize and use the cache information */
 +struct lock_file;
  extern int read_index(struct index_state *);
  extern int read_index_preload(struct index_state *, const struct pathspec *pathspec);
 +extern int do_read_index(struct index_state *istate, const char *path,
 +			 int must_exist); /* for testting only! */
  extern int read_index_from(struct index_state *, const char *path);
  extern int is_index_unborn(struct index_state *);
  extern int read_index_unmerged(struct index_state *);
 -extern int write_index(struct index_state *, int newfd);
 +#define COMMIT_LOCK		(1 << 0)
 +#define CLOSE_LOCK		(1 << 1)
 +extern int write_locked_index(struct index_state *, struct lock_file *lock, unsigned flags);
  extern int discard_index(struct index_state *);
  extern int unmerged_index(const struct index_state *);
  extern int verify_path(const char *path);
@@@ -528,7 -474,6 +528,7 @@@ extern int index_name_pos(const struct 
  #define ADD_CACHE_SKIP_DFCHECK 4	/* Ok to skip DF conflict checks */
  #define ADD_CACHE_JUST_APPEND 8		/* Append only; tree.c::read_tree() */
  #define ADD_CACHE_NEW_ONLY 16		/* Do not replace existing ones */
 +#define ADD_CACHE_KEEP_CACHE_TREE 32	/* Do not invalidate cache-tree */
  extern int add_index_entry(struct index_state *, struct cache_entry *ce, int option);
  extern void rename_index_entry_at(struct index_state *, int pos, const char *new_name);
  extern int remove_index_entry_at(struct index_state *, int pos);
@@@ -578,8 -523,6 +578,8 @@@ extern void fill_stat_data(struct stat_
   * INODE_CHANGED, and DATA_CHANGED.
   */
  extern int match_stat_data(const struct stat_data *sd, struct stat *st);
 +extern int match_stat_data_racy(const struct index_state *istate,
 +				const struct stat_data *sd, struct stat *st);
  
  extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st);
  
@@@ -591,10 -534,28 +591,10 @@@
  #define REFRESH_IN_PORCELAIN	0x0020	/* user friendly output, not "needs update" */
  extern int refresh_index(struct index_state *, unsigned int flags, const struct pathspec *pathspec, char *seen, const char *header_msg);
  
 -struct lock_file {
 -	struct lock_file *next;
 -	int fd;
 -	pid_t owner;
 -	char on_list;
 -	char filename[PATH_MAX];
 -};
 -#define LOCK_DIE_ON_ERROR 1
 -#define LOCK_NODEREF 2
 -extern int unable_to_lock_error(const char *path, int err);
 -extern NORETURN void unable_to_lock_index_die(const char *path, int err);
 -extern int hold_lock_file_for_update(struct lock_file *, const char *path, int);
 -extern int hold_lock_file_for_append(struct lock_file *, const char *path, int);
 -extern int commit_lock_file(struct lock_file *);
  extern void update_index_if_able(struct index_state *, struct lock_file *);
  
  extern int hold_locked_index(struct lock_file *, int);
 -extern int commit_locked_index(struct lock_file *);
  extern void set_alternate_index_output(const char *);
 -extern int close_lock_file(struct lock_file *);
 -extern void rollback_lock_file(struct lock_file *);
 -extern int delete_ref(const char *, const unsigned char *sha1, int delopt);
  
  /* Environment bits from configuration mechanism */
  extern int trust_executable_bit;
@@@ -631,7 -592,6 +631,7 @@@ extern unsigned long pack_size_limit_cf
   * been sought but there were none.
   */
  extern int check_replace_refs;
 +extern char *git_replace_ref_base;
  
  extern int fsync_object_files;
  extern int core_preload_index;
@@@ -639,22 -599,12 +639,22 @@@ extern int core_apply_sparse_checkout
  extern int precomposed_unicode;
  extern int protect_hfs;
  extern int protect_ntfs;
 +extern int git_db_env, git_index_env, git_graft_env, git_common_dir_env;
 +
 +/*
 + * Include broken refs in all ref iterations, which will
 + * generally choke dangerous operations rather than letting
 + * them silently proceed without taking the broken ref into
 + * account.
 + */
 +extern int ref_paranoia;
  
  /*
   * The character that begins a commented line in user-editable file
   * that is subject to stripspace.
   */
  extern char comment_line_char;
 +extern int auto_comment_line_char;
  
  enum branch_track {
  	BRANCH_TRACK_UNSPECIFIED = -1,
@@@ -710,19 -660,18 +710,19 @@@ extern int check_repository_format(void
  
  extern char *mksnpath(char *buf, size_t n, const char *fmt, ...)
  	__attribute__((format (printf, 3, 4)));
 -extern char *git_snpath(char *buf, size_t n, const char *fmt, ...)
 -	__attribute__((format (printf, 3, 4)));
 +extern void strbuf_git_path(struct strbuf *sb, const char *fmt, ...)
 +	__attribute__((format (printf, 2, 3)));
  extern char *git_pathdup(const char *fmt, ...)
  	__attribute__((format (printf, 1, 2)));
  extern char *mkpathdup(const char *fmt, ...)
  	__attribute__((format (printf, 1, 2)));
  
  /* Return a statically allocated filename matching the sha1 signature */
 -extern char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
 -extern char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
 -extern char *git_path_submodule(const char *path, const char *fmt, ...)
 +extern const char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
 +extern const char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
 +extern const char *git_path_submodule(const char *path, const char *fmt, ...)
  	__attribute__((format (printf, 2, 3)));
 +extern void report_linked_checkout_garbage(void);
  
  /*
   * Return the name of the file in the local object database that would
@@@ -747,13 -696,13 +747,13 @@@ extern char *sha1_pack_name(const unsig
  extern char *sha1_pack_index_name(const unsigned char *sha1);
  
  extern const char *find_unique_abbrev(const unsigned char *sha1, int);
 -extern const unsigned char null_sha1[20];
 +extern const unsigned char null_sha1[GIT_SHA1_RAWSZ];
  
  static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2)
  {
  	int i;
  
 -	for (i = 0; i < 20; i++, sha1++, sha2++) {
 +	for (i = 0; i < GIT_SHA1_RAWSZ; i++, sha1++, sha2++) {
  		if (*sha1 != *sha2)
  			return *sha1 - *sha2;
  	}
@@@ -761,42 -710,20 +761,42 @@@
  	return 0;
  }
  
 +static inline int oidcmp(const struct object_id *oid1, const struct object_id *oid2)
 +{
 +	return hashcmp(oid1->hash, oid2->hash);
 +}
 +
  static inline int is_null_sha1(const unsigned char *sha1)
  {
  	return !hashcmp(sha1, null_sha1);
  }
  
 +static inline int is_null_oid(const struct object_id *oid)
 +{
 +	return !hashcmp(oid->hash, null_sha1);
 +}
 +
  static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src)
  {
 -	memcpy(sha_dst, sha_src, 20);
 +	memcpy(sha_dst, sha_src, GIT_SHA1_RAWSZ);
 +}
 +
 +static inline void oidcpy(struct object_id *dst, const struct object_id *src)
 +{
 +	hashcpy(dst->hash, src->hash);
  }
 +
  static inline void hashclr(unsigned char *hash)
  {
 -	memset(hash, 0, 20);
 +	memset(hash, 0, GIT_SHA1_RAWSZ);
 +}
 +
 +static inline void oidclr(struct object_id *oid)
 +{
 +	hashclr(oid->hash);
  }
  
 +
  #define EMPTY_TREE_SHA1_HEX \
  	"4b825dc642cb6eb9a060e54bf8d69288fbee4904"
  #define EMPTY_TREE_SHA1_BIN_LITERAL \
@@@ -867,6 -794,7 +867,6 @@@ enum scld_error safe_create_leading_dir
  enum scld_error safe_create_leading_directories_const(const char *path);
  
  int mkdir_in_gitdir(const char *path);
 -extern void home_config_paths(char **global, char **xdg, char *file);
  extern char *expand_user_path(const char *path);
  const char *enter_repo(const char *path, int strict);
  static inline int is_absolute_path(const char *path)
@@@ -884,18 -812,11 +884,18 @@@ int normalize_path_copy(char *dst, cons
  int longest_ancestor_length(const char *path, struct string_list *prefixes);
  char *strip_path_suffix(const char *path, const char *suffix);
  int daemon_avoid_alias(const char *path);
 -int offset_1st_component(const char *path);
  extern int is_ntfs_dotgit(const char *name);
  
 +/**
 + * Return a newly allocated string with the evaluation of
 + * "$XDG_CONFIG_HOME/git/$filename" if $XDG_CONFIG_HOME is non-empty, otherwise
 + * "$HOME/.config/git/$filename". Return NULL upon error.
 + */
 +extern char *xdg_config_home(const char *filename);
 +
  /* object replacement */
  #define LOOKUP_REPLACE_OBJECT 1
 +#define LOOKUP_UNKNOWN_OBJECT 2
  extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag);
  static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size)
  {
@@@ -932,7 -853,6 +932,7 @@@ static inline const unsigned char *look
  extern int sha1_object_info(const unsigned char *, unsigned long *);
  extern int hash_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1);
  extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1);
 +extern int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type, unsigned char *sha1, unsigned flags);
  extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *);
  extern int force_object_loose(const unsigned char *sha1, time_t mtime);
  extern int git_open_noatime(const char *name);
@@@ -945,7 -865,7 +945,7 @@@ extern int do_check_packed_object_crc
  
  extern int check_sha1_signature(const unsigned char *sha1, void *buf, unsigned long size, const char *type);
  
- extern int move_temp_to_file(const char *tmpfile, const char *filename);
+ extern int finalize_object_file(const char *tmpfile, const char *filename);
  
  extern int has_sha1_pack(const unsigned char *sha1);
  
@@@ -953,17 -873,8 +953,17 @@@
   * Return true iff we have an object named sha1, whether local or in
   * an alternate object database, and whether packed or loose.  This
   * function does not respect replace references.
 + *
 + * If the QUICK flag is set, do not re-check the pack directory
 + * when we cannot find the object (this means we may give a false
 + * negative answer if another process is simultaneously repacking).
   */
 -extern int has_sha1_file(const unsigned char *sha1);
 +#define HAS_SHA1_QUICK 0x1
 +extern int has_sha1_file_with_flags(const unsigned char *sha1, int flags);
 +static inline int has_sha1_file(const unsigned char *sha1)
 +{
 +	return has_sha1_file_with_flags(sha1, 0);
 +}
  
  /*
   * Return true iff an alternate object database has a loose object
@@@ -990,21 -901,15 +990,21 @@@ struct object_context 
  	unsigned char tree[20];
  	char path[PATH_MAX];
  	unsigned mode;
 +	/*
 +	 * symlink_path is only used by get_tree_entry_follow_symlinks,
 +	 * and only for symlinks that point outside the repository.
 +	 */
 +	struct strbuf symlink_path;
  };
  
 -#define GET_SHA1_QUIETLY        01
 -#define GET_SHA1_COMMIT         02
 -#define GET_SHA1_COMMITTISH     04
 -#define GET_SHA1_TREE          010
 -#define GET_SHA1_TREEISH       020
 -#define GET_SHA1_BLOB	       040
 -#define GET_SHA1_ONLY_TO_DIE 04000
 +#define GET_SHA1_QUIETLY           01
 +#define GET_SHA1_COMMIT            02
 +#define GET_SHA1_COMMITTISH        04
 +#define GET_SHA1_TREE             010
 +#define GET_SHA1_TREEISH          020
 +#define GET_SHA1_BLOB             040
 +#define GET_SHA1_FOLLOW_SYMLINKS 0100
 +#define GET_SHA1_ONLY_TO_DIE    04000
  
  extern int get_sha1(const char *str, unsigned char *sha1);
  extern int get_sha1_commit(const char *str, unsigned char *sha1);
@@@ -1026,19 -931,63 +1026,19 @@@ extern int for_each_abbrev(const char *
   * null-terminated string.
   */
  extern int get_sha1_hex(const char *hex, unsigned char *sha1);
 +extern int get_oid_hex(const char *hex, struct object_id *sha1);
  
  extern char *sha1_to_hex(const unsigned char *sha1);	/* static buffer result! */
 -extern int read_ref_full(const char *refname, unsigned char *sha1,
 -			 int reading, int *flags);
 -extern int read_ref(const char *refname, unsigned char *sha1);
 -
 -/*
 - * Resolve a reference, recursively following symbolic refererences.
 - *
 - * Store the referred-to object's name in sha1 and return the name of
 - * the non-symbolic reference that ultimately pointed at it.  The
 - * return value, if not NULL, is a pointer into either a static buffer
 - * or the input ref.
 - *
 - * If the reference cannot be resolved to an object, the behavior
 - * depends on the "reading" argument:
 - *
 - * - If reading is set, return NULL.
 - *
 - * - If reading is not set, clear sha1 and return the name of the last
 - *   reference name in the chain, which will either be a non-symbolic
 - *   reference or an undefined reference.  If this is a prelude to
 - *   "writing" to the ref, the return value is the name of the ref
 - *   that will actually be created or changed.
 - *
 - * If flag is non-NULL, set the value that it points to the
 - * combination of REF_ISPACKED (if the reference was found among the
 - * packed references) and REF_ISSYMREF (if the initial reference was a
 - * symbolic reference).
 - *
 - * If ref is not a properly-formatted, normalized reference, return
 - * NULL.  If more than MAXDEPTH recursive symbolic lookups are needed,
 - * give up and return NULL.
 - *
 - * errno is sometimes set on errors, but not always.
 - */
 -extern const char *resolve_ref_unsafe(const char *ref, unsigned char *sha1, int reading, int *flag);
 -extern char *resolve_refdup(const char *ref, unsigned char *sha1, int reading, int *flag);
 +extern char *oid_to_hex(const struct object_id *oid);	/* same static buffer as sha1_to_hex */
  
 -extern int dwim_ref(const char *str, int len, unsigned char *sha1, char **ref);
 -extern int dwim_log(const char *str, int len, unsigned char *sha1, char **ref);
  extern int interpret_branch_name(const char *str, int len, struct strbuf *);
  extern int get_sha1_mb(const char *str, unsigned char *sha1);
  
 -/*
 - * Return true iff abbrev_name is a possible abbreviation for
 - * full_name according to the rules defined by ref_rev_parse_rules in
 - * refs.c.
 - */
 -extern int refname_match(const char *abbrev_name, const char *full_name);
 -
 -extern int create_symref(const char *ref, const char *refs_heads_master, const char *logmsg);
  extern int validate_headref(const char *ref);
  
  extern int base_name_compare(const char *name1, int len1, int mode1, const char *name2, int len2, int mode2);
  extern int df_name_compare(const char *name1, int len1, int mode1, const char *name2, int len2, int mode2);
 -extern int cache_name_compare(const char *name1, int len1, const char *name2, int len2);
 +extern int name_compare(const char *name1, size_t len1, const char *name2, size_t len2);
  extern int cache_name_stage_compare(const char *name1, int len1, int stage1, const char *name2, int len2, int stage2);
  
  extern void *read_object_with_reference(const unsigned char *sha1,
@@@ -1049,40 -998,27 +1049,40 @@@
  extern struct object *peel_to_type(const char *name, int namelen,
  				   struct object *o, enum object_type);
  
 -enum date_mode {
 -	DATE_NORMAL = 0,
 -	DATE_RELATIVE,
 -	DATE_SHORT,
 -	DATE_LOCAL,
 -	DATE_ISO8601,
 -	DATE_RFC2822,
 -	DATE_RAW
 +struct date_mode {
 +	enum date_mode_type {
 +		DATE_NORMAL = 0,
 +		DATE_RELATIVE,
 +		DATE_SHORT,
 +		DATE_LOCAL,
 +		DATE_ISO8601,
 +		DATE_ISO8601_STRICT,
 +		DATE_RFC2822,
 +		DATE_STRFTIME,
 +		DATE_RAW
 +	} type;
 +	const char *strftime_fmt;
  };
  
 -const char *show_date(unsigned long time, int timezone, enum date_mode mode);
 +/*
 + * Convenience helper for passing a constant type, like:
 + *
 + *   show_date(t, tz, DATE_MODE(NORMAL));
 + */
 +#define DATE_MODE(t) date_mode_from_type(DATE_##t)
 +struct date_mode *date_mode_from_type(enum date_mode_type type);
 +
 +const char *show_date(unsigned long time, int timezone, const struct date_mode *mode);
  void show_date_relative(unsigned long time, int tz, const struct timeval *now,
  			struct strbuf *timebuf);
 -int parse_date(const char *date, char *buf, int bufsize);
 +int parse_date(const char *date, struct strbuf *out);
  int parse_date_basic(const char *date, unsigned long *timestamp, int *offset);
  int parse_expiry_date(const char *date, unsigned long *timestamp);
 -void datestamp(char *buf, int bufsize);
 +void datestamp(struct strbuf *out);
  #define approxidate(s) approxidate_careful((s), NULL)
  unsigned long approxidate_careful(const char *, int *);
  unsigned long approxidate_relative(const char *date, const struct timeval *now);
 -enum date_mode parse_date_format(const char *format);
 +void parse_date_format(const char *format, struct date_mode *mode);
  int date_overflows(unsigned long date);
  
  #define IDENT_STRICT	       1
@@@ -1092,7 -1028,6 +1092,7 @@@ extern const char *git_author_info(int)
  extern const char *git_committer_info(int);
  extern const char *fmt_ident(const char *name, const char *email, const char *date_str, int);
  extern const char *fmt_name(const char *name, const char *email);
 +extern const char *ident_default_name(void);
  extern const char *ident_default_email(void);
  extern const char *git_editor(void);
  extern const char *git_pager(int stdout_is_tty);
@@@ -1114,14 -1049,6 +1114,14 @@@ struct ident_split 
   */
  extern int split_ident_line(struct ident_split *, const char *, int);
  
 +/*
 + * Like show_date, but pull the timestamp and tz parameters from
 + * the ident_split. It will also sanity-check the values and produce
 + * a well-known sentinel date if they appear bogus.
 + */
 +const char *show_ident_date(const struct ident_split *id,
 +			    const struct date_mode *mode);
 +
  /*
   * Compare split idents for equality or strict ordering. Note that we
   * compare only the ident part of the line, ignoring any timestamp.
@@@ -1132,7 -1059,6 +1132,7 @@@
  extern int ident_cmp(const struct ident_split *, const struct ident_split *);
  
  struct checkout {
 +	struct index_state *istate;
  	const char *base_dir;
  	int base_dir_len;
  	unsigned force:1,
@@@ -1145,16 -1071,12 +1145,16 @@@
  extern int checkout_entry(struct cache_entry *ce, const struct checkout *state, char *topath);
  
  struct cache_def {
 -	char path[PATH_MAX + 1];
 -	int len;
 +	struct strbuf path;
  	int flags;
  	int track_flags;
  	int prefix_len_stat_func;
  };
 +#define CACHE_DEF_INIT { STRBUF_INIT, 0, 0, 0 }
 +static inline void cache_def_clear(struct cache_def *cache)
 +{
 +	strbuf_release(&cache->path);
 +}
  
  extern int has_symlink_leading_path(const char *name, int len);
  extern int threaded_has_symlink_leading_path(struct cache_def *, const char *, int);
@@@ -1172,7 -1094,7 +1172,7 @@@ extern void prepare_alt_odb(void)
  extern void read_info_alternates(const char * relative_base, int depth);
  extern void add_to_alternates_file(const char *reference);
  typedef int alt_odb_fn(struct alternate_object_database *, void *);
 -extern void foreach_alt_odb(alt_odb_fn, void*);
 +extern int foreach_alt_odb(alt_odb_fn, void*);
  
  struct pack_window {
  	struct pack_window *next;
@@@ -1197,7 -1119,6 +1197,7 @@@ extern struct packed_git 
  	int pack_fd;
  	unsigned pack_local:1,
  		 pack_keep:1,
 +		 freshened:1,
  		 do_not_close:1;
  	unsigned char sha1[20];
  	/* something like ".git/objects/pack/xxxxx.pack" */
@@@ -1269,68 -1190,12 +1269,68 @@@ extern unsigned long unpack_object_head
  extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t);
  extern int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *);
  
 +/*
 + * Iterate over the files in the loose-object parts of the object
 + * directory "path", triggering the following callbacks:
 + *
 + *  - loose_object is called for each loose object we find.
 + *
 + *  - loose_cruft is called for any files that do not appear to be
 + *    loose objects. Note that we only look in the loose object
 + *    directories "objects/[0-9a-f]{2}/", so we will not report
 + *    "objects/foobar" as cruft.
 + *
 + *  - loose_subdir is called for each top-level hashed subdirectory
 + *    of the object directory (e.g., "$OBJDIR/f0"). It is called
 + *    after the objects in the directory are processed.
 + *
 + * Any callback that is NULL will be ignored. Callbacks returning non-zero
 + * will end the iteration.
 + *
 + * In the "buf" variant, "path" is a strbuf which will also be used as a
 + * scratch buffer, but restored to its original contents before
 + * the function returns.
 + */
 +typedef int each_loose_object_fn(const unsigned char *sha1,
 +				 const char *path,
 +				 void *data);
 +typedef int each_loose_cruft_fn(const char *basename,
 +				const char *path,
 +				void *data);
 +typedef int each_loose_subdir_fn(int nr,
 +				 const char *path,
 +				 void *data);
 +int for_each_loose_file_in_objdir(const char *path,
 +				  each_loose_object_fn obj_cb,
 +				  each_loose_cruft_fn cruft_cb,
 +				  each_loose_subdir_fn subdir_cb,
 +				  void *data);
 +int for_each_loose_file_in_objdir_buf(struct strbuf *path,
 +				      each_loose_object_fn obj_cb,
 +				      each_loose_cruft_fn cruft_cb,
 +				      each_loose_subdir_fn subdir_cb,
 +				      void *data);
 +
 +/*
 + * Iterate over loose and packed objects in both the local
 + * repository and any alternates repositories (unless the
 + * LOCAL_ONLY flag is set).
 + */
 +#define FOR_EACH_OBJECT_LOCAL_ONLY 0x1
 +typedef int each_packed_object_fn(const unsigned char *sha1,
 +				  struct packed_git *pack,
 +				  uint32_t pos,
 +				  void *data);
 +extern int for_each_loose_object(each_loose_object_fn, void *, unsigned flags);
 +extern int for_each_packed_object(each_packed_object_fn, void *, unsigned flags);
 +
  struct object_info {
  	/* Request */
  	enum object_type *typep;
  	unsigned long *sizep;
  	unsigned long *disk_sizep;
  	unsigned char *delta_base_sha1;
 +	struct strbuf *typename;
  
  	/* Response */
  	enum {
@@@ -1371,8 -1236,6 +1371,8 @@@ extern int update_server_info(int)
  #define CONFIG_INVALID_PATTERN 6
  #define CONFIG_GENERIC_ERROR 7
  
 +#define CONFIG_REGEX_NONE ((void *)1)
 +
  struct git_config_source {
  	unsigned int use_stdin:1;
  	const char *file;
@@@ -1386,7 -1249,7 +1386,7 @@@ extern int git_config_from_buf(config_f
  			       const char *buf, size_t len, void *data);
  extern void git_config_push_parameter(const char *text);
  extern int git_config_from_parameters(config_fn_t fn, void *data);
 -extern int git_config(config_fn_t fn, void *);
 +extern void git_config(config_fn_t fn, void *);
  extern int git_config_with_options(config_fn_t fn, void *,
  				   struct git_config_source *config_source,
  				   int respect_includes);
@@@ -1410,11 -1273,10 +1410,11 @@@ extern int git_config_rename_section_in
  extern const char *git_etc_gitconfig(void);
  extern int check_repository_format_version(const char *var, const char *value, void *cb);
  extern int git_env_bool(const char *, int);
 +extern unsigned long git_env_ulong(const char *, unsigned long);
  extern int git_config_system(void);
  extern int config_error_nonbool(const char *);
 -#if defined(__GNUC__) && ! defined(__clang__)
 -#define config_error_nonbool(s) (config_error_nonbool(s), -1)
 +#if defined(__GNUC__)
 +#define config_error_nonbool(s) (config_error_nonbool(s), const_error())
  #endif
  extern const char *get_log_output_encoding(void);
  extern const char *get_commit_output_encoding(void);
@@@ -1444,69 -1306,6 +1444,69 @@@ extern int parse_config_key(const char 
  			    const char **subsection, int *subsection_len,
  			    const char **key);
  
 +struct config_set_element {
 +	struct hashmap_entry ent;
 +	char *key;
 +	struct string_list value_list;
 +};
 +
 +struct configset_list_item {
 +	struct config_set_element *e;
 +	int value_index;
 +};
 +
 +/*
 + * the contents of the list are ordered according to their
 + * position in the config files and order of parsing the files.
 + * (i.e. key-value pair at the last position of .git/config will
 + * be at the last item of the list)
 + */
 +struct configset_list {
 +	struct configset_list_item *items;
 +	unsigned int nr, alloc;
 +};
 +
 +struct config_set {
 +	struct hashmap config_hash;
 +	int hash_initialized;
 +	struct configset_list list;
 +};
 +
 +extern void git_configset_init(struct config_set *cs);
 +extern int git_configset_add_file(struct config_set *cs, const char *filename);
 +extern int git_configset_get_value(struct config_set *cs, const char *key, const char **value);
 +extern const struct string_list *git_configset_get_value_multi(struct config_set *cs, const char *key);
 +extern void git_configset_clear(struct config_set *cs);
 +extern int git_configset_get_string_const(struct config_set *cs, const char *key, const char **dest);
 +extern int git_configset_get_string(struct config_set *cs, const char *key, char **dest);
 +extern int git_configset_get_int(struct config_set *cs, const char *key, int *dest);
 +extern int git_configset_get_ulong(struct config_set *cs, const char *key, unsigned long *dest);
 +extern int git_configset_get_bool(struct config_set *cs, const char *key, int *dest);
 +extern int git_configset_get_bool_or_int(struct config_set *cs, const char *key, int *is_bool, int *dest);
 +extern int git_configset_get_maybe_bool(struct config_set *cs, const char *key, int *dest);
 +extern int git_configset_get_pathname(struct config_set *cs, const char *key, const char **dest);
 +
 +extern int git_config_get_value(const char *key, const char **value);
 +extern const struct string_list *git_config_get_value_multi(const char *key);
 +extern void git_config_clear(void);
 +extern void git_config_iter(config_fn_t fn, void *data);
 +extern int git_config_get_string_const(const char *key, const char **dest);
 +extern int git_config_get_string(const char *key, char **dest);
 +extern int git_config_get_int(const char *key, int *dest);
 +extern int git_config_get_ulong(const char *key, unsigned long *dest);
 +extern int git_config_get_bool(const char *key, int *dest);
 +extern int git_config_get_bool_or_int(const char *key, int *is_bool, int *dest);
 +extern int git_config_get_maybe_bool(const char *key, int *dest);
 +extern int git_config_get_pathname(const char *key, const char **dest);
 +
 +struct key_value_info {
 +	const char *filename;
 +	int linenr;
 +};
 +
 +extern NORETURN void git_die_config(const char *key, const char *err, ...) __attribute__((format(printf, 2, 3)));
 +extern NORETURN void git_die_config_linenr(const char *key, const char *filename, int linenr);
 +
  extern int committer_ident_sufficiently_given(void);
  extern int author_ident_sufficiently_given(void);
  
@@@ -1517,15 -1316,9 +1517,15 @@@ extern const char *git_mailmap_blob
  
  /* IO helper functions */
  extern void maybe_flush_or_die(FILE *, const char *);
 +__attribute__((format (printf, 2, 3)))
 +extern void fprintf_or_die(FILE *, const char *fmt, ...);
 +
 +#define COPY_READ_ERROR (-2)
 +#define COPY_WRITE_ERROR (-3)
  extern int copy_fd(int ifd, int ofd);
  extern int copy_file(const char *dst, const char *src, int mode);
  extern int copy_file_with_time(const char *dst, const char *src, int mode);
 +
  extern void write_or_die(int fd, const void *buf, size_t count);
  extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg);
  extern int write_or_whine_pipe(int fd, const void *buf, size_t count, const char *msg);
@@@ -1539,8 -1332,6 +1539,8 @@@ static inline ssize_t write_str_in_full
  {
  	return write_in_full(fd, str, strlen(str));
  }
 +__attribute__((format (printf, 3, 4)))
 +extern int write_file(const char *path, int fatal, const char *fmt, ...);
  
  /* pager.c */
  extern void setup_pager(void);
@@@ -1548,7 -1339,7 +1548,7 @@@ extern const char *pager_program
  extern int pager_in_use(void);
  extern int pager_use_color;
  extern int term_columns(void);
 -extern int decimal_width(int);
 +extern int decimal_width(uintmax_t);
  extern int check_pager_config(const char *cmd);
  
  extern const char *editor_program;
@@@ -1568,7 -1359,17 +1568,7 @@@ extern void *alloc_object_node(void)
  extern void alloc_report(void);
  extern unsigned int alloc_commit_index(void);
  
 -/* trace.c */
 -__attribute__((format (printf, 1, 2)))
 -extern void trace_printf(const char *format, ...);
 -__attribute__((format (printf, 2, 3)))
 -extern void trace_argv_printf(const char **argv, const char *format, ...);
 -extern void trace_repo_setup(const char *prefix);
 -extern int trace_want(const char *key);
 -__attribute__((format (printf, 2, 3)))
 -extern void trace_printf_key(const char *key, const char *fmt, ...);
 -extern void trace_strbuf(const char *key, const struct strbuf *buf);
 -
 +/* pkt-line.c */
  void packet_trace_identity(const char *prog);
  
  /* add */
@@@ -1610,6 -1411,7 +1610,6 @@@ extern int ws_blank_line(const char *li
  #define ws_tab_width(rule)     ((rule) & WS_TAB_WIDTH_MASK)
  
  /* ls-files */
 -int report_path_error(const char *ps_matched, const struct pathspec *pathspec, const char *prefix);
  void overlay_tree_on_cache(const char *tree_name, const char *prefix);
  
  char *alias_lookup(const char *alias);
@@@ -1664,6 -1466,5 +1664,6 @@@ int stat_validity_check(struct stat_val
  void stat_validity_update(struct stat_validity *sv, int fd);
  
  int versioncmp(const char *s1, const char *s2);
 +void sleep_millisec(int millisec);
  
  #endif /* CACHE_H */
diff --combined fast-import.c
index 2ad4fee07e,9479ce6d0f..79d2bff2bf
--- a/fast-import.c
+++ b/fast-import.c
@@@ -134,17 -134,16 +134,17 @@@ Format of STDIN stream
    ts    ::= # time since the epoch in seconds, ascii base10 notation;
    tz    ::= # GIT style timezone;
  
 -     # note: comments, ls and cat requests may appear anywhere
 -     # in the input, except within a data command.  Any form
 -     # of the data command always escapes the related input
 -     # from comment processing.
 +     # note: comments, get-mark, ls-tree, and cat-blob requests may
 +     # appear anywhere in the input, except within a data command. Any
 +     # form of the data command always escapes the related input from
 +     # comment processing.
       #
       # In case it is not clear, the '#' that starts the comment
       # must be the first character on that line (an lf
       # preceded it).
       #
  
 +  get_mark ::= 'get-mark' sp idnum lf;
    cat_blob ::= 'cat-blob' sp (hexsha1 | idnum) lf;
    ls_tree  ::= 'ls' sp (hexsha1 | idnum) sp path_str lf;
  
@@@ -154,7 -153,6 +154,7 @@@
  
  #include "builtin.h"
  #include "cache.h"
 +#include "lockfile.h"
  #include "object.h"
  #include "blob.h"
  #include "tree.h"
@@@ -250,7 -248,6 +250,7 @@@ struct branch 
  	uintmax_t last_commit;
  	uintmax_t num_notes;
  	unsigned active : 1;
 +	unsigned delete : 1;
  	unsigned pack_id : PACK_ID_BITS;
  	unsigned char sha1[20];
  };
@@@ -373,9 -370,8 +373,9 @@@ static volatile sig_atomic_t checkpoint
  static int cat_blob_fd = STDOUT_FILENO;
  
  static void parse_argv(void);
 -static void parse_cat_blob(void);
 -static void parse_ls(struct branch *b);
 +static void parse_get_mark(const char *p);
 +static void parse_cat_blob(const char *p);
 +static void parse_ls(const char *p, struct branch *b);
  
  static void write_branch_report(FILE *rpt, struct branch *b)
  {
@@@ -407,7 -403,7 +407,7 @@@ static void dump_marks_helper(FILE *, u
  
  static void write_crash_report(const char *err)
  {
 -	char *loc = git_path("fast_import_crash_%"PRIuMAX, (uintmax_t) getpid());
 +	const char *loc = git_path("fast_import_crash_%"PRIuMAX, (uintmax_t) getpid());
  	FILE *rpt = fopen(loc, "w");
  	struct branch *b;
  	unsigned long lu;
@@@ -423,7 -419,7 +423,7 @@@
  	fprintf(rpt, "fast-import crash report:\n");
  	fprintf(rpt, "    fast-import process: %"PRIuMAX"\n", (uintmax_t) getpid());
  	fprintf(rpt, "    parent process     : %"PRIuMAX"\n", (uintmax_t) getppid());
 -	fprintf(rpt, "    at %s\n", show_date(time(NULL), 0, DATE_LOCAL));
 +	fprintf(rpt, "    at %s\n", show_date(time(NULL), 0, DATE_MODE(LOCAL)));
  	fputc('\n', rpt);
  
  	fputs("fatal: ", rpt);
@@@ -881,7 -877,7 +881,7 @@@ static void start_packfile(void
  	pack_size = sizeof(hdr);
  	object_count = 0;
  
 -	all_packs = xrealloc(all_packs, sizeof(*all_packs) * (pack_id + 1));
 +	REALLOC_ARRAY(all_packs, pack_id + 1);
  	all_packs[pack_id] = p;
  }
  
@@@ -923,12 -919,12 +923,12 @@@ static char *keep_pack(const char *curr
  
  	snprintf(name, sizeof(name), "%s/pack/pack-%s.pack",
  		 get_object_directory(), sha1_to_hex(pack_data->sha1));
- 	if (move_temp_to_file(pack_data->pack_name, name))
+ 	if (finalize_object_file(pack_data->pack_name, name))
  		die("cannot store pack file");
  
  	snprintf(name, sizeof(name), "%s/pack/pack-%s.idx",
  		 get_object_directory(), sha1_to_hex(pack_data->sha1));
- 	if (move_temp_to_file(curr_index_name, name))
+ 	if (finalize_object_file(curr_index_name, name))
  		die("cannot store index file");
  	free((void *)curr_index_name);
  	return name;
@@@ -949,15 -945,10 +949,15 @@@ static void unkeep_all_packs(void
  
  static void end_packfile(void)
  {
 -	struct packed_git *old_p = pack_data, *new_p;
 +	static int running;
  
 +	if (running || !pack_data)
 +		return;
 +
 +	running = 1;
  	clear_delta_base_cache();
  	if (object_count) {
 +		struct packed_git *new_p;
  		unsigned char cur_pack_sha1[20];
  		char *idx_name;
  		int i;
@@@ -999,12 -990,10 +999,12 @@@
  		pack_id++;
  	}
  	else {
 -		close(old_p->pack_fd);
 -		unlink_or_warn(old_p->pack_name);
 +		close(pack_data->pack_fd);
 +		unlink_or_warn(pack_data->pack_name);
  	}
 -	free(old_p);
 +	free(pack_data);
 +	pack_data = NULL;
 +	running = 0;
  
  	/* We can't carry a delta across packfiles. */
  	strbuf_release(&last_blob.data);
@@@ -1064,6 -1053,7 +1064,6 @@@ static int store_object
  	} else
  		delta = NULL;
  
 -	memset(&s, 0, sizeof(s));
  	git_deflate_init(&s, pack_compression_level);
  	if (delta) {
  		s.next_in = delta;
@@@ -1091,6 -1081,7 +1091,6 @@@
  			free(delta);
  			delta = NULL;
  
 -			memset(&s, 0, sizeof(s));
  			git_deflate_init(&s, pack_compression_level);
  			s.next_in = (void *)dat->buf;
  			s.avail_in = dat->len;
@@@ -1190,6 -1181,7 +1190,6 @@@ static void stream_blob(uintmax_t len, 
  
  	crc32_begin(pack_file);
  
 -	memset(&s, 0, sizeof(s));
  	git_deflate_init(&s, pack_compression_level);
  
  	hdrlen = encode_in_pack_object_header(OBJ_BLOB, len, out_buf);
@@@ -1426,7 -1418,7 +1426,7 @@@ static void mktree(struct tree_content 
  
  static void store_tree(struct tree_entry *root)
  {
 -	struct tree_content *t = root->tree;
 +	struct tree_content *t;
  	unsigned int i, j, del;
  	struct last_object lo = { STRBUF_INIT, 0, 0, /* no_swap */ 1 };
  	struct object_entry *le = NULL;
@@@ -1434,10 -1426,6 +1434,10 @@@
  	if (!is_null_sha1(root->versions[1].sha1))
  		return;
  
 +	if (!root->tree)
 +		load_tree(root);
 +	t = root->tree;
 +
  	for (i = 0; i < t->entry_count; i++) {
  		if (t->entries[i]->tree)
  			store_tree(t->entries[i]);
@@@ -1690,44 -1678,36 +1690,44 @@@ found_entry
  static int update_branch(struct branch *b)
  {
  	static const char *msg = "fast-import";
 -	struct ref_lock *lock;
 +	struct ref_transaction *transaction;
  	unsigned char old_sha1[20];
 +	struct strbuf err = STRBUF_INIT;
  
 -	if (is_null_sha1(b->sha1))
 +	if (is_null_sha1(b->sha1)) {
 +		if (b->delete)
 +			delete_ref(b->name, NULL, 0);
  		return 0;
 +	}
  	if (read_ref(b->name, old_sha1))
  		hashclr(old_sha1);
 -	lock = lock_any_ref_for_update(b->name, old_sha1, 0, NULL);
 -	if (!lock)
 -		return error("Unable to lock %s", b->name);
  	if (!force_update && !is_null_sha1(old_sha1)) {
  		struct commit *old_cmit, *new_cmit;
  
  		old_cmit = lookup_commit_reference_gently(old_sha1, 0);
  		new_cmit = lookup_commit_reference_gently(b->sha1, 0);
 -		if (!old_cmit || !new_cmit) {
 -			unlock_ref(lock);
 +		if (!old_cmit || !new_cmit)
  			return error("Branch %s is missing commits.", b->name);
 -		}
  
  		if (!in_merge_bases(old_cmit, new_cmit)) {
 -			unlock_ref(lock);
  			warning("Not updating %s"
  				" (new tip %s does not contain %s)",
  				b->name, sha1_to_hex(b->sha1), sha1_to_hex(old_sha1));
  			return -1;
  		}
  	}
 -	if (write_ref_sha1(lock, b->sha1, msg) < 0)
 -		return error("Unable to update %s", b->name);
 +	transaction = ref_transaction_begin(&err);
 +	if (!transaction ||
 +	    ref_transaction_update(transaction, b->name, b->sha1, old_sha1,
 +				   0, msg, &err) ||
 +	    ref_transaction_commit(transaction, &err)) {
 +		ref_transaction_free(transaction);
 +		error("%s", err.buf);
 +		strbuf_release(&err);
 +		return -1;
 +	}
 +	ref_transaction_free(transaction);
 +	strbuf_release(&err);
  	return 0;
  }
  
@@@ -1746,32 -1726,15 +1746,32 @@@ static void dump_tags(void
  {
  	static const char *msg = "fast-import";
  	struct tag *t;
 -	struct ref_lock *lock;
 -	char ref_name[PATH_MAX];
 +	struct strbuf ref_name = STRBUF_INIT;
 +	struct strbuf err = STRBUF_INIT;
 +	struct ref_transaction *transaction;
  
 +	transaction = ref_transaction_begin(&err);
 +	if (!transaction) {
 +		failure |= error("%s", err.buf);
 +		goto cleanup;
 +	}
  	for (t = first_tag; t; t = t->next_tag) {
 -		sprintf(ref_name, "tags/%s", t->name);
 -		lock = lock_ref_sha1(ref_name, NULL);
 -		if (!lock || write_ref_sha1(lock, t->sha1, msg) < 0)
 -			failure |= error("Unable to update %s", ref_name);
 +		strbuf_reset(&ref_name);
 +		strbuf_addf(&ref_name, "refs/tags/%s", t->name);
 +
 +		if (ref_transaction_update(transaction, ref_name.buf,
 +					   t->sha1, NULL, 0, msg, &err)) {
 +			failure |= error("%s", err.buf);
 +			goto cleanup;
 +		}
  	}
 +	if (ref_transaction_commit(transaction, &err))
 +		failure |= error("%s", err.buf);
 +
 + cleanup:
 +	ref_transaction_free(transaction);
 +	strbuf_release(&ref_name);
 +	strbuf_release(&err);
  }
  
  static void dump_marks_helper(FILE *f,
@@@ -1797,18 -1760,20 +1797,18 @@@
  static void dump_marks(void)
  {
  	static struct lock_file mark_lock;
 -	int mark_fd;
  	FILE *f;
  
  	if (!export_marks_file)
  		return;
  
 -	mark_fd = hold_lock_file_for_update(&mark_lock, export_marks_file, 0);
 -	if (mark_fd < 0) {
 +	if (hold_lock_file_for_update(&mark_lock, export_marks_file, 0) < 0) {
  		failure |= error("Unable to write marks file %s: %s",
  			export_marks_file, strerror(errno));
  		return;
  	}
  
 -	f = fdopen(mark_fd, "w");
 +	f = fdopen_lock_file(&mark_lock, "w");
  	if (!f) {
  		int saved_errno = errno;
  		rollback_lock_file(&mark_lock);
@@@ -1817,10 -1782,27 +1817,10 @@@
  		return;
  	}
  
 -	/*
 -	 * Since the lock file was fdopen()'ed, it should not be close()'ed.
 -	 * Assign -1 to the lock file descriptor so that commit_lock_file()
 -	 * won't try to close() it.
 -	 */
 -	mark_lock.fd = -1;
 -
  	dump_marks_helper(f, 0, marks);
 -	if (ferror(f) || fclose(f)) {
 -		int saved_errno = errno;
 -		rollback_lock_file(&mark_lock);
 -		failure |= error("Unable to write marks file %s: %s",
 -			export_marks_file, strerror(saved_errno));
 -		return;
 -	}
 -
  	if (commit_lock_file(&mark_lock)) {
 -		int saved_errno = errno;
 -		rollback_lock_file(&mark_lock);
  		failure |= error("Unable to commit marks file %s: %s",
 -			export_marks_file, strerror(saved_errno));
 +			export_marks_file, strerror(errno));
  		return;
  	}
  }
@@@ -1875,8 -1857,6 +1875,8 @@@ static int read_next_command(void
  	}
  
  	for (;;) {
 +		const char *p;
 +
  		if (unread_command_buf) {
  			unread_command_buf = 0;
  		} else {
@@@ -1909,12 -1889,8 +1909,12 @@@
  			rc->prev->next = rc;
  			cmd_tail = rc;
  		}
 -		if (starts_with(command_buf.buf, "cat-blob ")) {
 -			parse_cat_blob();
 +		if (skip_prefix(command_buf.buf, "get-mark ", &p)) {
 +			parse_get_mark(p);
 +			continue;
 +		}
 +		if (skip_prefix(command_buf.buf, "cat-blob ", &p)) {
 +			parse_cat_blob(p);
  			continue;
  		}
  		if (command_buf.buf[0] == '#')
@@@ -1932,9 -1908,8 +1932,9 @@@ static void skip_optional_lf(void
  
  static void parse_mark(void)
  {
 -	if (starts_with(command_buf.buf, "mark :")) {
 -		next_mark = strtoumax(command_buf.buf + 6, NULL, 10);
 +	const char *v;
 +	if (skip_prefix(command_buf.buf, "mark :", &v)) {
 +		next_mark = strtoumax(v, NULL, 10);
  		read_next_command();
  	}
  	else
@@@ -1943,15 -1918,14 +1943,15 @@@
  
  static int parse_data(struct strbuf *sb, uintmax_t limit, uintmax_t *len_res)
  {
 +	const char *data;
  	strbuf_reset(sb);
  
 -	if (!starts_with(command_buf.buf, "data "))
 +	if (!skip_prefix(command_buf.buf, "data ", &data))
  		die("Expected 'data n' command, found: %s", command_buf.buf);
  
 -	if (starts_with(command_buf.buf + 5, "<<")) {
 -		char *term = xstrdup(command_buf.buf + 5 + 2);
 -		size_t term_len = command_buf.len - 5 - 2;
 +	if (skip_prefix(data, "<<", &data)) {
 +		char *term = xstrdup(data);
 +		size_t term_len = command_buf.len - (data - command_buf.buf);
  
  		strbuf_detach(&command_buf, NULL);
  		for (;;) {
@@@ -1966,7 -1940,7 +1966,7 @@@
  		free(term);
  	}
  	else {
 -		uintmax_t len = strtoumax(command_buf.buf + 5, NULL, 10);
 +		uintmax_t len = strtoumax(data, NULL, 10);
  		size_t n = 0, length = (size_t)len;
  
  		if (limit && limit < len) {
@@@ -1989,7 -1963,7 +1989,7 @@@
  	return 1;
  }
  
 -static int validate_raw_date(const char *src, char *result, int maxlen)
 +static int validate_raw_date(const char *src, struct strbuf *result)
  {
  	const char *orig_src = src;
  	char *endp;
@@@ -2007,10 -1981,11 +2007,10 @@@
  		return -1;
  
  	num = strtoul(src + 1, &endp, 10);
 -	if (errno || endp == src + 1 || *endp || (endp - orig_src) >= maxlen ||
 -	    1400 < num)
 +	if (errno || endp == src + 1 || *endp || 1400 < num)
  		return -1;
  
 -	strcpy(result, orig_src);
 +	strbuf_addstr(result, orig_src);
  	return 0;
  }
  
@@@ -2018,7 -1993,7 +2018,7 @@@ static char *parse_ident(const char *bu
  {
  	const char *ltgt;
  	size_t name_len;
 -	char *ident;
 +	struct strbuf ident = STRBUF_INIT;
  
  	/* ensure there is a space delimiter even if there is no name */
  	if (*buf == '<')
@@@ -2037,25 -2012,26 +2037,25 @@@
  		die("Missing space after > in ident string: %s", buf);
  	ltgt++;
  	name_len = ltgt - buf;
 -	ident = xmalloc(name_len + 24);
 -	strncpy(ident, buf, name_len);
 +	strbuf_add(&ident, buf, name_len);
  
  	switch (whenspec) {
  	case WHENSPEC_RAW:
 -		if (validate_raw_date(ltgt, ident + name_len, 24) < 0)
 +		if (validate_raw_date(ltgt, &ident) < 0)
  			die("Invalid raw date \"%s\" in ident: %s", ltgt, buf);
  		break;
  	case WHENSPEC_RFC2822:
 -		if (parse_date(ltgt, ident + name_len, 24) < 0)
 +		if (parse_date(ltgt, &ident) < 0)
  			die("Invalid rfc2822 date \"%s\" in ident: %s", ltgt, buf);
  		break;
  	case WHENSPEC_NOW:
  		if (strcmp("now", ltgt))
  			die("Date in ident must be 'now': %s", buf);
 -		datestamp(ident + name_len, 24);
 +		datestamp(&ident);
  		break;
  	}
  
 -	return ident;
 +	return strbuf_detach(&ident, NULL);
  }
  
  static void parse_and_store_blob(
@@@ -2285,14 -2261,15 +2285,14 @@@ static uintmax_t parse_mark_ref_space(c
  	char *end;
  
  	mark = parse_mark_ref(*p, &end);
 -	if (*end != ' ')
 +	if (*end++ != ' ')
  		die("Missing space after mark: %s", command_buf.buf);
  	*p = end;
  	return mark;
  }
  
 -static void file_change_m(struct branch *b)
 +static void file_change_m(const char *p, struct branch *b)
  {
 -	const char *p = command_buf.buf + 2;
  	static struct strbuf uq = STRBUF_INIT;
  	const char *endp;
  	struct object_entry *oe;
@@@ -2320,17 -2297,20 +2320,17 @@@
  	if (*p == ':') {
  		oe = find_mark(parse_mark_ref_space(&p));
  		hashcpy(sha1, oe->idx.sha1);
 -	} else if (starts_with(p, "inline ")) {
 +	} else if (skip_prefix(p, "inline ", &p)) {
  		inline_data = 1;
  		oe = NULL; /* not used with inline_data, but makes gcc happy */
 -		p += strlen("inline");  /* advance to space */
  	} else {
  		if (get_sha1_hex(p, sha1))
  			die("Invalid dataref: %s", command_buf.buf);
  		oe = find_object(sha1);
  		p += 40;
 -		if (*p != ' ')
 +		if (*p++ != ' ')
  			die("Missing space after SHA1: %s", command_buf.buf);
  	}
 -	assert(*p == ' ');
 -	p++;  /* skip space */
  
  	strbuf_reset(&uq);
  	if (!unquote_c_style(&uq, p, &endp)) {
@@@ -2340,7 -2320,7 +2340,7 @@@
  	}
  
  	/* Git does not track empty, non-toplevel directories. */
 -	if (S_ISDIR(mode) && !memcmp(sha1, EMPTY_TREE_SHA1_BIN, 20) && *p) {
 +	if (S_ISDIR(mode) && !hashcmp(sha1, EMPTY_TREE_SHA1_BIN) && *p) {
  		tree_content_remove(&b->branch_tree, p, NULL, 0);
  		return;
  	}
@@@ -2390,8 -2370,9 +2390,8 @@@
  	tree_content_set(&b->branch_tree, p, sha1, mode, NULL);
  }
  
 -static void file_change_d(struct branch *b)
 +static void file_change_d(const char *p, struct branch *b)
  {
 -	const char *p = command_buf.buf + 2;
  	static struct strbuf uq = STRBUF_INIT;
  	const char *endp;
  
@@@ -2404,14 -2385,15 +2404,14 @@@
  	tree_content_remove(&b->branch_tree, p, NULL, 1);
  }
  
 -static void file_change_cr(struct branch *b, int rename)
 +static void file_change_cr(const char *s, struct branch *b, int rename)
  {
 -	const char *s, *d;
 +	const char *d;
  	static struct strbuf s_uq = STRBUF_INIT;
  	static struct strbuf d_uq = STRBUF_INIT;
  	const char *endp;
  	struct tree_entry leaf;
  
 -	s = command_buf.buf + 2;
  	strbuf_reset(&s_uq);
  	if (!unquote_c_style(&s_uq, s, &endp)) {
  		if (*endp != ' ')
@@@ -2456,8 -2438,9 +2456,8 @@@
  		leaf.tree);
  }
  
 -static void note_change_n(struct branch *b, unsigned char *old_fanout)
 +static void note_change_n(const char *p, struct branch *b, unsigned char *old_fanout)
  {
 -	const char *p = command_buf.buf + 2;
  	static struct strbuf uq = STRBUF_INIT;
  	struct object_entry *oe;
  	struct branch *s;
@@@ -2487,17 -2470,20 +2487,17 @@@
  	if (*p == ':') {
  		oe = find_mark(parse_mark_ref_space(&p));
  		hashcpy(sha1, oe->idx.sha1);
 -	} else if (starts_with(p, "inline ")) {
 +	} else if (skip_prefix(p, "inline ", &p)) {
  		inline_data = 1;
  		oe = NULL; /* not used with inline_data, but makes gcc happy */
 -		p += strlen("inline");  /* advance to space */
  	} else {
  		if (get_sha1_hex(p, sha1))
  			die("Invalid dataref: %s", command_buf.buf);
  		oe = find_object(sha1);
  		p += 40;
 -		if (*p != ' ')
 +		if (*p++ != ' ')
  			die("Missing space after SHA1: %s", command_buf.buf);
  	}
 -	assert(*p == ' ');
 -	p++;  /* skip space */
  
  	/* <commit-ish> */
  	s = lookup_branch(p);
@@@ -2594,13 -2580,16 +2594,13 @@@ static int parse_from(struct branch *b
  {
  	const char *from;
  	struct branch *s;
 +	unsigned char sha1[20];
  
 -	if (!starts_with(command_buf.buf, "from "))
 +	if (!skip_prefix(command_buf.buf, "from ", &from))
  		return 0;
  
 -	if (b->branch_tree.tree) {
 -		release_tree_content_recursive(b->branch_tree.tree);
 -		b->branch_tree.tree = NULL;
 -	}
 +	hashcpy(sha1, b->branch_tree.versions[1].sha1);
  
 -	from = strchr(command_buf.buf, ' ') + 1;
  	s = lookup_branch(from);
  	if (b == s)
  		die("Can't create a branch from itself: %s", b->name);
@@@ -2614,29 -2603,19 +2614,29 @@@
  		struct object_entry *oe = find_mark(idnum);
  		if (oe->type != OBJ_COMMIT)
  			die("Mark :%" PRIuMAX " not a commit", idnum);
 -		hashcpy(b->sha1, oe->idx.sha1);
 -		if (oe->pack_id != MAX_PACK_ID) {
 -			unsigned long size;
 -			char *buf = gfi_unpack_entry(oe, &size);
 -			parse_from_commit(b, buf, size);
 -			free(buf);
 -		} else
 -			parse_from_existing(b);
 -	} else if (!get_sha1(from, b->sha1))
 +		if (hashcmp(b->sha1, oe->idx.sha1)) {
 +			hashcpy(b->sha1, oe->idx.sha1);
 +			if (oe->pack_id != MAX_PACK_ID) {
 +				unsigned long size;
 +				char *buf = gfi_unpack_entry(oe, &size);
 +				parse_from_commit(b, buf, size);
 +				free(buf);
 +			} else
 +				parse_from_existing(b);
 +		}
 +	} else if (!get_sha1(from, b->sha1)) {
  		parse_from_existing(b);
 +		if (is_null_sha1(b->sha1))
 +			b->delete = 1;
 +	}
  	else
  		die("Invalid ref name or SHA1 expression: %s", from);
  
 +	if (b->branch_tree.tree && hashcmp(sha1, b->branch_tree.versions[1].sha1)) {
 +		release_tree_content_recursive(b->branch_tree.tree);
 +		b->branch_tree.tree = NULL;
 +	}
 +
  	read_next_command();
  	return 1;
  }
@@@ -2648,7 -2627,8 +2648,7 @@@ static struct hash_list *parse_merge(un
  	struct branch *s;
  
  	*count = 0;
 -	while (starts_with(command_buf.buf, "merge ")) {
 -		from = strchr(command_buf.buf, ' ') + 1;
 +	while (skip_prefix(command_buf.buf, "merge ", &from)) {
  		n = xmalloc(sizeof(*n));
  		s = lookup_branch(from);
  		if (s)
@@@ -2679,29 -2659,31 +2679,29 @@@
  	return list;
  }
  
 -static void parse_new_commit(void)
 +static void parse_new_commit(const char *arg)
  {
  	static struct strbuf msg = STRBUF_INIT;
  	struct branch *b;
 -	char *sp;
  	char *author = NULL;
  	char *committer = NULL;
  	struct hash_list *merge_list = NULL;
  	unsigned int merge_count;
  	unsigned char prev_fanout, new_fanout;
 +	const char *v;
  
 -	/* Obtain the branch name from the rest of our command */
 -	sp = strchr(command_buf.buf, ' ') + 1;
 -	b = lookup_branch(sp);
 +	b = lookup_branch(arg);
  	if (!b)
 -		b = new_branch(sp);
 +		b = new_branch(arg);
  
  	read_next_command();
  	parse_mark();
 -	if (starts_with(command_buf.buf, "author ")) {
 -		author = parse_ident(command_buf.buf + 7);
 +	if (skip_prefix(command_buf.buf, "author ", &v)) {
 +		author = parse_ident(v);
  		read_next_command();
  	}
 -	if (starts_with(command_buf.buf, "committer ")) {
 -		committer = parse_ident(command_buf.buf + 10);
 +	if (skip_prefix(command_buf.buf, "committer ", &v)) {
 +		committer = parse_ident(v);
  		read_next_command();
  	}
  	if (!committer)
@@@ -2721,20 -2703,20 +2721,20 @@@
  
  	/* file_change* */
  	while (command_buf.len > 0) {
 -		if (starts_with(command_buf.buf, "M "))
 -			file_change_m(b);
 -		else if (starts_with(command_buf.buf, "D "))
 -			file_change_d(b);
 -		else if (starts_with(command_buf.buf, "R "))
 -			file_change_cr(b, 1);
 -		else if (starts_with(command_buf.buf, "C "))
 -			file_change_cr(b, 0);
 -		else if (starts_with(command_buf.buf, "N "))
 -			note_change_n(b, &prev_fanout);
 +		if (skip_prefix(command_buf.buf, "M ", &v))
 +			file_change_m(v, b);
 +		else if (skip_prefix(command_buf.buf, "D ", &v))
 +			file_change_d(v, b);
 +		else if (skip_prefix(command_buf.buf, "R ", &v))
 +			file_change_cr(v, b, 1);
 +		else if (skip_prefix(command_buf.buf, "C ", &v))
 +			file_change_cr(v, b, 0);
 +		else if (skip_prefix(command_buf.buf, "N ", &v))
 +			note_change_n(v, b, &prev_fanout);
  		else if (!strcmp("deleteall", command_buf.buf))
  			file_change_deleteall(b);
 -		else if (starts_with(command_buf.buf, "ls "))
 -			parse_ls(b);
 +		else if (skip_prefix(command_buf.buf, "ls ", &v))
 +			parse_ls(v, b);
  		else {
  			unread_command_buf = 1;
  			break;
@@@ -2777,9 -2759,10 +2777,9 @@@
  	b->last_commit = object_count_by_type[OBJ_COMMIT];
  }
  
 -static void parse_new_tag(void)
 +static void parse_new_tag(const char *arg)
  {
  	static struct strbuf msg = STRBUF_INIT;
 -	char *sp;
  	const char *from;
  	char *tagger;
  	struct branch *s;
@@@ -2787,11 -2770,12 +2787,11 @@@
  	uintmax_t from_mark = 0;
  	unsigned char sha1[20];
  	enum object_type type;
 +	const char *v;
  
 -	/* Obtain the new tag name from the rest of our command */
 -	sp = strchr(command_buf.buf, ' ') + 1;
  	t = pool_alloc(sizeof(struct tag));
  	memset(t, 0, sizeof(struct tag));
 -	t->name = pool_strdup(sp);
 +	t->name = pool_strdup(arg);
  	if (last_tag)
  		last_tag->next_tag = t;
  	else
@@@ -2800,8 -2784,9 +2800,8 @@@
  	read_next_command();
  
  	/* from ... */
 -	if (!starts_with(command_buf.buf, "from "))
 +	if (!skip_prefix(command_buf.buf, "from ", &from))
  		die("Expected from command, got %s", command_buf.buf);
 -	from = strchr(command_buf.buf, ' ') + 1;
  	s = lookup_branch(from);
  	if (s) {
  		if (is_null_sha1(s->sha1))
@@@ -2827,8 -2812,8 +2827,8 @@@
  	read_next_command();
  
  	/* tagger ... */
 -	if (starts_with(command_buf.buf, "tagger ")) {
 -		tagger = parse_ident(command_buf.buf + 7);
 +	if (skip_prefix(command_buf.buf, "tagger ", &v)) {
 +		tagger = parse_ident(v);
  		read_next_command();
  	} else
  		tagger = NULL;
@@@ -2857,11 -2842,14 +2857,11 @@@
  		t->pack_id = pack_id;
  }
  
 -static void parse_reset_branch(void)
 +static void parse_reset_branch(const char *arg)
  {
  	struct branch *b;
 -	char *sp;
  
 -	/* Obtain the branch name from the rest of our command */
 -	sp = strchr(command_buf.buf, ' ') + 1;
 -	b = lookup_branch(sp);
 +	b = lookup_branch(arg);
  	if (b) {
  		hashclr(b->sha1);
  		hashclr(b->branch_tree.versions[0].sha1);
@@@ -2872,7 -2860,7 +2872,7 @@@
  		}
  	}
  	else
 -		b = new_branch(sp);
 +		b = new_branch(arg);
  	read_next_command();
  	parse_from(b);
  	if (command_buf.len > 0)
@@@ -2930,29 -2918,14 +2930,29 @@@ static void cat_blob(struct object_entr
  		free(buf);
  }
  
 -static void parse_cat_blob(void)
 +static void parse_get_mark(const char *p)
 +{
 +	struct object_entry *oe = oe;
 +	char output[42];
 +
 +	/* get-mark SP <object> LF */
 +	if (*p != ':')
 +		die("Not a mark: %s", p);
 +
 +	oe = find_mark(parse_mark_ref_eol(p));
 +	if (!oe)
 +		die("Unknown mark: %s", command_buf.buf);
 +
 +	snprintf(output, sizeof(output), "%s\n", sha1_to_hex(oe->idx.sha1));
 +	cat_blob_write(output, 41);
 +}
 +
 +static void parse_cat_blob(const char *p)
  {
 -	const char *p;
  	struct object_entry *oe = oe;
  	unsigned char sha1[20];
  
  	/* cat-blob SP <object> LF */
 -	p = command_buf.buf + strlen("cat-blob ");
  	if (*p == ':') {
  		oe = find_mark(parse_mark_ref_eol(p));
  		if (!oe)
@@@ -3035,8 -3008,6 +3035,8 @@@ static struct object_entry *parse_treei
  			die("Invalid dataref: %s", command_buf.buf);
  		e = find_object(sha1);
  		*p += 40;
 +		if (*(*p)++ != ' ')
 +			die("Missing space after tree-ish: %s", command_buf.buf);
  	}
  
  	while (!e || e->type != OBJ_TREE)
@@@ -3071,12 -3042,14 +3071,12 @@@ static void print_ls(int mode, const un
  	cat_blob_write(line.buf, line.len);
  }
  
 -static void parse_ls(struct branch *b)
 +static void parse_ls(const char *p, struct branch *b)
  {
 -	const char *p;
  	struct tree_entry *root = NULL;
  	struct tree_entry leaf = {NULL};
  
  	/* ls SP (<tree-ish> SP)? <path> */
 -	p = command_buf.buf + strlen("ls ");
  	if (*p == '"') {
  		if (!b)
  			die("Not in a commit: %s", command_buf.buf);
@@@ -3088,6 -3061,8 +3088,6 @@@
  		if (!is_null_sha1(root->versions[1].sha1))
  			root->versions[1].mode = S_IFDIR;
  		load_tree(root);
 -		if (*p++ != ' ')
 -			die("Missing space after tree-ish: %s", command_buf.buf);
  	}
  	if (*p == '"') {
  		static struct strbuf uq = STRBUF_INIT;
@@@ -3141,9 -3116,12 +3141,9 @@@ static void parse_progress(void
  
  static char* make_fast_import_path(const char *path)
  {
 -	struct strbuf abs_path = STRBUF_INIT;
 -
  	if (!relative_marks_paths || is_absolute_path(path))
  		return xstrdup(path);
 -	strbuf_addf(&abs_path, "%s/info/fast-import/%s", get_git_dir(), path);
 -	return strbuf_detach(&abs_path, NULL);
 +	return xstrdup(git_path("info/fast-import/%s", path));
  }
  
  static void option_import_marks(const char *marks,
@@@ -3222,9 -3200,9 +3222,9 @@@ static void option_export_pack_edges(co
  
  static int parse_one_option(const char *option)
  {
 -	if (starts_with(option, "max-pack-size=")) {
 +	if (skip_prefix(option, "max-pack-size=", &option)) {
  		unsigned long v;
 -		if (!git_parse_ulong(option + 14, &v))
 +		if (!git_parse_ulong(option, &v))
  			return 0;
  		if (v < 8192) {
  			warning("max-pack-size is now in bytes, assuming --max-pack-size=%lum", v);
@@@ -3234,17 -3212,17 +3234,17 @@@
  			v = 1024 * 1024;
  		}
  		max_packsize = v;
 -	} else if (starts_with(option, "big-file-threshold=")) {
 +	} else if (skip_prefix(option, "big-file-threshold=", &option)) {
  		unsigned long v;
 -		if (!git_parse_ulong(option + 19, &v))
 +		if (!git_parse_ulong(option, &v))
  			return 0;
  		big_file_threshold = v;
 -	} else if (starts_with(option, "depth=")) {
 -		option_depth(option + 6);
 -	} else if (starts_with(option, "active-branches=")) {
 -		option_active_branches(option + 16);
 -	} else if (starts_with(option, "export-pack-edges=")) {
 -		option_export_pack_edges(option + 18);
 +	} else if (skip_prefix(option, "depth=", &option)) {
 +		option_depth(option);
 +	} else if (skip_prefix(option, "active-branches=", &option)) {
 +		option_active_branches(option);
 +	} else if (skip_prefix(option, "export-pack-edges=", &option)) {
 +		option_export_pack_edges(option);
  	} else if (starts_with(option, "quiet")) {
  		show_stats = 0;
  	} else if (starts_with(option, "stats")) {
@@@ -3258,18 -3236,15 +3258,18 @@@
  
  static int parse_one_feature(const char *feature, int from_stream)
  {
 -	if (starts_with(feature, "date-format=")) {
 -		option_date_format(feature + 12);
 -	} else if (starts_with(feature, "import-marks=")) {
 -		option_import_marks(feature + 13, from_stream, 0);
 -	} else if (starts_with(feature, "import-marks-if-exists=")) {
 -		option_import_marks(feature + strlen("import-marks-if-exists="),
 -					from_stream, 1);
 -	} else if (starts_with(feature, "export-marks=")) {
 -		option_export_marks(feature + 13);
 +	const char *arg;
 +
 +	if (skip_prefix(feature, "date-format=", &arg)) {
 +		option_date_format(arg);
 +	} else if (skip_prefix(feature, "import-marks=", &arg)) {
 +		option_import_marks(arg, from_stream, 0);
 +	} else if (skip_prefix(feature, "import-marks-if-exists=", &arg)) {
 +		option_import_marks(arg, from_stream, 1);
 +	} else if (skip_prefix(feature, "export-marks=", &arg)) {
 +		option_export_marks(arg);
 +	} else if (!strcmp(feature, "get-mark")) {
 +		; /* Don't die - this feature is supported */
  	} else if (!strcmp(feature, "cat-blob")) {
  		; /* Don't die - this feature is supported */
  	} else if (!strcmp(feature, "relative-marks")) {
@@@ -3289,8 -3264,10 +3289,8 @@@
  	return 1;
  }
  
 -static void parse_feature(void)
 +static void parse_feature(const char *feature)
  {
 -	char *feature = command_buf.buf + 8;
 -
  	if (seen_data_command)
  		die("Got feature command '%s' after data command", feature);
  
@@@ -3300,8 -3277,10 +3300,8 @@@
  	die("This version of fast-import does not support feature %s.", feature);
  }
  
 -static void parse_option(void)
 +static void parse_option(const char *option)
  {
 -	char *option = command_buf.buf + 11;
 -
  	if (seen_data_command)
  		die("Got option command '%s' after data command", option);
  
@@@ -3311,34 -3290,36 +3311,34 @@@
  	die("This version of fast-import does not support option: %s", option);
  }
  
 -static int git_pack_config(const char *k, const char *v, void *cb)
 +static void git_pack_config(void)
  {
 -	if (!strcmp(k, "pack.depth")) {
 -		max_depth = git_config_int(k, v);
 +	int indexversion_value;
 +	unsigned long packsizelimit_value;
 +
 +	if (!git_config_get_ulong("pack.depth", &max_depth)) {
  		if (max_depth > MAX_DEPTH)
  			max_depth = MAX_DEPTH;
 -		return 0;
  	}
 -	if (!strcmp(k, "pack.compression")) {
 -		int level = git_config_int(k, v);
 -		if (level == -1)
 -			level = Z_DEFAULT_COMPRESSION;
 -		else if (level < 0 || level > Z_BEST_COMPRESSION)
 -			die("bad pack compression level %d", level);
 -		pack_compression_level = level;
 +	if (!git_config_get_int("pack.compression", &pack_compression_level)) {
 +		if (pack_compression_level == -1)
 +			pack_compression_level = Z_DEFAULT_COMPRESSION;
 +		else if (pack_compression_level < 0 ||
 +			 pack_compression_level > Z_BEST_COMPRESSION)
 +			git_die_config("pack.compression",
 +					"bad pack compression level %d", pack_compression_level);
  		pack_compression_seen = 1;
 -		return 0;
  	}
 -	if (!strcmp(k, "pack.indexversion")) {
 -		pack_idx_opts.version = git_config_int(k, v);
 +	if (!git_config_get_int("pack.indexversion", &indexversion_value)) {
 +		pack_idx_opts.version = indexversion_value;
  		if (pack_idx_opts.version > 2)
 -			die("bad pack.indexversion=%"PRIu32,
 -			    pack_idx_opts.version);
 -		return 0;
 +			git_die_config("pack.indexversion",
 +					"bad pack.indexversion=%"PRIu32, pack_idx_opts.version);
  	}
 -	if (!strcmp(k, "pack.packsizelimit")) {
 -		max_packsize = git_config_ulong(k, v);
 -		return 0;
 -	}
 -	return git_default_config(k, v, cb);
 +	if (!git_config_get_ulong("pack.packsizelimit", &packsizelimit_value))
 +		max_packsize = packsizelimit_value;
 +
 +	git_config(git_default_config, NULL);
  }
  
  static const char fast_import_usage[] =
@@@ -3354,21 -3335,18 +3354,21 @@@ static void parse_argv(void
  		if (*a != '-' || !strcmp(a, "--"))
  			break;
  
 -		if (parse_one_option(a + 2))
 +		if (!skip_prefix(a, "--", &a))
 +			die("unknown option %s", a);
 +
 +		if (parse_one_option(a))
  			continue;
  
 -		if (parse_one_feature(a + 2, 0))
 +		if (parse_one_feature(a, 0))
  			continue;
  
 -		if (starts_with(a + 2, "cat-blob-fd=")) {
 -			option_cat_blob_fd(a + 2 + strlen("cat-blob-fd="));
 +		if (skip_prefix(a, "cat-blob-fd=", &a)) {
 +			option_cat_blob_fd(a);
  			continue;
  		}
  
 -		die("unknown option %s", a);
 +		die("unknown option --%s", a);
  	}
  	if (i != global_argc)
  		usage(fast_import_usage);
@@@ -3391,7 -3369,7 +3391,7 @@@ int main(int argc, char **argv
  
  	setup_git_directory();
  	reset_pack_idx_option(&pack_idx_opts);
 -	git_config(git_pack_config, NULL);
 +	git_pack_config();
  	if (!pack_compression_seen && core_compression_seen)
  		pack_compression_level = core_compression_level;
  
@@@ -3415,27 -3393,26 +3415,27 @@@
  	set_die_routine(die_nicely);
  	set_checkpoint_signal();
  	while (read_next_command() != EOF) {
 +		const char *v;
  		if (!strcmp("blob", command_buf.buf))
  			parse_new_blob();
 -		else if (starts_with(command_buf.buf, "ls "))
 -			parse_ls(NULL);
 -		else if (starts_with(command_buf.buf, "commit "))
 -			parse_new_commit();
 -		else if (starts_with(command_buf.buf, "tag "))
 -			parse_new_tag();
 -		else if (starts_with(command_buf.buf, "reset "))
 -			parse_reset_branch();
 +		else if (skip_prefix(command_buf.buf, "ls ", &v))
 +			parse_ls(v, NULL);
 +		else if (skip_prefix(command_buf.buf, "commit ", &v))
 +			parse_new_commit(v);
 +		else if (skip_prefix(command_buf.buf, "tag ", &v))
 +			parse_new_tag(v);
 +		else if (skip_prefix(command_buf.buf, "reset ", &v))
 +			parse_reset_branch(v);
  		else if (!strcmp("checkpoint", command_buf.buf))
  			parse_checkpoint();
  		else if (!strcmp("done", command_buf.buf))
  			break;
  		else if (starts_with(command_buf.buf, "progress "))
  			parse_progress();
 -		else if (starts_with(command_buf.buf, "feature "))
 -			parse_feature();
 -		else if (starts_with(command_buf.buf, "option git "))
 -			parse_option();
 +		else if (skip_prefix(command_buf.buf, "feature ", &v))
 +			parse_feature(v);
 +		else if (skip_prefix(command_buf.buf, "option git ", &v))
 +			parse_option(v);
  		else if (starts_with(command_buf.buf, "option "))
  			/* ignore non-git options*/;
  		else
diff --combined http.c
index e9c6fdd835,030329627d..8cd59f7f36
--- a/http.c
+++ b/http.c
@@@ -1,4 -1,3 +1,4 @@@
 +#include "git-compat-util.h"
  #include "http.h"
  #include "pack.h"
  #include "sideband.h"
@@@ -8,7 -7,6 +8,7 @@@
  #include "credential.h"
  #include "version.h"
  #include "pkt-line.h"
 +#include "gettext.h"
  
  int active_requests;
  int http_is_verbose;
@@@ -36,7 -34,6 +36,7 @@@ char curl_errorstr[CURL_ERROR_SIZE]
  static int curl_ssl_verify = -1;
  static int curl_ssl_try;
  static const char *ssl_cert;
 +static const char *ssl_cipherlist;
  #if LIBCURL_VERSION_NUM >= 0x070903
  static const char *ssl_key;
  #endif
@@@ -64,17 -61,12 +64,17 @@@ static const char *user_agent
  
  static struct credential cert_auth = CREDENTIAL_INIT;
  static int ssl_cert_password_required;
 +#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
 +static unsigned long http_auth_methods = CURLAUTH_ANY;
 +#endif
  
  static struct curl_slist *pragma_header;
  static struct curl_slist *no_pragma_header;
  
  static struct active_request_slot *active_queue_head;
  
 +static char *cached_accept_language;
 +
  size_t fread_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
  {
  	size_t size = eltsize * nmemb;
@@@ -121,37 -113,6 +121,37 @@@ size_t fwrite_null(char *ptr, size_t el
  	return eltsize * nmemb;
  }
  
 +static void closedown_active_slot(struct active_request_slot *slot)
 +{
 +	active_requests--;
 +	slot->in_use = 0;
 +}
 +
 +static void finish_active_slot(struct active_request_slot *slot)
 +{
 +	closedown_active_slot(slot);
 +	curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
 +
 +	if (slot->finished != NULL)
 +		(*slot->finished) = 1;
 +
 +	/* Store slot results so they can be read after the slot is reused */
 +	if (slot->results != NULL) {
 +		slot->results->curl_result = slot->curl_result;
 +		slot->results->http_code = slot->http_code;
 +#if LIBCURL_VERSION_NUM >= 0x070a08
 +		curl_easy_getinfo(slot->curl, CURLINFO_HTTPAUTH_AVAIL,
 +				  &slot->results->auth_avail);
 +#else
 +		slot->results->auth_avail = 0;
 +#endif
 +	}
 +
 +	/* Run callback if appropriate */
 +	if (slot->callback_func != NULL)
 +		slot->callback_func(slot->callback_data);
 +}
 +
  #ifdef USE_CURL_MULTI
  static void process_curl_messages(void)
  {
@@@ -188,8 -149,6 +188,8 @@@ static int http_options(const char *var
  		curl_ssl_verify = git_config_bool(var, value);
  		return 0;
  	}
 +	if (!strcmp("http.sslcipherlist", var))
 +		return git_config_string(&ssl_cipherlist, var, value);
  	if (!strcmp("http.sslcert", var))
  		return git_config_string(&ssl_cert, var, value);
  #if LIBCURL_VERSION_NUM >= 0x070903
@@@ -341,9 -300,6 +341,9 @@@ static CURL *get_curl_handle(void
  {
  	CURL *result = curl_easy_init();
  
 +	if (!result)
 +		die("curl_easy_init failed");
 +
  	if (!curl_ssl_verify) {
  		curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
  		curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
@@@ -364,13 -320,6 +364,13 @@@
  	if (http_proactive_auth)
  		init_curl_http_auth(result);
  
 +	if (getenv("GIT_SSL_CIPHER_LIST"))
 +		ssl_cipherlist = getenv("GIT_SSL_CIPHER_LIST");
 +
 +	if (ssl_cipherlist != NULL && *ssl_cipherlist)
 +		curl_easy_setopt(result, CURLOPT_SSL_CIPHER_LIST,
 +				ssl_cipherlist);
 +
  	if (ssl_cert != NULL)
  		curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
  	if (has_cert_password())
@@@ -416,10 -365,8 +416,10 @@@
  
  	if (curl_http_proxy) {
  		curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy);
 -		curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
  	}
 +#if LIBCURL_VERSION_NUM >= 0x070a07
 +	curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
 +#endif
  
  	set_curl_keepalive(result);
  
@@@ -452,8 -399,7 +452,8 @@@ void http_init(struct remote *remote, c
  	git_config(urlmatch_config_entry, &config);
  	free(normalized_url);
  
 -	curl_global_init(CURL_GLOBAL_ALL);
 +	if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
 +		die("curl_global_init failed");
  
  	http_proactive_auth = proactive_auth;
  
@@@ -471,8 -417,10 +471,8 @@@
  	}
  
  	curlm = curl_multi_init();
 -	if (curlm == NULL) {
 -		fprintf(stderr, "Error creating curl multi handle.\n");
 -		exit(1);
 -	}
 +	if (!curlm)
 +		die("curl_multi_init failed");
  #endif
  
  	if (getenv("GIT_SSL_NO_VERIFY"))
@@@ -564,9 -512,6 +564,9 @@@ void http_cleanup(void
  		cert_auth.password = NULL;
  	}
  	ssl_cert_password_required = 0;
 +
 +	free(cached_accept_language);
 +	cached_accept_language = NULL;
  }
  
  struct active_request_slot *get_active_slot(void)
@@@ -632,9 -577,6 +632,9 @@@
  	curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
  	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
  	curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1);
 +#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
 +	curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods);
 +#endif
  	if (http_auth.password)
  		init_curl_http_auth(slot->curl);
  
@@@ -785,6 -727,12 +785,6 @@@ void run_active_slot(struct active_requ
  #endif
  }
  
 -static void closedown_active_slot(struct active_request_slot *slot)
 -{
 -	active_requests--;
 -	slot->in_use = 0;
 -}
 -
  static void release_active_slot(struct active_request_slot *slot)
  {
  	closedown_active_slot(slot);
@@@ -801,6 -749,31 +801,6 @@@
  #endif
  }
  
 -void finish_active_slot(struct active_request_slot *slot)
 -{
 -	closedown_active_slot(slot);
 -	curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
 -
 -	if (slot->finished != NULL)
 -		(*slot->finished) = 1;
 -
 -	/* Store slot results so they can be read after the slot is reused */
 -	if (slot->results != NULL) {
 -		slot->results->curl_result = slot->curl_result;
 -		slot->results->http_code = slot->http_code;
 -#if LIBCURL_VERSION_NUM >= 0x070a08
 -		curl_easy_getinfo(slot->curl, CURLINFO_HTTPAUTH_AVAIL,
 -				  &slot->results->auth_avail);
 -#else
 -		slot->results->auth_avail = 0;
 -#endif
 -	}
 -
 -	/* Run callback if appropriate */
 -	if (slot->callback_func != NULL)
 -		slot->callback_func(slot->callback_data);
 -}
 -
  void finish_all_active_slots(void)
  {
  	struct active_request_slot *slot = active_queue_head;
@@@ -863,7 -836,7 +863,7 @@@ char *get_remote_object_url(const char 
  	return strbuf_detach(&buf, NULL);
  }
  
 -int handle_curl_result(struct slot_results *results)
 +static int handle_curl_result(struct slot_results *results)
  {
  	/*
  	 * If we see a failing http code with CURLE_OK, we have turned off
@@@ -894,9 -867,6 +894,9 @@@
  			credential_reject(&http_auth);
  			return HTTP_NOAUTH;
  		} else {
 +#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
 +			http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE;
 +#endif
  			return HTTP_REAUTH;
  		}
  	} else {
@@@ -936,194 -906,6 +936,194 @@@ static CURLcode curlinfo_strbuf(CURL *c
  	return ret;
  }
  
 +/*
 + * Check for and extract a content-type parameter. "raw"
 + * should be positioned at the start of the potential
 + * parameter, with any whitespace already removed.
 + *
 + * "name" is the name of the parameter. The value is appended
 + * to "out".
 + */
 +static int extract_param(const char *raw, const char *name,
 +			 struct strbuf *out)
 +{
 +	size_t len = strlen(name);
 +
 +	if (strncasecmp(raw, name, len))
 +		return -1;
 +	raw += len;
 +
 +	if (*raw != '=')
 +		return -1;
 +	raw++;
 +
 +	while (*raw && !isspace(*raw) && *raw != ';')
 +		strbuf_addch(out, *raw++);
 +	return 0;
 +}
 +
 +/*
 + * Extract a normalized version of the content type, with any
 + * spaces suppressed, all letters lowercased, and no trailing ";"
 + * or parameters.
 + *
 + * Note that we will silently remove even invalid whitespace. For
 + * example, "text / plain" is specifically forbidden by RFC 2616,
 + * but "text/plain" is the only reasonable output, and this keeps
 + * our code simple.
 + *
 + * If the "charset" argument is not NULL, store the value of any
 + * charset parameter there.
 + *
 + * Example:
 + *   "TEXT/PLAIN; charset=utf-8" -> "text/plain", "utf-8"
 + *   "text / plain" -> "text/plain"
 + */
 +static void extract_content_type(struct strbuf *raw, struct strbuf *type,
 +				 struct strbuf *charset)
 +{
 +	const char *p;
 +
 +	strbuf_reset(type);
 +	strbuf_grow(type, raw->len);
 +	for (p = raw->buf; *p; p++) {
 +		if (isspace(*p))
 +			continue;
 +		if (*p == ';') {
 +			p++;
 +			break;
 +		}
 +		strbuf_addch(type, tolower(*p));
 +	}
 +
 +	if (!charset)
 +		return;
 +
 +	strbuf_reset(charset);
 +	while (*p) {
 +		while (isspace(*p) || *p == ';')
 +			p++;
 +		if (!extract_param(p, "charset", charset))
 +			return;
 +		while (*p && !isspace(*p))
 +			p++;
 +	}
 +
 +	if (!charset->len && starts_with(type->buf, "text/"))
 +		strbuf_addstr(charset, "ISO-8859-1");
 +}
 +
 +static void write_accept_language(struct strbuf *buf)
 +{
 +	/*
 +	 * MAX_DECIMAL_PLACES must not be larger than 3. If it is larger than
 +	 * that, q-value will be smaller than 0.001, the minimum q-value the
 +	 * HTTP specification allows. See
 +	 * http://tools.ietf.org/html/rfc7231#section-5.3.1 for q-value.
 +	 */
 +	const int MAX_DECIMAL_PLACES = 3;
 +	const int MAX_LANGUAGE_TAGS = 1000;
 +	const int MAX_ACCEPT_LANGUAGE_HEADER_SIZE = 4000;
 +	char **language_tags = NULL;
 +	int num_langs = 0;
 +	const char *s = get_preferred_languages();
 +	int i;
 +	struct strbuf tag = STRBUF_INIT;
 +
 +	/* Don't add Accept-Language header if no language is preferred. */
 +	if (!s)
 +		return;
 +
 +	/*
 +	 * Split the colon-separated string of preferred languages into
 +	 * language_tags array.
 +	 */
 +	do {
 +		/* collect language tag */
 +		for (; *s && (isalnum(*s) || *s == '_'); s++)
 +			strbuf_addch(&tag, *s == '_' ? '-' : *s);
 +
 +		/* skip .codeset, @modifier and any other unnecessary parts */
 +		while (*s && *s != ':')
 +			s++;
 +
 +		if (tag.len) {
 +			num_langs++;
 +			REALLOC_ARRAY(language_tags, num_langs);
 +			language_tags[num_langs - 1] = strbuf_detach(&tag, NULL);
 +			if (num_langs >= MAX_LANGUAGE_TAGS - 1) /* -1 for '*' */
 +				break;
 +		}
 +	} while (*s++);
 +
 +	/* write Accept-Language header into buf */
 +	if (num_langs) {
 +		int last_buf_len = 0;
 +		int max_q;
 +		int decimal_places;
 +		char q_format[32];
 +
 +		/* add '*' */
 +		REALLOC_ARRAY(language_tags, num_langs + 1);
 +		language_tags[num_langs++] = "*"; /* it's OK; this won't be freed */
 +
 +		/* compute decimal_places */
 +		for (max_q = 1, decimal_places = 0;
 +		     max_q < num_langs && decimal_places <= MAX_DECIMAL_PLACES;
 +		     decimal_places++, max_q *= 10)
 +			;
 +
 +		sprintf(q_format, ";q=0.%%0%dd", decimal_places);
 +
 +		strbuf_addstr(buf, "Accept-Language: ");
 +
 +		for (i = 0; i < num_langs; i++) {
 +			if (i > 0)
 +				strbuf_addstr(buf, ", ");
 +
 +			strbuf_addstr(buf, language_tags[i]);
 +
 +			if (i > 0)
 +				strbuf_addf(buf, q_format, max_q - i);
 +
 +			if (buf->len > MAX_ACCEPT_LANGUAGE_HEADER_SIZE) {
 +				strbuf_remove(buf, last_buf_len, buf->len - last_buf_len);
 +				break;
 +			}
 +
 +			last_buf_len = buf->len;
 +		}
 +	}
 +
 +	/* free language tags -- last one is a static '*' */
 +	for (i = 0; i < num_langs - 1; i++)
 +		free(language_tags[i]);
 +	free(language_tags);
 +}
 +
 +/*
 + * Get an Accept-Language header which indicates user's preferred languages.
 + *
 + * Examples:
 + *   LANGUAGE= -> ""
 + *   LANGUAGE=ko:en -> "Accept-Language: ko, en; q=0.9, *; q=0.1"
 + *   LANGUAGE=ko_KR.UTF-8:sr@latin -> "Accept-Language: ko-KR, sr; q=0.9, *; q=0.1"
 + *   LANGUAGE=ko LANG=en_US.UTF-8 -> "Accept-Language: ko, *; q=0.1"
 + *   LANGUAGE= LANG=en_US.UTF-8 -> "Accept-Language: en-US, *; q=0.1"
 + *   LANGUAGE= LANG=C -> ""
 + */
 +static const char *get_accept_language(void)
 +{
 +	if (!cached_accept_language) {
 +		struct strbuf buf = STRBUF_INIT;
 +		write_accept_language(&buf);
 +		if (buf.len > 0)
 +			cached_accept_language = strbuf_detach(&buf, NULL);
 +	}
 +
 +	return cached_accept_language;
 +}
 +
  /* http_request() targets */
  #define HTTP_REQUEST_STRBUF	0
  #define HTTP_REQUEST_FILE	1
@@@ -1136,7 -918,6 +1136,7 @@@ static int http_request(const char *url
  	struct slot_results results;
  	struct curl_slist *headers = NULL;
  	struct strbuf buf = STRBUF_INIT;
 +	const char *accept_language;
  	int ret;
  
  	slot = get_active_slot();
@@@ -1162,11 -943,6 +1162,11 @@@
  					 fwrite_buffer);
  	}
  
 +	accept_language = get_accept_language();
 +
 +	if (accept_language)
 +		headers = curl_slist_append(headers, accept_language);
 +
  	strbuf_addstr(&buf, "Pragma:");
  	if (options && options->no_cache)
  		strbuf_addstr(&buf, " no-cache");
@@@ -1181,13 -957,9 +1181,13 @@@
  
  	ret = run_one_slot(slot, &results);
  
 -	if (options && options->content_type)
 -		curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE,
 -				options->content_type);
 +	if (options && options->content_type) {
 +		struct strbuf raw = STRBUF_INIT;
 +		curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, &raw);
 +		extract_content_type(&raw, options->content_type,
 +				     options->charset);
 +		strbuf_release(&raw);
 +	}
  
  	if (options && options->effective_url)
  		curlinfo_strbuf(slot->curl, CURLINFO_EFFECTIVE_URL,
@@@ -1234,10 -1006,11 +1234,10 @@@ static int update_url_from_redirect(str
  	if (!strcmp(asked, got->buf))
  		return 0;
  
 -	if (!starts_with(asked, base->buf))
 +	if (!skip_prefix(asked, base->buf, &tail))
  		die("BUG: update_url_from_redirect: %s is not a superset of %s",
  		    asked, base->buf);
  
 -	tail = asked + base->len;
  	tail_len = strlen(tail);
  
  	if (got->len < tail_len ||
@@@ -1318,7 -1091,7 +1318,7 @@@ static int http_get_file(const char *ur
  	ret = http_request_reauth(url, result, HTTP_REQUEST_FILE, options);
  	fclose(result);
  
- 	if (ret == HTTP_OK && move_temp_to_file(tmpfile.buf, filename))
+ 	if (ret == HTTP_OK && finalize_object_file(tmpfile.buf, filename))
  		ret = HTTP_ERROR;
  cleanup:
  	strbuf_release(&tmpfile);
@@@ -1384,7 -1157,7 +1384,7 @@@ static int fetch_and_setup_pack_index(s
  	int ret;
  
  	if (has_pack_index(sha1)) {
 -		new_pack = parse_pack_index(sha1, NULL);
 +		new_pack = parse_pack_index(sha1, sha1_pack_index_name(sha1));
  		if (!new_pack)
  			return -1; /* parse_pack_index() already issued error message */
  		goto add_pack;
@@@ -1405,7 -1178,7 +1405,7 @@@
  	ret = verify_pack_index(new_pack);
  	if (!ret) {
  		close_pack_index(new_pack);
- 		ret = move_temp_to_file(tmp_idx, sha1_pack_index_name(sha1));
+ 		ret = finalize_object_file(tmp_idx, sha1_pack_index_name(sha1));
  	}
  	free(tmp_idx);
  	if (ret)
@@@ -1472,7 -1245,6 +1472,7 @@@ void release_http_pack_request(struct h
  	}
  	preq->slot = NULL;
  	free(preq->url);
 +	free(preq);
  }
  
  int finish_http_pack_request(struct http_pack_request *preq)
@@@ -1480,7 -1252,7 +1480,7 @@@
  	struct packed_git **lst;
  	struct packed_git *p = preq->target;
  	char *tmp_idx;
 -	struct child_process ip;
 +	struct child_process ip = CHILD_PROCESS_INIT;
  	const char *ip_argv[8];
  
  	close_pack_index(p);
@@@ -1503,6 -1275,7 +1503,6 @@@
  	ip_argv[3] = preq->tmpfile;
  	ip_argv[4] = NULL;
  
 -	memset(&ip, 0, sizeof(ip));
  	ip.argv = ip_argv;
  	ip.git_cmd = 1;
  	ip.no_stdin = 1;
@@@ -1517,8 -1290,8 +1517,8 @@@
  
  	unlink(sha1_pack_index_name(p->sha1));
  
- 	if (move_temp_to_file(preq->tmpfile, sha1_pack_name(p->sha1))
- 	 || move_temp_to_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
+ 	if (finalize_object_file(preq->tmpfile, sha1_pack_name(p->sha1))
+ 	 || finalize_object_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
  		free(tmp_idx);
  		return -1;
  	}
@@@ -1782,7 -1555,7 +1782,7 @@@ int finish_http_object_request(struct h
  		return -1;
  	}
  	freq->rename =
- 		move_temp_to_file(freq->tmpfile, sha1_file_name(freq->sha1));
+ 		finalize_object_file(freq->tmpfile, sha1_file_name(freq->sha1));
  
  	return freq->rename;
  }
diff --combined sha1_file.c
index 1cee438422,da22a120cb..1081b95713
--- a/sha1_file.c
+++ b/sha1_file.c
@@@ -8,7 -8,6 +8,7 @@@
   */
  #include "cache.h"
  #include "string-list.h"
 +#include "lockfile.h"
  #include "delta.h"
  #include "pack.h"
  #include "blob.h"
@@@ -37,6 -36,9 +37,6 @@@ static inline uintmax_t sz_fmt(size_t s
  
  const unsigned char null_sha1[20];
  
 -static const char *no_log_pack_access = "no_log_pack_access";
 -static const char *log_pack_access;
 -
  /*
   * This is meant to hold a *small* number of objects that you would
   * want read_sha1_file() to be able to return, but yet you do not want
@@@ -266,9 -268,9 +266,9 @@@ static struct alternate_object_databas
   * SHA1, an extra slash for the first level indirection, and the
   * terminating NUL.
   */
 -static int link_alt_odb_entry(const char *entry, const char *relative_base, int depth)
 +static int link_alt_odb_entry(const char *entry, const char *relative_base,
 +	int depth, const char *normalized_objdir)
  {
 -	const char *objdir = get_object_directory();
  	struct alternate_object_database *ent;
  	struct alternate_object_database *alt;
  	int pfxlen, entlen;
@@@ -319,7 -321,7 +319,7 @@@
  			return -1;
  		}
  	}
 -	if (!strcmp(ent->base, objdir)) {
 +	if (!strcmp_icase(ent->base, normalized_objdir)) {
  		free(ent);
  		return -1;
  	}
@@@ -343,7 -345,6 +343,7 @@@ static void link_alt_odb_entries(const 
  	struct string_list entries = STRING_LIST_INIT_NODUP;
  	char *alt_copy;
  	int i;
 +	struct strbuf objdirbuf = STRBUF_INIT;
  
  	if (depth > 5) {
  		error("%s: ignoring alternate object stores, nesting too deep.",
@@@ -351,9 -352,6 +351,9 @@@
  		return;
  	}
  
 +	strbuf_add_absolute_path(&objdirbuf, get_object_directory());
 +	normalize_path_copy(objdirbuf.buf, objdirbuf.buf);
 +
  	alt_copy = xmemdupz(alt, len);
  	string_list_split_in_place(&entries, alt_copy, sep, -1);
  	for (i = 0; i < entries.nr; i++) {
@@@ -364,12 -362,11 +364,12 @@@
  			error("%s: ignoring relative alternate object store %s",
  					relative_base, entry);
  		} else {
 -			link_alt_odb_entry(entry, relative_base, depth);
 +			link_alt_odb_entry(entry, relative_base, depth, objdirbuf.buf);
  		}
  	}
  	string_list_clear(&entries, 0);
  	free(alt_copy);
 +	strbuf_release(&objdirbuf);
  }
  
  void read_info_alternates(const char * relative_base, int depth)
@@@ -405,7 -402,7 +405,7 @@@ void add_to_alternates_file(const char 
  {
  	struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
  	int fd = hold_lock_file_for_append(lock, git_path("objects/info/alternates"), LOCK_DIE_ON_ERROR);
 -	char *alt = mkpath("%s\n", reference);
 +	const char *alt = mkpath("%s\n", reference);
  	write_or_die(fd, alt, strlen(alt));
  	if (commit_lock_file(lock))
  		die("could not close alternates file");
@@@ -413,18 -410,14 +413,18 @@@
  		link_alt_odb_entries(alt, strlen(alt), '\n', NULL, 0);
  }
  
 -void foreach_alt_odb(alt_odb_fn fn, void *cb)
 +int foreach_alt_odb(alt_odb_fn fn, void *cb)
  {
  	struct alternate_object_database *ent;
 +	int r = 0;
  
  	prepare_alt_odb();
 -	for (ent = alt_odb_list; ent; ent = ent->next)
 -		if (fn(ent, cb))
 -			return;
 +	for (ent = alt_odb_list; ent; ent = ent->next) {
 +		r = fn(ent, cb);
 +		if (r)
 +			break;
 +	}
 +	return r;
  }
  
  void prepare_alt_odb(void)
@@@ -443,61 -436,27 +443,61 @@@
  	read_info_alternates(get_object_directory(), 0);
  }
  
 -static int has_loose_object_local(const unsigned char *sha1)
 +/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
 +static int freshen_file(const char *fn)
  {
 -	return !access(sha1_file_name(sha1), F_OK);
 +	struct utimbuf t;
 +	t.actime = t.modtime = time(NULL);
 +	return !utime(fn, &t);
  }
  
 -int has_loose_object_nonlocal(const unsigned char *sha1)
 +/*
 + * All of the check_and_freshen functions return 1 if the file exists and was
 + * freshened (if freshening was requested), 0 otherwise. If they return
 + * 0, you should not assume that it is safe to skip a write of the object (it
 + * either does not exist on disk, or has a stale mtime and may be subject to
 + * pruning).
 + */
 +static int check_and_freshen_file(const char *fn, int freshen)
 +{
 +	if (access(fn, F_OK))
 +		return 0;
 +	if (freshen && !freshen_file(fn))
 +		return 0;
 +	return 1;
 +}
 +
 +static int check_and_freshen_local(const unsigned char *sha1, int freshen)
 +{
 +	return check_and_freshen_file(sha1_file_name(sha1), freshen);
 +}
 +
 +static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
  {
  	struct alternate_object_database *alt;
  	prepare_alt_odb();
  	for (alt = alt_odb_list; alt; alt = alt->next) {
  		fill_sha1_path(alt->name, sha1);
 -		if (!access(alt->base, F_OK))
 +		if (check_and_freshen_file(alt->base, freshen))
  			return 1;
  	}
  	return 0;
  }
  
 +static int check_and_freshen(const unsigned char *sha1, int freshen)
 +{
 +	return check_and_freshen_local(sha1, freshen) ||
 +	       check_and_freshen_nonlocal(sha1, freshen);
 +}
 +
 +int has_loose_object_nonlocal(const unsigned char *sha1)
 +{
 +	return check_and_freshen_nonlocal(sha1, 0);
 +}
 +
  static int has_loose_object(const unsigned char *sha1)
  {
 -	return has_loose_object_local(sha1) ||
 -	       has_loose_object_nonlocal(sha1);
 +	return check_and_freshen(sha1, 0);
  }
  
  static unsigned int pack_used_ctr;
@@@ -702,44 -661,21 +702,44 @@@ void release_pack_memory(size_t need
  		; /* nothing */
  }
  
 -void *xmmap(void *start, size_t length,
 -	int prot, int flags, int fd, off_t offset)
 +static void mmap_limit_check(size_t length)
  {
 -	void *ret = mmap(start, length, prot, flags, fd, offset);
 +	static size_t limit = 0;
 +	if (!limit) {
 +		limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
 +		if (!limit)
 +			limit = SIZE_MAX;
 +	}
 +	if (length > limit)
 +		die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX,
 +		    (uintmax_t)length, (uintmax_t)limit);
 +}
 +
 +void *xmmap_gently(void *start, size_t length,
 +		  int prot, int flags, int fd, off_t offset)
 +{
 +	void *ret;
 +
 +	mmap_limit_check(length);
 +	ret = mmap(start, length, prot, flags, fd, offset);
  	if (ret == MAP_FAILED) {
  		if (!length)
  			return NULL;
  		release_pack_memory(length);
  		ret = mmap(start, length, prot, flags, fd, offset);
 -		if (ret == MAP_FAILED)
 -			die_errno("Out of memory? mmap failed");
  	}
  	return ret;
  }
  
 +void *xmmap(void *start, size_t length,
 +	int prot, int flags, int fd, off_t offset)
 +{
 +	void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
 +	if (ret == MAP_FAILED)
 +		die_errno("mmap failed");
 +	return ret;
 +}
 +
  void close_pack_windows(struct packed_git *p)
  {
  	while (p->windows) {
@@@ -1213,7 -1149,7 +1213,7 @@@ static void report_pack_garbage(struct 
  	if (!report_garbage)
  		return;
  
 -	sort_string_list(list);
 +	string_list_sort(list);
  
  	for (i = 0; i < list->nr; i++) {
  		const char *path = list->items[i].string;
@@@ -1242,42 -1178,48 +1242,42 @@@
  
  static void prepare_packed_git_one(char *objdir, int local)
  {
 -	/* Ensure that this buffer is large enough so that we can
 -	   append "/pack/" without clobbering the stack even if
 -	   strlen(objdir) were PATH_MAX.  */
 -	char path[PATH_MAX + 1 + 4 + 1 + 1];
 -	int len;
 +	struct strbuf path = STRBUF_INIT;
 +	size_t dirnamelen;
  	DIR *dir;
  	struct dirent *de;
  	struct string_list garbage = STRING_LIST_INIT_DUP;
  
 -	sprintf(path, "%s/pack", objdir);
 -	len = strlen(path);
 -	dir = opendir(path);
 +	strbuf_addstr(&path, objdir);
 +	strbuf_addstr(&path, "/pack");
 +	dir = opendir(path.buf);
  	if (!dir) {
  		if (errno != ENOENT)
  			error("unable to open object pack directory: %s: %s",
 -			      path, strerror(errno));
 +			      path.buf, strerror(errno));
 +		strbuf_release(&path);
  		return;
  	}
 -	path[len++] = '/';
 +	strbuf_addch(&path, '/');
 +	dirnamelen = path.len;
  	while ((de = readdir(dir)) != NULL) {
 -		int namelen = strlen(de->d_name);
  		struct packed_git *p;
 -
 -		if (len + namelen + 1 > sizeof(path)) {
 -			if (report_garbage) {
 -				struct strbuf sb = STRBUF_INIT;
 -				strbuf_addf(&sb, "%.*s/%s", len - 1, path, de->d_name);
 -				report_garbage("path too long", sb.buf);
 -				strbuf_release(&sb);
 -			}
 -			continue;
 -		}
 +		size_t base_len;
  
  		if (is_dot_or_dotdot(de->d_name))
  			continue;
  
 -		strcpy(path + len, de->d_name);
 +		strbuf_setlen(&path, dirnamelen);
 +		strbuf_addstr(&path, de->d_name);
  
 -		if (has_extension(de->d_name, ".idx")) {
 +		base_len = path.len;
 +		if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
  			/* Don't reopen a pack we already have. */
  			for (p = packed_git; p; p = p->next) {
 -				if (!memcmp(path, p->pack_name, len + namelen - 4))
 +				size_t len;
 +				if (strip_suffix(p->pack_name, ".pack", &len) &&
 +				    len == base_len &&
 +				    !memcmp(p->pack_name, path.buf, len))
  					break;
  			}
  			if (p == NULL &&
@@@ -1285,25 -1227,24 +1285,25 @@@
  			     * See if it really is a valid .idx file with
  			     * corresponding .pack file that we can map.
  			     */
 -			    (p = add_packed_git(path, len + namelen, local)) != NULL)
 +			    (p = add_packed_git(path.buf, path.len, local)) != NULL)
  				install_packed_git(p);
  		}
  
  		if (!report_garbage)
  			continue;
  
 -		if (has_extension(de->d_name, ".idx") ||
 -		    has_extension(de->d_name, ".pack") ||
 -		    has_extension(de->d_name, ".bitmap") ||
 -		    has_extension(de->d_name, ".keep"))
 -			string_list_append(&garbage, path);
 +		if (ends_with(de->d_name, ".idx") ||
 +		    ends_with(de->d_name, ".pack") ||
 +		    ends_with(de->d_name, ".bitmap") ||
 +		    ends_with(de->d_name, ".keep"))
 +			string_list_append(&garbage, path.buf);
  		else
 -			report_garbage("garbage found", path);
 +			report_garbage("garbage found", path.buf);
  	}
  	closedir(dir);
  	report_pack_garbage(&garbage);
  	string_list_clear(&garbage, 0);
 +	strbuf_release(&path);
  }
  
  static int sort_pack(const void *a_, const void *b_)
@@@ -1579,40 -1520,6 +1579,40 @@@ int unpack_sha1_header(git_zstream *str
  	return git_inflate(stream, 0);
  }
  
 +static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map,
 +					unsigned long mapsize, void *buffer,
 +					unsigned long bufsiz, struct strbuf *header)
 +{
 +	int status;
 +
 +	status = unpack_sha1_header(stream, map, mapsize, buffer, bufsiz);
 +
 +	/*
 +	 * Check if entire header is unpacked in the first iteration.
 +	 */
 +	if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
 +		return 0;
 +
 +	/*
 +	 * buffer[0..bufsiz] was not large enough.  Copy the partial
 +	 * result out to header, and then append the result of further
 +	 * reading the stream.
 +	 */
 +	strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
 +	stream->next_out = buffer;
 +	stream->avail_out = bufsiz;
 +
 +	do {
 +		status = git_inflate(stream, 0);
 +		strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
 +		if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
 +			return 0;
 +		stream->next_out = buffer;
 +		stream->avail_out = bufsiz;
 +	} while (status != Z_STREAM_END);
 +	return -1;
 +}
 +
  static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
  {
  	int bytes = strlen(buffer) + 1;
@@@ -1663,38 -1570,27 +1663,38 @@@
   * too permissive for what we want to check. So do an anal
   * object header parse by hand.
   */
 -int parse_sha1_header(const char *hdr, unsigned long *sizep)
 +static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
 +			       unsigned int flags)
  {
 -	char type[10];
 -	int i;
 +	const char *type_buf = hdr;
  	unsigned long size;
 +	int type, type_len = 0;
  
  	/*
 -	 * The type can be at most ten bytes (including the
 -	 * terminating '\0' that we add), and is followed by
 +	 * The type can be of any size but is followed by
  	 * a space.
  	 */
 -	i = 0;
  	for (;;) {
  		char c = *hdr++;
  		if (c == ' ')
  			break;
 -		type[i++] = c;
 -		if (i >= sizeof(type))
 -			return -1;
 +		type_len++;
  	}
 -	type[i] = 0;
 +
 +	type = type_from_string_gently(type_buf, type_len, 1);
 +	if (oi->typename)
 +		strbuf_add(oi->typename, type_buf, type_len);
 +	/*
 +	 * Set type to 0 if its an unknown object and
 +	 * we're obtaining the type using '--allow-unkown-type'
 +	 * option.
 +	 */
 +	if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0))
 +		type = 0;
 +	else if (type < 0)
 +		die("invalid object type");
 +	if (oi->typep)
 +		*oi->typep = type;
  
  	/*
  	 * The length must follow immediately, and be in canonical
@@@ -1712,24 -1608,12 +1712,24 @@@
  			size = size * 10 + c;
  		}
  	}
 -	*sizep = size;
 +
 +	if (oi->sizep)
 +		*oi->sizep = size;
  
  	/*
  	 * The length must be followed by a zero byte
  	 */
 -	return *hdr ? -1 : type_from_string(type);
 +	return *hdr ? -1 : type;
 +}
 +
 +int parse_sha1_header(const char *hdr, unsigned long *sizep)
 +{
 +	struct object_info oi;
 +
 +	oi.sizep = sizep;
 +	oi.typename = NULL;
 +	oi.typep = NULL;
 +	return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT);
  }
  
  static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
@@@ -2042,9 -1926,7 +2042,9 @@@ static void *unpack_compressed_entry(st
  	git_zstream stream;
  	unsigned char *buffer, *in;
  
 -	buffer = xmallocz(size);
 +	buffer = xmallocz_gently(size);
 +	if (!buffer)
 +		return NULL;
  	memset(&stream, 0, sizeof(stream));
  	stream.next_out = buffer;
  	stream.avail_out = size + 1;
@@@ -2204,9 -2086,27 +2204,9 @@@ static void *read_object(const unsigne
  
  static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
  {
 -	static FILE *log_file;
 -
 -	if (!log_pack_access)
 -		log_pack_access = getenv("GIT_TRACE_PACK_ACCESS");
 -	if (!log_pack_access)
 -		log_pack_access = no_log_pack_access;
 -	if (log_pack_access == no_log_pack_access)
 -		return;
 -
 -	if (!log_file) {
 -		log_file = fopen(log_pack_access, "w");
 -		if (!log_file) {
 -			error("cannot open pack access log '%s' for writing: %s",
 -			      log_pack_access, strerror(errno));
 -			log_pack_access = no_log_pack_access;
 -			return;
 -		}
 -	}
 -	fprintf(log_file, "%s %"PRIuMAX"\n",
 -		p->pack_name, (uintmax_t)obj_offset);
 -	fflush(log_file);
 +	static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
 +	trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
 +			 p->pack_name, (uintmax_t)obj_offset);
  }
  
  int do_check_packed_object_crc;
@@@ -2231,7 -2131,8 +2231,7 @@@ void *unpack_entry(struct packed_git *p
  	int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
  	int base_from_cache = 0;
  
 -	if (log_pack_access != no_log_pack_access)
 -		write_pack_access_log(p, obj_offset);
 +	write_pack_access_log(p, obj_offset);
  
  	/* PHASE 1: drill down to the innermost base object */
  	for (;;) {
@@@ -2545,8 -2446,10 +2545,8 @@@ static int fill_pack_entry(const unsign
  	 * answer, as it may have been deleted since the index was
  	 * loaded!
  	 */
 -	if (!is_pack_valid(p)) {
 -		warning("packfile %s cannot be accessed", p->pack_name);
 +	if (!is_pack_valid(p))
  		return 0;
 -	}
  	e->offset = offset;
  	e->p = p;
  	hashcpy(e->sha1, sha1);
@@@ -2594,15 -2497,13 +2594,15 @@@ struct packed_git *find_sha1_pack(cons
  }
  
  static int sha1_loose_object_info(const unsigned char *sha1,
 -				  struct object_info *oi)
 +				  struct object_info *oi,
 +				  int flags)
  {
 -	int status;
 -	unsigned long mapsize, size;
 +	int status = 0;
 +	unsigned long mapsize;
  	void *map;
  	git_zstream stream;
  	char hdr[32];
 +	struct strbuf hdrbuf = STRBUF_INIT;
  
  	if (oi->delta_base_sha1)
  		hashclr(oi->delta_base_sha1);
@@@ -2615,7 -2516,7 +2615,7 @@@
  	 * return value implicitly indicates whether the
  	 * object even exists.
  	 */
 -	if (!oi->typep && !oi->sizep) {
 +	if (!oi->typep && !oi->typename && !oi->sizep) {
  		struct stat st;
  		if (stat_sha1_file(sha1, &st) < 0)
  			return -1;
@@@ -2629,26 -2530,17 +2629,26 @@@
  		return -1;
  	if (oi->disk_sizep)
  		*oi->disk_sizep = mapsize;
 -	if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
 +	if ((flags & LOOKUP_UNKNOWN_OBJECT)) {
 +		if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
 +			status = error("unable to unpack %s header with --allow-unknown-type",
 +				       sha1_to_hex(sha1));
 +	} else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
  		status = error("unable to unpack %s header",
  			       sha1_to_hex(sha1));
 -	else if ((status = parse_sha1_header(hdr, &size)) < 0)
 +	if (status < 0)
 +		; /* Do nothing */
 +	else if (hdrbuf.len) {
 +		if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0)
 +			status = error("unable to parse %s header with --allow-unknown-type",
 +				       sha1_to_hex(sha1));
 +	} else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0)
  		status = error("unable to parse %s header", sha1_to_hex(sha1));
 -	else if (oi->sizep)
 -		*oi->sizep = size;
  	git_inflate_end(&stream);
  	munmap(map, mapsize);
 -	if (oi->typep)
 +	if (status && oi->typep)
  		*oi->typep = status;
 +	strbuf_release(&hdrbuf);
  	return 0;
  }
  
@@@ -2657,7 -2549,6 +2657,7 @@@ int sha1_object_info_extended(const uns
  	struct cached_object *co;
  	struct pack_entry e;
  	int rtype;
 +	enum object_type real_type;
  	const unsigned char *real = lookup_replace_object_extended(sha1, flags);
  
  	co = find_cached_object(real);
@@@ -2670,15 -2561,13 +2670,15 @@@
  			*(oi->disk_sizep) = 0;
  		if (oi->delta_base_sha1)
  			hashclr(oi->delta_base_sha1);
 +		if (oi->typename)
 +			strbuf_addstr(oi->typename, typename(co->type));
  		oi->whence = OI_CACHED;
  		return 0;
  	}
  
  	if (!find_pack_entry(real, &e)) {
  		/* Most likely it's a loose object. */
 -		if (!sha1_loose_object_info(real, oi)) {
 +		if (!sha1_loose_object_info(real, oi, flags)) {
  			oi->whence = OI_LOOSE;
  			return 0;
  		}
@@@ -2689,18 -2578,9 +2689,18 @@@
  			return -1;
  	}
  
 +	/*
 +	 * packed_object_info() does not follow the delta chain to
 +	 * find out the real type, unless it is given oi->typep.
 +	 */
 +	if (oi->typename && !oi->typep)
 +		oi->typep = &real_type;
 +
  	rtype = packed_object_info(e.p, e.offset, oi);
  	if (rtype < 0) {
  		mark_bad_packed_object(e.p, real);
 +		if (oi->typep == &real_type)
 +			oi->typep = NULL;
  		return sha1_object_info_extended(real, oi, 0);
  	} else if (in_delta_base_cache(e.p, e.offset)) {
  		oi->whence = OI_DBCACHED;
@@@ -2711,10 -2591,6 +2711,10 @@@
  		oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
  					 rtype == OBJ_OFS_DELTA);
  	}
 +	if (oi->typename)
 +		strbuf_addstr(oi->typename, typename(*oi->typep));
 +	if (oi->typep == &real_type)
 +		oi->typep = NULL;
  
  	return 0;
  }
@@@ -2908,11 -2784,8 +2908,8 @@@ static void write_sha1_file_prepare(con
  
  /*
   * Move the just written object into its final resting place.
-  * NEEDSWORK: this should be renamed to finalize_temp_file() as
-  * "moving" is only a part of what it does, when no patch between
-  * master to pu changes the call sites of this function.
   */
- int move_temp_to_file(const char *tmpfile, const char *filename)
+ int finalize_object_file(const char *tmpfile, const char *filename)
  {
  	int ret = 0;
  
@@@ -3040,6 -2913,7 +3037,6 @@@ static int write_loose_object(const uns
  	}
  
  	/* Set it up */
 -	memset(&stream, 0, sizeof(stream));
  	git_deflate_init(&stream, zlib_compression_level);
  	stream.next_out = compressed;
  	stream.avail_out = sizeof(compressed);
@@@ -3085,29 -2959,12 +3082,29 @@@
  				tmp_file, strerror(errno));
  	}
  
- 	return move_temp_to_file(tmp_file, filename);
+ 	return finalize_object_file(tmp_file, filename);
  }
  
 -int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
 +static int freshen_loose_object(const unsigned char *sha1)
 +{
 +	return check_and_freshen(sha1, 1);
 +}
 +
 +static int freshen_packed_object(const unsigned char *sha1)
 +{
 +	struct pack_entry e;
 +	if (!find_pack_entry(sha1, &e))
 +		return 0;
 +	if (e.p->freshened)
 +		return 1;
 +	if (!freshen_file(e.p->pack_name))
 +		return 0;
 +	e.p->freshened = 1;
 +	return 1;
 +}
 +
 +int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)
  {
 -	unsigned char sha1[20];
  	char hdr[32];
  	int hdrlen;
  
@@@ -3115,32 -2972,13 +3112,32 @@@
  	 * it out into .git/objects/??/?{38} file.
  	 */
  	write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
 -	if (returnsha1)
 -		hashcpy(returnsha1, sha1);
 -	if (has_sha1_file(sha1))
 +	if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
  		return 0;
  	return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
  }
  
 +int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
 +			     unsigned char *sha1, unsigned flags)
 +{
 +	char *header;
 +	int hdrlen, status = 0;
 +
 +	/* type string, SP, %lu of the length plus NUL must fit this */
 +	header = xmalloc(strlen(type) + 32);
 +	write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen);
 +
 +	if (!(flags & HASH_WRITE_OBJECT))
 +		goto cleanup;
 +	if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
 +		goto cleanup;
 +	status = write_loose_object(sha1, header, hdrlen, buf, len, 0);
 +
 +cleanup:
 +	free(header);
 +	return status;
 +}
 +
  int force_object_loose(const unsigned char *sha1, time_t mtime)
  {
  	void *buf;
@@@ -3176,7 -3014,7 +3173,7 @@@ int has_sha1_pack(const unsigned char *
  	return find_pack_entry(sha1, &e);
  }
  
 -int has_sha1_file(const unsigned char *sha1)
 +int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
  {
  	struct pack_entry e;
  
@@@ -3184,8 -3022,6 +3181,8 @@@
  		return 1;
  	if (has_loose_object(sha1))
  		return 1;
 +	if (flags & HAS_SHA1_QUICK)
 +		return 0;
  	reprepare_packed_git();
  	return find_pack_entry(sha1, &e);
  }
@@@ -3257,29 -3093,6 +3254,29 @@@ static int index_mem(unsigned char *sha
  	return ret;
  }
  
 +static int index_stream_convert_blob(unsigned char *sha1, int fd,
 +				     const char *path, unsigned flags)
 +{
 +	int ret;
 +	const int write_object = flags & HASH_WRITE_OBJECT;
 +	struct strbuf sbuf = STRBUF_INIT;
 +
 +	assert(path);
 +	assert(would_convert_to_git_filter_fd(path));
 +
 +	convert_to_git_filter_fd(path, fd, &sbuf,
 +				 write_object ? safe_crlf : SAFE_CRLF_FALSE);
 +
 +	if (write_object)
 +		ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
 +				      sha1);
 +	else
 +		ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
 +				     sha1);
 +	strbuf_release(&sbuf);
 +	return ret;
 +}
 +
  static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
  		      const char *path, unsigned flags)
  {
@@@ -3303,7 -3116,7 +3300,7 @@@ static int index_core(unsigned char *sh
  	int ret;
  
  	if (!size) {
 -		ret = index_mem(sha1, NULL, size, type, path, flags);
 +		ret = index_mem(sha1, "", size, type, path, flags);
  	} else if (size <= SMALL_FILE_SIZE) {
  		char *buf = xmalloc(size);
  		if (size == read_in_full(fd, buf, size))
@@@ -3345,22 -3158,15 +3342,22 @@@ int index_fd(unsigned char *sha1, int f
  	     enum object_type type, const char *path, unsigned flags)
  {
  	int ret;
 -	size_t size = xsize_t(st->st_size);
  
 -	if (!S_ISREG(st->st_mode))
 +	/*
 +	 * Call xsize_t() only when needed to avoid potentially unnecessary
 +	 * die() for large files.
 +	 */
 +	if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path))
 +		ret = index_stream_convert_blob(sha1, fd, path, flags);
 +	else if (!S_ISREG(st->st_mode))
  		ret = index_pipe(sha1, fd, type, path, flags);
 -	else if (size <= big_file_threshold || type != OBJ_BLOB ||
 -		 (path && would_convert_to_git(path, NULL, 0, 0)))
 -		ret = index_core(sha1, fd, size, type, path, flags);
 +	else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
 +		 (path && would_convert_to_git(path)))
 +		ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
 +				 flags);
  	else
 -		ret = index_stream(sha1, fd, size, type, path, flags);
 +		ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path,
 +				   flags);
  	close(fd);
  	return ret;
  }
@@@ -3425,177 -3231,3 +3422,177 @@@ void assert_sha1_type(const unsigned ch
  		die("%s is not a valid '%s' object", sha1_to_hex(sha1),
  		    typename(expect));
  }
 +
 +static int for_each_file_in_obj_subdir(int subdir_nr,
 +				       struct strbuf *path,
 +				       each_loose_object_fn obj_cb,
 +				       each_loose_cruft_fn cruft_cb,
 +				       each_loose_subdir_fn subdir_cb,
 +				       void *data)
 +{
 +	size_t baselen = path->len;
 +	DIR *dir = opendir(path->buf);
 +	struct dirent *de;
 +	int r = 0;
 +
 +	if (!dir) {
 +		if (errno == ENOENT)
 +			return 0;
 +		return error("unable to open %s: %s", path->buf, strerror(errno));
 +	}
 +
 +	while ((de = readdir(dir))) {
 +		if (is_dot_or_dotdot(de->d_name))
 +			continue;
 +
 +		strbuf_setlen(path, baselen);
 +		strbuf_addf(path, "/%s", de->d_name);
 +
 +		if (strlen(de->d_name) == 38)  {
 +			char hex[41];
 +			unsigned char sha1[20];
 +
 +			snprintf(hex, sizeof(hex), "%02x%s",
 +				 subdir_nr, de->d_name);
 +			if (!get_sha1_hex(hex, sha1)) {
 +				if (obj_cb) {
 +					r = obj_cb(sha1, path->buf, data);
 +					if (r)
 +						break;
 +				}
 +				continue;
 +			}
 +		}
 +
 +		if (cruft_cb) {
 +			r = cruft_cb(de->d_name, path->buf, data);
 +			if (r)
 +				break;
 +		}
 +	}
 +	strbuf_setlen(path, baselen);
 +
 +	if (!r && subdir_cb)
 +		r = subdir_cb(subdir_nr, path->buf, data);
 +
 +	closedir(dir);
 +	return r;
 +}
 +
 +int for_each_loose_file_in_objdir_buf(struct strbuf *path,
 +			    each_loose_object_fn obj_cb,
 +			    each_loose_cruft_fn cruft_cb,
 +			    each_loose_subdir_fn subdir_cb,
 +			    void *data)
 +{
 +	size_t baselen = path->len;
 +	int r = 0;
 +	int i;
 +
 +	for (i = 0; i < 256; i++) {
 +		strbuf_addf(path, "/%02x", i);
 +		r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
 +						subdir_cb, data);
 +		strbuf_setlen(path, baselen);
 +		if (r)
 +			break;
 +	}
 +
 +	return r;
 +}
 +
 +int for_each_loose_file_in_objdir(const char *path,
 +				  each_loose_object_fn obj_cb,
 +				  each_loose_cruft_fn cruft_cb,
 +				  each_loose_subdir_fn subdir_cb,
 +				  void *data)
 +{
 +	struct strbuf buf = STRBUF_INIT;
 +	int r;
 +
 +	strbuf_addstr(&buf, path);
 +	r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb,
 +					      subdir_cb, data);
 +	strbuf_release(&buf);
 +
 +	return r;
 +}
 +
 +struct loose_alt_odb_data {
 +	each_loose_object_fn *cb;
 +	void *data;
 +};
 +
 +static int loose_from_alt_odb(struct alternate_object_database *alt,
 +			      void *vdata)
 +{
 +	struct loose_alt_odb_data *data = vdata;
 +	struct strbuf buf = STRBUF_INIT;
 +	int r;
 +
 +	/* copy base not including trailing '/' */
 +	strbuf_add(&buf, alt->base, alt->name - alt->base - 1);
 +	r = for_each_loose_file_in_objdir_buf(&buf,
 +					      data->cb, NULL, NULL,
 +					      data->data);
 +	strbuf_release(&buf);
 +	return r;
 +}
 +
 +int for_each_loose_object(each_loose_object_fn cb, void *data, unsigned flags)
 +{
 +	struct loose_alt_odb_data alt;
 +	int r;
 +
 +	r = for_each_loose_file_in_objdir(get_object_directory(),
 +					  cb, NULL, NULL, data);
 +	if (r)
 +		return r;
 +
 +	if (flags & FOR_EACH_OBJECT_LOCAL_ONLY)
 +		return 0;
 +
 +	alt.cb = cb;
 +	alt.data = data;
 +	return foreach_alt_odb(loose_from_alt_odb, &alt);
 +}
 +
 +static int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn cb, void *data)
 +{
 +	uint32_t i;
 +	int r = 0;
 +
 +	for (i = 0; i < p->num_objects; i++) {
 +		const unsigned char *sha1 = nth_packed_object_sha1(p, i);
 +
 +		if (!sha1)
 +			return error("unable to get sha1 of object %u in %s",
 +				     i, p->pack_name);
 +
 +		r = cb(sha1, p, i, data);
 +		if (r)
 +			break;
 +	}
 +	return r;
 +}
 +
 +int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
 +{
 +	struct packed_git *p;
 +	int r = 0;
 +	int pack_errors = 0;
 +
 +	prepare_packed_git();
 +	for (p = packed_git; p; p = p->next) {
 +		if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
 +			continue;
 +		if (open_pack_index(p)) {
 +			pack_errors = 1;
 +			continue;
 +		}
 +		r = for_each_object_in_pack(p, cb, data);
 +		if (r)
 +			break;
 +	}
 +	return r ? r : pack_errors;
 +}