From: Junio C Hamano <gitster@pobox.com>
Date: Wed, 5 Jul 2017 20:32:56 +0000 (-0700)
Subject: Merge branch 'rs/sha1-name-readdir-optim'
X-Git-Tag: v2.14.0-rc0~37
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/5ab148dda0076a136b4afb385d96bd9cdc4d2590?hp=-c

Merge branch 'rs/sha1-name-readdir-optim'

Optimize "what are the object names already taken in an alternate
object database?" query that is used to derive the length of prefix
an object name is uniquely abbreviated to.

* rs/sha1-name-readdir-optim:
sha1_file: guard against invalid loose subdirectory numbers
sha1_file: let for_each_file_in_obj_subdir() handle subdir names
p4205: add perf test script for pretty log formats
sha1_name: cache readdir(3) results in find_short_object_filename()
---

5ab148dda0076a136b4afb385d96bd9cdc4d2590
diff --combined builtin/fsck.c
index 87c6756899,2686951381..99dea7adf6
--- a/builtin/fsck.c
+++ b/builtin/fsck.c
@@@ -1,6 -1,5 +1,6 @@@
  #include "builtin.h"
  #include "cache.h"
 +#include "config.h"
  #include "commit.h"
  #include "tree.h"
  #include "blob.h"
@@@ -281,7 -280,8 +281,7 @@@ static void check_unreachable_object(st
  				free(filename);
  				return;
  			}
 -			if (!(f = fopen(filename, "w")))
 -				die_errno("Could not open '%s'", filename);
 +			f = xfopen(filename, "w");
  			if (obj->type == OBJ_BLOB) {
  				if (stream_blob_to_fd(fileno(f), &obj->oid, NULL, 1))
  					die_errno("Could not write '%s'", filename);
@@@ -377,7 -377,7 +377,7 @@@ static int fsck_obj(struct object *obj
  	return 0;
  }
  
 -static int fsck_obj_buffer(const unsigned char *sha1, enum object_type type,
 +static int fsck_obj_buffer(const struct object_id *oid, enum object_type type,
  			   unsigned long size, void *buffer, int *eaten)
  {
  	/*
@@@ -385,10 -385,10 +385,10 @@@
  	 * verify_packfile(), data_valid variable for details.
  	 */
  	struct object *obj;
 -	obj = parse_object_buffer(sha1, type, size, buffer, eaten);
 +	obj = parse_object_buffer(oid, type, size, buffer, eaten);
  	if (!obj) {
  		errors_found |= ERROR_OBJECT;
 -		return error("%s: object corrupt or missing", sha1_to_hex(sha1));
 +		return error("%s: object corrupt or missing", oid_to_hex(oid));
  	}
  	obj->flags = HAS_OBJ;
  	return fsck_obj(obj);
@@@ -397,7 -397,7 +397,7 @@@
  static int default_refs;
  
  static void fsck_handle_reflog_oid(const char *refname, struct object_id *oid,
 -	unsigned long timestamp)
 +	timestamp_t timestamp)
  {
  	struct object *obj;
  
@@@ -407,7 -407,7 +407,7 @@@
  			if (timestamp && name_objects)
  				add_decoration(fsck_walk_options.object_names,
  					obj,
 -					xstrfmt("%s@{%ld}", refname, timestamp));
 +					xstrfmt("%s@{%"PRItime"}", refname, timestamp));
  			obj->used = 1;
  			mark_object_reachable(obj);
  		} else {
@@@ -418,7 -418,7 +418,7 @@@
  }
  
  static int fsck_handle_reflog_ent(struct object_id *ooid, struct object_id *noid,
 -		const char *email, unsigned long timestamp, int tz,
 +		const char *email, timestamp_t timestamp, int tz,
  		const char *message, void *cb_data)
  {
  	const char *refname = cb_data;
@@@ -444,7 -444,7 +444,7 @@@ static int fsck_handle_ref(const char *
  {
  	struct object *obj;
  
 -	obj = parse_object(oid->hash);
 +	obj = parse_object(oid);
  	if (!obj) {
  		error("%s: invalid sha1 pointer %s", refname, oid_to_hex(oid));
  		errors_found |= ERROR_REACHABLE;
@@@ -506,7 -506,7 +506,7 @@@ static struct object *parse_loose_objec
  	if (!contents && type != OBJ_BLOB)
  		die("BUG: read_loose_object streamed a non-blob");
  
 -	obj = parse_object_buffer(oid->hash, type, size, contents, &eaten);
 +	obj = parse_object_buffer(oid, type, size, contents, &eaten);
  
  	if (!eaten)
  		free(contents);
@@@ -537,7 -537,7 +537,7 @@@ static int fsck_cruft(const char *basen
  	return 0;
  }
  
- static int fsck_subdir(int nr, const char *path, void *progress)
+ static int fsck_subdir(unsigned int nr, const char *path, void *progress)
  {
  	display_progress(progress, nr + 1);
  	return 0;
@@@ -599,10 -599,10 +599,10 @@@ static int fsck_cache_tree(struct cache
  		fprintf(stderr, "Checking cache tree\n");
  
  	if (0 <= it->entry_count) {
 -		struct object *obj = parse_object(it->sha1);
 +		struct object *obj = parse_object(&it->oid);
  		if (!obj) {
  			error("%s: invalid sha1 pointer in cache-tree",
 -			      sha1_to_hex(it->sha1));
 +			      oid_to_hex(&it->oid));
  			errors_found |= ERROR_REFS;
  			return 1;
  		}
@@@ -781,7 -781,7 +781,7 @@@ int cmd_fsck(int argc, const char **arg
  			mode = active_cache[i]->ce_mode;
  			if (S_ISGITLINK(mode))
  				continue;
 -			blob = lookup_blob(active_cache[i]->oid.hash);
 +			blob = lookup_blob(&active_cache[i]->oid);
  			if (!blob)
  				continue;
  			obj = &blob->object;
diff --combined builtin/prune.c
index f0e2bff04c,ea208c97f8..c378690545
--- a/builtin/prune.c
+++ b/builtin/prune.c
@@@ -13,7 -13,7 +13,7 @@@ static const char * const prune_usage[
  };
  static int show_only;
  static int verbose;
 -static unsigned long expire;
 +static timestamp_t expire;
  static int show_progress = -1;
  
  static int prune_tmp_file(const char *fullpath)
@@@ -68,7 -68,7 +68,7 @@@ static int prune_cruft(const char *base
  	return 0;
  }
  
- static int prune_subdir(int nr, const char *path, void *data)
+ static int prune_subdir(unsigned int nr, const char *path, void *data)
  {
  	if (!show_only)
  		rmdir(path);
@@@ -111,7 -111,7 +111,7 @@@ int cmd_prune(int argc, const char **ar
  	};
  	char *s;
  
 -	expire = ULONG_MAX;
 +	expire = TIME_MAX;
  	save_commit_buffer = 0;
  	check_replace_refs = 0;
  	ref_paranoia = 1;
@@@ -123,12 -123,11 +123,12 @@@
  		die(_("cannot prune in a precious-objects repo"));
  
  	while (argc--) {
 -		unsigned char sha1[20];
 +		struct object_id oid;
  		const char *name = *argv++;
  
 -		if (!get_sha1(name, sha1)) {
 -			struct object *object = parse_object_or_die(sha1, name);
 +		if (!get_oid(name, &oid)) {
 +			struct object *object = parse_object_or_die(&oid,
 +								    name);
  			add_pending_object(&revs, object, "");
  		}
  		else
diff --combined cache.h
index c958fc3ce5,7f7ec5d56d..c12f452890
--- a/cache.h
+++ b/cache.h
@@@ -11,7 -11,7 +11,8 @@@
  #include "string-list.h"
  #include "pack-revindex.h"
  #include "hash.h"
 +#include "path.h"
+ #include "sha1-array.h"
  
  #ifndef platform_SHA_CTX
  /*
@@@ -463,8 -463,6 +464,8 @@@ static inline enum object_type object_t
   */
  extern const char * const local_repo_env[];
  
 +extern void setup_git_env(void);
 +
  /*
   * Returns true iff we have a configured git repository (either via
   * setup_git_directory, or in the environment via $GIT_DIR).
@@@ -528,15 -526,12 +529,15 @@@ extern void set_git_work_tree(const cha
  
  extern void setup_work_tree(void);
  /*
 - * Find GIT_DIR of the repository that contains the current working directory,
 - * without changing the working directory or other global state. The result is
 - * appended to gitdir. The return value is either NULL if no repository was
 - * found, or pointing to the path inside gitdir's buffer.
 + * Find the commondir and gitdir of the repository that contains the current
 + * working directory, without changing the working directory or other global
 + * state. The result is appended to commondir and gitdir.  If the discovered
 + * gitdir does not correspond to a worktree, then 'commondir' and 'gitdir' will
 + * both have the same result appended to the buffer.  The return value is
 + * either 0 upon success and non-zero if no repository was found.
   */
 -extern const char *discover_git_directory(struct strbuf *gitdir);
 +extern int discover_git_directory(struct strbuf *commondir,
 +				  struct strbuf *gitdir);
  extern const char *setup_git_directory_gently(int *);
  extern const char *setup_git_directory(void);
  extern char *prefix_path(const char *prefix, int len, const char *path);
@@@ -603,7 -598,6 +604,7 @@@ extern int read_index_unmerged(struct i
  #define CLOSE_LOCK		(1 << 1)
  extern int write_locked_index(struct index_state *, struct lock_file *lock, unsigned flags);
  extern int discard_index(struct index_state *);
 +extern void move_index_extensions(struct index_state *dst, struct index_state *src);
  extern int unmerged_index(const struct index_state *);
  extern int verify_path(const char *path);
  extern int strcmp_offset(const char *s1, const char *s2, size_t *first_change);
@@@ -772,6 -766,7 +773,6 @@@ extern int core_apply_sparse_checkout
  extern int precomposed_unicode;
  extern int protect_hfs;
  extern int protect_ntfs;
 -extern int git_db_env, git_index_env, git_graft_env, git_common_dir_env;
  
  /*
   * Include broken refs in all ref iterations, which will
@@@ -893,6 -888,64 +894,6 @@@ extern void check_repository_format(voi
  #define DATA_CHANGED    0x0020
  #define TYPE_CHANGED    0x0040
  
 -/*
 - * Return a statically allocated filename, either generically (mkpath), in
 - * the repository directory (git_path), or in a submodule's repository
 - * directory (git_path_submodule). In all cases, note that the result
 - * may be overwritten by another call to _any_ of the functions. Consider
 - * using the safer "dup" or "strbuf" formats below (in some cases, the
 - * unsafe versions have already been removed).
 - */
 -extern const char *mkpath(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
 -extern const char *git_path(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
 -extern const char *git_common_path(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
 -
 -extern char *mksnpath(char *buf, size_t n, const char *fmt, ...)
 -	__attribute__((format (printf, 3, 4)));
 -extern void strbuf_git_path(struct strbuf *sb, const char *fmt, ...)
 -	__attribute__((format (printf, 2, 3)));
 -extern void strbuf_git_common_path(struct strbuf *sb, const char *fmt, ...)
 -	__attribute__((format (printf, 2, 3)));
 -extern char *git_path_buf(struct strbuf *buf, const char *fmt, ...)
 -	__attribute__((format (printf, 2, 3)));
 -extern int strbuf_git_path_submodule(struct strbuf *sb, const char *path,
 -				     const char *fmt, ...)
 -	__attribute__((format (printf, 3, 4)));
 -extern char *git_pathdup(const char *fmt, ...)
 -	__attribute__((format (printf, 1, 2)));
 -extern char *mkpathdup(const char *fmt, ...)
 -	__attribute__((format (printf, 1, 2)));
 -extern char *git_pathdup_submodule(const char *path, const char *fmt, ...)
 -	__attribute__((format (printf, 2, 3)));
 -
 -extern void report_linked_checkout_garbage(void);
 -
 -/*
 - * You can define a static memoized git path like:
 - *
 - *    static GIT_PATH_FUNC(git_path_foo, "FOO");
 - *
 - * or use one of the global ones below.
 - */
 -#define GIT_PATH_FUNC(func, filename) \
 -	const char *func(void) \
 -	{ \
 -		static char *ret; \
 -		if (!ret) \
 -			ret = git_pathdup(filename); \
 -		return ret; \
 -	}
 -
 -const char *git_path_cherry_pick_head(void);
 -const char *git_path_revert_head(void);
 -const char *git_path_squash_msg(void);
 -const char *git_path_merge_msg(void);
 -const char *git_path_merge_rr(void);
 -const char *git_path_merge_mode(void);
 -const char *git_path_merge_head(void);
 -const char *git_path_fetch_head(void);
 -const char *git_path_shallow(void);
 -
  /*
   * Return the name of the file in the local object database that would
   * be used to store a loose object with the specified sha1.  The
@@@ -973,13 -1026,6 +974,13 @@@ static inline void oidcpy(struct object
  	hashcpy(dst->hash, src->hash);
  }
  
 +static inline struct object_id *oiddup(const struct object_id *src)
 +{
 +	struct object_id *dst = xmalloc(sizeof(struct object_id));
 +	oidcpy(dst, src);
 +	return dst;
 +}
 +
  static inline void hashclr(unsigned char *hash)
  {
  	memset(hash, 0, GIT_SHA1_RAWSZ);
@@@ -1288,18 -1334,13 +1289,18 @@@ static inline int hex2chr(const char *s
  
  struct object_context {
  	unsigned char tree[20];
 -	char path[PATH_MAX];
  	unsigned mode;
  	/*
  	 * symlink_path is only used by get_tree_entry_follow_symlinks,
  	 * and only for symlinks that point outside the repository.
  	 */
  	struct strbuf symlink_path;
 +	/*
 +	 * If GET_SHA1_RECORD_PATH is set, this will record path (if any)
 +	 * found when resolving the name. The caller is responsible for
 +	 * releasing the memory.
 +	 */
 +	char *path;
  };
  
  #define GET_SHA1_QUIETLY           01
@@@ -1309,7 -1350,6 +1310,7 @@@
  #define GET_SHA1_TREEISH          020
  #define GET_SHA1_BLOB             040
  #define GET_SHA1_FOLLOW_SYMLINKS 0100
 +#define GET_SHA1_RECORD_PATH     0200
  #define GET_SHA1_ONLY_TO_DIE    04000
  
  #define GET_SHA1_DISAMBIGUATORS \
@@@ -1324,7 -1364,7 +1325,7 @@@ extern int get_sha1_tree(const char *st
  extern int get_sha1_treeish(const char *str, unsigned char *sha1);
  extern int get_sha1_blob(const char *str, unsigned char *sha1);
  extern void maybe_die_on_misspelt_object_name(const char *name, const char *prefix);
 -extern int get_sha1_with_context(const char *str, unsigned flags, unsigned char *sha1, struct object_context *orc);
 +extern int get_sha1_with_context(const char *str, unsigned flags, unsigned char *sha1, struct object_context *oc);
  
  extern int get_oid(const char *str, struct object_id *oid);
  
@@@ -1440,18 -1480,18 +1441,18 @@@ struct date_mode 
  #define DATE_MODE(t) date_mode_from_type(DATE_##t)
  struct date_mode *date_mode_from_type(enum date_mode_type type);
  
 -const char *show_date(unsigned long time, int timezone, const struct date_mode *mode);
 -void show_date_relative(unsigned long time, int tz, const struct timeval *now,
 +const char *show_date(timestamp_t time, int timezone, const struct date_mode *mode);
 +void show_date_relative(timestamp_t time, int tz, const struct timeval *now,
  			struct strbuf *timebuf);
  int parse_date(const char *date, struct strbuf *out);
 -int parse_date_basic(const char *date, unsigned long *timestamp, int *offset);
 -int parse_expiry_date(const char *date, unsigned long *timestamp);
 +int parse_date_basic(const char *date, timestamp_t *timestamp, int *offset);
 +int parse_expiry_date(const char *date, timestamp_t *timestamp);
  void datestamp(struct strbuf *out);
  #define approxidate(s) approxidate_careful((s), NULL)
 -unsigned long approxidate_careful(const char *, int *);
 -unsigned long approxidate_relative(const char *date, const struct timeval *now);
 +timestamp_t approxidate_careful(const char *, int *);
 +timestamp_t approxidate_relative(const char *date, const struct timeval *now);
  void parse_date_format(const char *format, struct date_mode *mode);
 -int date_overflows(unsigned long date);
 +int date_overflows(timestamp_t date);
  
  #define IDENT_STRICT	       1
  #define IDENT_NO_DATE	       2
@@@ -1540,6 -1580,16 +1541,16 @@@ extern struct alternate_object_databas
  	struct strbuf scratch;
  	size_t base_len;
  
+ 	/*
+ 	 * Used to store the results of readdir(3) calls when searching
+ 	 * for unique abbreviated hashes.  This cache is never
+ 	 * invalidated, thus it's racy and not necessarily accurate.
+ 	 * That's fine for its purpose; don't use it for tasks requiring
+ 	 * greater accuracy!
+ 	 */
+ 	char loose_objects_subdir_seen[256];
+ 	struct oid_array loose_objects_cache;
+ 
  	char path[FLEX_ARRAY];
  } *alt_odb_list;
  extern void prepare_alt_odb(void);
@@@ -1755,9 -1805,15 +1766,15 @@@ typedef int each_loose_object_fn(const 
  typedef int each_loose_cruft_fn(const char *basename,
  				const char *path,
  				void *data);
- typedef int each_loose_subdir_fn(int nr,
+ typedef int each_loose_subdir_fn(unsigned int nr,
  				 const char *path,
  				 void *data);
+ int for_each_file_in_obj_subdir(unsigned int subdir_nr,
+ 				struct strbuf *path,
+ 				each_loose_object_fn obj_cb,
+ 				each_loose_cruft_fn cruft_cb,
+ 				each_loose_subdir_fn subdir_cb,
+ 				void *data);
  int for_each_loose_file_in_objdir(const char *path,
  				  each_loose_object_fn obj_cb,
  				  each_loose_cruft_fn cruft_cb,
@@@ -1826,9 -1882,188 +1843,9 @@@ extern int packed_object_info(struct pa
  /* Dumb servers support */
  extern int update_server_info(int);
  
 -/* git_config_parse_key() returns these negated: */
 -#define CONFIG_INVALID_KEY 1
 -#define CONFIG_NO_SECTION_OR_NAME 2
 -/* git_config_set_gently(), git_config_set_multivar_gently() return the above or these: */
 -#define CONFIG_NO_LOCK -1
 -#define CONFIG_INVALID_FILE 3
 -#define CONFIG_NO_WRITE 4
 -#define CONFIG_NOTHING_SET 5
 -#define CONFIG_INVALID_PATTERN 6
 -#define CONFIG_GENERIC_ERROR 7
 -
 -#define CONFIG_REGEX_NONE ((void *)1)
 -
 -struct git_config_source {
 -	unsigned int use_stdin:1;
 -	const char *file;
 -	const char *blob;
 -};
 -
 -enum config_origin_type {
 -	CONFIG_ORIGIN_BLOB,
 -	CONFIG_ORIGIN_FILE,
 -	CONFIG_ORIGIN_STDIN,
 -	CONFIG_ORIGIN_SUBMODULE_BLOB,
 -	CONFIG_ORIGIN_CMDLINE
 -};
 -
 -struct config_options {
 -	unsigned int respect_includes : 1;
 -	const char *git_dir;
 -};
 -
 -typedef int (*config_fn_t)(const char *, const char *, void *);
 -extern int git_default_config(const char *, const char *, void *);
 -extern int git_config_from_file(config_fn_t fn, const char *, void *);
 -extern int git_config_from_mem(config_fn_t fn, const enum config_origin_type,
 -					const char *name, const char *buf, size_t len, void *data);
 -extern int git_config_from_blob_sha1(config_fn_t fn, const char *name,
 -				     const unsigned char *sha1, void *data);
 -extern void git_config_push_parameter(const char *text);
 -extern int git_config_from_parameters(config_fn_t fn, void *data);
 -extern void read_early_config(config_fn_t cb, void *data);
 -extern void git_config(config_fn_t fn, void *);
 -extern int git_config_with_options(config_fn_t fn, void *,
 -				   struct git_config_source *config_source,
 -				   const struct config_options *opts);
 -extern int git_parse_ulong(const char *, unsigned long *);
 -extern int git_parse_maybe_bool(const char *);
 -extern int git_config_int(const char *, const char *);
 -extern int64_t git_config_int64(const char *, const char *);
 -extern unsigned long git_config_ulong(const char *, const char *);
 -extern ssize_t git_config_ssize_t(const char *, const char *);
 -extern int git_config_bool_or_int(const char *, const char *, int *);
 -extern int git_config_bool(const char *, const char *);
 -extern int git_config_maybe_bool(const char *, const char *);
 -extern int git_config_string(const char **, const char *, const char *);
 -extern int git_config_pathname(const char **, const char *, const char *);
 -extern int git_config_set_in_file_gently(const char *, const char *, const char *);
 -extern void git_config_set_in_file(const char *, const char *, const char *);
 -extern int git_config_set_gently(const char *, const char *);
 -extern void git_config_set(const char *, const char *);
 -extern int git_config_parse_key(const char *, char **, int *);
 -extern int git_config_key_is_valid(const char *key);
 -extern int git_config_set_multivar_gently(const char *, const char *, const char *, int);
 -extern void git_config_set_multivar(const char *, const char *, const char *, int);
 -extern int git_config_set_multivar_in_file_gently(const char *, const char *, const char *, const char *, int);
 -extern void git_config_set_multivar_in_file(const char *, const char *, const char *, const char *, int);
 -extern int git_config_rename_section(const char *, const char *);
 -extern int git_config_rename_section_in_file(const char *, const char *, const char *);
 -extern const char *git_etc_gitconfig(void);
 -extern int git_env_bool(const char *, int);
 -extern unsigned long git_env_ulong(const char *, unsigned long);
 -extern int git_config_system(void);
 -extern int config_error_nonbool(const char *);
 -#if defined(__GNUC__)
 -#define config_error_nonbool(s) (config_error_nonbool(s), const_error())
 -#endif
  extern const char *get_log_output_encoding(void);
  extern const char *get_commit_output_encoding(void);
  
 -extern int git_config_parse_parameter(const char *, config_fn_t fn, void *data);
 -
 -enum config_scope {
 -	CONFIG_SCOPE_UNKNOWN = 0,
 -	CONFIG_SCOPE_SYSTEM,
 -	CONFIG_SCOPE_GLOBAL,
 -	CONFIG_SCOPE_REPO,
 -	CONFIG_SCOPE_CMDLINE,
 -};
 -
 -extern enum config_scope current_config_scope(void);
 -extern const char *current_config_origin_type(void);
 -extern const char *current_config_name(void);
 -
 -struct config_include_data {
 -	int depth;
 -	config_fn_t fn;
 -	void *data;
 -	const struct config_options *opts;
 -};
 -#define CONFIG_INCLUDE_INIT { 0 }
 -extern int git_config_include(const char *name, const char *value, void *data);
 -
 -/*
 - * Match and parse a config key of the form:
 - *
 - *   section.(subsection.)?key
 - *
 - * (i.e., what gets handed to a config_fn_t). The caller provides the section;
 - * we return -1 if it does not match, 0 otherwise. The subsection and key
 - * out-parameters are filled by the function (and *subsection is NULL if it is
 - * missing).
 - *
 - * If the subsection pointer-to-pointer passed in is NULL, returns 0 only if
 - * there is no subsection at all.
 - */
 -extern int parse_config_key(const char *var,
 -			    const char *section,
 -			    const char **subsection, int *subsection_len,
 -			    const char **key);
 -
 -struct config_set_element {
 -	struct hashmap_entry ent;
 -	char *key;
 -	struct string_list value_list;
 -};
 -
 -struct configset_list_item {
 -	struct config_set_element *e;
 -	int value_index;
 -};
 -
 -/*
 - * the contents of the list are ordered according to their
 - * position in the config files and order of parsing the files.
 - * (i.e. key-value pair at the last position of .git/config will
 - * be at the last item of the list)
 - */
 -struct configset_list {
 -	struct configset_list_item *items;
 -	unsigned int nr, alloc;
 -};
 -
 -struct config_set {
 -	struct hashmap config_hash;
 -	int hash_initialized;
 -	struct configset_list list;
 -};
 -
 -extern void git_configset_init(struct config_set *cs);
 -extern int git_configset_add_file(struct config_set *cs, const char *filename);
 -extern int git_configset_get_value(struct config_set *cs, const char *key, const char **value);
 -extern const struct string_list *git_configset_get_value_multi(struct config_set *cs, const char *key);
 -extern void git_configset_clear(struct config_set *cs);
 -extern int git_configset_get_string_const(struct config_set *cs, const char *key, const char **dest);
 -extern int git_configset_get_string(struct config_set *cs, const char *key, char **dest);
 -extern int git_configset_get_int(struct config_set *cs, const char *key, int *dest);
 -extern int git_configset_get_ulong(struct config_set *cs, const char *key, unsigned long *dest);
 -extern int git_configset_get_bool(struct config_set *cs, const char *key, int *dest);
 -extern int git_configset_get_bool_or_int(struct config_set *cs, const char *key, int *is_bool, int *dest);
 -extern int git_configset_get_maybe_bool(struct config_set *cs, const char *key, int *dest);
 -extern int git_configset_get_pathname(struct config_set *cs, const char *key, const char **dest);
 -
 -extern int git_config_get_value(const char *key, const char **value);
 -extern const struct string_list *git_config_get_value_multi(const char *key);
 -extern void git_config_clear(void);
 -extern void git_config_iter(config_fn_t fn, void *data);
 -extern int git_config_get_string_const(const char *key, const char **dest);
 -extern int git_config_get_string(const char *key, char **dest);
 -extern int git_config_get_int(const char *key, int *dest);
 -extern int git_config_get_ulong(const char *key, unsigned long *dest);
 -extern int git_config_get_bool(const char *key, int *dest);
 -extern int git_config_get_bool_or_int(const char *key, int *is_bool, int *dest);
 -extern int git_config_get_maybe_bool(const char *key, int *dest);
 -extern int git_config_get_pathname(const char *key, const char **dest);
 -extern int git_config_get_untracked_cache(void);
 -extern int git_config_get_split_index(void);
 -extern int git_config_get_max_percent_split_change(void);
 -
 -/* This dies if the configured or default date is in the future */
 -extern int git_config_get_expiry(const char *key, const char **output);
 -
  /*
   * This is a hack for test programs like test-dump-untracked-cache to
   * ensure that they do not modify the untracked cache when reading it.
@@@ -1836,6 -2071,16 +1853,6 @@@
   */
  extern int ignore_untracked_cache_config;
  
 -struct key_value_info {
 -	const char *filename;
 -	int linenr;
 -	enum config_origin_type origin_type;
 -	enum config_scope scope;
 -};
 -
 -extern NORETURN void git_die_config(const char *key, const char *err, ...) __attribute__((format(printf, 2, 3)));
 -extern NORETURN void git_die_config_linenr(const char *key, const char *filename, int linenr);
 -
  extern int committer_ident_sufficiently_given(void);
  extern int author_ident_sufficiently_given(void);
  
@@@ -1951,8 -2196,7 +1968,8 @@@ extern int ws_blank_line(const char *li
  #define ws_tab_width(rule)     ((rule) & WS_TAB_WIDTH_MASK)
  
  /* ls-files */
 -void overlay_tree_on_cache(const char *tree_name, const char *prefix);
 +void overlay_tree_on_index(struct index_state *istate,
 +			   const char *tree_name, const char *prefix);
  
  char *alias_lookup(const char *alias);
  int split_cmdline(char *cmdline, const char ***argv);
@@@ -1971,8 -2215,8 +1988,8 @@@ struct commit_list
  int try_merge_command(const char *strategy, size_t xopts_nr,
  		const char **xopts, struct commit_list *common,
  		const char *head_arg, struct commit_list *remotes);
 -int checkout_fast_forward(const unsigned char *from,
 -			  const unsigned char *to,
 +int checkout_fast_forward(const struct object_id *from,
 +			  const struct object_id *to,
  			  int overwrite_ignore);
  
  
diff --combined sha1_file.c
index fb1fd809dc,77050a3801..9a9f7f7bcc
--- a/sha1_file.c
+++ b/sha1_file.c
@@@ -7,7 -7,6 +7,7 @@@
   * creation etc.
   */
  #include "cache.h"
 +#include "config.h"
  #include "string-list.h"
  #include "lockfile.h"
  #include "delta.h"
@@@ -611,7 -610,8 +611,7 @@@ char *compute_alternate_path(const cha
  
  out:
  	if (seen_error) {
 -		free(ref_git);
 -		ref_git = NULL;
 +		FREE_AND_NULL(ref_git);
  	}
  
  	return ref_git;
@@@ -3546,7 -3546,7 +3546,7 @@@ static int index_mem(unsigned char *sha
  	 */
  	if ((type == OBJ_BLOB) && path) {
  		struct strbuf nbuf = STRBUF_INIT;
 -		if (convert_to_git(path, buf, size, &nbuf,
 +		if (convert_to_git(&the_index, path, buf, size, &nbuf,
  				   write_object ? safe_crlf : SAFE_CRLF_FALSE)) {
  			buf = strbuf_detach(&nbuf, &size);
  			re_allocated = 1;
@@@ -3580,7 -3580,7 +3580,7 @@@ static int index_stream_convert_blob(un
  	assert(path);
  	assert(would_convert_to_git_filter_fd(path));
  
 -	convert_to_git_filter_fd(path, fd, &sbuf,
 +	convert_to_git_filter_fd(&the_index, path, fd, &sbuf,
  				 write_object ? safe_crlf : SAFE_CRLF_FALSE);
  
  	if (write_object)
@@@ -3668,7 -3668,7 +3668,7 @@@ int index_fd(unsigned char *sha1, int f
  	else if (!S_ISREG(st->st_mode))
  		ret = index_pipe(sha1, fd, type, path, flags);
  	else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
 -		 (path && would_convert_to_git(path)))
 +		 (path && would_convert_to_git(&the_index, path)))
  		ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
  				 flags);
  	else
@@@ -3735,22 -3735,32 +3735,32 @@@ void assert_sha1_type(const unsigned ch
  		    typename(expect));
  }
  
- static int for_each_file_in_obj_subdir(int subdir_nr,
- 				       struct strbuf *path,
- 				       each_loose_object_fn obj_cb,
- 				       each_loose_cruft_fn cruft_cb,
- 				       each_loose_subdir_fn subdir_cb,
- 				       void *data)
- {
- 	size_t baselen = path->len;
- 	DIR *dir = opendir(path->buf);
+ int for_each_file_in_obj_subdir(unsigned int subdir_nr,
+ 				struct strbuf *path,
+ 				each_loose_object_fn obj_cb,
+ 				each_loose_cruft_fn cruft_cb,
+ 				each_loose_subdir_fn subdir_cb,
+ 				void *data)
+ {
+ 	size_t origlen, baselen;
+ 	DIR *dir;
  	struct dirent *de;
  	int r = 0;
  
+ 	if (subdir_nr > 0xff)
+ 		BUG("invalid loose object subdirectory: %x", subdir_nr);
+ 
+ 	origlen = path->len;
+ 	strbuf_complete(path, '/');
+ 	strbuf_addf(path, "%02x", subdir_nr);
+ 	baselen = path->len;
+ 
+ 	dir = opendir(path->buf);
  	if (!dir) {
- 		if (errno == ENOENT)
- 			return 0;
- 		return error_errno("unable to open %s", path->buf);
+ 		if (errno != ENOENT)
+ 			r = error_errno("unable to open %s", path->buf);
+ 		strbuf_setlen(path, origlen);
+ 		return r;
  	}
  
  	while ((de = readdir(dir))) {
@@@ -3788,6 -3798,8 +3798,8 @@@
  	if (!r && subdir_cb)
  		r = subdir_cb(subdir_nr, path->buf, data);
  
+ 	strbuf_setlen(path, origlen);
+ 
  	return r;
  }
  
@@@ -3797,15 -3809,12 +3809,12 @@@ int for_each_loose_file_in_objdir_buf(s
  			    each_loose_subdir_fn subdir_cb,
  			    void *data)
  {
- 	size_t baselen = path->len;
  	int r = 0;
  	int i;
  
  	for (i = 0; i < 256; i++) {
- 		strbuf_addf(path, "/%02x", i);
  		r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
  						subdir_cb, data);
- 		strbuf_setlen(path, baselen);
  		if (r)
  			break;
  	}
diff --combined sha1_name.c
index d2d732c19b,8de0e2d3b4..e7f7b12ceb
--- a/sha1_name.c
+++ b/sha1_name.c
@@@ -1,5 -1,4 +1,5 @@@
  #include "cache.h"
 +#include "config.h"
  #include "tag.h"
  #include "commit.h"
  #include "tree.h"
@@@ -78,10 -77,19 +78,19 @@@ static void update_candidates(struct di
  	/* otherwise, current can be discarded and candidate is still good */
  }
  
+ static int append_loose_object(const struct object_id *oid, const char *path,
+ 			       void *data)
+ {
+ 	oid_array_append(data, oid);
+ 	return 0;
+ }
+ 
+ static int match_sha(unsigned, const unsigned char *, const unsigned char *);
+ 
  static void find_short_object_filename(struct disambiguate_state *ds)
  {
+ 	int subdir_nr = ds->bin_pfx.hash[0];
  	struct alternate_object_database *alt;
- 	char hex[GIT_MAX_HEXSZ];
  	static struct alternate_object_database *fakeent;
  
  	if (!fakeent) {
@@@ -96,29 -104,29 +105,29 @@@
  	}
  	fakeent->next = alt_odb_list;
  
- 	xsnprintf(hex, sizeof(hex), "%.2s", ds->hex_pfx);
  	for (alt = fakeent; alt && !ds->ambiguous; alt = alt->next) {
- 		struct strbuf *buf = alt_scratch_buf(alt);
- 		struct dirent *de;
- 		DIR *dir;
- 
- 		strbuf_addf(buf, "%.2s/", ds->hex_pfx);
- 		dir = opendir(buf->buf);
- 		if (!dir)
- 			continue;
- 
- 		while (!ds->ambiguous && (de = readdir(dir)) != NULL) {
- 			struct object_id oid;
+ 		int pos;
+ 
+ 		if (!alt->loose_objects_subdir_seen[subdir_nr]) {
+ 			struct strbuf *buf = alt_scratch_buf(alt);
+ 			for_each_file_in_obj_subdir(subdir_nr, buf,
+ 						    append_loose_object,
+ 						    NULL, NULL,
+ 						    &alt->loose_objects_cache);
+ 			alt->loose_objects_subdir_seen[subdir_nr] = 1;
+ 		}
  
- 			if (strlen(de->d_name) != GIT_SHA1_HEXSZ - 2)
- 				continue;
- 			if (memcmp(de->d_name, ds->hex_pfx + 2, ds->len - 2))
- 				continue;
- 			memcpy(hex + 2, de->d_name, GIT_SHA1_HEXSZ - 2);
- 			if (!get_oid_hex(hex, &oid))
- 				update_candidates(ds, &oid);
+ 		pos = oid_array_lookup(&alt->loose_objects_cache, &ds->bin_pfx);
+ 		if (pos < 0)
+ 			pos = -1 - pos;
+ 		while (!ds->ambiguous && pos < alt->loose_objects_cache.nr) {
+ 			const struct object_id *oid;
+ 			oid = alt->loose_objects_cache.oid + pos;
+ 			if (!match_sha(ds->len, ds->bin_pfx.hash, oid->hash))
+ 				break;
+ 			update_candidates(ds, oid);
+ 			pos++;
  		}
- 		closedir(dir);
  	}
  }
  
@@@ -242,7 -250,7 +251,7 @@@ static int disambiguate_committish_only
  		return 0;
  
  	/* We need to do this the hard way... */
 -	obj = deref_tag(parse_object(oid->hash), NULL, 0);
 +	obj = deref_tag(parse_object(oid), NULL, 0);
  	if (obj && obj->type == OBJ_COMMIT)
  		return 1;
  	return 0;
@@@ -266,7 -274,7 +275,7 @@@ static int disambiguate_treeish_only(co
  		return 0;
  
  	/* We need to do this the hard way... */
 -	obj = deref_tag(parse_object(oid->hash), NULL, 0);
 +	obj = deref_tag(parse_object(oid), NULL, 0);
  	if (obj && (obj->type == OBJ_TREE || obj->type == OBJ_COMMIT))
  		return 1;
  	return 0;
@@@ -355,14 -363,14 +364,14 @@@ static int show_ambiguous_object(const 
  
  	type = sha1_object_info(oid->hash, NULL);
  	if (type == OBJ_COMMIT) {
 -		struct commit *commit = lookup_commit(oid->hash);
 +		struct commit *commit = lookup_commit(oid);
  		if (commit) {
  			struct pretty_print_context pp = {0};
  			pp.date_mode.type = DATE_SHORT;
  			format_commit_message(commit, " %ad - %s", &desc, &pp);
  		}
  	} else if (type == OBJ_TAG) {
 -		struct tag *tag = lookup_tag(oid->hash);
 +		struct tag *tag = lookup_tag(oid);
  		if (!parse_tag(tag) && tag->tag)
  			strbuf_addf(&desc, " %s", tag->tag);
  	}
@@@ -661,8 -669,8 +670,8 @@@ static int get_sha1_basic(const char *s
  
  	if (reflog_len) {
  		int nth, i;
 -		unsigned long at_time;
 -		unsigned long co_time;
 +		timestamp_t at_time;
 +		timestamp_t co_time;
  		int co_tz, co_cnt;
  
  		/* Is it asking for N-th entry, or approxidate? */
@@@ -723,14 -731,14 +732,14 @@@
  static int get_parent(const char *name, int len,
  		      unsigned char *result, int idx)
  {
 -	unsigned char sha1[20];
 -	int ret = get_sha1_1(name, len, sha1, GET_SHA1_COMMITTISH);
 +	struct object_id oid;
 +	int ret = get_sha1_1(name, len, oid.hash, GET_SHA1_COMMITTISH);
  	struct commit *commit;
  	struct commit_list *p;
  
  	if (ret)
  		return ret;
 -	commit = lookup_commit_reference(sha1);
 +	commit = lookup_commit_reference(&oid);
  	if (parse_commit(commit))
  		return -1;
  	if (!idx) {
@@@ -751,14 -759,14 +760,14 @@@
  static int get_nth_ancestor(const char *name, int len,
  			    unsigned char *result, int generation)
  {
 -	unsigned char sha1[20];
 +	struct object_id oid;
  	struct commit *commit;
  	int ret;
  
 -	ret = get_sha1_1(name, len, sha1, GET_SHA1_COMMITTISH);
 +	ret = get_sha1_1(name, len, oid.hash, GET_SHA1_COMMITTISH);
  	if (ret)
  		return ret;
 -	commit = lookup_commit_reference(sha1);
 +	commit = lookup_commit_reference(&oid);
  	if (!commit)
  		return -1;
  
@@@ -777,7 -785,7 +786,7 @@@ struct object *peel_to_type(const char 
  	if (name && !namelen)
  		namelen = strlen(name);
  	while (1) {
 -		if (!o || (!o->parsed && !parse_object(o->oid.hash)))
 +		if (!o || (!o->parsed && !parse_object(&o->oid)))
  			return NULL;
  		if (expected_type == OBJ_ANY || o->type == expected_type)
  			return o;
@@@ -799,7 -807,7 +808,7 @@@
  static int peel_onion(const char *name, int len, unsigned char *sha1,
  		      unsigned lookup_flags)
  {
 -	unsigned char outer[20];
 +	struct object_id outer;
  	const char *sp;
  	unsigned int expected_type = 0;
  	struct object *o;
@@@ -847,15 -855,15 +856,15 @@@
  	else if (expected_type == OBJ_TREE)
  		lookup_flags |= GET_SHA1_TREEISH;
  
 -	if (get_sha1_1(name, sp - name - 2, outer, lookup_flags))
 +	if (get_sha1_1(name, sp - name - 2, outer.hash, lookup_flags))
  		return -1;
  
 -	o = parse_object(outer);
 +	o = parse_object(&outer);
  	if (!o)
  		return -1;
  	if (!expected_type) {
  		o = deref_tag(o, name, sp - name - 2);
 -		if (!o || (!o->parsed && !parse_object(o->oid.hash)))
 +		if (!o || (!o->parsed && !parse_object(&o->oid)))
  			return -1;
  		hashcpy(sha1, o->oid.hash);
  		return 0;
@@@ -982,7 -990,7 +991,7 @@@ static int handle_one_ref(const char *p
  			  int flag, void *cb_data)
  {
  	struct commit_list **list = cb_data;
 -	struct object *object = parse_object(oid->hash);
 +	struct object *object = parse_object(oid);
  	if (!object)
  		return 0;
  	if (object->type == OBJ_TAG) {
@@@ -1028,7 -1036,7 +1037,7 @@@ static int get_sha1_oneline(const char 
  		int matches;
  
  		commit = pop_most_recent_commit(&list, ONELINE_SEEN);
 -		if (!parse_object(commit->object.oid.hash))
 +		if (!parse_object(&commit->object.oid))
  			continue;
  		buf = get_commit_buffer(commit, NULL);
  		p = strstr(buf, "\n\n");
@@@ -1055,7 -1063,7 +1064,7 @@@ struct grab_nth_branch_switch_cbdata 
  };
  
  static int grab_nth_branch_switch(struct object_id *ooid, struct object_id *noid,
 -				  const char *email, unsigned long timestamp, int tz,
 +				  const char *email, timestamp_t timestamp, int tz,
  				  const char *message, void *cb_data)
  {
  	struct grab_nth_branch_switch_cbdata *cb = cb_data;
@@@ -1137,13 -1145,13 +1146,13 @@@ int get_oid_mb(const char *name, struc
  	}
  	if (st)
  		return st;
 -	one = lookup_commit_reference_gently(oid_tmp.hash, 0);
 +	one = lookup_commit_reference_gently(&oid_tmp, 0);
  	if (!one)
  		return -1;
  
  	if (get_sha1_committish(dots[3] ? (dots + 3) : "HEAD", oid_tmp.hash))
  		return -1;
 -	two = lookup_commit_reference_gently(oid_tmp.hash, 0);
 +	two = lookup_commit_reference_gently(&oid_tmp, 0);
  	if (!two)
  		return -1;
  	mbs = get_merge_bases(one, two);
@@@ -1409,7 -1417,7 +1418,7 @@@ static void diagnose_invalid_sha1_path(
  	if (file_exists(filename))
  		die("Path '%s' exists on disk, but not in '%.*s'.",
  		    filename, object_name_len, object_name);
 -	if (errno == ENOENT || errno == ENOTDIR) {
 +	if (is_missing_file_error(errno)) {
  		char *fullname = xstrfmt("%s%s", prefix, filename);
  
  		if (!get_tree_entry(tree_sha1, fullname,
@@@ -1474,7 -1482,7 +1483,7 @@@ static void diagnose_invalid_index_path
  
  	if (file_exists(filename))
  		die("Path '%s' exists on disk, but not in the index.", filename);
 -	if (errno == ENOENT || errno == ENOTDIR)
 +	if (is_missing_file_error(errno))
  		die("Path '%s' does not exist (neither on disk nor in the index).",
  		    filename);
  
@@@ -1512,7 -1520,6 +1521,7 @@@ static int get_sha1_with_context_1(cons
  
  	memset(oc, 0, sizeof(*oc));
  	oc->mode = S_IFINVALID;
 +	strbuf_init(&oc->symlink_path, 0);
  	ret = get_sha1_1(name, namelen, sha1, flags);
  	if (!ret)
  		return ret;
@@@ -1551,8 -1558,7 +1560,8 @@@
  			namelen = strlen(cp);
  		}
  
 -		strlcpy(oc->path, cp, sizeof(oc->path));
 +		if (flags & GET_SHA1_RECORD_PATH)
 +			oc->path = xstrdup(cp);
  
  		if (!active_cache)
  			read_cache();
@@@ -1615,8 -1621,7 +1624,8 @@@
  				}
  			}
  			hashcpy(oc->tree, tree_sha1);
 -			strlcpy(oc->path, filename, sizeof(oc->path));
 +			if (flags & GET_SHA1_RECORD_PATH)
 +				oc->path = xstrdup(filename);
  
  			free(new_filename);
  			return ret;
@@@ -1642,9 -1647,9 +1651,9 @@@ void maybe_die_on_misspelt_object_name(
  	get_sha1_with_context_1(name, GET_SHA1_ONLY_TO_DIE, prefix, sha1, &oc);
  }
  
 -int get_sha1_with_context(const char *str, unsigned flags, unsigned char *sha1, struct object_context *orc)
 +int get_sha1_with_context(const char *str, unsigned flags, unsigned char *sha1, struct object_context *oc)
  {
  	if (flags & GET_SHA1_FOLLOW_SYMLINKS && flags & GET_SHA1_ONLY_TO_DIE)
  		die("BUG: incompatible flags for get_sha1_with_context");
 -	return get_sha1_with_context_1(str, flags, NULL, sha1, orc);
 +	return get_sha1_with_context_1(str, flags, NULL, sha1, oc);
  }