From: Junio C Hamano <gitster@pobox.com>
Date: Wed, 8 Oct 2014 20:05:32 +0000 (-0700)
Subject: Merge branch 'sp/stream-clean-filter'
X-Git-Tag: v2.2.0-rc0~61
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/f0d89001750a27c1c447b2eb3149b998521fa52c?ds=inline;hp=-c

Merge branch 'sp/stream-clean-filter'

When running a required clean filter, we do not have to mmap the
original before feeding the filter.  Instead, stream the file
contents directly to the filter and process its output.

* sp/stream-clean-filter:
sha1_file: don't convert off_t to size_t too early to avoid potential die()
convert: stream from fd to required clean filter to reduce used address space
copy_fd(): do not close the input file descriptor
mmap_limit: introduce GIT_MMAP_LIMIT to allow testing expected mmap size
memory_limit: use git_env_ulong() to parse GIT_ALLOC_LIMIT
config.c: add git_env_ulong() to parse environment variable
convert: drop arguments other than 'path' from would_convert_to_git()
---

f0d89001750a27c1c447b2eb3149b998521fa52c
diff --combined cache.h
index 8206039cfe,b820b6adc1..3e6a914dba
--- a/cache.h
+++ b/cache.h
@@@ -8,7 -8,6 +8,7 @@@
  #include "gettext.h"
  #include "convert.h"
  #include "trace.h"
 +#include "string-list.h"
  
  #include SHA1_HEADER
  #ifndef git_SHA_CTX
@@@ -482,7 -481,7 +482,7 @@@ extern int daemonize(void)
  				alloc = (nr); \
  			else \
  				alloc = alloc_nr(alloc); \
 -			x = xrealloc((x), alloc * sizeof(*(x))); \
 +			REALLOC_ARRAY(x, alloc); \
  		} \
  	} while (0)
  
@@@ -586,7 -585,6 +586,7 @@@ extern NORETURN void unable_to_lock_ind
  extern int hold_lock_file_for_update(struct lock_file *, const char *path, int);
  extern int hold_lock_file_for_append(struct lock_file *, const char *path, int);
  extern int commit_lock_file(struct lock_file *);
 +extern int reopen_lock_file(struct lock_file *);
  extern void update_index_if_able(struct index_state *, struct lock_file *);
  
  extern int hold_locked_index(struct lock_file *, int);
@@@ -1039,7 -1037,6 +1039,7 @@@ enum date_mode 
  	DATE_SHORT,
  	DATE_LOCAL,
  	DATE_ISO8601,
 +	DATE_ISO8601_STRICT,
  	DATE_RFC2822,
  	DATE_RAW
  };
@@@ -1047,10 -1044,10 +1047,10 @@@
  const char *show_date(unsigned long time, int timezone, enum date_mode mode);
  void show_date_relative(unsigned long time, int tz, const struct timeval *now,
  			struct strbuf *timebuf);
 -int parse_date(const char *date, char *buf, int bufsize);
 +int parse_date(const char *date, struct strbuf *out);
  int parse_date_basic(const char *date, unsigned long *timestamp, int *offset);
  int parse_expiry_date(const char *date, unsigned long *timestamp);
 -void datestamp(char *buf, int bufsize);
 +void datestamp(struct strbuf *out);
  #define approxidate(s) approxidate_careful((s), NULL)
  unsigned long approxidate_careful(const char *, int *);
  unsigned long approxidate_relative(const char *date, const struct timeval *now);
@@@ -1064,7 -1061,6 +1064,7 @@@ extern const char *git_author_info(int)
  extern const char *git_committer_info(int);
  extern const char *fmt_ident(const char *name, const char *email, const char *date_str, int);
  extern const char *fmt_name(const char *name, const char *email);
 +extern const char *ident_default_name(void);
  extern const char *ident_default_email(void);
  extern const char *git_editor(void);
  extern const char *git_pager(int stdout_is_tty);
@@@ -1285,8 -1281,6 +1285,8 @@@ extern int update_server_info(int)
  #define CONFIG_INVALID_PATTERN 6
  #define CONFIG_GENERIC_ERROR 7
  
 +#define CONFIG_REGEX_NONE ((void *)1)
 +
  struct git_config_source {
  	unsigned int use_stdin:1;
  	const char *file;
@@@ -1300,7 -1294,7 +1300,7 @@@ extern int git_config_from_buf(config_f
  			       const char *buf, size_t len, void *data);
  extern void git_config_push_parameter(const char *text);
  extern int git_config_from_parameters(config_fn_t fn, void *data);
 -extern int git_config(config_fn_t fn, void *);
 +extern void git_config(config_fn_t fn, void *);
  extern int git_config_with_options(config_fn_t fn, void *,
  				   struct git_config_source *config_source,
  				   int respect_includes);
@@@ -1324,6 -1318,7 +1324,7 @@@ extern int git_config_rename_section_in
  extern const char *git_etc_gitconfig(void);
  extern int check_repository_format_version(const char *var, const char *value, void *cb);
  extern int git_env_bool(const char *, int);
+ extern unsigned long git_env_ulong(const char *, unsigned long);
  extern int git_config_system(void);
  extern int config_error_nonbool(const char *);
  #if defined(__GNUC__)
@@@ -1357,69 -1352,6 +1358,69 @@@ extern int parse_config_key(const char 
  			    const char **subsection, int *subsection_len,
  			    const char **key);
  
 +struct config_set_element {
 +	struct hashmap_entry ent;
 +	char *key;
 +	struct string_list value_list;
 +};
 +
 +struct configset_list_item {
 +	struct config_set_element *e;
 +	int value_index;
 +};
 +
 +/*
 + * the contents of the list are ordered according to their
 + * position in the config files and order of parsing the files.
 + * (i.e. key-value pair at the last position of .git/config will
 + * be at the last item of the list)
 + */
 +struct configset_list {
 +	struct configset_list_item *items;
 +	unsigned int nr, alloc;
 +};
 +
 +struct config_set {
 +	struct hashmap config_hash;
 +	int hash_initialized;
 +	struct configset_list list;
 +};
 +
 +extern void git_configset_init(struct config_set *cs);
 +extern int git_configset_add_file(struct config_set *cs, const char *filename);
 +extern int git_configset_get_value(struct config_set *cs, const char *key, const char **value);
 +extern const struct string_list *git_configset_get_value_multi(struct config_set *cs, const char *key);
 +extern void git_configset_clear(struct config_set *cs);
 +extern int git_configset_get_string_const(struct config_set *cs, const char *key, const char **dest);
 +extern int git_configset_get_string(struct config_set *cs, const char *key, char **dest);
 +extern int git_configset_get_int(struct config_set *cs, const char *key, int *dest);
 +extern int git_configset_get_ulong(struct config_set *cs, const char *key, unsigned long *dest);
 +extern int git_configset_get_bool(struct config_set *cs, const char *key, int *dest);
 +extern int git_configset_get_bool_or_int(struct config_set *cs, const char *key, int *is_bool, int *dest);
 +extern int git_configset_get_maybe_bool(struct config_set *cs, const char *key, int *dest);
 +extern int git_configset_get_pathname(struct config_set *cs, const char *key, const char **dest);
 +
 +extern int git_config_get_value(const char *key, const char **value);
 +extern const struct string_list *git_config_get_value_multi(const char *key);
 +extern void git_config_clear(void);
 +extern void git_config_iter(config_fn_t fn, void *data);
 +extern int git_config_get_string_const(const char *key, const char **dest);
 +extern int git_config_get_string(const char *key, char **dest);
 +extern int git_config_get_int(const char *key, int *dest);
 +extern int git_config_get_ulong(const char *key, unsigned long *dest);
 +extern int git_config_get_bool(const char *key, int *dest);
 +extern int git_config_get_bool_or_int(const char *key, int *is_bool, int *dest);
 +extern int git_config_get_maybe_bool(const char *key, int *dest);
 +extern int git_config_get_pathname(const char *key, const char **dest);
 +
 +struct key_value_info {
 +	const char *filename;
 +	int linenr;
 +};
 +
 +extern NORETURN void git_die_config(const char *key, const char *err, ...) __attribute__((format(printf, 2, 3)));
 +extern NORETURN void git_die_config_linenr(const char *key, const char *filename, int linenr);
 +
  extern int committer_ident_sufficiently_given(void);
  extern int author_ident_sufficiently_given(void);
  
@@@ -1430,8 -1362,6 +1431,8 @@@ extern const char *git_mailmap_blob
  
  /* IO helper functions */
  extern void maybe_flush_or_die(FILE *, const char *);
 +__attribute__((format (printf, 2, 3)))
 +extern void fprintf_or_die(FILE *, const char *fmt, ...);
  extern int copy_fd(int ifd, int ofd);
  extern int copy_file(const char *dst, const char *src, int mode);
  extern int copy_file_with_time(const char *dst, const char *src, int mode);
diff --combined config.c
index a677eb6c59,adab91909e..039647d247
--- a/config.c
+++ b/config.c
@@@ -9,8 -9,6 +9,8 @@@
  #include "exec_cmd.h"
  #include "strbuf.h"
  #include "quote.h"
 +#include "hashmap.h"
 +#include "string-list.h"
  
  struct config_source {
  	struct config_source *prev;
@@@ -39,13 -37,6 +39,13 @@@ static struct config_source *cf
  
  static int zlib_compression_seen;
  
 +/*
 + * Default config_set that contains key-value pairs from the usual set of config
 + * config files (i.e repo specific .git/config, user wide ~/.gitconfig, XDG
 + * config file and the global /etc/gitconfig)
 + */
 +static struct config_set the_config_set;
 +
  static int config_file_fgetc(struct config_source *conf)
  {
  	return fgetc(conf->u.file);
@@@ -136,6 -127,7 +136,6 @@@ static int handle_path_include(const ch
  int git_config_include(const char *var, const char *value, void *data)
  {
  	struct config_include_data *inc = data;
 -	const char *type;
  	int ret;
  
  	/*
@@@ -146,7 -138,10 +146,7 @@@
  	if (ret < 0)
  		return ret;
  
 -	if (!skip_prefix(var, "include.", &type))
 -		return ret;
 -
 -	if (!strcmp(type, "path"))
 +	if (!strcmp(var, "include.path"))
  		ret = handle_path_include(value, inc);
  	return ret;
  }
@@@ -167,27 -162,19 +167,27 @@@ void git_config_push_parameter(const ch
  int git_config_parse_parameter(const char *text,
  			       config_fn_t fn, void *data)
  {
 +	const char *value;
  	struct strbuf **pair;
 +
  	pair = strbuf_split_str(text, '=', 2);
  	if (!pair[0])
  		return error("bogus config parameter: %s", text);
 -	if (pair[0]->len && pair[0]->buf[pair[0]->len - 1] == '=')
 +
 +	if (pair[0]->len && pair[0]->buf[pair[0]->len - 1] == '=') {
  		strbuf_setlen(pair[0], pair[0]->len - 1);
 +		value = pair[1] ? pair[1]->buf : "";
 +	} else {
 +		value = NULL;
 +	}
 +
  	strbuf_trim(pair[0]);
  	if (!pair[0]->len) {
  		strbuf_list_free(pair);
  		return error("bogus config parameter: %s", text);
  	}
  	strbuf_tolower(pair[0]);
 -	if (fn(pair[0]->buf, pair[1] ? pair[1]->buf : NULL, data) < 0) {
 +	if (fn(pair[0]->buf, value, data) < 0) {
  		strbuf_list_free(pair);
  		return -1;
  	}
@@@ -242,7 -229,6 +242,7 @@@ static int get_next_char(void
  		cf->linenr++;
  	if (c == EOF) {
  		cf->eof = 1;
 +		cf->linenr++;
  		c = '\n';
  	}
  	return c;
@@@ -318,7 -304,6 +318,7 @@@ static int get_value(config_fn_t fn, vo
  {
  	int c;
  	char *value;
 +	int ret;
  
  	/* Get the full name */
  	for (;;) {
@@@ -341,15 -326,7 +341,15 @@@
  		if (!value)
  			return -1;
  	}
 -	return fn(name->buf, value, data);
 +	/*
 +	 * We already consumed the \n, but we need linenr to point to
 +	 * the line we just parsed during the call to fn to get
 +	 * accurate line number in error messages.
 +	 */
 +	cf->linenr--;
 +	ret = fn(name->buf, value, data);
 +	cf->linenr++;
 +	return ret;
  }
  
  static int get_extended_base_var(struct strbuf *name, int c)
@@@ -465,9 -442,9 +465,9 @@@ static int git_parse_source(config_fn_
  			break;
  	}
  	if (cf->die_on_error)
 -		die("bad config file line %d in %s", cf->linenr, cf->name);
 +		die(_("bad config file line %d in %s"), cf->linenr, cf->name);
  	else
 -		return error("bad config file line %d in %s", cf->linenr, cf->name);
 +		return error(_("bad config file line %d in %s"), cf->linenr, cf->name);
  }
  
  static int parse_unit_factor(const char *end, uintmax_t *val)
@@@ -583,9 -560,9 +583,9 @@@ static void die_bad_number(const char *
  		value = "";
  
  	if (cf && cf->name)
 -		die("bad numeric config value '%s' for '%s' in %s: %s",
 +		die(_("bad numeric config value '%s' for '%s' in %s: %s"),
  		    value, name, cf->name, reason);
 -	die("bad numeric config value '%s' for '%s': %s", value, name, reason);
 +	die(_("bad numeric config value '%s' for '%s': %s"), value, name, reason);
  }
  
  int git_config_int(const char *name, const char *value)
@@@ -670,7 -647,7 +670,7 @@@ int git_config_pathname(const char **de
  		return config_error_nonbool(var);
  	*dest = expand_user_path(value);
  	if (!*dest)
 -		die("Failed to expand user dir in: '%s'", value);
 +		die(_("failed to expand user dir in: '%s'"), value);
  	return 0;
  }
  
@@@ -748,7 -725,7 +748,7 @@@ static int git_default_core_config(cons
  		if (level == -1)
  			level = Z_DEFAULT_COMPRESSION;
  		else if (level < 0 || level > Z_BEST_COMPRESSION)
 -			die("bad zlib compression level %d", level);
 +			die(_("bad zlib compression level %d"), level);
  		zlib_compression_level = level;
  		zlib_compression_seen = 1;
  		return 0;
@@@ -759,7 -736,7 +759,7 @@@
  		if (level == -1)
  			level = Z_DEFAULT_COMPRESSION;
  		else if (level < 0 || level > Z_BEST_COMPRESSION)
 -			die("bad zlib compression level %d", level);
 +			die(_("bad zlib compression level %d"), level);
  		core_compression_level = level;
  		core_compression_seen = 1;
  		if (!zlib_compression_seen)
@@@ -881,7 -858,7 +881,7 @@@
  		else if (!strcmp(value, "link"))
  			object_creation_mode = OBJECT_CREATION_USES_HARDLINKS;
  		else
 -			die("Invalid mode for object creation: %s", value);
 +			die(_("invalid mode for object creation: %s"), value);
  		return 0;
  	}
  
@@@ -1139,12 -1116,28 +1139,28 @@@ const char *git_etc_gitconfig(void
  	return system_wide;
  }
  
+ /*
+  * Parse environment variable 'k' as a boolean (in various
+  * possible spellings); if missing, use the default value 'def'.
+  */
  int git_env_bool(const char *k, int def)
  {
  	const char *v = getenv(k);
  	return v ? git_config_bool(k, v) : def;
  }
  
+ /*
+  * Parse environment variable 'k' as ulong with possibly a unit
+  * suffix; if missing, use the default value 'val'.
+  */
+ unsigned long git_env_ulong(const char *k, unsigned long val)
+ {
+ 	const char *v = getenv(k);
+ 	if (v && !git_parse_ulong(v, &val))
+ 		die("failed to parse %s", k);
+ 	return val;
+ }
+ 
  int git_config_system(void)
  {
  	return !git_env_bool("GIT_CONFIG_NOSYSTEM", 0);
@@@ -1181,7 -1174,7 +1197,7 @@@ int git_config_early(config_fn_t fn, vo
  
  	switch (git_config_from_parameters(fn, data)) {
  	case -1: /* error */
 -		die("unable to parse command-line config");
 +		die(_("unable to parse command-line config"));
  		break;
  	case 0: /* found nothing */
  		break;
@@@ -1228,365 -1221,9 +1244,365 @@@ int git_config_with_options(config_fn_
  	return ret;
  }
  
 -int git_config(config_fn_t fn, void *data)
 +static void git_config_raw(config_fn_t fn, void *data)
 +{
 +	if (git_config_with_options(fn, data, NULL, 1) < 0)
 +		/*
 +		 * git_config_with_options() normally returns only
 +		 * positive values, as most errors are fatal, and
 +		 * non-fatal potential errors are guarded by "if"
 +		 * statements that are entered only when no error is
 +		 * possible.
 +		 *
 +		 * If we ever encounter a non-fatal error, it means
 +		 * something went really wrong and we should stop
 +		 * immediately.
 +		 */
 +		die(_("unknown error occured while reading the configuration files"));
 +}
 +
 +static void configset_iter(struct config_set *cs, config_fn_t fn, void *data)
 +{
 +	int i, value_index;
 +	struct string_list *values;
 +	struct config_set_element *entry;
 +	struct configset_list *list = &cs->list;
 +	struct key_value_info *kv_info;
 +
 +	for (i = 0; i < list->nr; i++) {
 +		entry = list->items[i].e;
 +		value_index = list->items[i].value_index;
 +		values = &entry->value_list;
 +		if (fn(entry->key, values->items[value_index].string, data) < 0) {
 +			kv_info = values->items[value_index].util;
 +			git_die_config_linenr(entry->key, kv_info->filename, kv_info->linenr);
 +		}
 +	}
 +}
 +
 +static void git_config_check_init(void);
 +
 +void git_config(config_fn_t fn, void *data)
 +{
 +	git_config_check_init();
 +	configset_iter(&the_config_set, fn, data);
 +}
 +
 +static struct config_set_element *configset_find_element(struct config_set *cs, const char *key)
 +{
 +	struct config_set_element k;
 +	struct config_set_element *found_entry;
 +	char *normalized_key;
 +	int ret;
 +	/*
 +	 * `key` may come from the user, so normalize it before using it
 +	 * for querying entries from the hashmap.
 +	 */
 +	ret = git_config_parse_key(key, &normalized_key, NULL);
 +
 +	if (ret)
 +		return NULL;
 +
 +	hashmap_entry_init(&k, strhash(normalized_key));
 +	k.key = normalized_key;
 +	found_entry = hashmap_get(&cs->config_hash, &k, NULL);
 +	free(normalized_key);
 +	return found_entry;
 +}
 +
 +static int configset_add_value(struct config_set *cs, const char *key, const char *value)
 +{
 +	struct config_set_element *e;
 +	struct string_list_item *si;
 +	struct configset_list_item *l_item;
 +	struct key_value_info *kv_info = xmalloc(sizeof(*kv_info));
 +
 +	e = configset_find_element(cs, key);
 +	/*
 +	 * Since the keys are being fed by git_config*() callback mechanism, they
 +	 * are already normalized. So simply add them without any further munging.
 +	 */
 +	if (!e) {
 +		e = xmalloc(sizeof(*e));
 +		hashmap_entry_init(e, strhash(key));
 +		e->key = xstrdup(key);
 +		string_list_init(&e->value_list, 1);
 +		hashmap_add(&cs->config_hash, e);
 +	}
 +	si = string_list_append_nodup(&e->value_list, value ? xstrdup(value) : NULL);
 +
 +	ALLOC_GROW(cs->list.items, cs->list.nr + 1, cs->list.alloc);
 +	l_item = &cs->list.items[cs->list.nr++];
 +	l_item->e = e;
 +	l_item->value_index = e->value_list.nr - 1;
 +
 +	if (cf) {
 +		kv_info->filename = strintern(cf->name);
 +		kv_info->linenr = cf->linenr;
 +	} else {
 +		/* for values read from `git_config_from_parameters()` */
 +		kv_info->filename = NULL;
 +		kv_info->linenr = -1;
 +	}
 +	si->util = kv_info;
 +
 +	return 0;
 +}
 +
 +static int config_set_element_cmp(const struct config_set_element *e1,
 +				 const struct config_set_element *e2, const void *unused)
 +{
 +	return strcmp(e1->key, e2->key);
 +}
 +
 +void git_configset_init(struct config_set *cs)
 +{
 +	hashmap_init(&cs->config_hash, (hashmap_cmp_fn)config_set_element_cmp, 0);
 +	cs->hash_initialized = 1;
 +	cs->list.nr = 0;
 +	cs->list.alloc = 0;
 +	cs->list.items = NULL;
 +}
 +
 +void git_configset_clear(struct config_set *cs)
 +{
 +	struct config_set_element *entry;
 +	struct hashmap_iter iter;
 +	if (!cs->hash_initialized)
 +		return;
 +
 +	hashmap_iter_init(&cs->config_hash, &iter);
 +	while ((entry = hashmap_iter_next(&iter))) {
 +		free(entry->key);
 +		string_list_clear(&entry->value_list, 1);
 +	}
 +	hashmap_free(&cs->config_hash, 1);
 +	cs->hash_initialized = 0;
 +	free(cs->list.items);
 +	cs->list.nr = 0;
 +	cs->list.alloc = 0;
 +	cs->list.items = NULL;
 +}
 +
 +static int config_set_callback(const char *key, const char *value, void *cb)
 +{
 +	struct config_set *cs = cb;
 +	configset_add_value(cs, key, value);
 +	return 0;
 +}
 +
 +int git_configset_add_file(struct config_set *cs, const char *filename)
 +{
 +	return git_config_from_file(config_set_callback, filename, cs);
 +}
 +
 +int git_configset_get_value(struct config_set *cs, const char *key, const char **value)
 +{
 +	const struct string_list *values = NULL;
 +	/*
 +	 * Follows "last one wins" semantic, i.e., if there are multiple matches for the
 +	 * queried key in the files of the configset, the value returned will be the last
 +	 * value in the value list for that key.
 +	 */
 +	values = git_configset_get_value_multi(cs, key);
 +
 +	if (!values)
 +		return 1;
 +	assert(values->nr > 0);
 +	*value = values->items[values->nr - 1].string;
 +	return 0;
 +}
 +
 +const struct string_list *git_configset_get_value_multi(struct config_set *cs, const char *key)
 +{
 +	struct config_set_element *e = configset_find_element(cs, key);
 +	return e ? &e->value_list : NULL;
 +}
 +
 +int git_configset_get_string_const(struct config_set *cs, const char *key, const char **dest)
 +{
 +	const char *value;
 +	if (!git_configset_get_value(cs, key, &value))
 +		return git_config_string(dest, key, value);
 +	else
 +		return 1;
 +}
 +
 +int git_configset_get_string(struct config_set *cs, const char *key, char **dest)
 +{
 +	return git_configset_get_string_const(cs, key, (const char **)dest);
 +}
 +
 +int git_configset_get_int(struct config_set *cs, const char *key, int *dest)
 +{
 +	const char *value;
 +	if (!git_configset_get_value(cs, key, &value)) {
 +		*dest = git_config_int(key, value);
 +		return 0;
 +	} else
 +		return 1;
 +}
 +
 +int git_configset_get_ulong(struct config_set *cs, const char *key, unsigned long *dest)
 +{
 +	const char *value;
 +	if (!git_configset_get_value(cs, key, &value)) {
 +		*dest = git_config_ulong(key, value);
 +		return 0;
 +	} else
 +		return 1;
 +}
 +
 +int git_configset_get_bool(struct config_set *cs, const char *key, int *dest)
 +{
 +	const char *value;
 +	if (!git_configset_get_value(cs, key, &value)) {
 +		*dest = git_config_bool(key, value);
 +		return 0;
 +	} else
 +		return 1;
 +}
 +
 +int git_configset_get_bool_or_int(struct config_set *cs, const char *key,
 +				int *is_bool, int *dest)
 +{
 +	const char *value;
 +	if (!git_configset_get_value(cs, key, &value)) {
 +		*dest = git_config_bool_or_int(key, value, is_bool);
 +		return 0;
 +	} else
 +		return 1;
 +}
 +
 +int git_configset_get_maybe_bool(struct config_set *cs, const char *key, int *dest)
 +{
 +	const char *value;
 +	if (!git_configset_get_value(cs, key, &value)) {
 +		*dest = git_config_maybe_bool(key, value);
 +		if (*dest == -1)
 +			return -1;
 +		return 0;
 +	} else
 +		return 1;
 +}
 +
 +int git_configset_get_pathname(struct config_set *cs, const char *key, const char **dest)
 +{
 +	const char *value;
 +	if (!git_configset_get_value(cs, key, &value))
 +		return git_config_pathname(dest, key, value);
 +	else
 +		return 1;
 +}
 +
 +static void git_config_check_init(void)
 +{
 +	if (the_config_set.hash_initialized)
 +		return;
 +	git_configset_init(&the_config_set);
 +	git_config_raw(config_set_callback, &the_config_set);
 +}
 +
 +void git_config_clear(void)
 +{
 +	if (!the_config_set.hash_initialized)
 +		return;
 +	git_configset_clear(&the_config_set);
 +}
 +
 +int git_config_get_value(const char *key, const char **value)
 +{
 +	git_config_check_init();
 +	return git_configset_get_value(&the_config_set, key, value);
 +}
 +
 +const struct string_list *git_config_get_value_multi(const char *key)
 +{
 +	git_config_check_init();
 +	return git_configset_get_value_multi(&the_config_set, key);
 +}
 +
 +int git_config_get_string_const(const char *key, const char **dest)
 +{
 +	int ret;
 +	git_config_check_init();
 +	ret = git_configset_get_string_const(&the_config_set, key, dest);
 +	if (ret < 0)
 +		git_die_config(key, NULL);
 +	return ret;
 +}
 +
 +int git_config_get_string(const char *key, char **dest)
 +{
 +	git_config_check_init();
 +	return git_config_get_string_const(key, (const char **)dest);
 +}
 +
 +int git_config_get_int(const char *key, int *dest)
 +{
 +	git_config_check_init();
 +	return git_configset_get_int(&the_config_set, key, dest);
 +}
 +
 +int git_config_get_ulong(const char *key, unsigned long *dest)
 +{
 +	git_config_check_init();
 +	return git_configset_get_ulong(&the_config_set, key, dest);
 +}
 +
 +int git_config_get_bool(const char *key, int *dest)
 +{
 +	git_config_check_init();
 +	return git_configset_get_bool(&the_config_set, key, dest);
 +}
 +
 +int git_config_get_bool_or_int(const char *key, int *is_bool, int *dest)
  {
 -	return git_config_with_options(fn, data, NULL, 1);
 +	git_config_check_init();
 +	return git_configset_get_bool_or_int(&the_config_set, key, is_bool, dest);
 +}
 +
 +int git_config_get_maybe_bool(const char *key, int *dest)
 +{
 +	git_config_check_init();
 +	return git_configset_get_maybe_bool(&the_config_set, key, dest);
 +}
 +
 +int git_config_get_pathname(const char *key, const char **dest)
 +{
 +	int ret;
 +	git_config_check_init();
 +	ret = git_configset_get_pathname(&the_config_set, key, dest);
 +	if (ret < 0)
 +		git_die_config(key, NULL);
 +	return ret;
 +}
 +
 +NORETURN
 +void git_die_config_linenr(const char *key, const char *filename, int linenr)
 +{
 +	if (!filename)
 +		die(_("unable to parse '%s' from command-line config"), key);
 +	else
 +		die(_("bad config variable '%s' in file '%s' at line %d"),
 +		    key, filename, linenr);
 +}
 +
 +NORETURN __attribute__((format(printf, 2, 3)))
 +void git_die_config(const char *key, const char *err, ...)
 +{
 +	const struct string_list *values;
 +	struct key_value_info *kv_info;
 +
 +	if (err) {
 +		va_list params;
 +		va_start(params, err);
 +		vreportf("error: ", err, params);
 +		va_end(params);
 +	}
 +	values = git_config_get_value_multi(key);
 +	kv_info = values->items[values->nr - 1].util;
 +	git_die_config_linenr(key, kv_info->filename, kv_info->linenr);
  }
  
  /*
@@@ -1607,15 -1244,10 +1623,15 @@@ static struct 
  
  static int matches(const char *key, const char *value)
  {
 -	return !strcmp(key, store.key) &&
 -		(store.value_regex == NULL ||
 -		 (store.do_not_match ^
 -		  !regexec(store.value_regex, value, 0, NULL, 0)));
 +	if (strcmp(key, store.key))
 +		return 0; /* not ours */
 +	if (!store.value_regex)
 +		return 1; /* always matches */
 +	if (store.value_regex == CONFIG_REGEX_NONE)
 +		return 0; /* never matches */
 +
 +	return store.do_not_match ^
 +		(value && !regexec(store.value_regex, value, 0, NULL, 0));
  }
  
  static int store_aux(const char *key, const char *value, void *cb)
@@@ -1627,7 -1259,7 +1643,7 @@@
  	case KEY_SEEN:
  		if (matches(key, value)) {
  			if (store.seen == 1 && store.multi_replace == 0) {
 -				warning("%s has multiple values", key);
 +				warning(_("%s has multiple values"), key);
  			}
  
  			ALLOC_GROW(store.offset, store.seen + 1,
@@@ -1877,8 -1509,6 +1893,8 @@@ out_free_ret_1
  /*
   * If value==NULL, unset in (remove from) config,
   * if value_regex!=NULL, disregard key/value pairs where value does not match.
 + * if value_regex==CONFIG_REGEX_NONE, do not match any existing values
 + *     (only add a new one)
   * if multi_replace==0, nothing, or only one matching key/value is replaced,
   *     else all matching key/values (regardless how many) are removed,
   *     before the new pair is written.
@@@ -1962,8 -1592,6 +1978,8 @@@ int git_config_set_multivar_in_file(con
  
  		if (value_regex == NULL)
  			store.value_regex = NULL;
 +		else if (value_regex == CONFIG_REGEX_NONE)
 +			store.value_regex = CONFIG_REGEX_NONE;
  		else {
  			if (value_regex[0] == '!') {
  				store.do_not_match = 1;
@@@ -1995,8 -1623,7 +2011,8 @@@
  		if (git_config_from_file(store_aux, config_filename, NULL)) {
  			error("invalid config file %s", config_filename);
  			free(store.key);
 -			if (store.value_regex != NULL) {
 +			if (store.value_regex != NULL &&
 +			    store.value_regex != CONFIG_REGEX_NONE) {
  				regfree(store.value_regex);
  				free(store.value_regex);
  			}
@@@ -2005,8 -1632,7 +2021,8 @@@
  		}
  
  		free(store.key);
 -		if (store.value_regex != NULL) {
 +		if (store.value_regex != NULL &&
 +		    store.value_regex != CONFIG_REGEX_NONE) {
  			regfree(store.value_regex);
  			free(store.value_regex);
  		}
@@@ -2095,9 -1721,6 +2111,9 @@@
  	lock = NULL;
  	ret = 0;
  
 +	/* Invalidate the config cache */
 +	git_config_clear();
 +
  out_free:
  	if (lock)
  		rollback_lock_file(lock);
diff --combined convert.c
index aa7a139bcf,677d339a8b..9a5612e93d
--- a/convert.c
+++ b/convert.c
@@@ -312,16 -312,17 +312,17 @@@ static int crlf_to_worktree(const char 
  struct filter_params {
  	const char *src;
  	unsigned long size;
+ 	int fd;
  	const char *cmd;
  	const char *path;
  };
  
- static int filter_buffer(int in, int out, void *data)
+ static int filter_buffer_or_fd(int in, int out, void *data)
  {
  	/*
  	 * Spawn cmd and feed the buffer contents through its stdin.
  	 */
 -	struct child_process child_process;
 +	struct child_process child_process = CHILD_PROCESS_INIT;
  	struct filter_params *params = (struct filter_params *)data;
  	int write_err, status;
  	const char *argv[] = { NULL, NULL };
@@@ -344,6 -345,7 +345,6 @@@
  
  	argv[0] = cmd.buf;
  
 -	memset(&child_process, 0, sizeof(child_process));
  	child_process.argv = argv;
  	child_process.use_shell = 1;
  	child_process.in = -1;
@@@ -354,7 -356,12 +355,12 @@@
  
  	sigchain_push(SIGPIPE, SIG_IGN);
  
- 	write_err = (write_in_full(child_process.in, params->src, params->size) < 0);
+ 	if (params->src) {
+ 		write_err = (write_in_full(child_process.in, params->src, params->size) < 0);
+ 	} else {
+ 		write_err = copy_fd(params->fd, child_process.in);
+ 	}
+ 
  	if (close(child_process.in))
  		write_err = 1;
  	if (write_err)
@@@ -370,7 -377,7 +376,7 @@@
  	return (write_err || status);
  }
  
- static int apply_filter(const char *path, const char *src, size_t len,
+ static int apply_filter(const char *path, const char *src, size_t len, int fd,
                          struct strbuf *dst, const char *cmd)
  {
  	/*
@@@ -391,11 -398,12 +397,12 @@@
  		return 1;
  
  	memset(&async, 0, sizeof(async));
- 	async.proc = filter_buffer;
+ 	async.proc = filter_buffer_or_fd;
  	async.data = &params;
  	async.out = -1;
  	params.src = src;
  	params.size = len;
+ 	params.fd = fd;
  	params.cmd = cmd;
  	params.path = path;
  
@@@ -746,6 -754,25 +753,25 @@@ static void convert_attrs(struct conv_a
  	}
  }
  
+ int would_convert_to_git_filter_fd(const char *path)
+ {
+ 	struct conv_attrs ca;
+ 
+ 	convert_attrs(&ca, path);
+ 	if (!ca.drv)
+ 		return 0;
+ 
+ 	/*
+ 	 * Apply a filter to an fd only if the filter is required to succeed.
+ 	 * We must die if the filter fails, because the original data before
+ 	 * filtering is not available.
+ 	 */
+ 	if (!ca.drv->required)
+ 		return 0;
+ 
+ 	return apply_filter(path, NULL, 0, -1, NULL, ca.drv->clean);
+ }
+ 
  int convert_to_git(const char *path, const char *src, size_t len,
                     struct strbuf *dst, enum safe_crlf checksafe)
  {
@@@ -760,7 -787,7 +786,7 @@@
  		required = ca.drv->required;
  	}
  
- 	ret |= apply_filter(path, src, len, dst, filter);
+ 	ret |= apply_filter(path, src, len, -1, dst, filter);
  	if (!ret && required)
  		die("%s: clean filter '%s' failed", path, ca.drv->name);
  
@@@ -777,6 -804,23 +803,23 @@@
  	return ret | ident_to_git(path, src, len, dst, ca.ident);
  }
  
+ void convert_to_git_filter_fd(const char *path, int fd, struct strbuf *dst,
+ 			      enum safe_crlf checksafe)
+ {
+ 	struct conv_attrs ca;
+ 	convert_attrs(&ca, path);
+ 
+ 	assert(ca.drv);
+ 	assert(ca.drv->clean);
+ 
+ 	if (!apply_filter(path, NULL, 0, fd, dst, ca.drv->clean))
+ 		die("%s: clean filter '%s' failed", path, ca.drv->name);
+ 
+ 	ca.crlf_action = input_crlf_action(ca.crlf_action, ca.eol_attr);
+ 	crlf_to_git(path, dst->buf, dst->len, dst, ca.crlf_action, checksafe);
+ 	ident_to_git(path, dst->buf, dst->len, dst, ca.ident);
+ }
+ 
  static int convert_to_working_tree_internal(const char *path, const char *src,
  					    size_t len, struct strbuf *dst,
  					    int normalizing)
@@@ -810,7 -854,7 +853,7 @@@
  		}
  	}
  
- 	ret_filter = apply_filter(path, src, len, dst, filter);
+ 	ret_filter = apply_filter(path, src, len, -1, dst, filter);
  	if (!ret_filter && required)
  		die("%s: smudge filter %s failed", path, ca.drv->name);
  
diff --combined lockfile.c
index 2a800cef33,2448d30cd0..d34a96df4f
--- a/lockfile.c
+++ b/lockfile.c
@@@ -224,8 -224,11 +224,11 @@@ int hold_lock_file_for_append(struct lo
  	} else if (copy_fd(orig_fd, fd)) {
  		if (flags & LOCK_DIE_ON_ERROR)
  			exit(128);
+ 		close(orig_fd);
  		close(fd);
  		return -1;
+ 	} else {
+ 		close(orig_fd);
  	}
  	return fd;
  }
@@@ -237,16 -240,6 +240,16 @@@ int close_lock_file(struct lock_file *l
  	return close(fd);
  }
  
 +int reopen_lock_file(struct lock_file *lk)
 +{
 +	if (0 <= lk->fd)
 +		die(_("BUG: reopen a lockfile that is still open"));
 +	if (!lk->filename[0])
 +		die(_("BUG: reopen a lockfile that has been committed"));
 +	lk->fd = open(lk->filename, O_WRONLY);
 +	return lk->fd;
 +}
 +
  int commit_lock_file(struct lock_file *lk)
  {
  	char result_file[PATH_MAX];
diff --combined sha1_file.c
index c08c0cbea8,7b2612f733..6f18c22ab1
--- a/sha1_file.c
+++ b/sha1_file.c
@@@ -350,7 -350,7 +350,7 @@@ static void link_alt_odb_entries(const 
  		return;
  	}
  
 -	strbuf_addstr(&objdirbuf, absolute_path(get_object_directory()));
 +	strbuf_add_absolute_path(&objdirbuf, get_object_directory());
  	normalize_path_copy(objdirbuf.buf, objdirbuf.buf);
  
  	alt_copy = xmemdupz(alt, len);
@@@ -663,10 -663,26 +663,26 @@@ void release_pack_memory(size_t need
  		; /* nothing */
  }
  
+ static void mmap_limit_check(size_t length)
+ {
+ 	static size_t limit = 0;
+ 	if (!limit) {
+ 		limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
+ 		if (!limit)
+ 			limit = SIZE_MAX;
+ 	}
+ 	if (length > limit)
+ 		die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX,
+ 		    (uintmax_t)length, (uintmax_t)limit);
+ }
+ 
  void *xmmap(void *start, size_t length,
  	int prot, int flags, int fd, off_t offset)
  {
- 	void *ret = mmap(start, length, prot, flags, fd, offset);
+ 	void *ret;
+ 
+ 	mmap_limit_check(length);
+ 	ret = mmap(start, length, prot, flags, fd, offset);
  	if (ret == MAP_FAILED) {
  		if (!length)
  			return NULL;
@@@ -1923,9 -1939,7 +1939,9 @@@ static void *unpack_compressed_entry(st
  	git_zstream stream;
  	unsigned char *buffer, *in;
  
 -	buffer = xmallocz(size);
 +	buffer = xmallocz_gently(size);
 +	if (!buffer)
 +		return NULL;
  	memset(&stream, 0, sizeof(stream));
  	stream.next_out = buffer;
  	stream.avail_out = size + 1;
@@@ -3076,6 -3090,29 +3092,29 @@@ static int index_mem(unsigned char *sha
  	return ret;
  }
  
+ static int index_stream_convert_blob(unsigned char *sha1, int fd,
+ 				     const char *path, unsigned flags)
+ {
+ 	int ret;
+ 	const int write_object = flags & HASH_WRITE_OBJECT;
+ 	struct strbuf sbuf = STRBUF_INIT;
+ 
+ 	assert(path);
+ 	assert(would_convert_to_git_filter_fd(path));
+ 
+ 	convert_to_git_filter_fd(path, fd, &sbuf,
+ 				 write_object ? safe_crlf : SAFE_CRLF_FALSE);
+ 
+ 	if (write_object)
+ 		ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
+ 				      sha1);
+ 	else
+ 		ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
+ 				     sha1);
+ 	strbuf_release(&sbuf);
+ 	return ret;
+ }
+ 
  static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
  		      const char *path, unsigned flags)
  {
@@@ -3141,15 -3178,22 +3180,22 @@@ int index_fd(unsigned char *sha1, int f
  	     enum object_type type, const char *path, unsigned flags)
  {
  	int ret;
- 	size_t size = xsize_t(st->st_size);
  
- 	if (!S_ISREG(st->st_mode))
+ 	/*
+ 	 * Call xsize_t() only when needed to avoid potentially unnecessary
+ 	 * die() for large files.
+ 	 */
+ 	if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path))
+ 		ret = index_stream_convert_blob(sha1, fd, path, flags);
+ 	else if (!S_ISREG(st->st_mode))
  		ret = index_pipe(sha1, fd, type, path, flags);
- 	else if (size <= big_file_threshold || type != OBJ_BLOB ||
- 		 (path && would_convert_to_git(path, NULL, 0, 0)))
- 		ret = index_core(sha1, fd, size, type, path, flags);
+ 	else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
+ 		 (path && would_convert_to_git(path)))
+ 		ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
+ 				 flags);
  	else
- 		ret = index_stream(sha1, fd, size, type, path, flags);
+ 		ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path,
+ 				   flags);
  	close(fd);
  	return ret;
  }
diff --combined t/t1050-large.sh
index 05a1e1d270,e7657ab323..f5a9119290
--- a/t/t1050-large.sh
+++ b/t/t1050-large.sh
@@@ -13,7 -13,7 +13,7 @@@ test_expect_success setup 
  	echo X | dd of=large2 bs=1k seek=2000 &&
  	echo X | dd of=large3 bs=1k seek=2000 &&
  	echo Y | dd of=huge bs=1k seek=2500 &&
- 	GIT_ALLOC_LIMIT=1500 &&
+ 	GIT_ALLOC_LIMIT=1500k &&
  	export GIT_ALLOC_LIMIT
  '
  
@@@ -112,20 -112,6 +112,20 @@@ test_expect_success 'diff --raw' 
  	git diff --raw HEAD^
  '
  
 +test_expect_success 'diff --stat' '
 +	git diff --stat HEAD^ HEAD
 +'
 +
 +test_expect_success 'diff' '
 +	git diff HEAD^ HEAD >actual &&
 +	grep "Binary files.*differ" actual
 +'
 +
 +test_expect_success 'diff --cached' '
 +	git diff --cached HEAD^ >actual &&
 +	grep "Binary files.*differ" actual
 +'
 +
  test_expect_success 'hash-object' '
  	git hash-object large1
  '
@@@ -177,10 -163,4 +177,10 @@@ test_expect_success 'zip achiving, defl
  	git archive --format=zip HEAD >/dev/null
  '
  
 +test_expect_success 'fsck' '
 +	test_must_fail git fsck 2>err &&
 +	n=$(grep "error: attempting to allocate .* over limit" err | wc -l) &&
 +	test "$n" -gt 1
 +'
 +
  test_done
diff --combined wrapper.c
index 25074d71b6,c5204f7a07..5b77d2a1ae
--- a/wrapper.c
+++ b/wrapper.c
@@@ -9,23 -9,17 +9,24 @@@ static void do_nothing(size_t size
  
  static void (*try_to_free_routine)(size_t size) = do_nothing;
  
 -static void memory_limit_check(size_t size)
 +static int memory_limit_check(size_t size, int gentle)
  {
- 	static int limit = -1;
- 	if (limit == -1) {
- 		const char *env = getenv("GIT_ALLOC_LIMIT");
- 		limit = env ? atoi(env) * 1024 : 0;
+ 	static size_t limit = 0;
+ 	if (!limit) {
+ 		limit = git_env_ulong("GIT_ALLOC_LIMIT", 0);
+ 		if (!limit)
+ 			limit = SIZE_MAX;
  	}
- 	if (limit && size > limit) {
 -	if (size > limit)
 -		die("attempting to allocate %"PRIuMAX" over limit %"PRIuMAX,
 -		    (uintmax_t)size, (uintmax_t)limit);
++	if (size > limit) {
 +		if (gentle) {
- 			error("attempting to allocate %"PRIuMAX" over limit %d",
- 			      (intmax_t)size, limit);
++			error("attempting to allocate %"PRIuMAX" over limit %"PRIuMAX,
++			      (uintmax_t)size, (uintmax_t)limit);
 +			return -1;
 +		} else
- 			die("attempting to allocate %"PRIuMAX" over limit %d",
- 			    (intmax_t)size, limit);
++			die("attempting to allocate %"PRIuMAX" over limit %"PRIuMAX,
++			    (uintmax_t)size, (uintmax_t)limit);
 +	}
 +	return 0;
  }
  
  try_to_free_t set_try_to_free_routine(try_to_free_t routine)
@@@ -49,12 -43,11 +50,12 @@@ char *xstrdup(const char *str
  	return ret;
  }
  
 -void *xmalloc(size_t size)
 +static void *do_xmalloc(size_t size, int gentle)
  {
  	void *ret;
  
 -	memory_limit_check(size);
 +	if (memory_limit_check(size, gentle))
 +		return NULL;
  	ret = malloc(size);
  	if (!ret && !size)
  		ret = malloc(1);
@@@ -63,16 -56,9 +64,16 @@@
  		ret = malloc(size);
  		if (!ret && !size)
  			ret = malloc(1);
 -		if (!ret)
 -			die("Out of memory, malloc failed (tried to allocate %lu bytes)",
 -			    (unsigned long)size);
 +		if (!ret) {
 +			if (!gentle)
 +				die("Out of memory, malloc failed (tried to allocate %lu bytes)",
 +				    (unsigned long)size);
 +			else {
 +				error("Out of memory, malloc failed (tried to allocate %lu bytes)",
 +				      (unsigned long)size);
 +				return NULL;
 +			}
 +		}
  	}
  #ifdef XMALLOC_POISON
  	memset(ret, 0xA5, size);
@@@ -80,37 -66,16 +81,37 @@@
  	return ret;
  }
  
 -void *xmallocz(size_t size)
 +void *xmalloc(size_t size)
 +{
 +	return do_xmalloc(size, 0);
 +}
 +
 +static void *do_xmallocz(size_t size, int gentle)
  {
  	void *ret;
 -	if (unsigned_add_overflows(size, 1))
 -		die("Data too large to fit into virtual memory space.");
 -	ret = xmalloc(size + 1);
 -	((char*)ret)[size] = 0;
 +	if (unsigned_add_overflows(size, 1)) {
 +		if (gentle) {
 +			error("Data too large to fit into virtual memory space.");
 +			return NULL;
 +		} else
 +			die("Data too large to fit into virtual memory space.");
 +	}
 +	ret = do_xmalloc(size + 1, gentle);
 +	if (ret)
 +		((char*)ret)[size] = 0;
  	return ret;
  }
  
 +void *xmallocz(size_t size)
 +{
 +	return do_xmallocz(size, 0);
 +}
 +
 +void *xmallocz_gently(size_t size)
 +{
 +	return do_xmallocz(size, 1);
 +}
 +
  /*
   * xmemdupz() allocates (len + 1) bytes of memory, duplicates "len" bytes of
   * "data" to the allocated memory, zero terminates the allocated memory,
@@@ -132,7 -97,7 +133,7 @@@ void *xrealloc(void *ptr, size_t size
  {
  	void *ret;
  
 -	memory_limit_check(size);
 +	memory_limit_check(size, 0);
  	ret = realloc(ptr, size);
  	if (!ret && !size)
  		ret = realloc(ptr, 1);
@@@ -151,7 -116,7 +152,7 @@@ void *xcalloc(size_t nmemb, size_t size
  {
  	void *ret;
  
 -	memory_limit_check(size * nmemb);
 +	memory_limit_check(size * nmemb, 0);
  	ret = calloc(nmemb, size);
  	if (!ret && (!nmemb || !size))
  		ret = calloc(1, 1);
@@@ -529,11 -494,3 +530,11 @@@ struct passwd *xgetpwuid_self(void
  		    errno ? strerror(errno) : _("no such user"));
  	return pw;
  }
 +
 +char *xgetcwd(void)
 +{
 +	struct strbuf sb = STRBUF_INIT;
 +	if (strbuf_getcwd(&sb))
 +		die_errno(_("unable to get current working directory"));
 +	return strbuf_detach(&sb, NULL);
 +}