From: Junio C Hamano <gitster@pobox.com>
Date: Tue, 16 Aug 2011 18:23:26 +0000 (-0700)
Subject: Merge branch 'jc/zlib-wrap' into maint
X-Git-Tag: v1.7.6.1~28
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/a35d78c0f483a65ea96c4f0c9a825bf28a386273?ds=inline;hp=-c

Merge branch 'jc/zlib-wrap' into maint

* jc/zlib-wrap:
zlib: allow feeding more than 4GB in one go
zlib: zlib can only process 4GB at a time
zlib: wrap deflateBound() too
zlib: wrap deflate side of the API
zlib: wrap inflateInit2 used to accept only for gzip format
zlib: wrap remaining calls to direct inflate/inflateEnd
zlib wrapper: refactor error message formatter
---

a35d78c0f483a65ea96c4f0c9a825bf28a386273
diff --combined builtin/index-pack.c
index e40451ffb4,f65cb373bf..81cdc28b30
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@@ -1,4 -1,4 +1,4 @@@
 -#include "cache.h"
 +#include "builtin.h"
  #include "delta.h"
  #include "pack.h"
  #include "csum-file.h"
@@@ -265,7 -265,7 +265,7 @@@ static void unlink_base_data(struct bas
  static void *unpack_entry_data(unsigned long offset, unsigned long size)
  {
  	int status;
- 	z_stream stream;
+ 	git_zstream stream;
  	void *buf = xmalloc(size);
  
  	memset(&stream, 0, sizeof(stream));
@@@ -355,7 -355,7 +355,7 @@@ static void *get_data_from_pack(struct 
  	off_t from = obj[0].idx.offset + obj[0].hdr_size;
  	unsigned long len = obj[1].idx.offset - from;
  	unsigned char *data, *inbuf;
- 	z_stream stream;
+ 	git_zstream stream;
  	int status;
  
  	data = xmalloc(obj->size);
@@@ -666,26 -666,26 +666,26 @@@ static void parse_pack_objects(unsigne
  
  static int write_compressed(struct sha1file *f, void *in, unsigned int size)
  {
- 	z_stream stream;
+ 	git_zstream stream;
  	int status;
  	unsigned char outbuf[4096];
  
  	memset(&stream, 0, sizeof(stream));
- 	deflateInit(&stream, zlib_compression_level);
+ 	git_deflate_init(&stream, zlib_compression_level);
  	stream.next_in = in;
  	stream.avail_in = size;
  
  	do {
  		stream.next_out = outbuf;
  		stream.avail_out = sizeof(outbuf);
- 		status = deflate(&stream, Z_FINISH);
+ 		status = git_deflate(&stream, Z_FINISH);
  		sha1write(f, outbuf, sizeof(outbuf) - stream.avail_out);
  	} while (status == Z_OK);
  
  	if (status != Z_STREAM_END)
  		die("unable to deflate appended object (%d)", status);
  	size = stream.total_out;
- 	deflateEnd(&stream);
+ 	git_deflate_end(&stream);
  	return size;
  }
  
diff --combined cache.h
index e11cf6ab1c,1784bab7d5..be506187f8
--- a/cache.h
+++ b/cache.h
@@@ -16,13 -16,27 +16,27 @@@
  #endif
  
  #include <zlib.h>
- #if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200
- #define deflateBound(c,s)  ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11)
- #endif
- 
- void git_inflate_init(z_streamp strm);
- void git_inflate_end(z_streamp strm);
- int git_inflate(z_streamp strm, int flush);
+ typedef struct git_zstream {
+ 	z_stream z;
+ 	unsigned long avail_in;
+ 	unsigned long avail_out;
+ 	unsigned long total_in;
+ 	unsigned long total_out;
+ 	unsigned char *next_in;
+ 	unsigned char *next_out;
+ } git_zstream;
+ 
+ void git_inflate_init(git_zstream *);
+ void git_inflate_init_gzip_only(git_zstream *);
+ void git_inflate_end(git_zstream *);
+ int git_inflate(git_zstream *, int flush);
+ 
+ void git_deflate_init(git_zstream *, int level);
+ void git_deflate_init_gzip(git_zstream *, int level);
+ void git_deflate_end(git_zstream *);
+ int git_deflate_end_gently(git_zstream *);
+ int git_deflate(git_zstream *, int flush);
+ unsigned long git_deflate_bound(git_zstream *, unsigned long);
  
  #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT)
  #define DTYPE(de)	((de)->d_type)
@@@ -511,18 -525,15 +525,18 @@@ struct pathspec 
  	struct pathspec_item {
  		const char *match;
  		int len;
 -		unsigned int has_wildcard:1;
 +		unsigned int use_wildcard:1;
  	} *items;
  };
  
  extern int init_pathspec(struct pathspec *, const char **);
  extern void free_pathspec(struct pathspec *);
  extern int ce_path_match(const struct cache_entry *ce, const struct pathspec *pathspec);
 -extern int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, enum object_type type, const char *path, int format_check);
 -extern int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object);
 +
 +#define HASH_WRITE_OBJECT 1
 +#define HASH_FORMAT_CHECK 2
 +extern int index_fd(unsigned char *sha1, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags);
 +extern int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags);
  extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st);
  
  #define REFRESH_REALLY		0x0001	/* ignore_valid */
@@@ -609,7 -620,7 +623,7 @@@ enum eol 
  #endif
  };
  
 -extern enum eol eol;
 +extern enum eol core_eol;
  
  enum branch_track {
  	BRANCH_TRACK_UNSPECIFIED = -1,
@@@ -679,24 -690,14 +693,24 @@@ extern char *sha1_pack_name(const unsig
  extern char *sha1_pack_index_name(const unsigned char *sha1);
  extern const char *find_unique_abbrev(const unsigned char *sha1, int);
  extern const unsigned char null_sha1[20];
 -static inline int is_null_sha1(const unsigned char *sha1)
 +
 +static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2)
  {
 -	return !memcmp(sha1, null_sha1, 20);
 +	int i;
 +
 +	for (i = 0; i < 20; i++, sha1++, sha2++) {
 +		if (*sha1 != *sha2)
 +			return *sha1 - *sha2;
 +	}
 +
 +	return 0;
  }
 -static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2)
 +
 +static inline int is_null_sha1(const unsigned char *sha1)
  {
 -	return memcmp(sha1, sha2, 20);
 +	return !hashcmp(sha1, null_sha1);
  }
 +
  static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src)
  {
  	memcpy(sha_dst, sha_src, 20);
@@@ -759,23 -760,13 +773,23 @@@ char *strip_path_suffix(const char *pat
  int daemon_avoid_alias(const char *path);
  int offset_1st_component(const char *path);
  
 -/* Read and unpack a sha1 file into memory, write memory to a sha1 file */
 -extern int sha1_object_info(const unsigned char *, unsigned long *);
 -extern void *read_sha1_file_repl(const unsigned char *sha1, enum object_type *type, unsigned long *size, const unsigned char **replacement);
 +/* object replacement */
 +#define READ_SHA1_FILE_REPLACE 1
 +extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag);
  static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size)
  {
 -	return read_sha1_file_repl(sha1, type, size, NULL);
 +	return read_sha1_file_extended(sha1, type, size, READ_SHA1_FILE_REPLACE);
 +}
 +extern const unsigned char *do_lookup_replace_object(const unsigned char *sha1);
 +static inline const unsigned char *lookup_replace_object(const unsigned char *sha1)
 +{
 +	if (!read_replace_refs)
 +		return sha1;
 +	return do_lookup_replace_object(sha1);
  }
 +
 +/* Read and unpack a sha1 file into memory, write memory to a sha1 file */
 +extern int sha1_object_info(const unsigned char *, unsigned long *);
  extern int hash_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1);
  extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1);
  extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *);
@@@ -813,15 -804,15 +827,15 @@@ struct object_context 
  };
  
  extern int get_sha1(const char *str, unsigned char *sha1);
 -extern int get_sha1_with_mode_1(const char *str, unsigned char *sha1, unsigned *mode, int gently, const char *prefix);
 +extern int get_sha1_with_mode_1(const char *str, unsigned char *sha1, unsigned *mode, int only_to_die, const char *prefix);
  static inline int get_sha1_with_mode(const char *str, unsigned char *sha1, unsigned *mode)
  {
 -	return get_sha1_with_mode_1(str, sha1, mode, 1, NULL);
 +	return get_sha1_with_mode_1(str, sha1, mode, 0, NULL);
  }
 -extern int get_sha1_with_context_1(const char *name, unsigned char *sha1, struct object_context *orc, int gently, const char *prefix);
 +extern int get_sha1_with_context_1(const char *name, unsigned char *sha1, struct object_context *orc, int only_to_die, const char *prefix);
  static inline int get_sha1_with_context(const char *str, unsigned char *sha1, struct object_context *orc)
  {
 -	return get_sha1_with_context_1(str, sha1, orc, 1, NULL);
 +	return get_sha1_with_context_1(str, sha1, orc, 0, NULL);
  }
  extern int get_sha1_hex(const char *hex, unsigned char *sha1);
  extern char *sha1_to_hex(const unsigned char *sha1);	/* static buffer result! */
@@@ -1009,7 -1000,7 +1023,7 @@@ extern struct packed_git *find_sha1_pac
  extern void pack_report(void);
  extern int open_pack_index(struct packed_git *);
  extern void close_pack_index(struct packed_git *);
- extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *);
+ extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *);
  extern void close_pack_windows(struct packed_git *);
  extern void unuse_pack(struct pack_window **);
  extern void free_pack_by_name(const char *);
@@@ -1026,16 -1017,6 +1040,16 @@@ extern const char *packed_object_info_d
  /* Dumb servers support */
  extern int update_server_info(int);
  
 +/* git_config_parse_key() returns these negated: */
 +#define CONFIG_INVALID_KEY 1
 +#define CONFIG_NO_SECTION_OR_NAME 2
 +/* git_config_set(), git_config_set_multivar() return the above or these: */
 +#define CONFIG_NO_LOCK -1
 +#define CONFIG_INVALID_FILE 3
 +#define CONFIG_NO_WRITE 4
 +#define CONFIG_NOTHING_SET 5
 +#define CONFIG_INVALID_PATTERN 6
 +
  typedef int (*config_fn_t)(const char *, const char *, void *);
  extern int git_default_config(const char *, const char *, void *);
  extern int git_config_from_file(config_fn_t fn, const char *, void *);
diff --combined diff.c
index 5dd9049c7d,431873d43f..9038f190ec
--- a/diff.c
+++ b/diff.c
@@@ -31,7 -31,6 +31,7 @@@ static const char *external_diff_cmd_cf
  int diff_auto_refresh_index = 1;
  static int diff_mnemonic_prefix;
  static int diff_no_prefix;
 +static int diff_dirstat_permille_default = 30;
  static struct diff_options default_diff_options;
  
  static char diff_colors[][COLOR_MAXLEN] = {
@@@ -67,58 -66,6 +67,58 @@@ static int parse_diff_color_slot(const 
  	return -1;
  }
  
 +static int parse_dirstat_params(struct diff_options *options, const char *params,
 +				struct strbuf *errmsg)
 +{
 +	const char *p = params;
 +	int p_len, ret = 0;
 +
 +	while (*p) {
 +		p_len = strchrnul(p, ',') - p;
 +		if (!memcmp(p, "changes", p_len)) {
 +			DIFF_OPT_CLR(options, DIRSTAT_BY_LINE);
 +			DIFF_OPT_CLR(options, DIRSTAT_BY_FILE);
 +		} else if (!memcmp(p, "lines", p_len)) {
 +			DIFF_OPT_SET(options, DIRSTAT_BY_LINE);
 +			DIFF_OPT_CLR(options, DIRSTAT_BY_FILE);
 +		} else if (!memcmp(p, "files", p_len)) {
 +			DIFF_OPT_CLR(options, DIRSTAT_BY_LINE);
 +			DIFF_OPT_SET(options, DIRSTAT_BY_FILE);
 +		} else if (!memcmp(p, "noncumulative", p_len)) {
 +			DIFF_OPT_CLR(options, DIRSTAT_CUMULATIVE);
 +		} else if (!memcmp(p, "cumulative", p_len)) {
 +			DIFF_OPT_SET(options, DIRSTAT_CUMULATIVE);
 +		} else if (isdigit(*p)) {
 +			char *end;
 +			int permille = strtoul(p, &end, 10) * 10;
 +			if (*end == '.' && isdigit(*++end)) {
 +				/* only use first digit */
 +				permille += *end - '0';
 +				/* .. and ignore any further digits */
 +				while (isdigit(*++end))
 +					; /* nothing */
 +			}
 +			if (end - p == p_len)
 +				options->dirstat_permille = permille;
 +			else {
 +				strbuf_addf(errmsg, _("  Failed to parse dirstat cut-off percentage '%.*s'\n"),
 +					    p_len, p);
 +				ret++;
 +			}
 +		} else {
 +			strbuf_addf(errmsg, _("  Unknown dirstat parameter '%.*s'\n"),
 +				    p_len, p);
 +			ret++;
 +		}
 +
 +		p += p_len;
 +
 +		if (*p)
 +			p++; /* more parameters, swallow separator */
 +	}
 +	return ret;
 +}
 +
  static int git_config_rename(const char *var, const char *value)
  {
  	if (!value)
@@@ -198,17 -145,6 +198,17 @@@ int git_diff_basic_config(const char *v
  		return 0;
  	}
  
 +	if (!strcmp(var, "diff.dirstat")) {
 +		struct strbuf errmsg = STRBUF_INIT;
 +		default_diff_options.dirstat_permille = diff_dirstat_permille_default;
 +		if (parse_dirstat_params(&default_diff_options, value, &errmsg))
 +			warning(_("Found errors in 'diff.dirstat' config variable:\n%s"),
 +				errmsg.buf);
 +		strbuf_release(&errmsg);
 +		diff_dirstat_permille_default = default_diff_options.dirstat_permille;
 +		return 0;
 +	}
 +
  	if (!prefixcmp(var, "submodule."))
  		return parse_submodule_config_option(var, value);
  
@@@ -645,14 -581,11 +645,14 @@@ static void emit_rewrite_diff(const cha
  		line_prefix, metainfo, a_name.buf, name_a_tab, reset,
  		line_prefix, metainfo, b_name.buf, name_b_tab, reset,
  		line_prefix, fraginfo);
 -	print_line_count(o->file, lc_a);
 +	if (!o->irreversible_delete)
 +		print_line_count(o->file, lc_a);
 +	else
 +		fprintf(o->file, "?,?");
  	fprintf(o->file, " +");
  	print_line_count(o->file, lc_b);
  	fprintf(o->file, " @@%s\n", reset);
 -	if (lc_a)
 +	if (lc_a && !o->irreversible_delete)
  		emit_rewrite_lines(&ecbdata, '-', data_one, size_one);
  	if (lc_b)
  		emit_rewrite_lines(&ecbdata, '+', data_two, size_two);
@@@ -1527,7 -1460,7 +1527,7 @@@ struct dirstat_file 
  
  struct dirstat_dir {
  	struct dirstat_file *files;
 -	int alloc, nr, percent, cumulative;
 +	int alloc, nr, permille, cumulative;
  };
  
  static long gather_dirstat(struct diff_options *opt, struct dirstat_dir *dir,
@@@ -1574,11 -1507,12 +1574,11 @@@
  	 *    under this directory (sources == 1).
  	 */
  	if (baselen && sources != 1) {
 -		int permille = this_dir * 1000 / changed;
 -		if (permille) {
 -			int percent = permille / 10;
 -			if (percent >= dir->percent) {
 +		if (this_dir) {
 +			int permille = this_dir * 1000 / changed;
 +			if (permille >= dir->permille) {
  				fprintf(opt->file, "%s%4d.%01d%% %.*s\n", line_prefix,
 -					percent, permille % 10, baselen, base);
 +					permille / 10, permille % 10, baselen, base);
  				if (!dir->cumulative)
  					return 0;
  			}
@@@ -1604,7 -1538,7 +1604,7 @@@ static void show_dirstat(struct diff_op
  	dir.files = NULL;
  	dir.alloc = 0;
  	dir.nr = 0;
 -	dir.percent = options->dirstat_percent;
 +	dir.permille = options->dirstat_permille;
  	dir.cumulative = DIFF_OPT_TST(options, DIRSTAT_CUMULATIVE);
  
  	changed = 0;
@@@ -1693,50 -1627,6 +1693,50 @@@ found_damage
  	gather_dirstat(options, &dir, changed, "", 0);
  }
  
 +static void show_dirstat_by_line(struct diffstat_t *data, struct diff_options *options)
 +{
 +	int i;
 +	unsigned long changed;
 +	struct dirstat_dir dir;
 +
 +	if (data->nr == 0)
 +		return;
 +
 +	dir.files = NULL;
 +	dir.alloc = 0;
 +	dir.nr = 0;
 +	dir.permille = options->dirstat_permille;
 +	dir.cumulative = DIFF_OPT_TST(options, DIRSTAT_CUMULATIVE);
 +
 +	changed = 0;
 +	for (i = 0; i < data->nr; i++) {
 +		struct diffstat_file *file = data->files[i];
 +		unsigned long damage = file->added + file->deleted;
 +		if (file->is_binary)
 +			/*
 +			 * binary files counts bytes, not lines. Must find some
 +			 * way to normalize binary bytes vs. textual lines.
 +			 * The following heuristic assumes that there are 64
 +			 * bytes per "line".
 +			 * This is stupid and ugly, but very cheap...
 +			 */
 +			damage = (damage + 63) / 64;
 +		ALLOC_GROW(dir.files, dir.nr + 1, dir.alloc);
 +		dir.files[dir.nr].name = file->name;
 +		dir.files[dir.nr].changed = damage;
 +		changed += damage;
 +		dir.nr++;
 +	}
 +
 +	/* This can happen even with many files, if everything was renames */
 +	if (!changed)
 +		return;
 +
 +	/* Show all directories with more than x% of the changes */
 +	qsort(dir.files, dir.nr, sizeof(dir.files[0]), dirstat_compare);
 +	gather_dirstat(options, &dir, changed, "", 0);
 +}
 +
  static void free_diffstat_info(struct diffstat_t *diffstat)
  {
  	int i;
@@@ -1839,20 -1729,20 +1839,20 @@@ static unsigned char *deflate_it(char *
  {
  	int bound;
  	unsigned char *deflated;
- 	z_stream stream;
+ 	git_zstream stream;
  
  	memset(&stream, 0, sizeof(stream));
- 	deflateInit(&stream, zlib_compression_level);
- 	bound = deflateBound(&stream, size);
+ 	git_deflate_init(&stream, zlib_compression_level);
+ 	bound = git_deflate_bound(&stream, size);
  	deflated = xmalloc(bound);
  	stream.next_out = deflated;
  	stream.avail_out = bound;
  
  	stream.next_in = (unsigned char *)data;
  	stream.avail_in = size;
- 	while (deflate(&stream, Z_FINISH) == Z_OK)
+ 	while (git_deflate(&stream, Z_FINISH) == Z_OK)
  		; /* nothing */
- 	deflateEnd(&stream);
+ 	git_deflate_end(&stream);
  	*result_size = stream.total_out;
  	return deflated;
  }
@@@ -1984,7 -1874,19 +1984,7 @@@ struct userdiff_driver *get_textconv(st
  		return NULL;
  
  	diff_filespec_load_driver(one);
 -	if (!one->driver->textconv)
 -		return NULL;
 -
 -	if (one->driver->textconv_want_cache && !one->driver->textconv_cache) {
 -		struct notes_cache *c = xmalloc(sizeof(*c));
 -		struct strbuf name = STRBUF_INIT;
 -
 -		strbuf_addf(&name, "textconv/%s", one->driver->name);
 -		notes_cache_init(c, name.buf, one->driver->textconv);
 -		one->driver->textconv_cache = c;
 -	}
 -
 -	return one->driver;
 +	return userdiff_get_textconv(one->driver);
  }
  
  static void builtin_diff(const char *name_a,
@@@ -2087,11 -1989,7 +2087,11 @@@
  		}
  	}
  
 -	if (!DIFF_OPT_TST(o, TEXT) &&
 +	if (o->irreversible_delete && lbl[1][0] == '/') {
 +		fprintf(o->file, "%s", header.buf);
 +		strbuf_reset(&header);
 +		goto free_ab_and_return;
 +	} else if (!DIFF_OPT_TST(o, TEXT) &&
  	    ( (!textconv_one && diff_filespec_is_binary(one)) ||
  	      (!textconv_two && diff_filespec_is_binary(two)) )) {
  		if (fill_mmfile(&mf1, one) < 0 || fill_mmfile(&mf2, two) < 0)
@@@ -2111,7 -2009,8 +2111,7 @@@
  			fprintf(o->file, "%sBinary files %s and %s differ\n",
  				line_prefix, lbl[0], lbl[1]);
  		o->found_changes = 1;
 -	}
 -	else {
 +	} else {
  		/* Crazy xdl interfaces.. */
  		const char *diffopts = getenv("GIT_DIFF_OPTS");
  		xpparam_t xpp;
@@@ -2994,7 -2893,7 +2994,7 @@@ void diff_setup(struct diff_options *op
  	options->line_termination = '\n';
  	options->break_opt = -1;
  	options->rename_limit = -1;
 -	options->dirstat_percent = 3;
 +	options->dirstat_permille = diff_dirstat_permille_default;
  	options->context = 3;
  
  	options->change = diff_change;
@@@ -3252,21 -3151,6 +3252,21 @@@ static int stat_opt(struct diff_option
  	return argcount;
  }
  
 +static int parse_dirstat_opt(struct diff_options *options, const char *params)
 +{
 +	struct strbuf errmsg = STRBUF_INIT;
 +	if (parse_dirstat_params(options, params, &errmsg))
 +		die(_("Failed to parse --dirstat/-X option parameter:\n%s"),
 +		    errmsg.buf);
 +	strbuf_release(&errmsg);
 +	/*
 +	 * The caller knows a dirstat-related option is given from the command
 +	 * line; allow it to say "return this_function();"
 +	 */
 +	options->output_format |= DIFF_FORMAT_DIRSTAT;
 +	return 1;
 +}
 +
  int diff_opt_parse(struct diff_options *options, const char **av, int ac)
  {
  	const char *arg = av[0];
@@@ -3286,19 -3170,15 +3286,19 @@@
  		options->output_format |= DIFF_FORMAT_NUMSTAT;
  	else if (!strcmp(arg, "--shortstat"))
  		options->output_format |= DIFF_FORMAT_SHORTSTAT;
 -	else if (opt_arg(arg, 'X', "dirstat", &options->dirstat_percent))
 -		options->output_format |= DIFF_FORMAT_DIRSTAT;
 -	else if (!strcmp(arg, "--cumulative")) {
 -		options->output_format |= DIFF_FORMAT_DIRSTAT;
 -		DIFF_OPT_SET(options, DIRSTAT_CUMULATIVE);
 -	} else if (opt_arg(arg, 0, "dirstat-by-file",
 -			   &options->dirstat_percent)) {
 -		options->output_format |= DIFF_FORMAT_DIRSTAT;
 -		DIFF_OPT_SET(options, DIRSTAT_BY_FILE);
 +	else if (!strcmp(arg, "-X") || !strcmp(arg, "--dirstat"))
 +		return parse_dirstat_opt(options, "");
 +	else if (!prefixcmp(arg, "-X"))
 +		return parse_dirstat_opt(options, arg + 2);
 +	else if (!prefixcmp(arg, "--dirstat="))
 +		return parse_dirstat_opt(options, arg + 10);
 +	else if (!strcmp(arg, "--cumulative"))
 +		return parse_dirstat_opt(options, "cumulative");
 +	else if (!strcmp(arg, "--dirstat-by-file"))
 +		return parse_dirstat_opt(options, "files");
 +	else if (!prefixcmp(arg, "--dirstat-by-file=")) {
 +		parse_dirstat_opt(options, "files");
 +		return parse_dirstat_opt(options, arg + 18);
  	}
  	else if (!strcmp(arg, "--check"))
  		options->output_format |= DIFF_FORMAT_CHECKDIFF;
@@@ -3328,9 -3208,6 +3328,9 @@@
  			return error("invalid argument to -M: %s", arg+2);
  		options->detect_rename = DIFF_DETECT_RENAME;
  	}
 +	else if (!strcmp(arg, "-D") || !strcmp(arg, "--irreversible-delete")) {
 +		options->irreversible_delete = 1;
 +	}
  	else if (!prefixcmp(arg, "-C") || !prefixcmp(arg, "--find-copies=") ||
  		 !strcmp(arg, "--find-copies")) {
  		if (options->detect_rename == DIFF_DETECT_COPY)
@@@ -4145,7 -4022,6 +4145,7 @@@ void diff_flush(struct diff_options *op
  	struct diff_queue_struct *q = &diff_queued_diff;
  	int i, output_format = options->output_format;
  	int separator = 0;
 +	int dirstat_by_line = 0;
  
  	/*
  	 * Order: raw, stat, summary, patch
@@@ -4166,11 -4042,7 +4166,11 @@@
  		separator++;
  	}
  
 -	if (output_format & (DIFF_FORMAT_DIFFSTAT|DIFF_FORMAT_SHORTSTAT|DIFF_FORMAT_NUMSTAT)) {
 +	if (output_format & DIFF_FORMAT_DIRSTAT && DIFF_OPT_TST(options, DIRSTAT_BY_LINE))
 +		dirstat_by_line = 1;
 +
 +	if (output_format & (DIFF_FORMAT_DIFFSTAT|DIFF_FORMAT_SHORTSTAT|DIFF_FORMAT_NUMSTAT) ||
 +	    dirstat_by_line) {
  		struct diffstat_t diffstat;
  
  		memset(&diffstat, 0, sizeof(struct diffstat_t));
@@@ -4185,12 -4057,10 +4185,12 @@@
  			show_stats(&diffstat, options);
  		if (output_format & DIFF_FORMAT_SHORTSTAT)
  			show_shortstats(&diffstat, options);
 +		if (output_format & DIFF_FORMAT_DIRSTAT)
 +			show_dirstat_by_line(&diffstat, options);
  		free_diffstat_info(&diffstat);
  		separator++;
  	}
 -	if (output_format & DIFF_FORMAT_DIRSTAT)
 +	if ((output_format & DIFF_FORMAT_DIRSTAT) && !dirstat_by_line)
  		show_dirstat(options);
  
  	if (output_format & DIFF_FORMAT_SUMMARY && !is_summary_empty(q)) {
@@@ -4444,13 -4314,6 +4444,13 @@@ int diff_result_code(struct diff_option
  	return result;
  }
  
 +int diff_can_quit_early(struct diff_options *opt)
 +{
 +	return (DIFF_OPT_TST(opt, QUICK) &&
 +		!opt->filter &&
 +		DIFF_OPT_TST(opt, HAS_CHANGES));
 +}
 +
  /*
   * Shall changes to this submodule be ignored?
   *
diff --combined http-push.c
index 28bfe768f7,1e8a830937..6e8f6d09ab
--- a/http-push.c
+++ b/http-push.c
@@@ -169,7 -169,7 +169,7 @@@ enum dav_header_flag 
  	DAV_HEADER_TIMEOUT = (1u << 2)
  };
  
 -static char *xml_entities(char *s)
 +static char *xml_entities(const char *s)
  {
  	struct strbuf buf = STRBUF_INIT;
  	while (*s) {
@@@ -197,34 -197,6 +197,34 @@@
  	return strbuf_detach(&buf, NULL);
  }
  
 +static void curl_setup_http_get(CURL *curl, const char *url,
 +		const char *custom_req)
 +{
 +	curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
 +	curl_easy_setopt(curl, CURLOPT_URL, url);
 +	curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, custom_req);
 +	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_null);
 +}
 +
 +static void curl_setup_http(CURL *curl, const char *url,
 +		const char *custom_req, struct buffer *buffer,
 +		curl_write_callback write_fn)
 +{
 +	curl_easy_setopt(curl, CURLOPT_PUT, 1);
 +	curl_easy_setopt(curl, CURLOPT_URL, url);
 +	curl_easy_setopt(curl, CURLOPT_INFILE, buffer);
 +	curl_easy_setopt(curl, CURLOPT_INFILESIZE, buffer->buf.len);
 +	curl_easy_setopt(curl, CURLOPT_READFUNCTION, fread_buffer);
 +#ifndef NO_CURL_IOCTL
 +	curl_easy_setopt(curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer);
 +	curl_easy_setopt(curl, CURLOPT_IOCTLDATA, &buffer);
 +#endif
 +	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_fn);
 +	curl_easy_setopt(curl, CURLOPT_NOBODY, 0);
 +	curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, custom_req);
 +	curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);
 +}
 +
  static struct curl_slist *get_dav_token_headers(struct remote_lock *lock, enum dav_header_flag options)
  {
  	struct strbuf buf = STRBUF_INIT;
@@@ -300,8 -272,11 +300,8 @@@ static void start_mkcol(struct transfer
  	slot = get_active_slot();
  	slot->callback_func = process_response;
  	slot->callback_data = request;
 -	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1); /* undo PUT setup */
 -	curl_easy_setopt(slot->curl, CURLOPT_URL, request->url);
 +	curl_setup_http_get(slot->curl, request->url, DAV_MKCOL);
  	curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
 -	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_MKCOL);
 -	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null);
  
  	if (start_active_slot(slot)) {
  		request->slot = slot;
@@@ -377,15 -352,15 +377,15 @@@ static void start_put(struct transfer_r
  	unsigned long len;
  	int hdrlen;
  	ssize_t size;
- 	z_stream stream;
+ 	git_zstream stream;
  
  	unpacked = read_sha1_file(request->obj->sha1, &type, &len);
  	hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1;
  
  	/* Set it up */
  	memset(&stream, 0, sizeof(stream));
- 	deflateInit(&stream, zlib_compression_level);
- 	size = deflateBound(&stream, len + hdrlen);
+ 	git_deflate_init(&stream, zlib_compression_level);
+ 	size = git_deflate_bound(&stream, len + hdrlen);
  	strbuf_init(&request->buffer.buf, size);
  	request->buffer.posn = 0;
  
@@@ -396,15 -371,15 +396,15 @@@
  	/* First header.. */
  	stream.next_in = (void *)hdr;
  	stream.avail_in = hdrlen;
- 	while (deflate(&stream, 0) == Z_OK)
- 		/* nothing */;
+ 	while (git_deflate(&stream, 0) == Z_OK)
+ 		; /* nothing */
  
  	/* Then the data itself.. */
  	stream.next_in = unpacked;
  	stream.avail_in = len;
- 	while (deflate(&stream, Z_FINISH) == Z_OK)
- 		/* nothing */;
- 	deflateEnd(&stream);
+ 	while (git_deflate(&stream, Z_FINISH) == Z_OK)
+ 		; /* nothing */
+ 	git_deflate_end(&stream);
  	free(unpacked);
  
  	request->buffer.buf.len = stream.total_out;
@@@ -420,8 -395,19 +420,8 @@@
  	slot = get_active_slot();
  	slot->callback_func = process_response;
  	slot->callback_data = request;
 -	curl_easy_setopt(slot->curl, CURLOPT_INFILE, &request->buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, request->buffer.buf.len);
 -	curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer);
 -#ifndef NO_CURL_IOCTL
 -	curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &request->buffer);
 -#endif
 -	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null);
 -	curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
 -	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PUT);
 -	curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1);
 -	curl_easy_setopt(slot->curl, CURLOPT_PUT, 1);
 -	curl_easy_setopt(slot->curl, CURLOPT_URL, request->url);
 +	curl_setup_http(slot->curl, request->url, DAV_PUT,
 +			&request->buffer, fwrite_null);
  
  	if (start_active_slot(slot)) {
  		request->slot = slot;
@@@ -441,10 -427,13 +441,10 @@@ static void start_move(struct transfer_
  	slot = get_active_slot();
  	slot->callback_func = process_response;
  	slot->callback_data = request;
 -	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1); /* undo PUT setup */
 -	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_MOVE);
 +	curl_setup_http_get(slot->curl, request->url, DAV_MOVE);
  	dav_headers = curl_slist_append(dav_headers, request->dest);
  	dav_headers = curl_slist_append(dav_headers, "Overwrite: T");
  	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
 -	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null);
 -	curl_easy_setopt(slot->curl, CURLOPT_URL, request->url);
  
  	if (start_active_slot(slot)) {
  		request->slot = slot;
@@@ -469,7 -458,10 +469,7 @@@ static int refresh_lock(struct remote_l
  
  	slot = get_active_slot();
  	slot->results = &results;
 -	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
 -	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null);
 -	curl_easy_setopt(slot->curl, CURLOPT_URL, lock->url);
 -	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_LOCK);
 +	curl_setup_http_get(slot->curl, lock->url, DAV_LOCK);
  	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
  
  	if (start_active_slot(slot)) {
@@@ -805,7 -797,7 +805,7 @@@ static void handle_new_lock_ctx(struct 
  	}
  }
  
 -static void one_remote_ref(char *refname);
 +static void one_remote_ref(const char *refname);
  
  static void
  xml_start_tag(void *userData, const char *name, const char **atts)
@@@ -884,7 -876,10 +884,7 @@@ static struct remote_lock *lock_remote(
  		ep[1] = '\0';
  		slot = get_active_slot();
  		slot->results = &results;
 -		curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
 -		curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 -		curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_MKCOL);
 -		curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null);
 +		curl_setup_http_get(slot->curl, url, DAV_MKCOL);
  		if (start_active_slot(slot)) {
  			run_active_slot(slot);
  			if (results.curl_result != CURLE_OK &&
@@@ -914,9 -909,19 +914,9 @@@
  
  	slot = get_active_slot();
  	slot->results = &results;
 -	curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.buf.len);
 -	curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer);
 -#ifndef NO_CURL_IOCTL
 -	curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &out_buffer);
 -#endif
 -	curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 -	curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1);
 -	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_LOCK);
 +	curl_setup_http(slot->curl, url, DAV_LOCK, &out_buffer, fwrite_buffer);
  	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
 +	curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer);
  
  	lock = xcalloc(1, sizeof(*lock));
  	lock->timeout = -1;
@@@ -982,7 -987,9 +982,7 @@@ static int unlock_remote(struct remote_
  
  	slot = get_active_slot();
  	slot->results = &results;
 -	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null);
 -	curl_easy_setopt(slot->curl, CURLOPT_URL, lock->url);
 -	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_UNLOCK);
 +	curl_setup_http_get(slot->curl, lock->url, DAV_UNLOCK);
  	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
  
  	if (start_active_slot(slot)) {
@@@ -1160,10 -1167,19 +1160,10 @@@ static void remote_ls(const char *path
  
  	slot = get_active_slot();
  	slot->results = &results;
 -	curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.buf.len);
 -	curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer);
 -#ifndef NO_CURL_IOCTL
 -	curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &out_buffer);
 -#endif
 -	curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 -	curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1);
 -	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PROPFIND);
 +	curl_setup_http(slot->curl, url, DAV_PROPFIND,
 +			&out_buffer, fwrite_buffer);
  	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
 +	curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer);
  
  	if (start_active_slot(slot)) {
  		run_active_slot(slot);
@@@ -1234,10 -1250,19 +1234,10 @@@ static int locking_available(void
  
  	slot = get_active_slot();
  	slot->results = &results;
 -	curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.buf.len);
 -	curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer);
 -#ifndef NO_CURL_IOCTL
 -	curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &out_buffer);
 -#endif
 -	curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_URL, repo->url);
 -	curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1);
 -	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PROPFIND);
 +	curl_setup_http(slot->curl, repo->url, DAV_PROPFIND,
 +			&out_buffer, fwrite_buffer);
  	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
 +	curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer);
  
  	if (start_active_slot(slot)) {
  		run_active_slot(slot);
@@@ -1411,9 -1436,19 +1411,9 @@@ static int update_remote(unsigned char 
  
  	slot = get_active_slot();
  	slot->results = &results;
 -	curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.buf.len);
 -	curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer);
 -#ifndef NO_CURL_IOCTL
 -	curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer);
 -	curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &out_buffer);
 -#endif
 -	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null);
 -	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PUT);
 +	curl_setup_http(slot->curl, lock->url, DAV_PUT,
 +			&out_buffer, fwrite_null);
  	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
 -	curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1);
 -	curl_easy_setopt(slot->curl, CURLOPT_PUT, 1);
 -	curl_easy_setopt(slot->curl, CURLOPT_URL, lock->url);
  
  	if (start_active_slot(slot)) {
  		run_active_slot(slot);
@@@ -1436,7 -1471,7 +1436,7 @@@
  
  static struct ref *remote_refs;
  
 -static void one_remote_ref(char *refname)
 +static void one_remote_ref(const char *refname)
  {
  	struct ref *ref;
  	struct object *obj;
@@@ -1537,9 -1572,19 +1537,9 @@@ static void update_remote_info_refs(str
  
  		slot = get_active_slot();
  		slot->results = &results;
 -		curl_easy_setopt(slot->curl, CURLOPT_INFILE, &buffer);
 -		curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, buffer.buf.len);
 -		curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer);
 -#ifndef NO_CURL_IOCTL
 -		curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer);
 -		curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &buffer);
 -#endif
 -		curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null);
 -		curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PUT);
 +		curl_setup_http(slot->curl, lock->url, DAV_PUT,
 +				&buffer, fwrite_null);
  		curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
 -		curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1);
 -		curl_easy_setopt(slot->curl, CURLOPT_PUT, 1);
 -		curl_easy_setopt(slot->curl, CURLOPT_URL, lock->url);
  
  		if (start_active_slot(slot)) {
  			run_active_slot(slot);
@@@ -1615,7 -1660,7 +1615,7 @@@ static int verify_merge_base(unsigned c
  	return (merge_bases && !merge_bases->next && merge_bases->item == branch);
  }
  
 -static int delete_remote_branch(char *pattern, int force)
 +static int delete_remote_branch(const char *pattern, int force)
  {
  	struct ref *refs = remote_refs;
  	struct ref *remote_ref = NULL;
@@@ -1697,7 -1742,10 +1697,7 @@@
  	sprintf(url, "%s%s", repo->url, remote_ref->name);
  	slot = get_active_slot();
  	slot->results = &results;
 -	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
 -	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null);
 -	curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 -	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_DELETE);
 +	curl_setup_http_get(slot->curl, url, DAV_DELETE);
  	if (start_active_slot(slot)) {
  		run_active_slot(slot);
  		free(url);
diff --combined http.h
index 19b7134fa5,a39304a7ab..0bf8592dc4
--- a/http.h
+++ b/http.h
@@@ -66,9 -66,9 +66,9 @@@ struct buffer 
  };
  
  /* Curl request read/write callbacks */
 -extern size_t fread_buffer(void *ptr, size_t eltsize, size_t nmemb, void *strbuf);
 -extern size_t fwrite_buffer(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf);
 -extern size_t fwrite_null(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf);
 +extern size_t fread_buffer(char *ptr, size_t eltsize, size_t nmemb, void *strbuf);
 +extern size_t fwrite_buffer(char *ptr, size_t eltsize, size_t nmemb, void *strbuf);
 +extern size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf);
  #ifndef NO_CURL_IOCTL
  extern curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp);
  #endif
@@@ -172,7 -172,7 +172,7 @@@ struct http_object_request 
  	unsigned char sha1[20];
  	unsigned char real_sha1[20];
  	git_SHA_CTX c;
- 	z_stream stream;
+ 	git_zstream stream;
  	int zret;
  	int rename;
  	struct active_request_slot *slot;
diff --combined remote-curl.c
index 8ac5028343,bc48a36ef5..69831e931a
--- a/remote-curl.c
+++ b/remote-curl.c
@@@ -227,8 -227,6 +227,8 @@@ static struct ref *parse_info_refs(stru
  		if (data[i] == '\t')
  			mid = &data[i];
  		if (data[i] == '\n') {
 +			if (mid - start != 40)
 +				die("%sinfo/refs not valid: is this a git repository?", url);
  			data[i] = 0;
  			ref_name = mid + 1;
  			ref = xmalloc(sizeof(struct ref) +
@@@ -349,7 -347,7 +349,7 @@@ static curlioerr rpc_ioctl(CURL *handle
  }
  #endif
  
 -static size_t rpc_in(const void *ptr, size_t eltsize,
 +static size_t rpc_in(char *ptr, size_t eltsize,
  		size_t nmemb, void *buffer_)
  {
  	size_t size = eltsize * nmemb;
@@@ -473,16 -471,12 +473,12 @@@ static int post_rpc(struct rpc_state *r
  		 * the transfer time.
  		 */
  		size_t size;
- 		z_stream stream;
+ 		git_zstream stream;
  		int ret;
  
  		memset(&stream, 0, sizeof(stream));
- 		ret = deflateInit2(&stream, Z_BEST_COMPRESSION,
- 				Z_DEFLATED, (15 + 16),
- 				8, Z_DEFAULT_STRATEGY);
- 		if (ret != Z_OK)
- 			die("cannot deflate request; zlib init error %d", ret);
- 		size = deflateBound(&stream, rpc->len);
+ 		git_deflate_init_gzip(&stream, Z_BEST_COMPRESSION);
+ 		size = git_deflate_bound(&stream, rpc->len);
  		gzip_body = xmalloc(size);
  
  		stream.next_in = (unsigned char *)rpc->buf;
@@@ -490,11 -484,11 +486,11 @@@
  		stream.next_out = (unsigned char *)gzip_body;
  		stream.avail_out = size;
  
- 		ret = deflate(&stream, Z_FINISH);
+ 		ret = git_deflate(&stream, Z_FINISH);
  		if (ret != Z_STREAM_END)
  			die("cannot deflate request; zlib deflate error %d", ret);
  
- 		ret = deflateEnd(&stream);
+ 		ret = git_deflate_end_gently(&stream);
  		if (ret != Z_OK)
  			die("cannot deflate request; zlib end error %d", ret);
  
@@@ -813,21 -807,19 +809,21 @@@ static void parse_push(struct strbuf *b
  
  		strbuf_reset(buf);
  		if (strbuf_getline(buf, stdin, '\n') == EOF)
 -			return;
 +			goto free_specs;
  		if (!*buf->buf)
  			break;
  	} while (1);
  
  	if (push(nr_spec, specs))
  		exit(128); /* error already reported */
 -	for (i = 0; i < nr_spec; i++)
 -		free(specs[i]);
 -	free(specs);
  
  	printf("\n");
  	fflush(stdout);
 +
 + free_specs:
 +	for (i = 0; i < nr_spec; i++)
 +		free(specs[i]);
 +	free(specs);
  }
  
  int main(int argc, const char **argv)
diff --combined sha1_file.c
index 697f4a43c5,94d431907c..92e87ee225
--- a/sha1_file.c
+++ b/sha1_file.c
@@@ -11,7 -11,6 +11,7 @@@
  #include "pack.h"
  #include "blob.h"
  #include "commit.h"
 +#include "run-command.h"
  #include "tag.h"
  #include "tree.h"
  #include "tree-walk.h"
@@@ -834,7 -833,7 +834,7 @@@ static int in_window(struct pack_windo
  unsigned char *use_pack(struct packed_git *p,
  		struct pack_window **w_cursor,
  		off_t offset,
- 		unsigned int *left)
+ 		unsigned long *left)
  {
  	struct pack_window *win = *w_cursor;
  
@@@ -1205,29 -1204,20 +1205,29 @@@ static void *map_sha1_file(const unsign
  	return map;
  }
  
 -static int legacy_loose_object(unsigned char *map)
 +/*
 + * There used to be a second loose object header format which
 + * was meant to mimic the in-pack format, allowing for direct
 + * copy of the object data.  This format turned up not to be
 + * really worth it and we no longer write loose objects in that
 + * format.
 + */
 +static int experimental_loose_object(unsigned char *map)
  {
  	unsigned int word;
  
  	/*
  	 * Is it a zlib-compressed buffer? If so, the first byte
  	 * must be 0x78 (15-bit window size, deflated), and the
 -	 * first 16-bit word is evenly divisible by 31
 +	 * first 16-bit word is evenly divisible by 31. If so,
 +	 * we are looking at the official format, not the experimental
 +	 * one.
  	 */
  	word = (map[0] << 8) + map[1];
  	if (map[0] == 0x78 && !(word % 31))
 -		return 1;
 -	else
  		return 0;
 +	else
 +		return 1;
  }
  
  unsigned long unpack_object_header_buffer(const unsigned char *buf,
@@@ -1254,7 -1244,7 +1254,7 @@@
  	return used;
  }
  
- static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
+ static int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
  {
  	unsigned long size, used;
  	static const char valid_loose_object_type[8] = {
@@@ -1271,32 -1261,37 +1271,32 @@@
  	stream->next_out = buffer;
  	stream->avail_out = bufsiz;
  
 -	if (legacy_loose_object(map)) {
 -		git_inflate_init(stream);
 -		return git_inflate(stream, 0);
 -	}
 -
 +	if (experimental_loose_object(map)) {
 +		/*
 +		 * The old experimental format we no longer produce;
 +		 * we can still read it.
 +		 */
 +		used = unpack_object_header_buffer(map, mapsize, &type, &size);
 +		if (!used || !valid_loose_object_type[type])
 +			return -1;
 +		map += used;
 +		mapsize -= used;
  
 -	/*
 -	 * There used to be a second loose object header format which
 -	 * was meant to mimic the in-pack format, allowing for direct
 -	 * copy of the object data.  This format turned up not to be
 -	 * really worth it and we don't write it any longer.  But we
 -	 * can still read it.
 -	 */
 -	used = unpack_object_header_buffer(map, mapsize, &type, &size);
 -	if (!used || !valid_loose_object_type[type])
 -		return -1;
 -	map += used;
 -	mapsize -= used;
 +		/* Set up the stream for the rest.. */
 +		stream->next_in = map;
 +		stream->avail_in = mapsize;
 +		git_inflate_init(stream);
  
 -	/* Set up the stream for the rest.. */
 -	stream->next_in = map;
 -	stream->avail_in = mapsize;
 +		/* And generate the fake traditional header */
 +		stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
 +						 typename(type), size);
 +		return 0;
 +	}
  	git_inflate_init(stream);
 -
 -	/* And generate the fake traditional header */
 -	stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
 -					 typename(type), size);
 -	return 0;
 +	return git_inflate(stream, 0);
  }
  
- static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
+ static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
  {
  	int bytes = strlen(buffer) + 1;
  	unsigned char *buf = xmallocz(size);
@@@ -1395,7 -1390,7 +1395,7 @@@ static int parse_sha1_header(const cha
  static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
  {
  	int ret;
- 	z_stream stream;
+ 	git_zstream stream;
  	char hdr[8192];
  
  	ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
@@@ -1411,7 -1406,7 +1411,7 @@@ unsigned long get_size_from_delta(struc
  {
  	const unsigned char *data;
  	unsigned char delta_head[20], *in;
- 	z_stream stream;
+ 	git_zstream stream;
  	int st;
  
  	memset(&stream, 0, sizeof(stream));
@@@ -1533,7 -1528,7 +1533,7 @@@ static int unpack_object_header(struct 
  				unsigned long *sizep)
  {
  	unsigned char *base;
- 	unsigned int left;
+ 	unsigned long left;
  	unsigned long used;
  	enum object_type type;
  
@@@ -1646,7 -1641,7 +1646,7 @@@ static void *unpack_compressed_entry(st
  				    unsigned long size)
  {
  	int st;
- 	z_stream stream;
+ 	git_zstream stream;
  	unsigned char *buffer, *in;
  
  	buffer = xmallocz(size);
@@@ -2079,7 -2074,7 +2079,7 @@@ static int sha1_loose_object_info(cons
  	int status;
  	unsigned long mapsize, size;
  	void *map;
- 	z_stream stream;
+ 	git_zstream stream;
  	char hdr[32];
  
  	map = map_sha1_file(sha1, &mapsize);
@@@ -2210,21 -2205,23 +2210,21 @@@ static void *read_object(const unsigne
   * deal with them should arrange to call read_object() and give error
   * messages themselves.
   */
 -void *read_sha1_file_repl(const unsigned char *sha1,
 -			  enum object_type *type,
 -			  unsigned long *size,
 -			  const unsigned char **replacement)
 +void *read_sha1_file_extended(const unsigned char *sha1,
 +			      enum object_type *type,
 +			      unsigned long *size,
 +			      unsigned flag)
  {
 -	const unsigned char *repl = lookup_replace_object(sha1);
  	void *data;
  	char *path;
  	const struct packed_git *p;
 +	const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE)
 +		? lookup_replace_object(sha1) : sha1;
  
  	errno = 0;
  	data = read_object(repl, type, size);
 -	if (data) {
 -		if (replacement)
 -			*replacement = repl;
 +	if (data)
  		return data;
 -	}
  
  	if (errno && errno != ENOENT)
  		die_errno("failed to read object %s", sha1_to_hex(sha1));
@@@ -2428,7 -2425,7 +2428,7 @@@ static int write_loose_object(const uns
  {
  	int fd, ret;
  	unsigned char compressed[4096];
- 	z_stream stream;
+ 	git_zstream stream;
  	git_SHA_CTX c;
  	unsigned char parano_sha1[20];
  	char *filename;
@@@ -2445,7 -2442,7 +2445,7 @@@
  
  	/* Set it up */
  	memset(&stream, 0, sizeof(stream));
- 	deflateInit(&stream, zlib_compression_level);
+ 	git_deflate_init(&stream, zlib_compression_level);
  	stream.next_out = compressed;
  	stream.avail_out = sizeof(compressed);
  	git_SHA1_Init(&c);
@@@ -2453,8 -2450,8 +2453,8 @@@
  	/* First header.. */
  	stream.next_in = (unsigned char *)hdr;
  	stream.avail_in = hdrlen;
- 	while (deflate(&stream, 0) == Z_OK)
- 		/* nothing */;
+ 	while (git_deflate(&stream, 0) == Z_OK)
+ 		; /* nothing */
  	git_SHA1_Update(&c, hdr, hdrlen);
  
  	/* Then the data itself.. */
@@@ -2462,7 -2459,7 +2462,7 @@@
  	stream.avail_in = len;
  	do {
  		unsigned char *in0 = stream.next_in;
- 		ret = deflate(&stream, Z_FINISH);
+ 		ret = git_deflate(&stream, Z_FINISH);
  		git_SHA1_Update(&c, in0, stream.next_in - in0);
  		if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
  			die("unable to write sha1 file");
@@@ -2472,7 -2469,7 +2472,7 @@@
  
  	if (ret != Z_STREAM_END)
  		die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
- 	ret = deflateEnd(&stream);
+ 	ret = git_deflate_end_gently(&stream);
  	if (ret != Z_OK)
  		die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
  	git_SHA1_Final(parano_sha1, &c);
@@@ -2583,11 -2580,10 +2583,11 @@@ static void check_tag(const void *buf, 
  }
  
  static int index_mem(unsigned char *sha1, void *buf, size_t size,
 -		     int write_object, enum object_type type,
 -		     const char *path, int format_check)
 +		     enum object_type type,
 +		     const char *path, unsigned flags)
  {
  	int ret, re_allocated = 0;
 +	int write_object = flags & HASH_WRITE_OBJECT;
  
  	if (!type)
  		type = OBJ_BLOB;
@@@ -2603,7 -2599,7 +2603,7 @@@
  			re_allocated = 1;
  		}
  	}
 -	if (format_check) {
 +	if (flags & HASH_FORMAT_CHECK) {
  		if (type == OBJ_TREE)
  			check_tree(buf, size);
  		if (type == OBJ_COMMIT)
@@@ -2621,141 -2617,44 +2621,141 @@@
  	return ret;
  }
  
 +static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
 +		      const char *path, unsigned flags)
 +{
 +	struct strbuf sbuf = STRBUF_INIT;
 +	int ret;
 +
 +	if (strbuf_read(&sbuf, fd, 4096) >= 0)
 +		ret = index_mem(sha1, sbuf.buf, sbuf.len, type,	path, flags);
 +	else
 +		ret = -1;
 +	strbuf_release(&sbuf);
 +	return ret;
 +}
 +
  #define SMALL_FILE_SIZE (32*1024)
  
 -int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object,
 -	     enum object_type type, const char *path, int format_check)
 +static int index_core(unsigned char *sha1, int fd, size_t size,
 +		      enum object_type type, const char *path,
 +		      unsigned flags)
  {
  	int ret;
 -	size_t size = xsize_t(st->st_size);
  
 -	if (!S_ISREG(st->st_mode)) {
 -		struct strbuf sbuf = STRBUF_INIT;
 -		if (strbuf_read(&sbuf, fd, 4096) >= 0)
 -			ret = index_mem(sha1, sbuf.buf, sbuf.len, write_object,
 -					type, path, format_check);
 -		else
 -			ret = -1;
 -		strbuf_release(&sbuf);
 -	} else if (!size) {
 -		ret = index_mem(sha1, NULL, size, write_object, type, path,
 -				format_check);
 +	if (!size) {
 +		ret = index_mem(sha1, NULL, size, type, path, flags);
  	} else if (size <= SMALL_FILE_SIZE) {
  		char *buf = xmalloc(size);
  		if (size == read_in_full(fd, buf, size))
 -			ret = index_mem(sha1, buf, size, write_object, type,
 -					path, format_check);
 +			ret = index_mem(sha1, buf, size, type, path, flags);
  		else
  			ret = error("short read %s", strerror(errno));
  		free(buf);
  	} else {
  		void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
 -		ret = index_mem(sha1, buf, size, write_object, type, path,
 -				format_check);
 +		ret = index_mem(sha1, buf, size, type, path, flags);
  		munmap(buf, size);
  	}
 +	return ret;
 +}
 +
 +/*
 + * This creates one packfile per large blob, because the caller
 + * immediately wants the result sha1, and fast-import can report the
 + * object name via marks mechanism only by closing the created
 + * packfile.
 + *
 + * This also bypasses the usual "convert-to-git" dance, and that is on
 + * purpose. We could write a streaming version of the converting
 + * functions and insert that before feeding the data to fast-import
 + * (or equivalent in-core API described above), but the primary
 + * motivation for trying to stream from the working tree file and to
 + * avoid mmaping it in core is to deal with large binary blobs, and
 + * by definition they do _not_ want to get any conversion.
 + */
 +static int index_stream(unsigned char *sha1, int fd, size_t size,
 +			enum object_type type, const char *path,
 +			unsigned flags)
 +{
 +	struct child_process fast_import;
 +	char export_marks[512];
 +	const char *argv[] = { "fast-import", "--quiet", export_marks, NULL };
 +	char tmpfile[512];
 +	char fast_import_cmd[512];
 +	char buf[512];
 +	int len, tmpfd;
 +
 +	strcpy(tmpfile, git_path("hashstream_XXXXXX"));
 +	tmpfd = git_mkstemp_mode(tmpfile, 0600);
 +	if (tmpfd < 0)
 +		die_errno("cannot create tempfile: %s", tmpfile);
 +	if (close(tmpfd))
 +		die_errno("cannot close tempfile: %s", tmpfile);
 +	sprintf(export_marks, "--export-marks=%s", tmpfile);
 +
 +	memset(&fast_import, 0, sizeof(fast_import));
 +	fast_import.in = -1;
 +	fast_import.argv = argv;
 +	fast_import.git_cmd = 1;
 +	if (start_command(&fast_import))
 +		die_errno("index-stream: git fast-import failed");
 +
 +	len = sprintf(fast_import_cmd, "blob\nmark :1\ndata %lu\n",
 +		      (unsigned long) size);
 +	write_or_whine(fast_import.in, fast_import_cmd, len,
 +		       "index-stream: feeding fast-import");
 +	while (size) {
 +		char buf[10240];
 +		size_t sz = size < sizeof(buf) ? size : sizeof(buf);
 +		size_t actual;
 +
 +		actual = read_in_full(fd, buf, sz);
 +		if (actual < 0)
 +			die_errno("index-stream: reading input");
 +		if (write_in_full(fast_import.in, buf, actual) != actual)
 +			die_errno("index-stream: feeding fast-import");
 +		size -= actual;
 +	}
 +	if (close(fast_import.in))
 +		die_errno("index-stream: closing fast-import");
 +	if (finish_command(&fast_import))
 +		die_errno("index-stream: finishing fast-import");
 +
 +	tmpfd = open(tmpfile, O_RDONLY);
 +	if (tmpfd < 0)
 +		die_errno("index-stream: cannot open fast-import mark");
 +	len = read(tmpfd, buf, sizeof(buf));
 +	if (len < 0)
 +		die_errno("index-stream: reading fast-import mark");
 +	if (close(tmpfd) < 0)
 +		die_errno("index-stream: closing fast-import mark");
 +	if (unlink(tmpfile))
 +		die_errno("index-stream: unlinking fast-import mark");
 +	if (len != 44 ||
 +	    memcmp(":1 ", buf, 3) ||
 +	    get_sha1_hex(buf + 3, sha1))
 +		die_errno("index-stream: unexpected fast-import mark: <%s>", buf);
 +	return 0;
 +}
 +
 +int index_fd(unsigned char *sha1, int fd, struct stat *st,
 +	     enum object_type type, const char *path, unsigned flags)
 +{
 +	int ret;
 +	size_t size = xsize_t(st->st_size);
 +
 +	if (!S_ISREG(st->st_mode))
 +		ret = index_pipe(sha1, fd, type, path, flags);
 +	else if (size <= big_file_threshold || type != OBJ_BLOB)
 +		ret = index_core(sha1, fd, size, type, path, flags);
 +	else
 +		ret = index_stream(sha1, fd, size, type, path, flags);
  	close(fd);
  	return ret;
  }
  
 -int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object)
 +int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags)
  {
  	int fd;
  	struct strbuf sb = STRBUF_INIT;
@@@ -2766,7 -2665,7 +2766,7 @@@
  		if (fd < 0)
  			return error("open(\"%s\"): %s", path,
  				     strerror(errno));
 -		if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path, 0) < 0)
 +		if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
  			return error("%s: failed to insert into database",
  				     path);
  		break;
@@@ -2776,7 -2675,7 +2776,7 @@@
  			return error("readlink(\"%s\"): %s", path,
  			             errstr);
  		}
 -		if (!write_object)
 +		if (!(flags & HASH_WRITE_OBJECT))
  			hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
  		else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
  			return error("%s: failed to insert into database",