From: Junio C Hamano <gitster@pobox.com>
Date: Mon, 31 Oct 2016 20:15:21 +0000 (-0700)
Subject: Merge branch 'ls/filter-process'
X-Git-Tag: v2.11.0-rc0~10
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/dbaa6bdce22914843e956e36d41d328547514342?ds=inline;hp=-c

Merge branch 'ls/filter-process'

The smudge/clean filter API expect an external process is spawned
to filter the contents for each path that has a filter defined.  A
new type of "process" filter API has been added to allow the first
request to run the filter for a path to spawn a single process, and
all filtering need is served by this single process for multiple
paths, reducing the process creation overhead.

* ls/filter-process:
contrib/long-running-filter: add long running filter example
convert: add filter.<driver>.process option
convert: prepare filter.<driver>.process option
convert: make apply_filter() adhere to standard Git error handling
pkt-line: add functions to read/write flush terminated packet streams
pkt-line: add packet_write_gently()
pkt-line: add packet_flush_gently()
pkt-line: add packet_write_fmt_gently()
pkt-line: extract set_packet_header()
pkt-line: rename packet_write() to packet_write_fmt()
run-command: add clean_on_exit_handler
run-command: move check_pipe() from write_or_die to run_command
convert: modernize tests
convert: quote filter names in error messages
---

dbaa6bdce22914843e956e36d41d328547514342
diff --combined builtin/receive-pack.c
index 680759d256,173d081647..e6b3879a5b
--- a/builtin/receive-pack.c
+++ b/builtin/receive-pack.c
@@@ -20,7 -20,6 +20,7 @@@
  #include "gpg-interface.h"
  #include "sigchain.h"
  #include "fsck.h"
 +#include "tmp-objdir.h"
  
  static const char * const receive_pack_usage[] = {
  	N_("git receive-pack <git-dir>"),
@@@ -87,8 -86,6 +87,8 @@@ static enum 
  } use_keepalive;
  static int keepalive_in_sec = 5;
  
 +static struct tmp_objdir *tmp_objdir;
 +
  static enum deny_action parse_deny_action(const char *var, const char *value)
  {
  	if (value) {
@@@ -227,7 -224,7 +227,7 @@@ static int receive_pack_config(const ch
  static void show_ref(const char *path, const unsigned char *sha1)
  {
  	if (sent_capabilities) {
- 		packet_write(1, "%s %s\n", sha1_to_hex(sha1), path);
+ 		packet_write_fmt(1, "%s %s\n", sha1_to_hex(sha1), path);
  	} else {
  		struct strbuf cap = STRBUF_INIT;
  
@@@ -242,7 -239,7 +242,7 @@@
  		if (advertise_push_options)
  			strbuf_addstr(&cap, " push-options");
  		strbuf_addf(&cap, " agent=%s", git_user_agent_sanitized());
- 		packet_write(1, "%s %s%c%s\n",
+ 		packet_write_fmt(1, "%s %s%c%s\n",
  			     sha1_to_hex(sha1), path, 0, cap.buf);
  		strbuf_release(&cap);
  		sent_capabilities = 1;
@@@ -271,10 -268,9 +271,10 @@@ static int show_ref_cb(const char *path
  	return 0;
  }
  
 -static void show_one_alternate_sha1(const unsigned char sha1[20], void *unused)
 +static int show_one_alternate_sha1(const unsigned char sha1[20], void *unused)
  {
  	show_ref(".have", sha1);
 +	return 0;
  }
  
  static void collect_one_alternate_ref(const struct ref *ref, void *data)
@@@ -667,9 -663,6 +667,9 @@@ static int run_and_feed_hook(const cha
  	} else
  		argv_array_pushf(&proc.env_array, "GIT_PUSH_OPTION_COUNT");
  
 +	if (tmp_objdir)
 +		argv_array_pushv(&proc.env_array, tmp_objdir_env(tmp_objdir));
 +
  	if (use_sideband) {
  		memset(&muxer, 0, sizeof(muxer));
  		muxer.proc = copy_to_sideband;
@@@ -769,7 -762,6 +769,7 @@@ static int run_update_hook(struct comma
  	proc.stdout_to_stderr = 1;
  	proc.err = use_sideband ? -1 : 0;
  	proc.argv = argv;
 +	proc.env = tmp_objdir_env(tmp_objdir);
  
  	code = start_command(&proc);
  	if (code)
@@@ -789,39 -781,47 +789,39 @@@ static int is_ref_checked_out(const cha
  	return !strcmp(head_name, ref);
  }
  
 -static char *refuse_unconfigured_deny_msg[] = {
 -	"By default, updating the current branch in a non-bare repository",
 -	"is denied, because it will make the index and work tree inconsistent",
 -	"with what you pushed, and will require 'git reset --hard' to match",
 -	"the work tree to HEAD.",
 -	"",
 -	"You can set 'receive.denyCurrentBranch' configuration variable to",
 -	"'ignore' or 'warn' in the remote repository to allow pushing into",
 -	"its current branch; however, this is not recommended unless you",
 -	"arranged to update its work tree to match what you pushed in some",
 -	"other way.",
 -	"",
 -	"To squelch this message and still keep the default behaviour, set",
 -	"'receive.denyCurrentBranch' configuration variable to 'refuse'."
 -};
 +static char *refuse_unconfigured_deny_msg =
 +	N_("By default, updating the current branch in a non-bare repository\n"
 +	   "is denied, because it will make the index and work tree inconsistent\n"
 +	   "with what you pushed, and will require 'git reset --hard' to match\n"
 +	   "the work tree to HEAD.\n"
 +	   "\n"
 +	   "You can set 'receive.denyCurrentBranch' configuration variable to\n"
 +	   "'ignore' or 'warn' in the remote repository to allow pushing into\n"
 +	   "its current branch; however, this is not recommended unless you\n"
 +	   "arranged to update its work tree to match what you pushed in some\n"
 +	   "other way.\n"
 +	   "\n"
 +	   "To squelch this message and still keep the default behaviour, set\n"
 +	   "'receive.denyCurrentBranch' configuration variable to 'refuse'.");
  
  static void refuse_unconfigured_deny(void)
  {
 -	int i;
 -	for (i = 0; i < ARRAY_SIZE(refuse_unconfigured_deny_msg); i++)
 -		rp_error("%s", refuse_unconfigured_deny_msg[i]);
 +	rp_error("%s", _(refuse_unconfigured_deny_msg));
  }
  
 -static char *refuse_unconfigured_deny_delete_current_msg[] = {
 -	"By default, deleting the current branch is denied, because the next",
 -	"'git clone' won't result in any file checked out, causing confusion.",
 -	"",
 -	"You can set 'receive.denyDeleteCurrent' configuration variable to",
 -	"'warn' or 'ignore' in the remote repository to allow deleting the",
 -	"current branch, with or without a warning message.",
 -	"",
 -	"To squelch this message, you can set it to 'refuse'."
 -};
 +static char *refuse_unconfigured_deny_delete_current_msg =
 +	N_("By default, deleting the current branch is denied, because the next\n"
 +	   "'git clone' won't result in any file checked out, causing confusion.\n"
 +	   "\n"
 +	   "You can set 'receive.denyDeleteCurrent' configuration variable to\n"
 +	   "'warn' or 'ignore' in the remote repository to allow deleting the\n"
 +	   "current branch, with or without a warning message.\n"
 +	   "\n"
 +	   "To squelch this message, you can set it to 'refuse'.");
  
  static void refuse_unconfigured_deny_delete_current(void)
  {
 -	int i;
 -	for (i = 0;
 -	     i < ARRAY_SIZE(refuse_unconfigured_deny_delete_current_msg);
 -	     i++)
 -		rp_error("%s", refuse_unconfigured_deny_delete_current_msg[i]);
 +	rp_error("%s", _(refuse_unconfigured_deny_delete_current_msg));
  }
  
  static int command_singleton_iterator(void *cb_data, unsigned char sha1[20]);
@@@ -841,7 -841,6 +841,7 @@@ static int update_shallow_ref(struct co
  		    !delayed_reachability_test(si, i))
  			sha1_array_append(&extra, si->shallow->sha1[i]);
  
 +	opt.env = tmp_objdir_env(tmp_objdir);
  	setup_alternate_shallow(&shallow_lock, &opt.shallow_file, &extra);
  	if (check_connected(command_singleton_iterator, cmd, &opt)) {
  		rollback_lock_file(&shallow_lock);
@@@ -1163,6 -1162,10 +1163,6 @@@ static void check_aliased_update(struc
  	struct string_list_item *item;
  	struct command *dst_cmd;
  	unsigned char sha1[GIT_SHA1_RAWSZ];
 -	char cmd_oldh[GIT_SHA1_HEXSZ + 1],
 -	     cmd_newh[GIT_SHA1_HEXSZ + 1],
 -	     dst_oldh[GIT_SHA1_HEXSZ + 1],
 -	     dst_newh[GIT_SHA1_HEXSZ + 1];
  	int flag;
  
  	strbuf_addf(&buf, "%s%s", get_git_namespace(), cmd->ref_name);
@@@ -1193,14 -1196,14 +1193,14 @@@
  
  	dst_cmd->skip_update = 1;
  
 -	find_unique_abbrev_r(cmd_oldh, cmd->old_sha1, DEFAULT_ABBREV);
 -	find_unique_abbrev_r(cmd_newh, cmd->new_sha1, DEFAULT_ABBREV);
 -	find_unique_abbrev_r(dst_oldh, dst_cmd->old_sha1, DEFAULT_ABBREV);
 -	find_unique_abbrev_r(dst_newh, dst_cmd->new_sha1, DEFAULT_ABBREV);
  	rp_error("refusing inconsistent update between symref '%s' (%s..%s) and"
  		 " its target '%s' (%s..%s)",
 -		 cmd->ref_name, cmd_oldh, cmd_newh,
 -		 dst_cmd->ref_name, dst_oldh, dst_newh);
 +		 cmd->ref_name,
 +		 find_unique_abbrev(cmd->old_sha1, DEFAULT_ABBREV),
 +		 find_unique_abbrev(cmd->new_sha1, DEFAULT_ABBREV),
 +		 dst_cmd->ref_name,
 +		 find_unique_abbrev(dst_cmd->old_sha1, DEFAULT_ABBREV),
 +		 find_unique_abbrev(dst_cmd->new_sha1, DEFAULT_ABBREV));
  
  	cmd->error_string = dst_cmd->error_string =
  		"inconsistent aliased update";
@@@ -1245,17 -1248,12 +1245,17 @@@ static void set_connectivity_errors(str
  
  	for (cmd = commands; cmd; cmd = cmd->next) {
  		struct command *singleton = cmd;
 +		struct check_connected_options opt = CHECK_CONNECTED_INIT;
 +
  		if (shallow_update && si->shallow_ref[cmd->index])
  			/* to be checked in update_shallow_ref() */
  			continue;
 +
 +		opt.env = tmp_objdir_env(tmp_objdir);
  		if (!check_connected(command_singleton_iterator, &singleton,
 -				     NULL))
 +				     &opt))
  			continue;
 +
  		cmd->error_string = "missing necessary objects";
  	}
  }
@@@ -1438,7 -1436,6 +1438,7 @@@ static void execute_commands(struct com
  	data.si = si;
  	opt.err_fd = err_fd;
  	opt.progress = err_fd && !quiet;
 +	opt.env = tmp_objdir_env(tmp_objdir);
  	if (check_connected(iterate_receive_command_list, &data, &opt))
  		set_connectivity_errors(commands, si);
  
@@@ -1455,19 -1452,6 +1455,19 @@@
  		return;
  	}
  
 +	/*
 +	 * Now we'll start writing out refs, which means the objects need
 +	 * to be in their final positions so that other processes can see them.
 +	 */
 +	if (tmp_objdir_migrate(tmp_objdir) < 0) {
 +		for (cmd = commands; cmd; cmd = cmd->next) {
 +			if (!cmd->error_string)
 +				cmd->error_string = "unable to migrate objects to permanent storage";
 +		}
 +		return;
 +	}
 +	tmp_objdir = NULL;
 +
  	check_aliased_updates(commands);
  
  	free(head_name_to_free);
@@@ -1663,18 -1647,6 +1663,18 @@@ static const char *unpack(int err_fd, s
  		argv_array_push(&child.args, alt_shallow_file);
  	}
  
 +	tmp_objdir = tmp_objdir_create();
 +	if (!tmp_objdir)
 +		return "unable to create temporary object directory";
 +	child.env = tmp_objdir_env(tmp_objdir);
 +
 +	/*
 +	 * Normally we just pass the tmp_objdir environment to the child
 +	 * processes that do the heavy lifting, but we may need to see these
 +	 * objects ourselves to set up shallow information.
 +	 */
 +	tmp_objdir_add_as_alternate(tmp_objdir);
 +
  	if (ntohl(hdr.hdr_entries) < unpack_limit) {
  		argv_array_pushl(&child.args, "unpack-objects", hdr_arg, NULL);
  		if (quiet)
diff --combined connect.c
index d99d6435fd,5330d9c162..8cb93b0720
--- a/connect.c
+++ b/connect.c
@@@ -43,14 -43,14 +43,14 @@@ int check_ref_type(const struct ref *re
  	return check_ref(ref->name, flags);
  }
  
 -static void die_initial_contact(int got_at_least_one_head)
 +static void die_initial_contact(int unexpected)
  {
 -	if (got_at_least_one_head)
 -		die("The remote end hung up upon initial contact");
 +	if (unexpected)
 +		die(_("The remote end hung up upon initial contact"));
  	else
 -		die("Could not read from remote repository.\n\n"
 -		    "Please make sure you have the correct access rights\n"
 -		    "and the repository exists.");
 +		die(_("Could not read from remote repository.\n\n"
 +		      "Please make sure you have the correct access rights\n"
 +		      "and the repository exists."));
  }
  
  static void parse_one_symref_info(struct string_list *symref, const char *val, int len)
@@@ -115,18 -115,10 +115,18 @@@ struct ref **get_remote_heads(int in, c
  			      struct sha1_array *shallow_points)
  {
  	struct ref **orig_list = list;
 -	int got_at_least_one_head = 0;
 +
 +	/*
 +	 * A hang-up after seeing some response from the other end
 +	 * means that it is unexpected, as we know the other end is
 +	 * willing to talk to us.  A hang-up before seeing any
 +	 * response does not necessarily mean an ACL problem, though.
 +	 */
 +	int saw_response;
 +	int got_dummy_ref_with_capabilities_declaration = 0;
  
  	*list = NULL;
 -	for (;;) {
 +	for (saw_response = 0; ; saw_response = 1) {
  		struct ref *ref;
  		struct object_id old_oid;
  		char *name;
@@@ -139,7 -131,7 +139,7 @@@
  				  PACKET_READ_GENTLE_ON_EOF |
  				  PACKET_READ_CHOMP_NEWLINE);
  		if (len < 0)
 -			die_initial_contact(got_at_least_one_head);
 +			die_initial_contact(saw_response);
  
  		if (!len)
  			break;
@@@ -173,25 -165,13 +173,25 @@@
  			continue;
  		}
  
 +		if (!strcmp(name, "capabilities^{}")) {
 +			if (saw_response)
 +				die("protocol error: unexpected capabilities^{}");
 +			if (got_dummy_ref_with_capabilities_declaration)
 +				die("protocol error: multiple capabilities^{}");
 +			got_dummy_ref_with_capabilities_declaration = 1;
 +			continue;
 +		}
 +
  		if (!check_ref(name, flags))
  			continue;
 +
 +		if (got_dummy_ref_with_capabilities_declaration)
 +			die("protocol error: unexpected ref after capabilities^{}");
 +
  		ref = alloc_ref(buffer + GIT_SHA1_HEXSZ + 1);
  		oidcpy(&ref->old_oid, &old_oid);
  		*list = ref;
  		list = &ref->next;
 -		got_at_least_one_head = 1;
  	}
  
  	annotate_refs_with_symref_info(*orig_list);
@@@ -750,7 -730,7 +750,7 @@@ struct child_process *git_connect(int f
  		 * Note: Do not add any other headers here!  Doing so
  		 * will cause older git-daemon servers to crash.
  		 */
- 		packet_write(fd[1],
+ 		packet_write_fmt(fd[1],
  			     "%s %s%chost=%s%c",
  			     prog, path, 0,
  			     target_host, 0);
diff --combined convert.c
index 0ad39b16cc,bc242276ff..be91358462
--- a/convert.c
+++ b/convert.c
@@@ -3,6 -3,7 +3,7 @@@
  #include "run-command.h"
  #include "quote.h"
  #include "sigchain.h"
+ #include "pkt-line.h"
  
  /*
   * convert.c - convert a file when checking it out and checking it in.
@@@ -197,21 -198,17 +198,21 @@@ static void check_safe_crlf(const char 
  		 * CRLFs would not be restored by checkout
  		 */
  		if (checksafe == SAFE_CRLF_WARN)
 -			warning("CRLF will be replaced by LF in %s.\nThe file will have its original line endings in your working directory.", path);
 +			warning(_("CRLF will be replaced by LF in %s.\n"
 +				  "The file will have its original line"
 +				  " endings in your working directory."), path);
  		else /* i.e. SAFE_CRLF_FAIL */
 -			die("CRLF would be replaced by LF in %s.", path);
 +			die(_("CRLF would be replaced by LF in %s."), path);
  	} else if (old_stats->lonelf && !new_stats->lonelf ) {
  		/*
  		 * CRLFs would be added by checkout
  		 */
  		if (checksafe == SAFE_CRLF_WARN)
 -			warning("LF will be replaced by CRLF in %s.\nThe file will have its original line endings in your working directory.", path);
 +			warning(_("LF will be replaced by CRLF in %s.\n"
 +				  "The file will have its original line"
 +				  " endings in your working directory."), path);
  		else /* i.e. SAFE_CRLF_FAIL */
 -			die("LF would be replaced by CRLF in %s", path);
 +			die(_("LF would be replaced by CRLF in %s"), path);
  	}
  }
  
@@@ -416,7 -413,7 +417,7 @@@ static int filter_buffer_or_fd(int in, 
  	child_process.out = out;
  
  	if (start_command(&child_process))
- 		return error("cannot fork to run external filter %s", params->cmd);
+ 		return error("cannot fork to run external filter '%s'", params->cmd);
  
  	sigchain_push(SIGPIPE, SIG_IGN);
  
@@@ -434,19 -431,19 +435,19 @@@
  	if (close(child_process.in))
  		write_err = 1;
  	if (write_err)
- 		error("cannot feed the input to external filter %s", params->cmd);
+ 		error("cannot feed the input to external filter '%s'", params->cmd);
  
  	sigchain_pop(SIGPIPE);
  
  	status = finish_command(&child_process);
  	if (status)
- 		error("external filter %s failed %d", params->cmd, status);
+ 		error("external filter '%s' failed %d", params->cmd, status);
  
  	strbuf_release(&cmd);
  	return (write_err || status);
  }
  
- static int apply_filter(const char *path, const char *src, size_t len, int fd,
+ static int apply_single_file_filter(const char *path, const char *src, size_t len, int fd,
                          struct strbuf *dst, const char *cmd)
  {
  	/*
@@@ -455,17 -452,11 +456,11 @@@
  	 *
  	 * (child --> cmd) --> us
  	 */
- 	int ret = 1;
+ 	int err = 0;
  	struct strbuf nbuf = STRBUF_INIT;
  	struct async async;
  	struct filter_params params;
  
- 	if (!cmd || !*cmd)
- 		return 0;
- 
- 	if (!dst)
- 		return 1;
- 
  	memset(&async, 0, sizeof(async));
  	async.proc = filter_buffer_or_fd;
  	async.data = &params;
@@@ -481,23 -472,304 +476,304 @@@
  		return 0;	/* error was already reported */
  
  	if (strbuf_read(&nbuf, async.out, len) < 0) {
- 		error("read from external filter %s failed", cmd);
- 		ret = 0;
+ 		err = error("read from external filter '%s' failed", cmd);
  	}
  	if (close(async.out)) {
- 		error("read from external filter %s failed", cmd);
- 		ret = 0;
+ 		err = error("read from external filter '%s' failed", cmd);
  	}
  	if (finish_async(&async)) {
- 		error("external filter %s failed", cmd);
- 		ret = 0;
+ 		err = error("external filter '%s' failed", cmd);
  	}
  
- 	if (ret) {
+ 	if (!err) {
  		strbuf_swap(dst, &nbuf);
  	}
  	strbuf_release(&nbuf);
- 	return ret;
+ 	return !err;
+ }
+ 
+ #define CAP_CLEAN    (1u<<0)
+ #define CAP_SMUDGE   (1u<<1)
+ 
+ struct cmd2process {
+ 	struct hashmap_entry ent; /* must be the first member! */
+ 	unsigned int supported_capabilities;
+ 	const char *cmd;
+ 	struct child_process process;
+ };
+ 
+ static int cmd_process_map_initialized;
+ static struct hashmap cmd_process_map;
+ 
+ static int cmd2process_cmp(const struct cmd2process *e1,
+ 			   const struct cmd2process *e2,
+ 			   const void *unused)
+ {
+ 	return strcmp(e1->cmd, e2->cmd);
+ }
+ 
+ static struct cmd2process *find_multi_file_filter_entry(struct hashmap *hashmap, const char *cmd)
+ {
+ 	struct cmd2process key;
+ 	hashmap_entry_init(&key, strhash(cmd));
+ 	key.cmd = cmd;
+ 	return hashmap_get(hashmap, &key, NULL);
+ }
+ 
+ static int packet_write_list(int fd, const char *line, ...)
+ {
+ 	va_list args;
+ 	int err;
+ 	va_start(args, line);
+ 	for (;;) {
+ 		if (!line)
+ 			break;
+ 		if (strlen(line) > LARGE_PACKET_DATA_MAX)
+ 			return -1;
+ 		err = packet_write_fmt_gently(fd, "%s\n", line);
+ 		if (err)
+ 			return err;
+ 		line = va_arg(args, const char*);
+ 	}
+ 	va_end(args);
+ 	return packet_flush_gently(fd);
+ }
+ 
+ static void read_multi_file_filter_status(int fd, struct strbuf *status)
+ {
+ 	struct strbuf **pair;
+ 	char *line;
+ 	for (;;) {
+ 		line = packet_read_line(fd, NULL);
+ 		if (!line)
+ 			break;
+ 		pair = strbuf_split_str(line, '=', 2);
+ 		if (pair[0] && pair[0]->len && pair[1]) {
+ 			/* the last "status=<foo>" line wins */
+ 			if (!strcmp(pair[0]->buf, "status=")) {
+ 				strbuf_reset(status);
+ 				strbuf_addbuf(status, pair[1]);
+ 			}
+ 		}
+ 		strbuf_list_free(pair);
+ 	}
+ }
+ 
+ static void kill_multi_file_filter(struct hashmap *hashmap, struct cmd2process *entry)
+ {
+ 	if (!entry)
+ 		return;
+ 
+ 	entry->process.clean_on_exit = 0;
+ 	kill(entry->process.pid, SIGTERM);
+ 	finish_command(&entry->process);
+ 
+ 	hashmap_remove(hashmap, entry, NULL);
+ 	free(entry);
+ }
+ 
+ static void stop_multi_file_filter(struct child_process *process)
+ {
+ 	sigchain_push(SIGPIPE, SIG_IGN);
+ 	/* Closing the pipe signals the filter to initiate a shutdown. */
+ 	close(process->in);
+ 	close(process->out);
+ 	sigchain_pop(SIGPIPE);
+ 	/* Finish command will wait until the shutdown is complete. */
+ 	finish_command(process);
+ }
+ 
+ static struct cmd2process *start_multi_file_filter(struct hashmap *hashmap, const char *cmd)
+ {
+ 	int err;
+ 	struct cmd2process *entry;
+ 	struct child_process *process;
+ 	const char *argv[] = { cmd, NULL };
+ 	struct string_list cap_list = STRING_LIST_INIT_NODUP;
+ 	char *cap_buf;
+ 	const char *cap_name;
+ 
+ 	entry = xmalloc(sizeof(*entry));
+ 	entry->cmd = cmd;
+ 	entry->supported_capabilities = 0;
+ 	process = &entry->process;
+ 
+ 	child_process_init(process);
+ 	process->argv = argv;
+ 	process->use_shell = 1;
+ 	process->in = -1;
+ 	process->out = -1;
+ 	process->clean_on_exit = 1;
+ 	process->clean_on_exit_handler = stop_multi_file_filter;
+ 
+ 	if (start_command(process)) {
+ 		error("cannot fork to run external filter '%s'", cmd);
+ 		return NULL;
+ 	}
+ 
+ 	hashmap_entry_init(entry, strhash(cmd));
+ 
+ 	sigchain_push(SIGPIPE, SIG_IGN);
+ 
+ 	err = packet_write_list(process->in, "git-filter-client", "version=2", NULL);
+ 	if (err)
+ 		goto done;
+ 
+ 	err = strcmp(packet_read_line(process->out, NULL), "git-filter-server");
+ 	if (err) {
+ 		error("external filter '%s' does not support filter protocol version 2", cmd);
+ 		goto done;
+ 	}
+ 	err = strcmp(packet_read_line(process->out, NULL), "version=2");
+ 	if (err)
+ 		goto done;
+ 	err = packet_read_line(process->out, NULL) != NULL;
+ 	if (err)
+ 		goto done;
+ 
+ 	err = packet_write_list(process->in, "capability=clean", "capability=smudge", NULL);
+ 
+ 	for (;;) {
+ 		cap_buf = packet_read_line(process->out, NULL);
+ 		if (!cap_buf)
+ 			break;
+ 		string_list_split_in_place(&cap_list, cap_buf, '=', 1);
+ 
+ 		if (cap_list.nr != 2 || strcmp(cap_list.items[0].string, "capability"))
+ 			continue;
+ 
+ 		cap_name = cap_list.items[1].string;
+ 		if (!strcmp(cap_name, "clean")) {
+ 			entry->supported_capabilities |= CAP_CLEAN;
+ 		} else if (!strcmp(cap_name, "smudge")) {
+ 			entry->supported_capabilities |= CAP_SMUDGE;
+ 		} else {
+ 			warning(
+ 				"external filter '%s' requested unsupported filter capability '%s'",
+ 				cmd, cap_name
+ 			);
+ 		}
+ 
+ 		string_list_clear(&cap_list, 0);
+ 	}
+ 
+ done:
+ 	sigchain_pop(SIGPIPE);
+ 
+ 	if (err || errno == EPIPE) {
+ 		error("initialization for external filter '%s' failed", cmd);
+ 		kill_multi_file_filter(hashmap, entry);
+ 		return NULL;
+ 	}
+ 
+ 	hashmap_add(hashmap, entry);
+ 	return entry;
+ }
+ 
+ static int apply_multi_file_filter(const char *path, const char *src, size_t len,
+ 				   int fd, struct strbuf *dst, const char *cmd,
+ 				   const unsigned int wanted_capability)
+ {
+ 	int err;
+ 	struct cmd2process *entry;
+ 	struct child_process *process;
+ 	struct strbuf nbuf = STRBUF_INIT;
+ 	struct strbuf filter_status = STRBUF_INIT;
+ 	const char *filter_type;
+ 
+ 	if (!cmd_process_map_initialized) {
+ 		cmd_process_map_initialized = 1;
+ 		hashmap_init(&cmd_process_map, (hashmap_cmp_fn) cmd2process_cmp, 0);
+ 		entry = NULL;
+ 	} else {
+ 		entry = find_multi_file_filter_entry(&cmd_process_map, cmd);
+ 	}
+ 
+ 	fflush(NULL);
+ 
+ 	if (!entry) {
+ 		entry = start_multi_file_filter(&cmd_process_map, cmd);
+ 		if (!entry)
+ 			return 0;
+ 	}
+ 	process = &entry->process;
+ 
+ 	if (!(wanted_capability & entry->supported_capabilities))
+ 		return 0;
+ 
+ 	if (CAP_CLEAN & wanted_capability)
+ 		filter_type = "clean";
+ 	else if (CAP_SMUDGE & wanted_capability)
+ 		filter_type = "smudge";
+ 	else
+ 		die("unexpected filter type");
+ 
+ 	sigchain_push(SIGPIPE, SIG_IGN);
+ 
+ 	assert(strlen(filter_type) < LARGE_PACKET_DATA_MAX - strlen("command=\n"));
+ 	err = packet_write_fmt_gently(process->in, "command=%s\n", filter_type);
+ 	if (err)
+ 		goto done;
+ 
+ 	err = strlen(path) > LARGE_PACKET_DATA_MAX - strlen("pathname=\n");
+ 	if (err) {
+ 		error("path name too long for external filter");
+ 		goto done;
+ 	}
+ 
+ 	err = packet_write_fmt_gently(process->in, "pathname=%s\n", path);
+ 	if (err)
+ 		goto done;
+ 
+ 	err = packet_flush_gently(process->in);
+ 	if (err)
+ 		goto done;
+ 
+ 	if (fd >= 0)
+ 		err = write_packetized_from_fd(fd, process->in);
+ 	else
+ 		err = write_packetized_from_buf(src, len, process->in);
+ 	if (err)
+ 		goto done;
+ 
+ 	read_multi_file_filter_status(process->out, &filter_status);
+ 	err = strcmp(filter_status.buf, "success");
+ 	if (err)
+ 		goto done;
+ 
+ 	err = read_packetized_to_strbuf(process->out, &nbuf) < 0;
+ 	if (err)
+ 		goto done;
+ 
+ 	read_multi_file_filter_status(process->out, &filter_status);
+ 	err = strcmp(filter_status.buf, "success");
+ 
+ done:
+ 	sigchain_pop(SIGPIPE);
+ 
+ 	if (err || errno == EPIPE) {
+ 		if (!strcmp(filter_status.buf, "error")) {
+ 			/* The filter signaled a problem with the file. */
+ 		} else if (!strcmp(filter_status.buf, "abort")) {
+ 			/*
+ 			 * The filter signaled a permanent problem. Don't try to filter
+ 			 * files with the same command for the lifetime of the current
+ 			 * Git process.
+ 			 */
+ 			 entry->supported_capabilities &= ~wanted_capability;
+ 		} else {
+ 			/*
+ 			 * Something went wrong with the protocol filter.
+ 			 * Force shutdown and restart if another blob requires filtering.
+ 			 */
+ 			error("external filter '%s' failed", cmd);
+ 			kill_multi_file_filter(&cmd_process_map, entry);
+ 		}
+ 	} else {
+ 		strbuf_swap(dst, &nbuf);
+ 	}
+ 	strbuf_release(&nbuf);
+ 	return !err;
  }
  
  static struct convert_driver {
@@@ -505,9 -777,35 +781,35 @@@
  	struct convert_driver *next;
  	const char *smudge;
  	const char *clean;
+ 	const char *process;
  	int required;
  } *user_convert, **user_convert_tail;
  
+ static int apply_filter(const char *path, const char *src, size_t len,
+ 			int fd, struct strbuf *dst, struct convert_driver *drv,
+ 			const unsigned int wanted_capability)
+ {
+ 	const char *cmd = NULL;
+ 
+ 	if (!drv)
+ 		return 0;
+ 
+ 	if (!dst)
+ 		return 1;
+ 
+ 	if ((CAP_CLEAN & wanted_capability) && !drv->process && drv->clean)
+ 		cmd = drv->clean;
+ 	else if ((CAP_SMUDGE & wanted_capability) && !drv->process && drv->smudge)
+ 		cmd = drv->smudge;
+ 
+ 	if (cmd && *cmd)
+ 		return apply_single_file_filter(path, src, len, fd, dst, cmd);
+ 	else if (drv->process && *drv->process)
+ 		return apply_multi_file_filter(path, src, len, fd, dst, drv->process, wanted_capability);
+ 
+ 	return 0;
+ }
+ 
  static int read_convert_config(const char *var, const char *value, void *cb)
  {
  	const char *key, *name;
@@@ -545,6 -843,9 +847,9 @@@
  	if (!strcmp("clean", key))
  		return git_config_string(&drv->clean, var, value);
  
+ 	if (!strcmp("process", key))
+ 		return git_config_string(&drv->process, var, value);
+ 
  	if (!strcmp("required", key)) {
  		drv->required = git_config_bool(var, value);
  		return 0;
@@@ -846,7 -1147,7 +1151,7 @@@ int would_convert_to_git_filter_fd(cons
  	if (!ca.drv->required)
  		return 0;
  
- 	return apply_filter(path, NULL, 0, -1, NULL, ca.drv->clean);
+ 	return apply_filter(path, NULL, 0, -1, NULL, ca.drv, CAP_CLEAN);
  }
  
  const char *get_convert_attr_ascii(const char *path)
@@@ -879,18 -1180,12 +1184,12 @@@ int convert_to_git(const char *path, co
                     struct strbuf *dst, enum safe_crlf checksafe)
  {
  	int ret = 0;
- 	const char *filter = NULL;
- 	int required = 0;
  	struct conv_attrs ca;
  
  	convert_attrs(&ca, path);
- 	if (ca.drv) {
- 		filter = ca.drv->clean;
- 		required = ca.drv->required;
- 	}
  
- 	ret |= apply_filter(path, src, len, -1, dst, filter);
- 	if (!ret && required)
+ 	ret |= apply_filter(path, src, len, -1, dst, ca.drv, CAP_CLEAN);
+ 	if (!ret && ca.drv && ca.drv->required)
  		die("%s: clean filter '%s' failed", path, ca.drv->name);
  
  	if (ret && dst) {
@@@ -912,9 -1207,9 +1211,9 @@@ void convert_to_git_filter_fd(const cha
  	convert_attrs(&ca, path);
  
  	assert(ca.drv);
- 	assert(ca.drv->clean);
+ 	assert(ca.drv->clean || ca.drv->process);
  
- 	if (!apply_filter(path, NULL, 0, fd, dst, ca.drv->clean))
+ 	if (!apply_filter(path, NULL, 0, fd, dst, ca.drv, CAP_CLEAN))
  		die("%s: clean filter '%s' failed", path, ca.drv->name);
  
  	crlf_to_git(path, dst->buf, dst->len, dst, ca.crlf_action, checksafe);
@@@ -926,15 -1221,9 +1225,9 @@@ static int convert_to_working_tree_inte
  					    int normalizing)
  {
  	int ret = 0, ret_filter = 0;
- 	const char *filter = NULL;
- 	int required = 0;
  	struct conv_attrs ca;
  
  	convert_attrs(&ca, path);
- 	if (ca.drv) {
- 		filter = ca.drv->smudge;
- 		required = ca.drv->required;
- 	}
  
  	ret |= ident_to_worktree(path, src, len, dst, ca.ident);
  	if (ret) {
@@@ -943,9 -1232,10 +1236,10 @@@
  	}
  	/*
  	 * CRLF conversion can be skipped if normalizing, unless there
- 	 * is a smudge filter.  The filter might expect CRLFs.
+ 	 * is a smudge or process filter (even if the process filter doesn't
+ 	 * support smudge).  The filters might expect CRLFs.
  	 */
- 	if (filter || !normalizing) {
+ 	if ((ca.drv && (ca.drv->smudge || ca.drv->process)) || !normalizing) {
  		ret |= crlf_to_worktree(path, src, len, dst, ca.crlf_action);
  		if (ret) {
  			src = dst->buf;
@@@ -953,8 -1243,8 +1247,8 @@@
  		}
  	}
  
- 	ret_filter = apply_filter(path, src, len, -1, dst, filter);
- 	if (!ret_filter && required)
+ 	ret_filter = apply_filter(path, src, len, -1, dst, ca.drv, CAP_SMUDGE);
+ 	if (!ret_filter && ca.drv && ca.drv->required)
  		die("%s: smudge filter %s failed", path, ca.drv->name);
  
  	return ret | ret_filter;
@@@ -1406,7 -1696,7 +1700,7 @@@ struct stream_filter *get_stream_filter
  	struct stream_filter *filter = NULL;
  
  	convert_attrs(&ca, path);
- 	if (ca.drv && (ca.drv->smudge || ca.drv->clean))
+ 	if (ca.drv && (ca.drv->process || ca.drv->smudge || ca.drv->clean))
  		return NULL;
  
  	if (ca.crlf_action == CRLF_AUTO || ca.crlf_action == CRLF_AUTO_CRLF)
diff --combined daemon.c
index ff0fa583b0,afce1b9b7f..473e6b6b63
--- a/daemon.c
+++ b/daemon.c
@@@ -160,7 -160,6 +160,7 @@@ static const char *path_ok(const char *
  {
  	static char rpath[PATH_MAX];
  	static char interp_path[PATH_MAX];
 +	size_t rlen;
  	const char *path;
  	const char *dir;
  
@@@ -188,12 -187,8 +188,12 @@@
  			namlen = slash - dir;
  			restlen -= namlen;
  			loginfo("userpath <%s>, request <%s>, namlen %d, restlen %d, slash <%s>", user_path, dir, namlen, restlen, slash);
 -			snprintf(rpath, PATH_MAX, "%.*s/%s%.*s",
 -				 namlen, dir, user_path, restlen, slash);
 +			rlen = snprintf(rpath, sizeof(rpath), "%.*s/%s%.*s",
 +					namlen, dir, user_path, restlen, slash);
 +			if (rlen >= sizeof(rpath)) {
 +				logerror("user-path too large: %s", rpath);
 +				return NULL;
 +			}
  			dir = rpath;
  		}
  	}
@@@ -212,15 -207,7 +212,15 @@@
  
  		strbuf_expand(&expanded_path, interpolated_path,
  			      expand_path, &context);
 -		strlcpy(interp_path, expanded_path.buf, PATH_MAX);
 +
 +		rlen = strlcpy(interp_path, expanded_path.buf,
 +			       sizeof(interp_path));
 +		if (rlen >= sizeof(interp_path)) {
 +			logerror("interpolated path too large: %s",
 +				 interp_path);
 +			return NULL;
 +		}
 +
  		strbuf_release(&expanded_path);
  		loginfo("Interpolated dir '%s'", interp_path);
  
@@@ -232,11 -219,7 +232,11 @@@
  			logerror("'%s': Non-absolute path denied (base-path active)", dir);
  			return NULL;
  		}
 -		snprintf(rpath, PATH_MAX, "%s%s", base_path, dir);
 +		rlen = snprintf(rpath, sizeof(rpath), "%s%s", base_path, dir);
 +		if (rlen >= sizeof(rpath)) {
 +			logerror("base-path too large: %s", rpath);
 +			return NULL;
 +		}
  		dir = rpath;
  	}
  
@@@ -298,7 -281,7 +298,7 @@@ static int daemon_error(const char *dir
  {
  	if (!informative_errors)
  		msg = "access denied or repository not exported";
- 	packet_write(1, "ERR %s: %s", msg, dir);
+ 	packet_write_fmt(1, "ERR %s: %s", msg, dir);
  	return -1;
  }
  
diff --combined shallow.c
index 2531e3af3b,d666e24f12..4d0b005d39
--- a/shallow.c
+++ b/shallow.c
@@@ -10,8 -10,6 +10,8 @@@
  #include "diff.h"
  #include "revision.h"
  #include "commit-slab.h"
 +#include "revision.h"
 +#include "list-objects.h"
  
  static int is_shallow = -1;
  static struct stat_validity shallow_stat;
@@@ -139,82 -137,6 +139,82 @@@ struct commit_list *get_shallow_commits
  	return result;
  }
  
 +static void show_commit(struct commit *commit, void *data)
 +{
 +	commit_list_insert(commit, data);
 +}
 +
 +/*
 + * Given rev-list arguments, run rev-list. All reachable commits
 + * except border ones are marked with not_shallow_flag. Border commits
 + * are marked with shallow_flag. The list of border/shallow commits
 + * are also returned.
 + */
 +struct commit_list *get_shallow_commits_by_rev_list(int ac, const char **av,
 +						    int shallow_flag,
 +						    int not_shallow_flag)
 +{
 +	struct commit_list *result = NULL, *p;
 +	struct commit_list *not_shallow_list = NULL;
 +	struct rev_info revs;
 +	int both_flags = shallow_flag | not_shallow_flag;
 +
 +	/*
 +	 * SHALLOW (excluded) and NOT_SHALLOW (included) should not be
 +	 * set at this point. But better be safe than sorry.
 +	 */
 +	clear_object_flags(both_flags);
 +
 +	is_repository_shallow(); /* make sure shallows are read */
 +
 +	init_revisions(&revs, NULL);
 +	save_commit_buffer = 0;
 +	setup_revisions(ac, av, &revs, NULL);
 +
 +	if (prepare_revision_walk(&revs))
 +		die("revision walk setup failed");
 +	traverse_commit_list(&revs, show_commit, NULL, &not_shallow_list);
 +
 +	/* Mark all reachable commits as NOT_SHALLOW */
 +	for (p = not_shallow_list; p; p = p->next)
 +		p->item->object.flags |= not_shallow_flag;
 +
 +	/*
 +	 * mark border commits SHALLOW + NOT_SHALLOW.
 +	 * We cannot clear NOT_SHALLOW right now. Imagine border
 +	 * commit A is processed first, then commit B, whose parent is
 +	 * A, later. If NOT_SHALLOW on A is cleared at step 1, B
 +	 * itself is considered border at step 2, which is incorrect.
 +	 */
 +	for (p = not_shallow_list; p; p = p->next) {
 +		struct commit *c = p->item;
 +		struct commit_list *parent;
 +
 +		if (parse_commit(c))
 +			die("unable to parse commit %s",
 +			    oid_to_hex(&c->object.oid));
 +
 +		for (parent = c->parents; parent; parent = parent->next)
 +			if (!(parent->item->object.flags & not_shallow_flag)) {
 +				c->object.flags |= shallow_flag;
 +				commit_list_insert(c, &result);
 +				break;
 +			}
 +	}
 +	free_commit_list(not_shallow_list);
 +
 +	/*
 +	 * Now we can clean up NOT_SHALLOW on border commits. Having
 +	 * both flags set can confuse the caller.
 +	 */
 +	for (p = result; p; p = p->next) {
 +		struct object *o = &p->item->object;
 +		if ((o->flags & both_flags) == both_flags)
 +			o->flags &= ~not_shallow_flag;
 +	}
 +	return result;
 +}
 +
  static void check_shallow_file_for_update(void)
  {
  	if (is_shallow == -1)
@@@ -338,7 -260,7 +338,7 @@@ static int advertise_shallow_grafts_cb(
  {
  	int fd = *(int *)cb;
  	if (graft->nr_parent == -1)
- 		packet_write(fd, "shallow %s\n", oid_to_hex(&graft->oid));
+ 		packet_write_fmt(fd, "shallow %s\n", oid_to_hex(&graft->oid));
  	return 0;
  }
  
diff --combined upload-pack.c
index d9e381f291,cd47de69ab..e0db8b42be
--- a/upload-pack.c
+++ b/upload-pack.c
@@@ -15,8 -15,6 +15,8 @@@
  #include "version.h"
  #include "string-list.h"
  #include "parse-options.h"
 +#include "argv-array.h"
 +#include "prio-queue.h"
  
  static const char * const upload_pack_usage[] = {
  	N_("git upload-pack [<options>] <dir>"),
@@@ -37,7 -35,6 +37,7 @@@
  
  static unsigned long oldest_have;
  
 +static int deepen_relative;
  static int multi_ack;
  static int no_done;
  static int use_thin_pack, use_ofs_delta, use_include_tag;
@@@ -284,7 -281,7 +284,7 @@@ static void create_pack_file(void
  	die("git upload-pack: %s", abort_msg);
  }
  
 -static int got_sha1(char *hex, unsigned char *sha1)
 +static int got_sha1(const char *hex, unsigned char *sha1)
  {
  	struct object *o;
  	int we_knew_they_have = 0;
@@@ -320,12 -317,12 +320,12 @@@
  
  static int reachable(struct commit *want)
  {
 -	struct commit_list *work = NULL;
 +	struct prio_queue work = { compare_commits_by_commit_date };
  
 -	commit_list_insert_by_date(want, &work);
 -	while (work) {
 +	prio_queue_put(&work, want);
 +	while (work.nr) {
  		struct commit_list *list;
 -		struct commit *commit = pop_commit(&work);
 +		struct commit *commit = prio_queue_get(&work);
  
  		if (commit->object.flags & THEY_HAVE) {
  			want->object.flags |= COMMON_KNOWN;
@@@ -341,12 -338,12 +341,12 @@@
  		for (list = commit->parents; list; list = list->next) {
  			struct commit *parent = list->item;
  			if (!(parent->object.flags & REACHABLE))
 -				commit_list_insert_by_date(parent, &work);
 +				prio_queue_put(&work, parent);
  		}
  	}
  	want->object.flags |= REACHABLE;
  	clear_commit_marks(want, REACHABLE);
 -	free_commit_list(work);
 +	clear_prio_queue(&work);
  	return (want->object.flags & COMMON_KNOWN);
  }
  
@@@ -390,21 -387,19 +390,21 @@@ static int get_common_commits(void
  
  	for (;;) {
  		char *line = packet_read_line(0, NULL);
 +		const char *arg;
 +
  		reset_timeout();
  
  		if (!line) {
  			if (multi_ack == 2 && got_common
  			    && !got_other && ok_to_give_up()) {
  				sent_ready = 1;
- 				packet_write(1, "ACK %s ready\n", last_hex);
+ 				packet_write_fmt(1, "ACK %s ready\n", last_hex);
  			}
  			if (have_obj.nr == 0 || multi_ack)
- 				packet_write(1, "NAK\n");
+ 				packet_write_fmt(1, "NAK\n");
  
  			if (no_done && sent_ready) {
- 				packet_write(1, "ACK %s\n", last_hex);
+ 				packet_write_fmt(1, "ACK %s\n", last_hex);
  				return 0;
  			}
  			if (stateless_rpc)
@@@ -413,28 -408,28 +413,28 @@@
  			got_other = 0;
  			continue;
  		}
 -		if (starts_with(line, "have ")) {
 -			switch (got_sha1(line+5, sha1)) {
 +		if (skip_prefix(line, "have ", &arg)) {
 +			switch (got_sha1(arg, sha1)) {
  			case -1: /* they have what we do not */
  				got_other = 1;
  				if (multi_ack && ok_to_give_up()) {
  					const char *hex = sha1_to_hex(sha1);
  					if (multi_ack == 2) {
  						sent_ready = 1;
- 						packet_write(1, "ACK %s ready\n", hex);
+ 						packet_write_fmt(1, "ACK %s ready\n", hex);
  					} else
- 						packet_write(1, "ACK %s continue\n", hex);
+ 						packet_write_fmt(1, "ACK %s continue\n", hex);
  				}
  				break;
  			default:
  				got_common = 1;
  				memcpy(last_hex, sha1_to_hex(sha1), 41);
  				if (multi_ack == 2)
- 					packet_write(1, "ACK %s common\n", last_hex);
+ 					packet_write_fmt(1, "ACK %s common\n", last_hex);
  				else if (multi_ack)
- 					packet_write(1, "ACK %s continue\n", last_hex);
+ 					packet_write_fmt(1, "ACK %s continue\n", last_hex);
  				else if (have_obj.nr == 1)
- 					packet_write(1, "ACK %s\n", last_hex);
+ 					packet_write_fmt(1, "ACK %s\n", last_hex);
  				break;
  			}
  			continue;
@@@ -442,10 -437,10 +442,10 @@@
  		if (!strcmp(line, "done")) {
  			if (have_obj.nr > 0) {
  				if (multi_ack)
- 					packet_write(1, "ACK %s\n", last_hex);
+ 					packet_write_fmt(1, "ACK %s\n", last_hex);
  				return 0;
  			}
- 			packet_write(1, "NAK\n");
+ 			packet_write_fmt(1, "NAK\n");
  			return -1;
  		}
  		die("git upload-pack: expected SHA1 list, got '%s'", line);
@@@ -459,136 -454,73 +459,136 @@@ static int is_our_ref(struct object *o
  	return o->flags & ((allow_hidden_ref ? HIDDEN_REF : 0) | OUR_REF);
  }
  
 -static void check_non_tip(void)
 +/*
 + * on successful case, it's up to the caller to close cmd->out
 + */
 +static int do_reachable_revlist(struct child_process *cmd,
 +				struct object_array *src,
 +				struct object_array *reachable)
  {
  	static const char *argv[] = {
  		"rev-list", "--stdin", NULL,
  	};
 -	static struct child_process cmd = CHILD_PROCESS_INIT;
  	struct object *o;
  	char namebuf[42]; /* ^ + SHA-1 + LF */
  	int i;
  
 -	/*
 -	 * In the normal in-process case without
 -	 * uploadpack.allowReachableSHA1InWant,
 -	 * non-tip requests can never happen.
 -	 */
 -	if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1))
 -		goto error;
 -
 -	cmd.argv = argv;
 -	cmd.git_cmd = 1;
 -	cmd.no_stderr = 1;
 -	cmd.in = -1;
 -	cmd.out = -1;
 -
 -	if (start_command(&cmd))
 -		goto error;
 +	cmd->argv = argv;
 +	cmd->git_cmd = 1;
 +	cmd->no_stderr = 1;
 +	cmd->in = -1;
 +	cmd->out = -1;
  
  	/*
 -	 * If rev-list --stdin encounters an unknown commit, it
 -	 * terminates, which will cause SIGPIPE in the write loop
 +	 * If the next rev-list --stdin encounters an unknown commit,
 +	 * it terminates, which will cause SIGPIPE in the write loop
  	 * below.
  	 */
  	sigchain_push(SIGPIPE, SIG_IGN);
  
 +	if (start_command(cmd))
 +		goto error;
 +
  	namebuf[0] = '^';
  	namebuf[41] = '\n';
  	for (i = get_max_object_index(); 0 < i; ) {
  		o = get_indexed_object(--i);
  		if (!o)
  			continue;
 +		if (reachable && o->type == OBJ_COMMIT)
 +			o->flags &= ~TMP_MARK;
  		if (!is_our_ref(o))
  			continue;
  		memcpy(namebuf + 1, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
 -		if (write_in_full(cmd.in, namebuf, 42) < 0)
 +		if (write_in_full(cmd->in, namebuf, 42) < 0)
  			goto error;
  	}
  	namebuf[40] = '\n';
 -	for (i = 0; i < want_obj.nr; i++) {
 -		o = want_obj.objects[i].item;
 -		if (is_our_ref(o))
 +	for (i = 0; i < src->nr; i++) {
 +		o = src->objects[i].item;
 +		if (is_our_ref(o)) {
 +			if (reachable)
 +				add_object_array(o, NULL, reachable);
  			continue;
 +		}
 +		if (reachable && o->type == OBJ_COMMIT)
 +			o->flags |= TMP_MARK;
  		memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
 -		if (write_in_full(cmd.in, namebuf, 41) < 0)
 +		if (write_in_full(cmd->in, namebuf, 41) < 0)
  			goto error;
  	}
 -	close(cmd.in);
 +	close(cmd->in);
 +	cmd->in = -1;
 +	sigchain_pop(SIGPIPE);
  
 +	return 0;
 +
 +error:
  	sigchain_pop(SIGPIPE);
  
 +	if (cmd->in >= 0)
 +		close(cmd->in);
 +	if (cmd->out >= 0)
 +		close(cmd->out);
 +	return -1;
 +}
 +
 +static int get_reachable_list(struct object_array *src,
 +			      struct object_array *reachable)
 +{
 +	struct child_process cmd = CHILD_PROCESS_INIT;
 +	int i;
 +	struct object *o;
 +	char namebuf[42]; /* ^ + SHA-1 + LF */
 +
 +	if (do_reachable_revlist(&cmd, src, reachable) < 0)
 +		return -1;
 +
 +	while ((i = read_in_full(cmd.out, namebuf, 41)) == 41) {
 +		struct object_id sha1;
 +
 +		if (namebuf[40] != '\n' || get_oid_hex(namebuf, &sha1))
 +			break;
 +
 +		o = lookup_object(sha1.hash);
 +		if (o && o->type == OBJ_COMMIT) {
 +			o->flags &= ~TMP_MARK;
 +		}
 +	}
 +	for (i = get_max_object_index(); 0 < i; i--) {
 +		o = get_indexed_object(i - 1);
 +		if (o && o->type == OBJ_COMMIT &&
 +		    (o->flags & TMP_MARK)) {
 +			add_object_array(o, NULL, reachable);
 +				o->flags &= ~TMP_MARK;
 +		}
 +	}
 +	close(cmd.out);
 +
 +	if (finish_command(&cmd))
 +		return -1;
 +
 +	return 0;
 +}
 +
 +static int has_unreachable(struct object_array *src)
 +{
 +	struct child_process cmd = CHILD_PROCESS_INIT;
 +	char buf[1];
 +	int i;
 +
 +	if (do_reachable_revlist(&cmd, src, NULL) < 0)
 +		return 1;
 +
  	/*
  	 * The commits out of the rev-list are not ancestors of
  	 * our ref.
  	 */
 -	i = read_in_full(cmd.out, namebuf, 1);
 +	i = read_in_full(cmd.out, buf, 1);
  	if (i)
  		goto error;
  	close(cmd.out);
 +	cmd.out = -1;
  
  	/*
  	 * rev-list may have died by encountering a bad commit
@@@ -599,142 -531,23 +599,142 @@@
  		goto error;
  
  	/* All the non-tip ones are ancestors of what we advertised */
 -	return;
 +	return 0;
 +
 +error:
 +	sigchain_pop(SIGPIPE);
 +	if (cmd.out >= 0)
 +		close(cmd.out);
 +	return 1;
 +}
 +
 +static void check_non_tip(void)
 +{
 +	int i;
 +
 +	/*
 +	 * In the normal in-process case without
 +	 * uploadpack.allowReachableSHA1InWant,
 +	 * non-tip requests can never happen.
 +	 */
 +	if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1))
 +		goto error;
 +	if (!has_unreachable(&want_obj))
 +		/* All the non-tip ones are ancestors of what we advertised */
 +		return;
  
  error:
  	/* Pick one of them (we know there at least is one) */
  	for (i = 0; i < want_obj.nr; i++) {
 -		o = want_obj.objects[i].item;
 +		struct object *o = want_obj.objects[i].item;
  		if (!is_our_ref(o))
  			die("git upload-pack: not our ref %s",
  			    oid_to_hex(&o->oid));
  	}
  }
  
 +static void send_shallow(struct commit_list *result)
 +{
 +	while (result) {
 +		struct object *object = &result->item->object;
 +		if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) {
- 			packet_write(1, "shallow %s",
- 				     oid_to_hex(&object->oid));
++			packet_write_fmt(1, "shallow %s",
++					 oid_to_hex(&object->oid));
 +			register_shallow(object->oid.hash);
 +			shallow_nr++;
 +		}
 +		result = result->next;
 +	}
 +}
 +
 +static void send_unshallow(const struct object_array *shallows)
 +{
 +	int i;
 +
 +	for (i = 0; i < shallows->nr; i++) {
 +		struct object *object = shallows->objects[i].item;
 +		if (object->flags & NOT_SHALLOW) {
 +			struct commit_list *parents;
- 			packet_write(1, "unshallow %s",
- 				     oid_to_hex(&object->oid));
++			packet_write_fmt(1, "unshallow %s",
++					 oid_to_hex(&object->oid));
 +			object->flags &= ~CLIENT_SHALLOW;
 +			/*
 +			 * We want to _register_ "object" as shallow, but we
 +			 * also need to traverse object's parents to deepen a
 +			 * shallow clone. Unregister it for now so we can
 +			 * parse and add the parents to the want list, then
 +			 * re-register it.
 +			 */
 +			unregister_shallow(object->oid.hash);
 +			object->parsed = 0;
 +			parse_commit_or_die((struct commit *)object);
 +			parents = ((struct commit *)object)->parents;
 +			while (parents) {
 +				add_object_array(&parents->item->object,
 +						 NULL, &want_obj);
 +				parents = parents->next;
 +			}
 +			add_object_array(object, NULL, &extra_edge_obj);
 +		}
 +		/* make sure commit traversal conforms to client */
 +		register_shallow(object->oid.hash);
 +	}
 +}
 +
 +static void deepen(int depth, int deepen_relative,
 +		   struct object_array *shallows)
 +{
 +	if (depth == INFINITE_DEPTH && !is_repository_shallow()) {
 +		int i;
 +
 +		for (i = 0; i < shallows->nr; i++) {
 +			struct object *object = shallows->objects[i].item;
 +			object->flags |= NOT_SHALLOW;
 +		}
 +	} else if (deepen_relative) {
 +		struct object_array reachable_shallows = OBJECT_ARRAY_INIT;
 +		struct commit_list *result;
 +
 +		get_reachable_list(shallows, &reachable_shallows);
 +		result = get_shallow_commits(&reachable_shallows,
 +					     depth + 1,
 +					     SHALLOW, NOT_SHALLOW);
 +		send_shallow(result);
 +		free_commit_list(result);
 +		object_array_clear(&reachable_shallows);
 +	} else {
 +		struct commit_list *result;
 +
 +		result = get_shallow_commits(&want_obj, depth,
 +					     SHALLOW, NOT_SHALLOW);
 +		send_shallow(result);
 +		free_commit_list(result);
 +	}
 +
 +	send_unshallow(shallows);
 +	packet_flush(1);
 +}
 +
 +static void deepen_by_rev_list(int ac, const char **av,
 +			       struct object_array *shallows)
 +{
 +	struct commit_list *result;
 +
 +	result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW);
 +	send_shallow(result);
 +	free_commit_list(result);
 +	send_unshallow(shallows);
 +	packet_flush(1);
 +}
 +
  static void receive_needs(void)
  {
  	struct object_array shallows = OBJECT_ARRAY_INIT;
 +	struct string_list deepen_not = STRING_LIST_INIT_DUP;
  	int depth = 0;
  	int has_non_tip = 0;
 +	unsigned long deepen_since = 0;
 +	int deepen_rev_list = 0;
  
  	shallow_nr = 0;
  	for (;;) {
@@@ -742,16 -555,14 +742,16 @@@
  		const char *features;
  		unsigned char sha1_buf[20];
  		char *line = packet_read_line(0, NULL);
 +		const char *arg;
 +
  		reset_timeout();
  		if (!line)
  			break;
  
 -		if (starts_with(line, "shallow ")) {
 +		if (skip_prefix(line, "shallow ", &arg)) {
  			unsigned char sha1[20];
  			struct object *object;
 -			if (get_sha1_hex(line + 8, sha1))
 +			if (get_sha1_hex(arg, sha1))
  				die("invalid shallow line: %s", line);
  			object = parse_object(sha1);
  			if (!object)
@@@ -764,42 -575,20 +764,42 @@@
  			}
  			continue;
  		}
 -		if (starts_with(line, "deepen ")) {
 -			char *end;
 -			depth = strtol(line + 7, &end, 0);
 -			if (end == line + 7 || depth <= 0)
 +		if (skip_prefix(line, "deepen ", &arg)) {
 +			char *end = NULL;
 +			depth = strtol(arg, &end, 0);
 +			if (!end || *end || depth <= 0)
  				die("Invalid deepen: %s", line);
  			continue;
  		}
 -		if (!starts_with(line, "want ") ||
 -		    get_sha1_hex(line+5, sha1_buf))
 +		if (skip_prefix(line, "deepen-since ", &arg)) {
 +			char *end = NULL;
 +			deepen_since = strtoul(arg, &end, 0);
 +			if (!end || *end || !deepen_since ||
 +			    /* revisions.c's max_age -1 is special */
 +			    deepen_since == -1)
 +				die("Invalid deepen-since: %s", line);
 +			deepen_rev_list = 1;
 +			continue;
 +		}
 +		if (skip_prefix(line, "deepen-not ", &arg)) {
 +			char *ref = NULL;
 +			unsigned char sha1[20];
 +			if (expand_ref(arg, strlen(arg), sha1, &ref) != 1)
 +				die("git upload-pack: ambiguous deepen-not: %s", line);
 +			string_list_append(&deepen_not, ref);
 +			free(ref);
 +			deepen_rev_list = 1;
 +			continue;
 +		}
 +		if (!skip_prefix(line, "want ", &arg) ||
 +		    get_sha1_hex(arg, sha1_buf))
  			die("git upload-pack: protocol error, "
  			    "expected to get sha, not '%s'", line);
  
 -		features = line + 45;
 +		features = arg + 40;
  
 +		if (parse_feature_request(features, "deepen-relative"))
 +			deepen_relative = 1;
  		if (parse_feature_request(features, "multi_ack_detailed"))
  			multi_ack = 2;
  		else if (parse_feature_request(features, "multi_ack"))
@@@ -844,35 -633,55 +844,35 @@@
  	if (!use_sideband && daemon_mode)
  		no_progress = 1;
  
 -	if (depth == 0 && shallows.nr == 0)
 +	if (depth == 0 && !deepen_rev_list && shallows.nr == 0)
  		return;
 -	if (depth > 0) {
 -		struct commit_list *result = NULL, *backup = NULL;
 +	if (depth > 0 && deepen_rev_list)
 +		die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together");
 +	if (depth > 0)
 +		deepen(depth, deepen_relative, &shallows);
 +	else if (deepen_rev_list) {
 +		struct argv_array av = ARGV_ARRAY_INIT;
  		int i;
 -		if (depth == INFINITE_DEPTH && !is_repository_shallow())
 -			for (i = 0; i < shallows.nr; i++) {
 -				struct object *object = shallows.objects[i].item;
 -				object->flags |= NOT_SHALLOW;
 -			}
 -		else
 -			backup = result =
 -				get_shallow_commits(&want_obj, depth,
 -						    SHALLOW, NOT_SHALLOW);
 -		while (result) {
 -			struct object *object = &result->item->object;
 -			if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) {
 -				packet_write_fmt(1, "shallow %s",
 -						oid_to_hex(&object->oid));
 -				register_shallow(object->oid.hash);
 -				shallow_nr++;
 +
 +		argv_array_push(&av, "rev-list");
 +		if (deepen_since)
 +			argv_array_pushf(&av, "--max-age=%lu", deepen_since);
 +		if (deepen_not.nr) {
 +			argv_array_push(&av, "--not");
 +			for (i = 0; i < deepen_not.nr; i++) {
 +				struct string_list_item *s = deepen_not.items + i;
 +				argv_array_push(&av, s->string);
  			}
 -			result = result->next;
 +			argv_array_push(&av, "--not");
  		}
 -		free_commit_list(backup);
 -		for (i = 0; i < shallows.nr; i++) {
 -			struct object *object = shallows.objects[i].item;
 -			if (object->flags & NOT_SHALLOW) {
 -				struct commit_list *parents;
 -				packet_write_fmt(1, "unshallow %s",
 -					oid_to_hex(&object->oid));
 -				object->flags &= ~CLIENT_SHALLOW;
 -				/* make sure the real parents are parsed */
 -				unregister_shallow(object->oid.hash);
 -				object->parsed = 0;
 -				parse_commit_or_die((struct commit *)object);
 -				parents = ((struct commit *)object)->parents;
 -				while (parents) {
 -					add_object_array(&parents->item->object,
 -							NULL, &want_obj);
 -					parents = parents->next;
 -				}
 -				add_object_array(object, NULL, &extra_edge_obj);
 -			}
 -			/* make sure commit traversal conforms to client */
 -			register_shallow(object->oid.hash);
 +		for (i = 0; i < want_obj.nr; i++) {
 +			struct object *o = want_obj.objects[i].item;
 +			argv_array_push(&av, oid_to_hex(&o->oid));
  		}
 -		packet_flush(1);
 -	} else
 +		deepen_by_rev_list(av.argc, av.argv, &shallows);
 +		argv_array_clear(&av);
 +	}
 +	else
  		if (shallows.nr > 0) {
  			int i;
  			for (i = 0; i < shallows.nr; i++)
@@@ -920,8 -729,8 +920,8 @@@ static int send_ref(const char *refname
  		    int flag, void *cb_data)
  {
  	static const char *capabilities = "multi_ack thin-pack side-band"
 -		" side-band-64k ofs-delta shallow no-progress"
 -		" include-tag multi_ack_detailed";
 +		" side-band-64k ofs-delta shallow deepen-since deepen-not"
 +		" deepen-relative no-progress include-tag multi_ack_detailed";
  	const char *refname_nons = strip_namespace(refname);
  	struct object_id peeled;
  
@@@ -932,7 -741,7 +932,7 @@@
  		struct strbuf symref_info = STRBUF_INIT;
  
  		format_symref_info(&symref_info, cb_data);
- 		packet_write(1, "%s %s%c%s%s%s%s%s agent=%s\n",
+ 		packet_write_fmt(1, "%s %s%c%s%s%s%s%s agent=%s\n",
  			     oid_to_hex(oid), refname_nons,
  			     0, capabilities,
  			     (allow_unadvertised_object_request & ALLOW_TIP_SHA1) ?
@@@ -944,11 -753,11 +944,11 @@@
  			     git_user_agent_sanitized());
  		strbuf_release(&symref_info);
  	} else {
- 		packet_write(1, "%s %s\n", oid_to_hex(oid), refname_nons);
+ 		packet_write_fmt(1, "%s %s\n", oid_to_hex(oid), refname_nons);
  	}
  	capabilities = NULL;
  	if (!peel_ref(refname, peeled.hash))
- 		packet_write(1, "%s %s^{}\n", oid_to_hex(&peeled), refname_nons);
+ 		packet_write_fmt(1, "%s %s^{}\n", oid_to_hex(&peeled), refname_nons);
  	return 0;
  }